From dc7166f1dd014ebeb6ec2db6241a502533211d7c Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Tue, 3 Mar 2026 07:50:17 -0500 Subject: [PATCH 001/326] [feature] Replace eXist-db native file module with EXPath File Module 4.0 Replace the custom file module (http://exist-db.org/xquery/file) with a spec-compliant EXPath File Module 4.0 (http://expath.org/ns/file) implementation, improving interoperability with other XQuery processors. The new module implements all 35+ functions from the EXPath File Module 4.0 specification including file properties, I/O, manipulation, path utilities, and system properties. The old module's eXist-specific file:sync function is relocated to util:file-sync. Includes 64 XQuery integration tests covering all major functions and error conditions. Co-Authored-By: Claude Opus 4.6 --- .../exist/xquery/functions/util/FileSync.java | 112 ++- .../xquery/functions/util/UtilModule.java | 3 +- .../org/exist/xquery/functions/util}/repo.xsl | 0 .../resources/org/exist/xmldb/allowAnyUri.xml | 2 +- exist-distribution/pom.xml | 6 - exist-distribution/src/main/config/conf.xml | 2 +- .../exist/file/ExpathFileErrorCode.java | 73 ++ .../expath/exist/file/ExpathFileModule.java | 141 ++++ .../exist/file/ExpathFileModuleHelper.java | 99 +++ .../org/expath/exist/file/FileAppend.java | 256 ++++++ .../java/org/expath/exist/file/FileIO.java | 230 ++++++ .../expath/exist/file/FileManipulation.java | 496 +++++++++++ .../java/org/expath/exist/file/FilePaths.java | 139 ++++ .../org/expath/exist/file/FileProperties.java | 184 +++++ .../exist/file/FileSystemProperties.java | 130 +++ .../java/org/expath/exist/file/FileWrite.java | 289 +++++++ .../expath/exist/file/ExpathFileTests.java} | 7 +- .../src/test/resources-filtered/conf.xml | 3 +- .../src/test/resources/util/fixtures.xqm | 0 .../src/test/resources/util/helper.xqm | 66 +- .../org/expath/exist/file/file-tests.xql | 547 ++++++++++++ .../org/expath/exist}/file/sync-serialize.xqm | 22 +- .../xquery/org/expath/exist}/file/sync.xqm | 71 +- .../xquery/org/expath/exist}/file/syncmod.xqm | 31 +- extensions/modules/file/pom.xml | 201 ----- .../exist/xquery/modules/file/Directory.java | 205 ----- .../xquery/modules/file/DirectoryCreate.java | 115 --- .../xquery/modules/file/DirectoryList.java | 191 ----- .../exist/xquery/modules/file/FileDelete.java | 82 -- .../xquery/modules/file/FileErrorCode.java | 34 - .../exist/xquery/modules/file/FileExists.java | 85 -- .../xquery/modules/file/FileIsDirectory.java | 85 -- .../xquery/modules/file/FileIsReadable.java | 84 -- .../xquery/modules/file/FileIsWriteable.java | 84 -- .../exist/xquery/modules/file/FileModule.java | 100 --- .../xquery/modules/file/FileModuleHelper.java | 63 -- .../exist/xquery/modules/file/FileMove.java | 101 --- .../exist/xquery/modules/file/FileRead.java | 110 --- .../xquery/modules/file/FileReadBinary.java | 80 -- .../xquery/modules/file/FileReadUnicode.java | 118 --- .../xquery/modules/file/SerializeToFile.java | 236 ------ .../xquery/modules/file/UnicodeReader.java | 181 ---- .../modules/file/AbstractBinariesTest.java | 140 ---- .../modules/file/EmbeddedBinariesTest.java | 164 ---- .../xquery/modules/file/RestBinariesTest.java | 219 ----- .../modules/file/XmldbBinariesTest.java | 199 ----- .../file/src/test/resources-filtered/conf.xml | 781 ------------------ .../file/src/test/resources/log4j2.xml | 36 - .../WEB-INF/controller-config.xml | 33 - .../standalone-webapp/WEB-INF/web.xml | 80 -- .../xquery/modules/file/directory-list.xqm | 47 -- .../test/xquery/modules/file/read-binary.xqm | 57 -- .../test/xquery/modules/file/serialize.xqm | 148 ---- extensions/modules/image/pom.xml | 6 +- .../src/test/resources-filtered/conf.xml | 2 +- extensions/modules/pom.xml | 1 - 56 files changed, 2750 insertions(+), 4227 deletions(-) rename extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java => exist-core/src/main/java/org/exist/xquery/functions/util/FileSync.java (86%) rename {extensions/modules/file/src/main/resources/org/exist/xquery/modules/file => exist-core/src/main/resources/org/exist/xquery/functions/util}/repo.xsl (100%) create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/ExpathFileErrorCode.java create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModuleHelper.java create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/FileAppend.java create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/FileIO.java create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/FilePaths.java create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/FileProperties.java create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/FileSystemProperties.java create mode 100644 extensions/expath/src/main/java/org/expath/exist/file/FileWrite.java rename extensions/{modules/file/src/test/java/xquery/modules/file/FileTests.java => expath/src/test/java/org/expath/exist/file/ExpathFileTests.java} (90%) rename extensions/{modules/file => expath}/src/test/resources/util/fixtures.xqm (100%) rename extensions/{modules/file => expath}/src/test/resources/util/helper.xqm (80%) create mode 100644 extensions/expath/src/test/xquery/org/expath/exist/file/file-tests.xql rename extensions/{modules/file/src/test/xquery/modules => expath/src/test/xquery/org/expath/exist}/file/sync-serialize.xqm (94%) rename extensions/{modules/file/src/test/xquery/modules => expath/src/test/xquery/org/expath/exist}/file/sync.xqm (90%) rename extensions/{modules/file/src/test/xquery/modules => expath/src/test/xquery/org/expath/exist}/file/syncmod.xqm (94%) delete mode 100644 extensions/modules/file/pom.xml delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Directory.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryCreate.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryList.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileDelete.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileErrorCode.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileExists.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsDirectory.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsReadable.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsWriteable.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModule.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModuleHelper.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileMove.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileRead.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadBinary.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadUnicode.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/SerializeToFile.java delete mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/UnicodeReader.java delete mode 100644 extensions/modules/file/src/test/java/org/exist/xquery/modules/file/AbstractBinariesTest.java delete mode 100644 extensions/modules/file/src/test/java/org/exist/xquery/modules/file/EmbeddedBinariesTest.java delete mode 100644 extensions/modules/file/src/test/java/org/exist/xquery/modules/file/RestBinariesTest.java delete mode 100644 extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java delete mode 100644 extensions/modules/file/src/test/resources-filtered/conf.xml delete mode 100644 extensions/modules/file/src/test/resources/log4j2.xml delete mode 100644 extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml delete mode 100644 extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/web.xml delete mode 100644 extensions/modules/file/src/test/xquery/modules/file/directory-list.xqm delete mode 100644 extensions/modules/file/src/test/xquery/modules/file/read-binary.xqm delete mode 100644 extensions/modules/file/src/test/xquery/modules/file/serialize.xqm diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java b/exist-core/src/main/java/org/exist/xquery/functions/util/FileSync.java similarity index 86% rename from extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java rename to exist-core/src/main/java/org/exist/xquery/functions/util/FileSync.java index 23d205c45b7..079f348885e 100644 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/util/FileSync.java @@ -19,18 +19,19 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -package org.exist.xquery.modules.file; +package org.exist.xquery.functions.util; import java.io.*; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; -import java.net.URISyntaxException; import java.util.*; import java.util.stream.Stream; @@ -74,21 +75,24 @@ import org.xml.sax.SAXException; import uk.ac.ic.doc.slurp.multilock.MultiLock; -public class Sync extends BasicFunction { +/** + * Synchronize a collection with a directory hierarchy. + * Relocated from the old file module (file:sync) to util:file-sync. + * This function is only available to the DBA role. + */ +public class FileSync extends BasicFunction { public static final String PRUNE_OPT = "prune"; public static final String AFTER_OPT = "after"; public static final String EXCLUDES_OPT = "excludes"; - public static final QName FILE_SYNC_ELEMENT = new QName("sync", FileModule.NAMESPACE_URI); - public static final QName FILE_UPDATE_ELEMENT = new QName("update", FileModule.NAMESPACE_URI); - public static final QName FILE_DELETE_ELEMENT = new QName("delete", FileModule.NAMESPACE_URI); - public static final QName FILE_ERROR_ELEMENT = new QName("error", FileModule.NAMESPACE_URI); + public static final QName SYNC_ELEMENT = new QName("sync", UtilModule.NAMESPACE_URI); + public static final QName UPDATE_ELEMENT = new QName("update", UtilModule.NAMESPACE_URI); + public static final QName DELETE_ELEMENT = new QName("delete", UtilModule.NAMESPACE_URI); + public static final QName ERROR_ELEMENT = new QName("error", UtilModule.NAMESPACE_URI); - // TODO(JL) Figure out which namespace all attributes should be in (possible breaking change) - // https://github.com/eXist-db/exist/issues/4207 - public static final QName FILE_COLLECTION_ATTRIBUTE = new QName("collection", FileModule.NAMESPACE_URI); - public static final QName FILE_DIR_ATTRIBUTE = new QName("dir", FileModule.NAMESPACE_URI); + public static final QName COLLECTION_ATTR = new QName("collection", UtilModule.NAMESPACE_URI); + public static final QName DIR_ATTR = new QName("dir", UtilModule.NAMESPACE_URI); public static final QName FILE_ATTRIBUTE = new QName("file", XMLConstants.NULL_NS_URI); public static final QName NAME_ATTRIBUTE = new QName("name", XMLConstants.NULL_NS_URI); @@ -98,22 +102,23 @@ public class Sync extends BasicFunction { public static final FunctionSignature signature = new FunctionSignature( - new QName("sync", FileModule.NAMESPACE_URI, FileModule.PREFIX), - "Synchronize a collection with a directory hierarchy." + - "This method is only available to the DBA role. ", + new QName("file-sync", UtilModule.NAMESPACE_URI, UtilModule.PREFIX), + "Synchronize a collection with a directory hierarchy. " + + "This method is only available to the DBA role.", new SequenceType[]{ new FunctionParameterSequenceType("collection", Type.STRING, Cardinality.EXACTLY_ONE, "Absolute path to the collection to synchronize to disk."), new FunctionParameterSequenceType("targetPath", Type.ITEM, Cardinality.EXACTLY_ONE, "The path or URI to the target directory. Relative paths resolve against EXIST_HOME."), new FunctionParameterSequenceType("dateTimeOrOptionsMap", Type.ITEM, Cardinality.ZERO_OR_ONE, - "Options as map(*). The available settings are:" + + "Options as map(*). The available settings are: " + "\"" + PRUNE_OPT + "\": delete any file/dir that does not correspond to a doc/collection in the DB. " + - "\"" + AFTER_OPT + "\": only resources modified after this date will be taken into account." + - "\"" + EXCLUDES_OPT + "\": files on the file system matching any of these patterns will be left untouched." + + "\"" + AFTER_OPT + "\": only resources modified after this date will be taken into account. " + + "\"" + EXCLUDES_OPT + "\": files on the file system matching any of these patterns will be left untouched. " + "(deprecated) If the third parameter is of type xs:dateTime, it is the same as setting the \"" + AFTER_OPT + "\" option.") }, - new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.EXACTLY_ONE, "A report (file:sync) which files and directories were updated (file:update) or deleted (file:delete).") + new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.EXACTLY_ONE, + "A report (util:sync) which files and directories were updated (util:update) or deleted (util:delete).") ); private static final Properties DEFAULT_PROPERTIES = new Properties(); @@ -127,7 +132,7 @@ public class Sync extends BasicFunction { private Properties outputProperties = new Properties(); - public Sync(final XQueryContext context, final FunctionSignature signature) { + public FileSync(final XQueryContext context, final FunctionSignature signature) { super(context, signature); } @@ -135,7 +140,7 @@ public Sync(final XQueryContext context, final FunctionSignature signature) { public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { if (!context.getSubject().hasDbaRole()) { - throw new XPathException(this, "Function file:sync is only available to the DBA role"); + throw new XPathException(this, "Function util:file-sync is only available to the DBA role"); } final String collectionPath = args[0].getStringValue(); @@ -163,8 +168,7 @@ private Map getOptions(final Sequence parameter) throws XPathE outputProperties = SerializerUtils.getSerializationOptions(this, optionsMap); - // override defaults set in SerializerUtils - for(String p : DEFAULT_PROPERTIES.stringPropertyNames()) { + for (String p : DEFAULT_PROPERTIES.stringPropertyNames()) { if (optionsMap.get(new StringValue(this, p)).isEmpty()) { outputProperties.setProperty(p, DEFAULT_PROPERTIES.getProperty(p)); } @@ -198,10 +202,10 @@ private void checkOption( final Sequence p = optionsMap.get(new StringValue(this, name)); if (p.isEmpty()) { - return; // nothing to do, continue + return; } - if (p.hasMany() || !Type.subTypeOf(p.getItemType(),expectedType)) { + if (p.hasMany() || !Type.subTypeOf(p.getItemType(), expectedType)) { throw new XPathException(this, ErrorCodes.XPTY0004, "Invalid value type for option \"" + name + "\", expected " + Type.getTypeName(expectedType) + " got " + @@ -225,7 +229,7 @@ private Sequence startSync( excludes.add(si.nextItem().getStringValue()); } - final Path p = FileModuleHelper.getFile(target, this); + final Path p = getFile(target); context.pushDocumentContext(); final MemTreeBuilder output = context.getDocumentBuilder(); final Path targetDir; @@ -238,9 +242,9 @@ private Sequence startSync( } output.startDocument(); - output.startElement(FILE_SYNC_ELEMENT, null); - output.addAttribute(FILE_COLLECTION_ATTRIBUTE, collectionPath); - output.addAttribute(FILE_DIR_ATTRIBUTE, targetDir.toAbsolutePath().toString()); + output.startElement(SYNC_ELEMENT, null); + output.addAttribute(COLLECTION_ATTR, collectionPath); + output.addAttribute(DIR_ATTR, targetDir.toAbsolutePath().toString()); final String rootTargetAbsPath = targetDir.toAbsolutePath().toString(); final String separator = rootTargetAbsPath.endsWith(File.separator) ? "" : File.separator; @@ -256,6 +260,18 @@ private Sequence startSync( return output.getDocument(); } + private Path getFile(final String path) throws XPathException { + if (path.startsWith("file:")) { + try { + return Paths.get(new URI(path)); + } catch (final Exception ex) { + throw new XPathException(this, path + " is not a valid URI: '" + ex.getMessage() + "'"); + } + } else { + return Paths.get(path); + } + } + private void syncCollection( final XmldbURI collectionPath, final String rootTargetAbsPath, @@ -329,7 +345,6 @@ private void pruneCollectionEntries( try (final Stream fileStream = Files.walk(targetDir, 1)) { fileStream.forEach(path -> { try { - // guard against deletion of output folder if (rootTargetAbsPath.startsWith(path.toString())) { return; } @@ -348,19 +363,17 @@ private void pruneCollectionEntries( return; } - // handle non-empty directories if (Files.isDirectory(path)) { deleteWithExcludes(rootTargetAbsPath, path, excludes, output); } else { Files.deleteIfExists(path); - // reporting - output.startElement(FILE_DELETE_ELEMENT, null); + output.startElement(DELETE_ELEMENT, null); output.addAttribute(FILE_ATTRIBUTE, path.toAbsolutePath().toString()); output.addAttribute(NAME_ATTRIBUTE, fileName); output.endElement(); } - } catch (final IOException | URISyntaxException + } catch (final IOException | java.net.URISyntaxException | PermissionDeniedException | LockException e) { reportError(output, e.getMessage()); } @@ -371,17 +384,15 @@ private void pruneCollectionEntries( } private void saveFile(final Path targetFile, final DocumentImpl doc, final Date startDate, final MemTreeBuilder output) throws LockException { - // the resource has not changed in the selected period if (startDate != null && doc.getLastModified() <= startDate.getTime()) { return; } try (final ManagedLock lock = context.getBroker().getBrokerPool().getLockManager().acquireDocumentReadLock(doc.getURI())) { - // the file on the disk appears to be up-to-date if (Files.exists(targetFile) && Files.getLastModifiedTime(targetFile).compareTo(FileTime.fromMillis(doc.getLastModified())) >= 0) { return; } - output.startElement(FILE_UPDATE_ELEMENT, null); + output.startElement(UPDATE_ELEMENT, null); output.addAttribute(FILE_ATTRIBUTE, targetFile.toAbsolutePath().toString()); output.addAttribute(NAME_ATTRIBUTE, doc.getFileURI().toString()); output.addAttribute(COLLECTION_ATTRIBUTE, doc.getCollection().getURI().toString()); @@ -429,11 +440,6 @@ private void saveXML(final Path targetFile, final DocumentImpl doc, final MemTre } } - /** - * Merge repo.xml modified by user with original file. This is necessary because we have to - * remove sensitive information during upload (default password) and need to restore it - * when the package is synchronized back to disk. - */ private void processRepoDesc(final Path targetFile, final DocumentImpl doc, final SAXSerializer sax, final MemTreeBuilder output) { try { final DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); @@ -444,7 +450,7 @@ private void processRepoDesc(final Path targetFile, final DocumentImpl doc, fina try (final Writer writer = new OutputStreamWriter(new BufferedOutputStream(Files.newOutputStream(targetFile)), StandardCharsets.UTF_8)) { sax.setOutput(writer, outputProperties); - final StreamSource styleSource = new StreamSource(Sync.class.getResourceAsStream("repo.xsl")); + final StreamSource styleSource = new StreamSource(FileSync.class.getResourceAsStream("repo.xsl")); final SAXTransformerFactory factory = TransformerFactoryAllocator.getTransformerFactory(context.getBroker().getBrokerPool()); final TransformerHandler handler = factory.newTransformerHandler(styleSource); @@ -479,27 +485,15 @@ private void saveBinary(final Path targetFile, final BinaryDocument binary, fina } private void reportError(final MemTreeBuilder output, final String msg) { - output.startElement(FILE_ERROR_ELEMENT, null); + output.startElement(ERROR_ELEMENT, null); output.characters(msg); output.endElement(); } - /** - * We need to convert to a relative path in relation to rootTargetAbsPath, - * as all the exclusion patterns are relative to rootTargetAbsPath. - * - * @param rootTargetAbsPath the root target (abs)path - * @param path (abs)path to check for being excluded. Should be subdir of rootTargetAbsPath - * @param excludes exclude patterns (in the convention of DirectoryScanner.match) - * @return true if the (rel)path in question is matched by some of the exclusion patterns - */ private static boolean isExcludedPath(final String rootTargetAbsPath, final Path path, final List excludes) { if (excludes.isEmpty()) { return false; } - // root folder cannot be excluded - // path will then also be one character shorter than rootTargetApsPath - // and throw when attempting to construct the relative path if (rootTargetAbsPath.startsWith(path.toString())) { return false; } @@ -513,9 +507,6 @@ private static boolean isExcludedPath(final String rootTargetAbsPath, final Path return matchAny(excludes, normalizedPath); } - /** - * Check if any of the patterns matches the path. - */ public static boolean matchAny(final Iterable patterns, final String path) { for (final String pattern : patterns) { if (DirectoryScanner.match(pattern, path)) { @@ -563,7 +554,7 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr } Files.deleteIfExists(file); - output.startElement(FILE_DELETE_ELEMENT, null); + output.startElement(DELETE_ELEMENT, null); output.addAttribute(FILE_ATTRIBUTE, file.toAbsolutePath().toString()); output.addAttribute(NAME_ATTRIBUTE, file.getFileName().toString()); output.endElement(); @@ -576,13 +567,12 @@ public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) if (exc != null) { throw exc; } - // deletion would fail due to non-empty directory if (hasExcludedChildren) { return FileVisitResult.CONTINUE; } Files.deleteIfExists(dir); - output.startElement(FILE_DELETE_ELEMENT, null); + output.startElement(DELETE_ELEMENT, null); output.addAttribute(FILE_ATTRIBUTE, dir.toAbsolutePath().toString()); output.addAttribute(NAME_ATTRIBUTE, dir.getFileName().toString()); output.endElement(); diff --git a/exist-core/src/main/java/org/exist/xquery/functions/util/UtilModule.java b/exist-core/src/main/java/org/exist/xquery/functions/util/UtilModule.java index b9e49a04b9b..a200850c4f0 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/util/UtilModule.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/util/UtilModule.java @@ -152,7 +152,8 @@ public class UtilModule extends AbstractInternalModule { new FunctionDef(Base64Functions.signatures[3], Base64Functions.class), new FunctionDef(BaseConversionFunctions.FNS_INT_TO_OCTAL, BaseConversionFunctions.class), new FunctionDef(BaseConversionFunctions.FNS_OCTAL_TO_INT, BaseConversionFunctions.class), - new FunctionDef(LineNumber.signature, LineNumber.class) + new FunctionDef(LineNumber.signature, LineNumber.class), + new FunctionDef(FileSync.signature, FileSync.class) }; static { diff --git a/extensions/modules/file/src/main/resources/org/exist/xquery/modules/file/repo.xsl b/exist-core/src/main/resources/org/exist/xquery/functions/util/repo.xsl similarity index 100% rename from extensions/modules/file/src/main/resources/org/exist/xquery/modules/file/repo.xsl rename to exist-core/src/main/resources/org/exist/xquery/functions/util/repo.xsl diff --git a/exist-core/src/test/resources/org/exist/xmldb/allowAnyUri.xml b/exist-core/src/test/resources/org/exist/xmldb/allowAnyUri.xml index 46f2ccd8e99..589e37370c4 100644 --- a/exist-core/src/test/resources/org/exist/xmldb/allowAnyUri.xml +++ b/exist-core/src/test/resources/org/exist/xmldb/allowAnyUri.xml @@ -1022,7 +1022,7 @@ - + diff --git a/exist-distribution/pom.xml b/exist-distribution/pom.xml index a1373fec076..6fa28396d37 100644 --- a/exist-distribution/pom.xml +++ b/exist-distribution/pom.xml @@ -198,12 +198,6 @@ ${project.version} runtime - - ${project.groupId} - exist-file - ${project.version} - runtime - ${project.groupId} exist-image diff --git a/exist-distribution/src/main/config/conf.xml b/exist-distribution/src/main/config/conf.xml index 026734ebf05..708b45097f5 100644 --- a/exist-distribution/src/main/config/conf.xml +++ b/exist-distribution/src/main/config/conf.xml @@ -1048,7 +1048,7 @@ - + diff --git a/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileErrorCode.java b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileErrorCode.java new file mode 100644 index 00000000000..38ba38e2cf1 --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileErrorCode.java @@ -0,0 +1,73 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import org.exist.dom.QName; +import org.exist.xquery.ErrorCodes.ErrorCode; + +/** + * Error codes defined by the EXPath File Module 4.0. + * + * @see EXPath File Module 4.0 + */ +public class ExpathFileErrorCode { + + public static final ErrorCode NOT_FOUND = new ErrorCode( + new QName("not-found", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "The specified path does not exist."); + + public static final ErrorCode INVALID_PATH = new ErrorCode( + new QName("invalid-path", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "The specified path is invalid."); + + public static final ErrorCode EXISTS = new ErrorCode( + new QName("exists", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "The specified path already exists."); + + public static final ErrorCode NO_DIR = new ErrorCode( + new QName("no-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "The specified path does not point to a directory."); + + public static final ErrorCode IS_DIR = new ErrorCode( + new QName("is-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "The specified path points to a directory."); + + public static final ErrorCode IS_RELATIVE = new ErrorCode( + new QName("is-relative", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "The specified path is relative."); + + public static final ErrorCode UNKNOWN_ENCODING = new ErrorCode( + new QName("unknown-encoding", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "The specified encoding is not supported."); + + public static final ErrorCode OUT_OF_RANGE = new ErrorCode( + new QName("out-of-range", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "The specified offset or length is out of range."); + + public static final ErrorCode IO_ERROR = new ErrorCode( + new QName("io-error", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "A generic file system error occurred."); + + private ExpathFileErrorCode() { + // no instances + } +} diff --git a/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java new file mode 100644 index 00000000000..bc0fa4ba42a --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java @@ -0,0 +1,141 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import java.util.List; +import java.util.Map; + +import org.exist.xquery.AbstractInternalModule; +import org.exist.xquery.FunctionDef; + +/** + * EXPath File Module 4.0 implementation for eXist-db. + * + * @see EXPath File Module 4.0 + */ +public class ExpathFileModule extends AbstractInternalModule { + + public static final String NAMESPACE_URI = "http://expath.org/ns/file"; + public static final String PREFIX = "file"; + public static final String INCLUSION_DATE = "2025-05-01"; + public static final String RELEASED_IN_VERSION = "7.0.0"; + + private static final FunctionDef[] functions = { + // FileProperties: exists, is-dir, is-file, is-absolute, last-modified, size(1), size(2) + new FunctionDef(FileProperties.signatures[0], FileProperties.class), + new FunctionDef(FileProperties.signatures[1], FileProperties.class), + new FunctionDef(FileProperties.signatures[2], FileProperties.class), + new FunctionDef(FileProperties.signatures[3], FileProperties.class), + new FunctionDef(FileProperties.signatures[4], FileProperties.class), + new FunctionDef(FileProperties.signatures[5], FileProperties.class), + new FunctionDef(FileProperties.signatures[6], FileProperties.class), + + // FileIO: read-text(1), read-text(2), read-text-lines(1), read-text-lines(2), + // read-binary(1), read-binary(2), read-binary(3) + new FunctionDef(FileIO.signatures[0], FileIO.class), + new FunctionDef(FileIO.signatures[1], FileIO.class), + new FunctionDef(FileIO.signatures[2], FileIO.class), + new FunctionDef(FileIO.signatures[3], FileIO.class), + new FunctionDef(FileIO.signatures[4], FileIO.class), + new FunctionDef(FileIO.signatures[5], FileIO.class), + new FunctionDef(FileIO.signatures[6], FileIO.class), + + // FileWrite: write(2), write(3), write-text(2), write-text(3), + // write-text-lines(2), write-text-lines(3), write-binary(2), write-binary(3) + new FunctionDef(FileWrite.signatures[0], FileWrite.class), + new FunctionDef(FileWrite.signatures[1], FileWrite.class), + new FunctionDef(FileWrite.signatures[2], FileWrite.class), + new FunctionDef(FileWrite.signatures[3], FileWrite.class), + new FunctionDef(FileWrite.signatures[4], FileWrite.class), + new FunctionDef(FileWrite.signatures[5], FileWrite.class), + new FunctionDef(FileWrite.signatures[6], FileWrite.class), + new FunctionDef(FileWrite.signatures[7], FileWrite.class), + + // FileAppend: append(2), append(3), append-binary, append-text(2), append-text(3), + // append-text-lines(2), append-text-lines(3) + new FunctionDef(FileAppend.signatures[0], FileAppend.class), + new FunctionDef(FileAppend.signatures[1], FileAppend.class), + new FunctionDef(FileAppend.signatures[2], FileAppend.class), + new FunctionDef(FileAppend.signatures[3], FileAppend.class), + new FunctionDef(FileAppend.signatures[4], FileAppend.class), + new FunctionDef(FileAppend.signatures[5], FileAppend.class), + new FunctionDef(FileAppend.signatures[6], FileAppend.class), + + // FileManipulation: copy, move, delete(1), delete(2), create-dir, + // create-temp-dir, create-temp-file, list(1), list(2), list(3), + // children, descendants, list-roots + new FunctionDef(FileManipulation.signatures[0], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[1], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[2], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[3], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[4], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[5], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[6], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[7], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[8], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[9], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[10], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[11], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[12], FileManipulation.class), + + // FilePaths: name, parent, path-to-native, path-to-uri, resolve-path(1), resolve-path(2) + new FunctionDef(FilePaths.signatures[0], FilePaths.class), + new FunctionDef(FilePaths.signatures[1], FilePaths.class), + new FunctionDef(FilePaths.signatures[2], FilePaths.class), + new FunctionDef(FilePaths.signatures[3], FilePaths.class), + new FunctionDef(FilePaths.signatures[4], FilePaths.class), + new FunctionDef(FilePaths.signatures[5], FilePaths.class), + + // FileSystemProperties: dir-separator, line-separator, path-separator, + // temp-dir, base-dir, current-dir + new FunctionDef(FileSystemProperties.signatures[0], FileSystemProperties.class), + new FunctionDef(FileSystemProperties.signatures[1], FileSystemProperties.class), + new FunctionDef(FileSystemProperties.signatures[2], FileSystemProperties.class), + new FunctionDef(FileSystemProperties.signatures[3], FileSystemProperties.class), + new FunctionDef(FileSystemProperties.signatures[4], FileSystemProperties.class), + new FunctionDef(FileSystemProperties.signatures[5], FileSystemProperties.class) + }; + + public ExpathFileModule(final Map> parameters) { + super(functions, parameters); + } + + @Override + public String getNamespaceURI() { + return NAMESPACE_URI; + } + + @Override + public String getDefaultPrefix() { + return PREFIX; + } + + @Override + public String getDescription() { + return "EXPath File Module 4.0 - http://expath.org/ns/file"; + } + + @Override + public String getReleaseVersion() { + return RELEASED_IN_VERSION; + } +} diff --git a/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModuleHelper.java b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModuleHelper.java new file mode 100644 index 00000000000..b6b48182471 --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModuleHelper.java @@ -0,0 +1,99 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import java.net.URI; +import java.nio.charset.Charset; +import java.nio.charset.UnsupportedCharsetException; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.exist.xquery.Expression; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; + +/** + * Helper utilities for the EXPath File Module. + */ +public class ExpathFileModuleHelper { + + private ExpathFileModuleHelper() { + // no instances + } + + /** + * Check that the calling user has DBA role. + * + * @param context the XQuery context + * @param expression the calling expression (for error reporting) + * @throws XPathException if the user is not a DBA + */ + public static void checkDbaRole(final XQueryContext context, final Expression expression) throws XPathException { + if (!context.getSubject().hasDbaRole()) { + throw new XPathException(expression, + "Permission denied, calling user '" + context.getSubject().getName() + + "' must be a DBA to call this function."); + } + } + + /** + * Resolve a path string (file: URI or native path) to a {@link Path}. + * + * @param path the path string or file: URI + * @param expression the calling expression (for error reporting) + * @return the resolved Path + * @throws XPathException if the path is invalid + */ + public static Path getPath(final String path, final Expression expression) throws XPathException { + try { + if (path.startsWith("file:")) { + return Paths.get(new URI(path)); + } else { + return Paths.get(path); + } + } catch (final InvalidPathException e) { + throw new XPathException(expression, ExpathFileErrorCode.INVALID_PATH, + "Invalid path: " + path + " - " + e.getMessage()); + } catch (final Exception e) { + throw new XPathException(expression, ExpathFileErrorCode.INVALID_PATH, + path + " is not a valid path or URI: " + e.getMessage()); + } + } + + /** + * Validate and return a {@link Charset} for the given encoding name. + * + * @param encoding the encoding name + * @param expression the calling expression (for error reporting) + * @return the Charset + * @throws XPathException if the encoding is not supported + */ + public static Charset getCharset(final String encoding, final Expression expression) throws XPathException { + try { + return Charset.forName(encoding); + } catch (final UnsupportedCharsetException e) { + throw new XPathException(expression, ExpathFileErrorCode.UNKNOWN_ENCODING, + "Unknown encoding: " + encoding); + } + } +} diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileAppend.java b/extensions/expath/src/main/java/org/expath/exist/file/FileAppend.java new file mode 100644 index 00000000000..88de70ab611 --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileAppend.java @@ -0,0 +1,256 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import java.io.*; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Properties; + +import org.exist.dom.QName; +import org.exist.storage.serializers.Serializer; +import org.exist.util.serializer.SAXSerializer; +import org.exist.util.serializer.SerializerPool; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.util.SerializerUtils; +import org.exist.xquery.value.*; +import org.xml.sax.SAXException; + +/** + * EXPath File Module 4.0 - Append functions. + *

+ * Implements: file:append, file:append-binary, file:append-text, file:append-text-lines + */ +public class FileAppend extends BasicFunction { + + private static final FunctionParameterSequenceType FILE_PARAM = + new FunctionParameterSequenceType("file", Type.STRING, Cardinality.EXACTLY_ONE, "The path to the file."); + + public static final FunctionSignature[] signatures = { + // file:append($file, $value) + new FunctionSignature( + new QName("append", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Appends a serialized sequence to a file. Creates the file if it does not exist.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.ITEM, Cardinality.ZERO_OR_MORE, "The items to serialize and append.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:append($file, $value, $options) + new FunctionSignature( + new QName("append", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Appends a serialized sequence to a file with serialization options.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.ITEM, Cardinality.ZERO_OR_MORE, "The items to serialize and append."), + new FunctionParameterSequenceType("options", Type.ITEM, Cardinality.ZERO_OR_ONE, "Serialization parameters as map(*) or element(output:serialization-parameters).") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:append-binary($file, $value) + new FunctionSignature( + new QName("append-binary", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Appends binary data to a file. Creates the file if it does not exist.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.BASE64_BINARY, Cardinality.EXACTLY_ONE, "The binary data to append.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:append-text($file, $value) + new FunctionSignature( + new QName("append-text", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Appends a string to a file. Creates the file if it does not exist.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.STRING, Cardinality.EXACTLY_ONE, "The string to append.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:append-text($file, $value, $encoding) + new FunctionSignature( + new QName("append-text", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Appends a string to a file with the specified encoding.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.STRING, Cardinality.EXACTLY_ONE, "The string to append."), + new FunctionParameterSequenceType("encoding", Type.STRING, Cardinality.ZERO_OR_ONE, "The character encoding. Default: UTF-8.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:append-text-lines($file, $lines) + new FunctionSignature( + new QName("append-text-lines", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Appends a sequence of strings as lines to a file, separated by the platform line separator.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("lines", Type.STRING, Cardinality.ZERO_OR_MORE, "The lines to append.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:append-text-lines($file, $lines, $encoding) + new FunctionSignature( + new QName("append-text-lines", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Appends a sequence of strings as lines to a file with the specified encoding.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("lines", Type.STRING, Cardinality.ZERO_OR_MORE, "The lines to append."), + new FunctionParameterSequenceType("encoding", Type.STRING, Cardinality.ZERO_OR_ONE, "The character encoding. Default: UTF-8.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ) + }; + + public FileAppend(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + ExpathFileModuleHelper.checkDbaRole(context, this); + + final String pathStr = args[0].getStringValue(); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + + checkParentDir(path); + + if (Files.isDirectory(path)) { + throw new XPathException(this, ExpathFileErrorCode.IS_DIR, + "Path is a directory: " + path.toAbsolutePath()); + } + + if (isCalledAs("append")) { + return append(path, args); + } else if (isCalledAs("append-binary")) { + return appendBinary(path, args); + } else if (isCalledAs("append-text")) { + return appendText(path, args); + } else if (isCalledAs("append-text-lines")) { + return appendTextLines(path, args); + } + + throw new XPathException(this, "Unknown function: " + getSignature().getName().getLocalPart()); + } + + private Sequence append(final Path path, final Sequence[] args) throws XPathException { + final Sequence value = args[1]; + final Properties outputProperties = new Properties(); + if (args.length > 2 && !args[2].isEmpty()) { + final Item optionsItem = args[2].itemAt(0); + if (optionsItem instanceof AbstractMapType) { + outputProperties.putAll(SerializerUtils.getSerializationOptions(this, (AbstractMapType) optionsItem)); + } else if (optionsItem instanceof NodeValue) { + SerializerUtils.getSerializationOptions(this, (NodeValue) optionsItem, outputProperties); + } + } + + final SAXSerializer sax = (SAXSerializer) SerializerPool.getInstance().borrowObject(SAXSerializer.class); + try { + final Serializer serializer = context.getBroker().borrowSerializer(); + try (final Writer writer = new OutputStreamWriter( + new BufferedOutputStream(Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.APPEND)), + StandardCharsets.UTF_8)) { + sax.setOutput(writer, outputProperties); + serializer.setProperties(outputProperties); + serializer.setSAXHandlers(sax, sax); + + for (final SequenceIterator i = value.iterate(); i.hasNext(); ) { + final Item item = i.nextItem(); + if (Type.subTypeOf(item.getType(), Type.NODE)) { + serializer.toSAX((NodeValue) item); + } else { + writer.write(item.getStringValue()); + } + } + } finally { + context.getBroker().returnSerializer(serializer); + } + } catch (final IOException | SAXException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } finally { + SerializerPool.getInstance().returnObject(sax); + } + return Sequence.EMPTY_SEQUENCE; + } + + private Sequence appendBinary(final Path path, final Sequence[] args) throws XPathException { + final BinaryValue binaryValue = (BinaryValue) args[1].itemAt(0); + try (final OutputStream os = Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.APPEND); + final InputStream is = binaryValue.getInputStream()) { + is.transferTo(os); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private Sequence appendText(final Path path, final Sequence[] args) throws XPathException { + final String text = args[1].getStringValue(); + final Charset encoding = getEncoding(args, 2); + try (final Writer writer = new OutputStreamWriter( + Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.APPEND), encoding)) { + writer.write(text); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private Sequence appendTextLines(final Path path, final Sequence[] args) throws XPathException { + final Charset encoding = getEncoding(args, 2); + try (final Writer writer = new OutputStreamWriter( + Files.newOutputStream(path, StandardOpenOption.CREATE, StandardOpenOption.APPEND), encoding)) { + final String lineSep = System.lineSeparator(); + for (final SequenceIterator i = args[1].iterate(); i.hasNext(); ) { + writer.write(i.nextItem().getStringValue()); + writer.write(lineSep); + } + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private void checkParentDir(final Path path) throws XPathException { + final Path parent = path.getParent(); + if (parent != null && !Files.isDirectory(parent)) { + throw new XPathException(this, ExpathFileErrorCode.NO_DIR, + "Parent directory does not exist: " + parent.toAbsolutePath()); + } + } + + private Charset getEncoding(final Sequence[] args, final int index) throws XPathException { + if (args.length > index && !args[index].isEmpty()) { + return ExpathFileModuleHelper.getCharset(args[index].getStringValue(), this); + } + return StandardCharsets.UTF_8; + } +} diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileIO.java b/extensions/expath/src/main/java/org/expath/exist/file/FileIO.java new file mode 100644 index 00000000000..a5c4e35be1b --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileIO.java @@ -0,0 +1,230 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.Base64BinaryValueType; +import org.exist.xquery.value.BinaryValueFromInputStream; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +/** + * EXPath File Module 4.0 - Input functions. + *

+ * Implements: file:read-text, file:read-text-lines, file:read-binary + */ +public class FileIO extends BasicFunction { + + private static final FunctionParameterSequenceType FILE_PARAM = + new FunctionParameterSequenceType("file", Type.STRING, Cardinality.EXACTLY_ONE, + "The path to the file."); + private static final FunctionParameterSequenceType ENCODING_PARAM = + new FunctionParameterSequenceType("encoding", Type.STRING, Cardinality.ZERO_OR_ONE, + "The character encoding. Default: UTF-8."); + + public static final FunctionSignature[] signatures = { + // file:read-text($file as xs:string) as xs:string + new FunctionSignature( + new QName("read-text", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Reads the contents of a file as text.", + new SequenceType[]{FILE_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the file contents as string.") + ), + // file:read-text($file as xs:string, $encoding as xs:string) as xs:string + new FunctionSignature( + new QName("read-text", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Reads the contents of a file as text with the specified encoding.", + new SequenceType[]{FILE_PARAM, ENCODING_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the file contents as string.") + ), + // file:read-text-lines($file as xs:string) as xs:string* + new FunctionSignature( + new QName("read-text-lines", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Reads the contents of a file as a sequence of lines.", + new SequenceType[]{FILE_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "the lines of the file.") + ), + // file:read-text-lines($file as xs:string, $encoding as xs:string) as xs:string* + new FunctionSignature( + new QName("read-text-lines", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Reads the contents of a file as a sequence of lines with the specified encoding.", + new SequenceType[]{FILE_PARAM, ENCODING_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "the lines of the file.") + ), + // file:read-binary($file as xs:string) as xs:base64Binary + new FunctionSignature( + new QName("read-binary", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Reads the contents of a file as binary.", + new SequenceType[]{FILE_PARAM}, + new FunctionReturnSequenceType(Type.BASE64_BINARY, Cardinality.EXACTLY_ONE, "the binary contents.") + ), + // file:read-binary($file as xs:string, $offset as xs:integer) as xs:base64Binary + new FunctionSignature( + new QName("read-binary", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Reads a portion of a file as binary starting at the given byte offset.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("offset", Type.INTEGER, Cardinality.ZERO_OR_ONE, + "The byte offset to start reading from. Default: 0.") + }, + new FunctionReturnSequenceType(Type.BASE64_BINARY, Cardinality.EXACTLY_ONE, "the binary contents.") + ), + // file:read-binary($file as xs:string, $offset as xs:integer, $length as xs:integer) as xs:base64Binary + new FunctionSignature( + new QName("read-binary", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Reads a portion of a file as binary with offset and length.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("offset", Type.INTEGER, Cardinality.ZERO_OR_ONE, + "The byte offset to start reading from. Default: 0."), + new FunctionParameterSequenceType("length", Type.INTEGER, Cardinality.ZERO_OR_ONE, + "The number of bytes to read.") + }, + new FunctionReturnSequenceType(Type.BASE64_BINARY, Cardinality.EXACTLY_ONE, "the binary contents.") + ) + }; + + public FileIO(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + ExpathFileModuleHelper.checkDbaRole(context, this); + + final String pathStr = args[0].getStringValue(); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + + if (!Files.exists(path)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "File not found: " + path.toAbsolutePath()); + } + if (Files.isDirectory(path)) { + throw new XPathException(this, ExpathFileErrorCode.IS_DIR, + "Path is a directory: " + path.toAbsolutePath()); + } + + if (isCalledAs("read-text")) { + return readText(path, args); + } else if (isCalledAs("read-text-lines")) { + return readTextLines(path, args); + } else if (isCalledAs("read-binary")) { + return readBinary(path, args); + } + + throw new XPathException(this, "Unknown function: " + getSignature().getName().getLocalPart()); + } + + private Sequence readText(final Path path, final Sequence[] args) throws XPathException { + final Charset encoding = getEncoding(args, 1); + try { + final String content = Files.readString(path, encoding); + // Normalize newlines per spec: CR or CRLF -> LF + final String normalized = content.replace("\r\n", "\n").replace("\r", "\n"); + return new StringValue(this, normalized); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } + + private Sequence readTextLines(final Path path, final Sequence[] args) throws XPathException { + final Charset encoding = getEncoding(args, 1); + try { + final String content = Files.readString(path, encoding); + // Split at newline boundaries per spec + final String[] lines = content.split("\r\n|\r|\n", -1); + final ValueSequence result = new ValueSequence(lines.length); + // If file ends with newline, last split element is empty - exclude it per spec + final int count = (lines.length > 0 && lines[lines.length - 1].isEmpty()) ? lines.length - 1 : lines.length; + for (int i = 0; i < count; i++) { + result.add(new StringValue(this, lines[i])); + } + return result; + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } + + private Sequence readBinary(final Path path, final Sequence[] args) throws XPathException { + final long offset = args.length > 1 && !args[1].isEmpty() ? args[1].itemAt(0).toJavaObject(Long.class) : 0; + final long length = args.length > 2 && !args[2].isEmpty() ? args[2].itemAt(0).toJavaObject(Long.class) : -1; + + try { + final long fileSize = Files.size(path); + if (offset < 0 || offset > fileSize) { + throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, + "Offset " + offset + " is out of range for file of size " + fileSize); + } + if (length < -1) { + throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, + "Length must not be negative: " + length); + } + if (length >= 0 && offset + length > fileSize) { + throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, + "Offset + length exceeds file size: " + (offset + length) + " > " + fileSize); + } + + if (offset == 0 && length < 0) { + // Read entire file + final InputStream is = Files.newInputStream(path); + return BinaryValueFromInputStream.getInstance(context, new Base64BinaryValueType(), is, this); + } + + // Partial read + try (final RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) { + raf.seek(offset); + final int readLen = length >= 0 ? (int) length : (int) (fileSize - offset); + final byte[] data = new byte[readLen]; + raf.readFully(data); + final InputStream bis = new java.io.ByteArrayInputStream(data); + return BinaryValueFromInputStream.getInstance(context, new Base64BinaryValueType(), bis, this); + } + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } + + private Charset getEncoding(final Sequence[] args, final int index) throws XPathException { + if (args.length > index && !args[index].isEmpty()) { + return ExpathFileModuleHelper.getCharset(args[index].getStringValue(), this); + } + return StandardCharsets.UTF_8; + } +} diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java b/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java new file mode 100644 index 00000000000..23f5821a8de --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java @@ -0,0 +1,496 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Comparator; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.*; + +/** + * EXPath File Module 4.0 - File/directory manipulation and listing functions. + *

+ * Implements: file:copy, file:move, file:delete, file:create-dir, file:create-temp-dir, + * file:create-temp-file, file:list, file:children, file:descendants + */ +public class FileManipulation extends BasicFunction { + + private static final FunctionParameterSequenceType PATH_PARAM = + new FunctionParameterSequenceType("path", Type.STRING, Cardinality.EXACTLY_ONE, "The file or directory path."); + + public static final FunctionSignature[] signatures = { + // file:copy($source, $target) + new FunctionSignature( + new QName("copy", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Copies a file or directory. If the target exists it is overwritten.", + new SequenceType[]{ + new FunctionParameterSequenceType("source", Type.STRING, Cardinality.EXACTLY_ONE, "Source path."), + new FunctionParameterSequenceType("target", Type.STRING, Cardinality.EXACTLY_ONE, "Target path.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:move($source, $target) + new FunctionSignature( + new QName("move", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Moves a file or directory.", + new SequenceType[]{ + new FunctionParameterSequenceType("source", Type.STRING, Cardinality.EXACTLY_ONE, "Source path."), + new FunctionParameterSequenceType("target", Type.STRING, Cardinality.EXACTLY_ONE, "Target path.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:delete($path) + new FunctionSignature( + new QName("delete", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Deletes a file or empty directory.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:delete($path, $recursive) + new FunctionSignature( + new QName("delete", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Deletes a file or directory. If $recursive is true, non-empty directories are removed recursively.", + new SequenceType[]{ + PATH_PARAM, + new FunctionParameterSequenceType("recursive", Type.BOOLEAN, Cardinality.ZERO_OR_ONE, + "If true, delete directories recursively. Default: false.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:create-dir($dir) + new FunctionSignature( + new QName("create-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Creates a directory, including any necessary parent directories.", + new SequenceType[]{ + new FunctionParameterSequenceType("dir", Type.STRING, Cardinality.EXACTLY_ONE, "The directory path.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:create-temp-dir($prefix, $suffix, $dir) + new FunctionSignature( + new QName("create-temp-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Creates a temporary directory.", + new SequenceType[]{ + new FunctionParameterSequenceType("prefix", Type.STRING, Cardinality.ZERO_OR_ONE, "Prefix for the directory name."), + new FunctionParameterSequenceType("suffix", Type.STRING, Cardinality.ZERO_OR_ONE, "Suffix for the directory name."), + new FunctionParameterSequenceType("dir", Type.STRING, Cardinality.ZERO_OR_ONE, "The parent directory. Default: system temp dir.") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "The path of the created temporary directory.") + ), + // file:create-temp-file($prefix, $suffix, $dir) + new FunctionSignature( + new QName("create-temp-file", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Creates a temporary file.", + new SequenceType[]{ + new FunctionParameterSequenceType("prefix", Type.STRING, Cardinality.ZERO_OR_ONE, "Prefix for the file name."), + new FunctionParameterSequenceType("suffix", Type.STRING, Cardinality.ZERO_OR_ONE, "Suffix for the file name."), + new FunctionParameterSequenceType("dir", Type.STRING, Cardinality.ZERO_OR_ONE, "The parent directory. Default: system temp dir.") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "The path of the created temporary file.") + ), + // file:list($dir) + new FunctionSignature( + new QName("list", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Lists the contents of a directory as relative paths.", + new SequenceType[]{ + new FunctionParameterSequenceType("dir", Type.STRING, Cardinality.EXACTLY_ONE, "The directory path.") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "The relative paths of directory contents.") + ), + // file:list($dir, $recursive) + new FunctionSignature( + new QName("list", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Lists the contents of a directory, optionally recursively.", + new SequenceType[]{ + new FunctionParameterSequenceType("dir", Type.STRING, Cardinality.EXACTLY_ONE, "The directory path."), + new FunctionParameterSequenceType("recursive", Type.BOOLEAN, Cardinality.ZERO_OR_ONE, "If true, list recursively. Default: false.") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "The relative paths of directory contents.") + ), + // file:list($dir, $recursive, $pattern) + new FunctionSignature( + new QName("list", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Lists the contents of a directory matching a glob pattern.", + new SequenceType[]{ + new FunctionParameterSequenceType("dir", Type.STRING, Cardinality.EXACTLY_ONE, "The directory path."), + new FunctionParameterSequenceType("recursive", Type.BOOLEAN, Cardinality.ZERO_OR_ONE, "If true, list recursively. Default: false."), + new FunctionParameterSequenceType("pattern", Type.STRING, Cardinality.ZERO_OR_ONE, "A glob pattern to filter results.") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "The relative paths of matching directory contents.") + ), + // file:children($path) + new FunctionSignature( + new QName("children", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the paths of immediate children of a directory.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "The absolute paths of children.") + ), + // file:descendants($path) + new FunctionSignature( + new QName("descendants", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the paths of all descendants of a directory recursively.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "The absolute paths of all descendants.") + ), + // file:list-roots() + new FunctionSignature( + new QName("list-roots", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the root directories of the file system.", + new SequenceType[]{}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "The root directories.") + ) + }; + + public FileManipulation(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + ExpathFileModuleHelper.checkDbaRole(context, this); + + if (isCalledAs("list-roots")) { + return listRoots(); + } + + final String pathStr = args[0].getStringValue(); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + + if (isCalledAs("copy")) { + return copy(path, args); + } else if (isCalledAs("move")) { + return move(path, args); + } else if (isCalledAs("delete")) { + return delete(path, args); + } else if (isCalledAs("create-dir")) { + return createDir(path); + } else if (isCalledAs("create-temp-dir")) { + return createTempDir(args); + } else if (isCalledAs("create-temp-file")) { + return createTempFile(args); + } else if (isCalledAs("list")) { + return list(path, args); + } else if (isCalledAs("children")) { + return children(path); + } else if (isCalledAs("descendants")) { + return descendants(path); + } + + throw new XPathException(this, "Unknown function: " + getSignature().getName().getLocalPart()); + } + + private Sequence copy(final Path source, final Sequence[] args) throws XPathException { + if (!Files.exists(source)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "Source does not exist: " + source.toAbsolutePath()); + } + final Path target = ExpathFileModuleHelper.getPath(args[1].getStringValue(), this); + try { + if (Files.isDirectory(source)) { + copyDirectory(source, target); + } else { + // If target is an existing directory, copy into it + final Path actualTarget = Files.isDirectory(target) ? target.resolve(source.getFileName()) : target; + Files.copy(source, actualTarget, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private void copyDirectory(final Path source, final Path target) throws IOException { + Files.walkFileTree(source, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException { + final Path targetDir = target.resolve(source.relativize(dir)); + Files.createDirectories(targetDir); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + Files.copy(file, target.resolve(source.relativize(file)), StandardCopyOption.REPLACE_EXISTING); + return FileVisitResult.CONTINUE; + } + }); + } + + private Sequence move(final Path source, final Sequence[] args) throws XPathException { + if (!Files.exists(source)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "Source does not exist: " + source.toAbsolutePath()); + } + final Path target = ExpathFileModuleHelper.getPath(args[1].getStringValue(), this); + try { + // If target is an existing directory, move into it + final Path actualTarget = Files.isDirectory(target) ? target.resolve(source.getFileName()) : target; + Files.move(source, actualTarget, StandardCopyOption.REPLACE_EXISTING); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private Sequence delete(final Path path, final Sequence[] args) throws XPathException { + if (!Files.exists(path)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "Path does not exist: " + path.toAbsolutePath()); + } + final boolean recursive = args.length > 1 && !args[1].isEmpty() + && args[1].itemAt(0).toJavaObject(Boolean.class); + + try { + if (Files.isDirectory(path)) { + if (recursive) { + try (final Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.delete(p); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + }); + } + } else { + // Attempt to delete; will fail if non-empty + try { + Files.delete(path); + } catch (final DirectoryNotEmptyException e) { + throw new XPathException(this, ExpathFileErrorCode.IS_DIR, + "Directory is not empty (use $recursive = true()): " + path.toAbsolutePath()); + } + } + } else { + Files.delete(path); + } + } catch (final UncheckedIOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getCause().getMessage()); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private Sequence createDir(final Path path) throws XPathException { + if (Files.exists(path) && !Files.isDirectory(path)) { + throw new XPathException(this, ExpathFileErrorCode.EXISTS, + "Path exists and is not a directory: " + path.toAbsolutePath()); + } + try { + Files.createDirectories(path); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private Sequence createTempDir(final Sequence[] args) throws XPathException { + final String prefix = args.length > 0 && !args[0].isEmpty() ? args[0].getStringValue() : ""; + final String suffix = args.length > 1 && !args[1].isEmpty() ? args[1].getStringValue() : ""; + final Path dir = args.length > 2 && !args[2].isEmpty() + ? ExpathFileModuleHelper.getPath(args[2].getStringValue(), this) + : Paths.get(System.getProperty("java.io.tmpdir")); + + if (!Files.isDirectory(dir)) { + throw new XPathException(this, ExpathFileErrorCode.NO_DIR, + "Parent is not a directory: " + dir.toAbsolutePath()); + } + + try { + // Java's createTempDirectory only supports prefix, so we append suffix manually + final Path tempDir = Files.createTempDirectory(dir, prefix); + if (!suffix.isEmpty()) { + final Path renamed = tempDir.resolveSibling(tempDir.getFileName().toString() + suffix); + Files.move(tempDir, renamed); + return new StringValue(this, renamed.toAbsolutePath().toString() + File.separator); + } + return new StringValue(this, tempDir.toAbsolutePath().toString() + File.separator); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } + + private Sequence createTempFile(final Sequence[] args) throws XPathException { + final String prefix = args.length > 0 && !args[0].isEmpty() ? args[0].getStringValue() : ""; + final String suffix = args.length > 1 && !args[1].isEmpty() ? args[1].getStringValue() : ""; + final Path dir = args.length > 2 && !args[2].isEmpty() + ? ExpathFileModuleHelper.getPath(args[2].getStringValue(), this) + : Paths.get(System.getProperty("java.io.tmpdir")); + + if (!Files.isDirectory(dir)) { + throw new XPathException(this, ExpathFileErrorCode.NO_DIR, + "Parent is not a directory: " + dir.toAbsolutePath()); + } + + try { + final Path tempFile = Files.createTempFile(dir, prefix, suffix.isEmpty() ? null : suffix); + return new StringValue(this, tempFile.toAbsolutePath().toString()); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } + + private Sequence list(final Path dir, final Sequence[] args) throws XPathException { + if (!Files.exists(dir)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "Directory does not exist: " + dir.toAbsolutePath()); + } + if (!Files.isDirectory(dir)) { + throw new XPathException(this, ExpathFileErrorCode.NO_DIR, + "Path is not a directory: " + dir.toAbsolutePath()); + } + + final boolean recursive = args.length > 1 && !args[1].isEmpty() + && args[1].itemAt(0).toJavaObject(Boolean.class); + final String pattern = args.length > 2 && !args[2].isEmpty() + ? args[2].getStringValue() : null; + + final Pattern regex = pattern != null ? globToRegex(pattern) : null; + + try { + final ValueSequence result = new ValueSequence(); + try (final Stream stream = recursive ? Files.walk(dir) : Files.list(dir)) { + stream.filter(p -> !p.equals(dir)) + .forEach(p -> { + final String relative = dir.relativize(p).toString(); + final String entry = Files.isDirectory(p) + ? relative + File.separator : relative; + if (regex == null || regex.matcher(entry.replace(File.separator, "/")).matches() + || regex.matcher(p.getFileName().toString()).matches()) { + result.add(new StringValue(this, entry)); + } + }); + } + return result; + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } + + private Sequence children(final Path path) throws XPathException { + if (!Files.exists(path)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "Path does not exist: " + path.toAbsolutePath()); + } + if (!Files.isDirectory(path)) { + throw new XPathException(this, ExpathFileErrorCode.NO_DIR, + "Path is not a directory: " + path.toAbsolutePath()); + } + + try { + final ValueSequence result = new ValueSequence(); + try (final Stream stream = Files.list(path)) { + stream.forEach(p -> { + final String absPath = p.toAbsolutePath().toString(); + final String entry = Files.isDirectory(p) ? absPath + File.separator : absPath; + result.add(new StringValue(this, entry)); + }); + } + return result; + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } + + private Sequence descendants(final Path path) throws XPathException { + if (!Files.exists(path)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "Path does not exist: " + path.toAbsolutePath()); + } + if (!Files.isDirectory(path)) { + throw new XPathException(this, ExpathFileErrorCode.NO_DIR, + "Path is not a directory: " + path.toAbsolutePath()); + } + + try { + final ValueSequence result = new ValueSequence(); + try (final Stream walk = Files.walk(path)) { + walk.filter(p -> !p.equals(path)) + .forEach(p -> { + final String absPath = p.toAbsolutePath().toString(); + final String entry = Files.isDirectory(p) ? absPath + File.separator : absPath; + result.add(new StringValue(this, entry)); + }); + } + return result; + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } + + private Sequence listRoots() { + final ValueSequence result = new ValueSequence(); + for (final File root : File.listRoots()) { + result.add(new StringValue(this, root.getAbsolutePath())); + } + return result; + } + + /** + * Convert a simple glob pattern (with * and ?) to a Java regex Pattern. + */ + private static Pattern globToRegex(final String glob) { + final StringBuilder regex = new StringBuilder(); + for (int i = 0; i < glob.length(); i++) { + final char c = glob.charAt(i); + switch (c) { + case '*': + regex.append(".*"); + break; + case '?': + regex.append('.'); + break; + case '.': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '\\': + case '^': + case '$': + case '|': + case '+': + regex.append('\\').append(c); + break; + default: + regex.append(c); + } + } + return Pattern.compile(regex.toString()); + } +} diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FilePaths.java b/extensions/expath/src/main/java/org/expath/exist/file/FilePaths.java new file mode 100644 index 00000000000..ee8c260bc96 --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/FilePaths.java @@ -0,0 +1,139 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.*; + +/** + * EXPath File Module 4.0 - Path functions. + *

+ * Implements: file:name, file:parent, file:path-to-native, file:path-to-uri, file:resolve-path + */ +public class FilePaths extends BasicFunction { + + private static final FunctionParameterSequenceType PATH_PARAM = + new FunctionParameterSequenceType("path", Type.STRING, Cardinality.EXACTLY_ONE, "The file path."); + + public static final FunctionSignature[] signatures = { + // file:name($path as xs:string) as xs:string + new FunctionSignature( + new QName("name", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the name of a file or directory.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the file or directory name.") + ), + // file:parent($path as xs:string) as xs:string? + new FunctionSignature( + new QName("parent", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the parent directory of a path.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_ONE, "the parent directory path, or empty for root.") + ), + // file:path-to-native($path as xs:string) as xs:string + new FunctionSignature( + new QName("path-to-native", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the native, canonical path.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the native path.") + ), + // file:path-to-uri($path as xs:string) as xs:anyURI + new FunctionSignature( + new QName("path-to-uri", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the path as a file:// URI.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.ANY_URI, Cardinality.EXACTLY_ONE, "the file:// URI.") + ), + // file:resolve-path($path as xs:string) as xs:string + new FunctionSignature( + new QName("resolve-path", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Resolves a relative path against the current working directory.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the resolved absolute path.") + ), + // file:resolve-path($path as xs:string, $base as xs:string) as xs:string + new FunctionSignature( + new QName("resolve-path", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Resolves a relative path against a base directory.", + new SequenceType[]{ + PATH_PARAM, + new FunctionParameterSequenceType("base", Type.STRING, Cardinality.ZERO_OR_ONE, "The base directory to resolve against.") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the resolved absolute path.") + ) + }; + + public FilePaths(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + ExpathFileModuleHelper.checkDbaRole(context, this); + + final String pathStr = args[0].getStringValue(); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + + if (isCalledAs("name")) { + final Path fileName = path.getFileName(); + return new StringValue(this, fileName != null ? fileName.toString() : ""); + } else if (isCalledAs("parent")) { + final Path absPath = path.toAbsolutePath().normalize(); + final Path parent = absPath.getParent(); + if (parent == null) { + return Sequence.EMPTY_SEQUENCE; + } + return new StringValue(this, parent.toString() + File.separator); + } else if (isCalledAs("path-to-native")) { + if (!Files.exists(path)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "Path does not exist: " + path.toAbsolutePath()); + } + try { + return new StringValue(this, path.toRealPath().toString()); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } else if (isCalledAs("path-to-uri")) { + final Path abs = path.toAbsolutePath().normalize(); + return new AnyURIValue(this, abs.toUri().toString()); + } else if (isCalledAs("resolve-path")) { + if (args.length > 1 && !args[1].isEmpty()) { + final Path base = ExpathFileModuleHelper.getPath(args[1].getStringValue(), this); + return new StringValue(this, base.resolve(path).toAbsolutePath().normalize().toString()); + } + return new StringValue(this, path.toAbsolutePath().normalize().toString()); + } + + throw new XPathException(this, "Unknown function: " + getSignature().getName().getLocalPart()); + } +} diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileProperties.java b/extensions/expath/src/main/java/org/expath/exist/file/FileProperties.java new file mode 100644 index 00000000000..d7027f06977 --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileProperties.java @@ -0,0 +1,184 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.util.Date; +import java.util.stream.Stream; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.DateTimeValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.IntegerValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * EXPath File Module 4.0 - File Properties functions. + *

+ * Implements: file:exists, file:is-dir, file:is-file, file:is-absolute, file:last-modified, file:size + */ +public class FileProperties extends BasicFunction { + + private static final FunctionParameterSequenceType PATH_PARAM = + new FunctionParameterSequenceType("path", Type.STRING, Cardinality.EXACTLY_ONE, + "The file path."); + + public static final FunctionSignature[] signatures = { + // file:exists($path as xs:string) as xs:boolean + new FunctionSignature( + new QName("exists", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Tests whether a path exists.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "true if the path exists.") + ), + // file:is-dir($path as xs:string) as xs:boolean + new FunctionSignature( + new QName("is-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Tests whether a path points to a directory.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "true if the path is a directory.") + ), + // file:is-file($path as xs:string) as xs:boolean + new FunctionSignature( + new QName("is-file", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Tests whether a path points to a regular file.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "true if the path is a regular file.") + ), + // file:is-absolute($path as xs:string) as xs:boolean + new FunctionSignature( + new QName("is-absolute", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Tests whether a path is absolute.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "true if the path is absolute.") + ), + // file:last-modified($path as xs:string) as xs:dateTime + new FunctionSignature( + new QName("last-modified", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the last modification time of a file or directory.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.DATE_TIME, Cardinality.EXACTLY_ONE, + "the last modification time.") + ), + // file:size($path as xs:string) as xs:integer + new FunctionSignature( + new QName("size", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the byte size of a file, or 0 for a directory.", + new SequenceType[]{PATH_PARAM}, + new FunctionReturnSequenceType(Type.INTEGER, Cardinality.EXACTLY_ONE, + "the file size in bytes.") + ), + // file:size($path as xs:string, $recursive as xs:boolean) as xs:integer + new FunctionSignature( + new QName("size", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the byte size of a file, or for a directory the recursive size if $recursive is true.", + new SequenceType[]{ + PATH_PARAM, + new FunctionParameterSequenceType("recursive", Type.BOOLEAN, Cardinality.ZERO_OR_ONE, + "If true and path is a directory, compute recursive size.") + }, + new FunctionReturnSequenceType(Type.INTEGER, Cardinality.EXACTLY_ONE, + "the file or directory size in bytes.") + ) + }; + + public FileProperties(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + ExpathFileModuleHelper.checkDbaRole(context, this); + + final String pathStr = args[0].getStringValue(); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + + if (isCalledAs("exists")) { + return BooleanValue.valueOf(Files.exists(path)); + } else if (isCalledAs("is-dir")) { + return BooleanValue.valueOf(Files.isDirectory(path)); + } else if (isCalledAs("is-file")) { + return BooleanValue.valueOf(Files.isRegularFile(path)); + } else if (isCalledAs("is-absolute")) { + return BooleanValue.valueOf(path.isAbsolute()); + } else if (isCalledAs("last-modified")) { + if (!Files.exists(path)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "Path does not exist: " + path.toAbsolutePath()); + } + try { + final FileTime ft = Files.getLastModifiedTime(path); + return new DateTimeValue(this, new Date(ft.toMillis())); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } else if (isCalledAs("size")) { + if (!Files.exists(path)) { + throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, + "Path does not exist: " + path.toAbsolutePath()); + } + try { + if (Files.isDirectory(path)) { + final boolean recursive = args.length > 1 && !args[1].isEmpty() + && args[1].itemAt(0).toJavaObject(Boolean.class); + if (recursive) { + try (final Stream walk = Files.walk(path)) { + final long total = walk + .filter(Files::isRegularFile) + .mapToLong(p -> { + try { + return Files.size(p); + } catch (final IOException e) { + return 0L; + } + }) + .sum(); + return new IntegerValue(this, total); + } + } + return new IntegerValue(this, 0); + } + return new IntegerValue(this, Files.size(path)); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } + + throw new XPathException(this, "Unknown function: " + getSignature().getName().getLocalPart()); + } +} diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileSystemProperties.java b/extensions/expath/src/main/java/org/expath/exist/file/FileSystemProperties.java new file mode 100644 index 00000000000..0af85e24717 --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileSystemProperties.java @@ -0,0 +1,130 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import java.io.File; +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; + +/** + * EXPath File Module 4.0 - System property functions. + *

+ * Implements: file:dir-separator, file:line-separator, file:path-separator, + * file:temp-dir, file:base-dir, file:current-dir + */ +public class FileSystemProperties extends BasicFunction { + + public static final FunctionSignature[] signatures = { + // file:dir-separator() as xs:string + new FunctionSignature( + new QName("dir-separator", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the directory separator used by the operating system.", + new SequenceType[]{}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the directory separator.") + ), + // file:line-separator() as xs:string + new FunctionSignature( + new QName("line-separator", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the line separator used by the operating system.", + new SequenceType[]{}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the line separator.") + ), + // file:path-separator() as xs:string + new FunctionSignature( + new QName("path-separator", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the path separator used by the operating system.", + new SequenceType[]{}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the path separator.") + ), + // file:temp-dir() as xs:string + new FunctionSignature( + new QName("temp-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the path of the temporary directory.", + new SequenceType[]{}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the temporary directory path.") + ), + // file:base-dir() as xs:string? + new FunctionSignature( + new QName("base-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the base directory of the current query, or empty if not available.", + new SequenceType[]{}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_ONE, "the base directory path.") + ), + // file:current-dir() as xs:string + new FunctionSignature( + new QName("current-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Returns the current working directory.", + new SequenceType[]{}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the current working directory.") + ) + }; + + public FileSystemProperties(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + ExpathFileModuleHelper.checkDbaRole(context, this); + + if (isCalledAs("dir-separator")) { + return new StringValue(this, File.separator); + } else if (isCalledAs("line-separator")) { + return new StringValue(this, System.lineSeparator()); + } else if (isCalledAs("path-separator")) { + return new StringValue(this, File.pathSeparator); + } else if (isCalledAs("temp-dir")) { + return new StringValue(this, System.getProperty("java.io.tmpdir") + File.separator); + } else if (isCalledAs("base-dir")) { + try { + final String baseURI = context.getBaseURI().getStringValue(); + if (baseURI != null && !baseURI.isEmpty() && baseURI.startsWith("file:")) { + final Path basePath = Paths.get(new URI(baseURI)); + final Path parent = basePath.getParent(); + if (parent != null) { + return new StringValue(this, parent.toString() + File.separator); + } + } + } catch (final Exception e) { + // Fall through to return empty + } + return Sequence.EMPTY_SEQUENCE; + } else if (isCalledAs("current-dir")) { + return new StringValue(this, System.getProperty("user.dir") + File.separator); + } + + throw new XPathException(this, "Unknown function: " + getSignature().getName().getLocalPart()); + } +} diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileWrite.java b/extensions/expath/src/main/java/org/expath/exist/file/FileWrite.java new file mode 100644 index 00000000000..d89eeb103a8 --- /dev/null +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileWrite.java @@ -0,0 +1,289 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.expath.exist.file; + +import java.io.*; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; + +import org.exist.dom.QName; +import org.exist.storage.serializers.Serializer; +import org.exist.util.serializer.SAXSerializer; +import org.exist.util.serializer.SerializerPool; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.util.SerializerUtils; +import org.exist.xquery.value.*; +import org.xml.sax.SAXException; + +/** + * EXPath File Module 4.0 - Write functions. + *

+ * Implements: file:write, file:write-text, file:write-text-lines, file:write-binary + */ +public class FileWrite extends BasicFunction { + + private static final FunctionParameterSequenceType FILE_PARAM = + new FunctionParameterSequenceType("file", Type.STRING, Cardinality.EXACTLY_ONE, "The path to the file."); + + public static final FunctionSignature[] signatures = { + // file:write($file, $value) + new FunctionSignature( + new QName("write", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Writes a serialized sequence to a file. Creates the file if it does not exist, overwrites it otherwise.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.ITEM, Cardinality.ZERO_OR_MORE, "The items to serialize and write.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:write($file, $value, $options) + new FunctionSignature( + new QName("write", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Writes a serialized sequence to a file with serialization options.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.ITEM, Cardinality.ZERO_OR_MORE, "The items to serialize and write."), + new FunctionParameterSequenceType("options", Type.ITEM, Cardinality.ZERO_OR_ONE, "Serialization parameters as map(*) or element(output:serialization-parameters).") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:write-text($file, $value) + new FunctionSignature( + new QName("write-text", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Writes a string to a file. Creates the file if it does not exist, overwrites it otherwise.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.STRING, Cardinality.EXACTLY_ONE, "The string to write.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:write-text($file, $value, $encoding) + new FunctionSignature( + new QName("write-text", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Writes a string to a file with the specified encoding.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.STRING, Cardinality.EXACTLY_ONE, "The string to write."), + new FunctionParameterSequenceType("encoding", Type.STRING, Cardinality.ZERO_OR_ONE, "The character encoding. Default: UTF-8.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:write-text-lines($file, $values) + new FunctionSignature( + new QName("write-text-lines", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Writes a sequence of strings as lines to a file, separated by the platform line separator.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("values", Type.STRING, Cardinality.ZERO_OR_MORE, "The lines to write.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:write-text-lines($file, $values, $encoding) + new FunctionSignature( + new QName("write-text-lines", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Writes a sequence of strings as lines to a file with the specified encoding.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("values", Type.STRING, Cardinality.ZERO_OR_MORE, "The lines to write."), + new FunctionParameterSequenceType("encoding", Type.STRING, Cardinality.ZERO_OR_ONE, "The character encoding. Default: UTF-8.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:write-binary($file, $value) + new FunctionSignature( + new QName("write-binary", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Writes binary data to a file. Creates the file if it does not exist, overwrites it otherwise.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.BASE64_BINARY, Cardinality.EXACTLY_ONE, "The binary data to write.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ), + // file:write-binary($file, $value, $offset) + new FunctionSignature( + new QName("write-binary", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Writes binary data to a file at the given offset.", + new SequenceType[]{ + FILE_PARAM, + new FunctionParameterSequenceType("value", Type.BASE64_BINARY, Cardinality.EXACTLY_ONE, "The binary data to write."), + new FunctionParameterSequenceType("offset", Type.INTEGER, Cardinality.ZERO_OR_ONE, "The byte offset at which to start writing. Default: 0.") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") + ) + }; + + public FileWrite(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + ExpathFileModuleHelper.checkDbaRole(context, this); + + final String pathStr = args[0].getStringValue(); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + + checkParentDir(path); + + if (Files.isDirectory(path)) { + throw new XPathException(this, ExpathFileErrorCode.IS_DIR, + "Path is a directory: " + path.toAbsolutePath()); + } + + if (isCalledAs("write")) { + return write(path, args); + } else if (isCalledAs("write-text")) { + return writeText(path, args); + } else if (isCalledAs("write-text-lines")) { + return writeTextLines(path, args); + } else if (isCalledAs("write-binary")) { + return writeBinary(path, args); + } + + throw new XPathException(this, "Unknown function: " + getSignature().getName().getLocalPart()); + } + + private Sequence write(final Path path, final Sequence[] args) throws XPathException { + final Sequence value = args[1]; + final Properties outputProperties = new Properties(); + if (args.length > 2 && !args[2].isEmpty()) { + final Item optionsItem = args[2].itemAt(0); + if (optionsItem instanceof AbstractMapType) { + outputProperties.putAll(SerializerUtils.getSerializationOptions(this, (AbstractMapType) optionsItem)); + } else if (optionsItem instanceof NodeValue) { + SerializerUtils.getSerializationOptions(this, (NodeValue) optionsItem, outputProperties); + } + } + + final SAXSerializer sax = (SAXSerializer) SerializerPool.getInstance().borrowObject(SAXSerializer.class); + try { + final Serializer serializer = context.getBroker().borrowSerializer(); + try (final Writer writer = new OutputStreamWriter( + new BufferedOutputStream(Files.newOutputStream(path)), StandardCharsets.UTF_8)) { + sax.setOutput(writer, outputProperties); + serializer.setProperties(outputProperties); + serializer.setSAXHandlers(sax, sax); + + for (final SequenceIterator i = value.iterate(); i.hasNext(); ) { + final Item item = i.nextItem(); + if (Type.subTypeOf(item.getType(), Type.NODE)) { + serializer.toSAX((NodeValue) item); + } else { + writer.write(item.getStringValue()); + } + } + } finally { + context.getBroker().returnSerializer(serializer); + } + } catch (final IOException | SAXException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } finally { + SerializerPool.getInstance().returnObject(sax); + } + return Sequence.EMPTY_SEQUENCE; + } + + private Sequence writeText(final Path path, final Sequence[] args) throws XPathException { + final String text = args[1].getStringValue(); + final Charset encoding = getEncoding(args, 2); + try { + Files.writeString(path, text, encoding); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private Sequence writeTextLines(final Path path, final Sequence[] args) throws XPathException { + final Charset encoding = getEncoding(args, 2); + try (final Writer writer = new OutputStreamWriter( + new BufferedOutputStream(Files.newOutputStream(path)), encoding)) { + final String lineSep = System.lineSeparator(); + for (final SequenceIterator i = args[1].iterate(); i.hasNext(); ) { + writer.write(i.nextItem().getStringValue()); + writer.write(lineSep); + } + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private Sequence writeBinary(final Path path, final Sequence[] args) throws XPathException { + final BinaryValue binaryValue = (BinaryValue) args[1].itemAt(0); + final long offset = args.length > 2 && !args[2].isEmpty() ? args[2].itemAt(0).toJavaObject(Long.class) : 0; + + try { + if (offset == 0) { + try (final OutputStream os = Files.newOutputStream(path); + final InputStream is = binaryValue.getInputStream()) { + is.transferTo(os); + } + } else { + if (offset < 0) { + throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, + "Offset must not be negative: " + offset); + } + try (final RandomAccessFile raf = new RandomAccessFile(path.toFile(), "rw"); + final InputStream is = binaryValue.getInputStream()) { + raf.seek(offset); + is.transferTo(new OutputStream() { + @Override + public void write(int b) throws IOException { + raf.write(b); + } + @Override + public void write(byte[] b, int off, int len) throws IOException { + raf.write(b, off, len); + } + }); + } + } + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + return Sequence.EMPTY_SEQUENCE; + } + + private void checkParentDir(final Path path) throws XPathException { + final Path parent = path.getParent(); + if (parent != null && !Files.isDirectory(parent)) { + throw new XPathException(this, ExpathFileErrorCode.NO_DIR, + "Parent directory does not exist: " + parent.toAbsolutePath()); + } + } + + private Charset getEncoding(final Sequence[] args, final int index) throws XPathException { + if (args.length > index && !args[index].isEmpty()) { + return ExpathFileModuleHelper.getCharset(args[index].getStringValue(), this); + } + return StandardCharsets.UTF_8; + } +} diff --git a/extensions/modules/file/src/test/java/xquery/modules/file/FileTests.java b/extensions/expath/src/test/java/org/expath/exist/file/ExpathFileTests.java similarity index 90% rename from extensions/modules/file/src/test/java/xquery/modules/file/FileTests.java rename to extensions/expath/src/test/java/org/expath/exist/file/ExpathFileTests.java index f3abd2ce42c..15efee29342 100644 --- a/extensions/modules/file/src/test/java/xquery/modules/file/FileTests.java +++ b/extensions/expath/src/test/java/org/expath/exist/file/ExpathFileTests.java @@ -19,15 +19,14 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -package xquery.modules.file; +package org.expath.exist.file; import org.exist.test.runner.XSuite; import org.junit.runner.RunWith; @RunWith(XSuite.class) @XSuite.XSuiteFiles({ - "src/test/xquery/modules/file" + "src/test/xquery/org/expath/exist/file" }) -public class FileTests { +public class ExpathFileTests { } - diff --git a/extensions/expath/src/test/resources-filtered/conf.xml b/extensions/expath/src/test/resources-filtered/conf.xml index a0e02a2c06d..ffcfb5d71b0 100644 --- a/extensions/expath/src/test/resources-filtered/conf.xml +++ b/extensions/expath/src/test/resources-filtered/conf.xml @@ -743,8 +743,9 @@ - + + diff --git a/extensions/modules/file/src/test/resources/util/fixtures.xqm b/extensions/expath/src/test/resources/util/fixtures.xqm similarity index 100% rename from extensions/modules/file/src/test/resources/util/fixtures.xqm rename to extensions/expath/src/test/resources/util/fixtures.xqm diff --git a/extensions/modules/file/src/test/resources/util/helper.xqm b/extensions/expath/src/test/resources/util/helper.xqm similarity index 80% rename from extensions/modules/file/src/test/resources/util/helper.xqm rename to extensions/expath/src/test/resources/util/helper.xqm index adddc4a214d..c34307815bb 100644 --- a/extensions/modules/file/src/test/resources/util/helper.xqm +++ b/extensions/expath/src/test/resources/util/helper.xqm @@ -23,10 +23,12 @@ xquery version "3.1"; module namespace helper="http://exist-db.org/xquery/test/util/helper"; import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "fixtures.xqm"; -import module namespace file="http://exist-db.org/xquery/file"; +import module namespace file="http://expath.org/ns/file"; import module namespace xmldb="http://exist-db.org/xquery/xmldb"; import module namespace util="http://exist-db.org/xquery/util"; +declare namespace utilns="http://exist-db.org/xquery/util"; + declare variable $helper:error := xs:QName("helper:assert-sync-error"); declare variable $helper:path-separator := util:system-property("file.separator"); @@ -68,18 +70,23 @@ declare function helper:modify-db-resource($collection as xs:string, $resource a }; declare function helper:clear-suite-fs ($suite as xs:string) as empty-sequence() { - let $_ := + let $dir := helper:glue-path(( util:system-property("java.io.tmpdir"), $suite )) - => file:delete() - return () + return + if (file:exists($dir)) then + let $_ := file:delete($dir, true()) + return () + else () }; declare function helper:clear-fs ($directory as xs:string) as empty-sequence() { - let $_ := file:delete($directory) - return () + if (file:exists($directory)) then + let $_ := file:delete($directory, true()) + return () + else () }; declare function helper:get-test-directory ($suite as xs:string) as xs:string { @@ -99,42 +106,47 @@ declare function helper:glue-path ($parts as xs:string+) as xs:string { : @returns given directory to allow use in pipeline (chain of arrow operators) :) declare function helper:setup-fs-extra ($directory as xs:string) as xs:string { - let $action1 := file:mkdirs($directory) - let $action2 := file:mkdirs(helper:glue-path(($directory, "test"))) - let $action3 := ( - (: cannot use fixtures here because this will lead to consumed input streams! :) - file:serialize-binary( - util:string-to-binary("SERVER_SECRET=123!"), - helper:glue-path(($directory, ".env"))), - file:serialize-binary( - util:string-to-binary("..."), - helper:glue-path(($directory, "test", "three.s"))) + let $_ := file:create-dir($directory) + let $_ := file:create-dir(helper:glue-path(($directory, "test"))) + let $_ := ( + file:write-binary( + helper:glue-path(($directory, ".env")), + util:string-to-binary("SERVER_SECRET=123!")), + file:write-binary( + helper:glue-path(($directory, "test", "three.s")), + util:string-to-binary("...")) ) return $directory }; -declare function helper:get-deleted-from-sync-result ($result as element(file:sync)) as xs:string* { - $result//file:delete/@name/string() +declare function helper:get-deleted-from-sync-result ($result as element(utilns:sync)) as xs:string* { + $result//utilns:delete/@name/string() }; -declare function helper:get-dir-from-sync-result ($result as element(file:sync)) as xs:string* { - $result/@file:dir/string() +declare function helper:get-dir-from-sync-result ($result as element(utilns:sync)) as xs:string* { + $result/@utilns:dir/string() }; -declare function helper:get-updated-from-sync-result ($result as element(file:sync)) as xs:string* { - $result//file:update/@name/string() +declare function helper:get-updated-from-sync-result ($result as element(utilns:sync)) as xs:string* { + $result//utilns:update/@name/string() }; declare function helper:list-files-and-directories ($directory as xs:string) as xs:string* { - file:list($directory)//(file:file|file:directory)/@name/string() + (: EXPath file:list returns relative path strings, directories end with separator :) + for $entry in file:list($directory) + return + (: strip trailing separator from directory names :) + if (ends-with($entry, file:dir-separator())) + then substring($entry, 1, string-length($entry) - string-length(file:dir-separator())) + else $entry }; -declare function helper:sync-with-options ($directory as xs:string, $options as item()?) as element(file:sync) { - file:sync($fixtures:collection, $directory, $options)/* +declare function helper:sync-with-options ($directory as xs:string, $options as item()?) as element(utilns:sync) { + util:file-sync($fixtures:collection, $directory, $options)/* }; declare function helper:assert-sync-result ( - $result as document-node(element(file:sync)), + $result as document-node(element(utilns:sync)), $expected as map(xs:string, xs:string*) ) as xs:boolean { helper:assert-permutation-of( @@ -202,7 +214,7 @@ declare function helper:maybe-remove-item-at-index($sequence as xs:anyAtomicType declare function helper:assert-file-contents($expected as xs:string, $path-parts as xs:string+) as xs:boolean { let $path := helper:glue-path($path-parts) - let $actual := file:read($path) + let $actual := file:read-text($path) return if ( diff --git a/extensions/expath/src/test/xquery/org/expath/exist/file/file-tests.xql b/extensions/expath/src/test/xquery/org/expath/exist/file/file-tests.xql new file mode 100644 index 00000000000..12b6233a274 --- /dev/null +++ b/extensions/expath/src/test/xquery/org/expath/exist/file/file-tests.xql @@ -0,0 +1,547 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace ft="http://exist-db.org/testsuite/expath-file"; + +import module namespace test="http://exist-db.org/xquery/xqsuite" + at "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql"; +import module namespace file="http://expath.org/ns/file"; + +(: ======== Helper variables ======== :) + +declare variable $ft:temp-dir := file:temp-dir(); +declare variable $ft:test-dir := $ft:temp-dir || "expath-file-test/"; + +(: ======== Setup / Teardown ======== :) + +declare + %test:setUp +function ft:setup() { + file:create-dir($ft:test-dir) +}; + +declare + %test:tearDown +function ft:teardown() { + if (file:exists($ft:test-dir)) then + file:delete($ft:test-dir, true()) + else + () +}; + +(: ======== System Properties ======== :) + +declare + %test:assertExists +function ft:dir-separator() { + file:dir-separator() +}; + +declare + %test:assertExists +function ft:line-separator() { + file:line-separator() +}; + +declare + %test:assertExists +function ft:path-separator() { + file:path-separator() +}; + +declare + %test:assertExists +function ft:temp-dir() { + file:temp-dir() +}; + +declare + %test:assertExists +function ft:current-dir() { + file:current-dir() +}; + +(: temp-dir and current-dir should end with separator :) +declare + %test:assertTrue +function ft:temp-dir-ends-with-separator() { + ends-with(file:temp-dir(), file:dir-separator()) +}; + +declare + %test:assertTrue +function ft:current-dir-ends-with-separator() { + ends-with(file:current-dir(), file:dir-separator()) +}; + +(: ======== File Properties ======== :) + +declare + %test:assertTrue +function ft:exists-temp-dir() { + file:exists(file:temp-dir()) +}; + +declare + %test:assertFalse +function ft:exists-nonexistent() { + file:exists($ft:test-dir || "nonexistent-file.txt") +}; + +declare + %test:assertTrue +function ft:is-dir-temp() { + file:is-dir(file:temp-dir()) +}; + +declare + %test:assertFalse +function ft:is-dir-nonexistent() { + file:is-dir($ft:test-dir || "nonexistent") +}; + +declare + %test:assertTrue +function ft:is-file-after-write() { + let $path := $ft:test-dir || "is-file-test.txt" + let $_ := file:write-text($path, "hello") + return file:is-file($path) +}; + +declare + %test:assertFalse +function ft:is-file-on-dir() { + file:is-file($ft:test-dir) +}; + +declare + %test:assertTrue +function ft:is-absolute-absolute-path() { + file:is-absolute(file:temp-dir()) +}; + +declare + %test:assertFalse +function ft:is-absolute-relative-path() { + file:is-absolute("relative/path") +}; + +declare + %test:assertExists +function ft:last-modified() { + let $path := $ft:test-dir || "last-mod-test.txt" + let $_ := file:write-text($path, "test") + return file:last-modified($path) +}; + +declare + %test:assertEquals(5) +function ft:size-file() { + let $path := $ft:test-dir || "size-test.txt" + let $_ := file:write-text($path, "hello") + return file:size($path) +}; + +declare + %test:assertEquals(0) +function ft:size-dir() { + file:size($ft:test-dir) +}; + +(: ======== Read / Write Text ======== :) + +declare + %test:assertEquals("hello world") +function ft:write-read-text() { + let $path := $ft:test-dir || "write-read.txt" + let $_ := file:write-text($path, "hello world") + return file:read-text($path) +}; + +declare + %test:assertEquals("héllo wörld") +function ft:write-read-text-utf8() { + let $path := $ft:test-dir || "write-read-utf8.txt" + let $_ := file:write-text($path, "héllo wörld") + return file:read-text($path) +}; + +declare + %test:assertEquals("line1", "line2", "line3") +function ft:write-read-text-lines() { + let $path := $ft:test-dir || "lines-test.txt" + let $_ := file:write-text-lines($path, ("line1", "line2", "line3")) + return file:read-text-lines($path) +}; + +declare + %test:assertEquals(0) +function ft:write-text-lines-empty() { + let $path := $ft:test-dir || "empty-lines.txt" + let $_ := file:write-text-lines($path, ()) + return count(file:read-text-lines($path)) +}; + +(: Verify newline normalization: CR and CRLF -> LF :) +declare + %test:assertEquals("a", "b", "c") +function ft:read-text-lines-normalization() { + let $path := $ft:test-dir || "crlf-test.txt" + (: Write raw bytes with CRLF line endings :) + let $_ := file:write-text($path, "a b c") + return file:read-text-lines($path) +}; + +(: ======== Read / Write Binary ======== :) + +declare + %test:assertExists +function ft:write-read-binary() { + let $path := $ft:test-dir || "binary-test.bin" + let $data := xs:base64Binary("SGVsbG8gV29ybGQ=") (: "Hello World" :) + let $_ := file:write-binary($path, $data) + return file:read-binary($path) +}; + +(: ======== Append ======== :) + +declare + %test:assertEquals("helloworld") +function ft:append-text() { + let $path := $ft:test-dir || "append-test.txt" + let $_ := file:write-text($path, "hello") + let $_ := file:append-text($path, "world") + return file:read-text($path) +}; + +declare + %test:assertEquals("line1", "line2", "line3", "line4") +function ft:append-text-lines() { + let $path := $ft:test-dir || "append-lines.txt" + let $_ := file:write-text-lines($path, ("line1", "line2")) + let $_ := file:append-text-lines($path, ("line3", "line4")) + return file:read-text-lines($path) +}; + +(: ======== Directory Operations ======== :) + +declare + %test:assertTrue +function ft:create-dir() { + let $dir := $ft:test-dir || "subdir/" + let $_ := file:create-dir($dir) + return file:is-dir($dir) +}; + +declare + %test:assertTrue +function ft:create-dir-nested() { + let $dir := $ft:test-dir || "a/b/c/" + let $_ := file:create-dir($dir) + return file:is-dir($dir) +}; + +declare + %test:assertExists +function ft:create-temp-dir() { + let $dir := file:create-temp-dir("test-", "-dir", $ft:test-dir) + return + if (file:is-dir($dir)) then + $dir + else + () +}; + +declare + %test:assertExists +function ft:create-temp-file() { + let $f := file:create-temp-file("test-", ".tmp", $ft:test-dir) + return + if (file:is-file($f)) then + $f + else + () +}; + +(: ======== List / Children / Descendants ======== :) + +declare + %test:assertExists +function ft:list-dir() { + let $_ := file:write-text($ft:test-dir || "list-a.txt", "a") + let $_ := file:write-text($ft:test-dir || "list-b.txt", "b") + return file:list($ft:test-dir) +}; + +declare + %test:assertTrue +function ft:list-contains-file() { + let $_ := file:write-text($ft:test-dir || "list-find.txt", "find me") + return "list-find.txt" = file:list($ft:test-dir) +}; + +declare + %test:assertTrue +function ft:list-dir-trailing-separator() { + let $_ := file:create-dir($ft:test-dir || "list-subdir") + let $entries := file:list($ft:test-dir) + return some $e in $entries satisfies + starts-with($e, "list-subdir") and ends-with($e, file:dir-separator()) +}; + +declare + %test:assertTrue +function ft:list-recursive() { + let $_ := file:create-dir($ft:test-dir || "rec-dir") + let $_ := file:write-text($ft:test-dir || "rec-dir/nested.txt", "nested") + let $entries := file:list($ft:test-dir, true()) + return some $e in $entries satisfies contains($e, "nested.txt") +}; + +declare + %test:assertTrue +function ft:list-pattern() { + let $_ := file:write-text($ft:test-dir || "pat-a.txt", "a") + let $_ := file:write-text($ft:test-dir || "pat-b.xml", "b") + let $entries := file:list($ft:test-dir, false(), "*.txt") + return + (some $e in $entries satisfies $e = "pat-a.txt") + and not(some $e in $entries satisfies $e = "pat-b.xml") +}; + +declare + %test:assertExists +function ft:children() { + let $_ := file:write-text($ft:test-dir || "child.txt", "x") + return file:children($ft:test-dir) +}; + +declare + %test:assertTrue +function ft:children-absolute-paths() { + let $_ := file:write-text($ft:test-dir || "child-abs.txt", "x") + let $children := file:children($ft:test-dir) + return every $c in $children satisfies file:is-absolute($c) +}; + +declare + %test:assertExists +function ft:descendants() { + let $_ := file:create-dir($ft:test-dir || "desc-dir") + let $_ := file:write-text($ft:test-dir || "desc-dir/deep.txt", "deep") + return file:descendants($ft:test-dir) +}; + +declare + %test:assertExists +function ft:list-roots() { + file:list-roots() +}; + +(: ======== Copy / Move / Delete ======== :) + +declare + %test:assertEquals("copy content") +function ft:copy-file() { + let $src := $ft:test-dir || "copy-src.txt" + let $dst := $ft:test-dir || "copy-dst.txt" + let $_ := file:write-text($src, "copy content") + let $_ := file:copy($src, $dst) + return file:read-text($dst) +}; + +declare + %test:assertTrue +function ft:move-file() { + let $src := $ft:test-dir || "move-src.txt" + let $dst := $ft:test-dir || "move-dst.txt" + let $_ := file:write-text($src, "move content") + let $_ := file:move($src, $dst) + return + file:exists($dst) and not(file:exists($src)) +}; + +declare + %test:assertFalse +function ft:delete-file() { + let $path := $ft:test-dir || "delete-me.txt" + let $_ := file:write-text($path, "bye") + let $_ := file:delete($path) + return file:exists($path) +}; + +declare + %test:assertFalse +function ft:delete-dir-recursive() { + let $dir := $ft:test-dir || "delete-dir/" + let $_ := file:create-dir($dir) + let $_ := file:write-text($dir || "inner.txt", "inner") + let $_ := file:delete($dir, true()) + return file:exists($dir) +}; + +(: ======== Path Functions ======== :) + +declare + %test:assertEquals("file.txt") +function ft:name() { + file:name("/some/path/file.txt") +}; + +declare + %test:assertEquals("") +function ft:name-root() { + file:name("/") +}; + +declare + %test:assertExists +function ft:parent() { + file:parent("/some/path/file.txt") +}; + +declare + %test:assertEmpty +function ft:parent-root() { + file:parent("/") +}; + +declare + %test:assertTrue +function ft:parent-ends-with-separator() { + ends-with(file:parent("/some/path/file.txt"), file:dir-separator()) +}; + +declare + %test:assertExists +function ft:path-to-native() { + (: temp-dir definitely exists :) + file:path-to-native(file:temp-dir()) +}; + +declare + %test:assertTrue +function ft:path-to-uri-starts-with-file() { + starts-with(string(file:path-to-uri("/tmp")), "file:/") +}; + +declare + %test:assertTrue +function ft:resolve-path-absolute() { + let $resolved := file:resolve-path("relative") + return file:is-absolute($resolved) +}; + +declare + %test:assertTrue +function ft:resolve-path-with-base() { + let $resolved := file:resolve-path("child.txt", "/some/base/") + return contains($resolved, "base") and contains($resolved, "child.txt") +}; + +(: ======== Serialized Write / Append ======== :) + +declare + %test:assertTrue +function ft:write-xml() { + let $path := $ft:test-dir || "write-xml.xml" + let $_ := file:write($path, text) + let $content := file:read-text($path) + return contains($content, "") and contains($content, "text") +}; + +declare + %test:assertTrue +function ft:append-xml() { + let $path := $ft:test-dir || "append-xml.xml" + let $_ := file:write($path, ) + let $_ := file:append($path, ) + let $content := file:read-text($path) + return contains($content, "first") and contains($content, "second") +}; + +(: ======== Error Conditions ======== :) + +declare + %test:assertError("file:not-found") +function ft:read-text-not-found() { + file:read-text($ft:test-dir || "does-not-exist.txt") +}; + +declare + %test:assertError("file:is-dir") +function ft:read-text-is-dir() { + file:read-text($ft:test-dir) +}; + +declare + %test:assertError("file:not-found") +function ft:last-modified-not-found() { + file:last-modified($ft:test-dir || "does-not-exist.txt") +}; + +declare + %test:assertError("file:not-found") +function ft:size-not-found() { + file:size($ft:test-dir || "does-not-exist.txt") +}; + +declare + %test:assertError("file:not-found") +function ft:delete-not-found() { + file:delete($ft:test-dir || "does-not-exist.txt") +}; + +declare + %test:assertError("file:not-found") +function ft:children-not-found() { + file:children($ft:test-dir || "does-not-exist/") +}; + +declare + %test:assertError("file:no-dir") +function ft:children-not-dir() { + let $path := $ft:test-dir || "not-a-dir.txt" + let $_ := file:write-text($path, "x") + return file:children($path) +}; + +declare + %test:assertError("file:no-dir") +function ft:write-text-no-parent-dir() { + file:write-text($ft:test-dir || "no-such-parent/file.txt", "hello") +}; + +declare + %test:assertError("file:unknown-encoding") +function ft:read-text-bad-encoding() { + let $path := $ft:test-dir || "encoding-test.txt" + let $_ := file:write-text($path, "hello") + return file:read-text($path, "not-a-real-encoding") +}; + +declare + %test:assertError("file:not-found") +function ft:path-to-native-not-found() { + file:path-to-native("/this/path/surely/does/not/exist/anywhere") +}; diff --git a/extensions/modules/file/src/test/xquery/modules/file/sync-serialize.xqm b/extensions/expath/src/test/xquery/org/expath/exist/file/sync-serialize.xqm similarity index 94% rename from extensions/modules/file/src/test/xquery/modules/file/sync-serialize.xqm rename to extensions/expath/src/test/xquery/org/expath/exist/file/sync-serialize.xqm index c26c201a77a..6c6cbfd2d12 100644 --- a/extensions/modules/file/src/test/xquery/modules/file/sync-serialize.xqm +++ b/extensions/expath/src/test/xquery/org/expath/exist/file/sync-serialize.xqm @@ -23,11 +23,11 @@ xquery version "3.1"; (: : test serialization defaults and setting different serialization options on - : file:sync#3 + : util:file-sync#3 :) -module namespace syse="http://exist-db.org/xquery/test/file/sync-serialize"; - +module namespace syse="http://exist-db.org/xquery/test/util/sync-serialize"; +import module namespace util="http://exist-db.org/xquery/util"; import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; @@ -61,7 +61,7 @@ declare %test:assertEquals("true", "true") function syse:defaults() { let $directory := helper:get-test-directory($syse:suite) - let $sync := file:sync( + let $sync := util:file-sync( $fixtures:collection, $directory, () @@ -85,7 +85,7 @@ declare %test:assertEquals("true", "true") function syse:indent-no() { let $directory := helper:get-test-directory($syse:suite) - let $sync := file:sync( + let $sync := util:file-sync( $fixtures:collection, $directory, map{"indent": false()} @@ -109,7 +109,7 @@ declare %test:assertEquals("true", "true") function syse:indent-yes() { let $directory := helper:get-test-directory($syse:suite) - let $sync := file:sync( + let $sync := util:file-sync( $fixtures:collection, $directory, map{"indent": true()} @@ -133,7 +133,7 @@ declare %test:assertEquals("true", "true") function syse:omit-xml-declaration-no() { let $directory := helper:get-test-directory($syse:suite) - let $sync := file:sync( + let $sync := util:file-sync( $fixtures:collection, $directory, map{"omit-xml-declaration": false()} @@ -157,7 +157,7 @@ declare %test:assertEquals("true", "true") function syse:omit-xml-declaration-yes() { let $directory := helper:get-test-directory($syse:suite) - let $sync := file:sync( + let $sync := util:file-sync( $fixtures:collection, $directory, map{"omit-xml-declaration": true()} @@ -179,7 +179,7 @@ declare %test:assertEquals("true", "true") function syse:unindented-no-declaration() { let $directory := helper:get-test-directory($syse:suite) - let $sync := file:sync( + let $sync := util:file-sync( $fixtures:collection, $directory, map{ @@ -204,7 +204,7 @@ declare %test:assertEquals("true", "true") function syse:insert-final-newline-yes() { let $directory := helper:get-test-directory($syse:suite) - let $sync := file:sync( + let $sync := util:file-sync( $fixtures:collection, $directory, map{ "exist:insert-final-newline": true() } @@ -228,7 +228,7 @@ declare %test:assertEquals("true", "true") function syse:insert-final-newline-no() { let $directory := helper:get-test-directory($syse:suite) - let $sync := file:sync( + let $sync := util:file-sync( $fixtures:collection, $directory, map{ "exist:insert-final-newline": false() } diff --git a/extensions/modules/file/src/test/xquery/modules/file/sync.xqm b/extensions/expath/src/test/xquery/org/expath/exist/file/sync.xqm similarity index 90% rename from extensions/modules/file/src/test/xquery/modules/file/sync.xqm rename to extensions/expath/src/test/xquery/org/expath/exist/file/sync.xqm index 645ee8bb024..f9ee5dd1c74 100644 --- a/extensions/modules/file/src/test/xquery/modules/file/sync.xqm +++ b/extensions/expath/src/test/xquery/org/expath/exist/file/sync.xqm @@ -21,9 +21,10 @@ :) xquery version "3.1"; -module namespace sync="http://exist-db.org/xquery/test/file/sync"; - +module namespace sync="http://exist-db.org/xquery/test/util/sync"; +import module namespace util="http://exist-db.org/xquery/util"; +import module namespace file="http://expath.org/ns/file"; import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; @@ -47,7 +48,7 @@ function sync:tear-down() { declare %test:assertTrue function sync:simple() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), () @@ -62,7 +63,7 @@ function sync:simple() { declare %test:assertTrue function sync:empty-options-map() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), map{} @@ -77,7 +78,7 @@ function sync:empty-options-map() { declare %test:assertError function sync:deprecated-options() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), $fixtures:mod-date @@ -92,7 +93,7 @@ function sync:deprecated-options() { declare %test:assertError("err:XPTY0004") function sync:bad-options-1() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), xs:date("2012-12-21") @@ -102,7 +103,7 @@ function sync:bad-options-1() { declare %test:assertError("err:XPTY0004") function sync:bad-options-2() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), "2012-12-21T10:12:21" @@ -112,7 +113,7 @@ function sync:bad-options-2() { declare %test:assertError("err:XPTY0004") function sync:bad-options-3() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), "lizard" @@ -122,7 +123,7 @@ function sync:bad-options-3() { declare %test:assertError("err:XPTY0004") function sync:bad-options-4() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), "" @@ -136,7 +137,7 @@ function sync:bad-options-4() { declare %test:assertError function sync:bad-options-5() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), (1, map{}, "") @@ -146,7 +147,7 @@ function sync:bad-options-5() { declare %test:assertError("err:XPTY0004") function sync:bad-options-6() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), map{ "prune": "true" } @@ -156,7 +157,7 @@ function sync:bad-options-6() { declare %test:assertError("err:XPTY0004") function sync:bad-options-7() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), map{ "prune": "no" } @@ -166,7 +167,7 @@ function sync:bad-options-7() { declare %test:assertError("err:XPTY0004") function sync:bad-options-8() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), map{ "after": 1234325 } @@ -176,7 +177,7 @@ function sync:bad-options-8() { declare %test:assertError("err:XPTY0004") function sync:bad-options-9() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($sync:suite), map{ "excludes": [] } @@ -190,7 +191,7 @@ function sync:do-not-prune() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "prune": false() } @@ -209,7 +210,7 @@ function sync:prune() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "prune": true() } @@ -228,7 +229,7 @@ function sync:prune-with-excludes-matching-none() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "prune": true(), "excludes": "*.txt" } @@ -247,7 +248,7 @@ function sync:after() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "after": $fixtures:mod-date } @@ -266,7 +267,7 @@ function sync:after-mod-date-2() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "after": $fixtures:mod-date-2 } @@ -285,7 +286,7 @@ function sync:after-with-excludes() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "after": $fixtures:mod-date, "excludes": ".env" } @@ -303,22 +304,22 @@ function sync:prune-with-after-and-excludes() { let $directory := helper:get-test-directory($sync:suite) let $_ := helper:setup-fs-extra($directory) let $_ := ( - file:serialize-binary( - util:string-to-binary("1"), - $directory || "/excluded.xq" + file:write-binary( + $directory || "/excluded.xq", + util:string-to-binary("1") ), - file:serialize-binary( - util:string-to-binary("1"), - $directory || "/pruned.xql" + file:write-binary( + $directory || "/pruned.xql", + util:string-to-binary("1") ), - file:serialize-binary( - util:string-to-binary("oh oh"), - $directory || "/readme.md" + file:write-binary( + $directory || "/readme.md", + util:string-to-binary("oh oh") ) ) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ @@ -341,7 +342,7 @@ function sync:prunes-a-directory() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "prune": true(), "excludes": ".*" } @@ -360,7 +361,7 @@ function sync:prunes-a-file() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "prune": true(), "excludes": "test" || $helper:path-separator || "*" } @@ -379,7 +380,7 @@ function sync:prunes-with-multiple-excludes() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ @@ -403,14 +404,14 @@ function sync:twice() { : syncing to disk, see https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8177809 :) let $_ := util:wait(1000) - let $_ := file:sync( + let $_ := util:file-sync( $fixtures:collection, $directory, () ) return - file:sync( + util:file-sync( $fixtures:collection, $directory, () diff --git a/extensions/modules/file/src/test/xquery/modules/file/syncmod.xqm b/extensions/expath/src/test/xquery/org/expath/exist/file/syncmod.xqm similarity index 94% rename from extensions/modules/file/src/test/xquery/modules/file/syncmod.xqm rename to extensions/expath/src/test/xquery/org/expath/exist/file/syncmod.xqm index 178010889f0..ee1859dcf5d 100644 --- a/extensions/modules/file/src/test/xquery/modules/file/syncmod.xqm +++ b/extensions/expath/src/test/xquery/org/expath/exist/file/syncmod.xqm @@ -21,9 +21,10 @@ :) xquery version "3.1"; -module namespace syncmod="http://exist-db.org/xquery/test/file/syncmod"; - +module namespace syncmod="http://exist-db.org/xquery/test/util/syncmod"; +import module namespace util="http://exist-db.org/xquery/util"; +import module namespace file="http://expath.org/ns/file"; import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; @@ -53,7 +54,7 @@ function syncmod:tearDown() { declare %test:assertTrue function syncmod:simple() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($syncmod:suite), () @@ -68,7 +69,7 @@ function syncmod:simple() { declare %test:assertTrue function syncmod:empty-options() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($syncmod:suite), map{} @@ -83,7 +84,7 @@ function syncmod:empty-options() { declare %test:assertError function syncmod:deprecated-options() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($syncmod:suite), $fixtures:mod-date @@ -102,7 +103,7 @@ function syncmod:do-not-prune() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "prune": false() } @@ -121,7 +122,7 @@ function syncmod:prune() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "prune": true() } @@ -140,7 +141,7 @@ function syncmod:prune-with-excludes-matching-none() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "prune": true(), "excludes": "*.txt" } @@ -159,7 +160,7 @@ function syncmod:after() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "after": $fixtures:mod-date } @@ -175,7 +176,7 @@ function syncmod:after() { declare %test:assertTrue function syncmod:after-mod-date-2() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($syncmod:suite), map{ "after": $fixtures:mod-date-2 } @@ -191,7 +192,7 @@ declare %test:pending("this would only work if exclude patterns would exclude DB resources from syncing") %test:assertTrue function syncmod:exclude-changed-files() { - file:sync( + util:file-sync( $fixtures:collection, helper:get-test-directory($syncmod:suite), map{ "excludes":("*.txt", "data/*"), "after": $fixtures:mod-date } @@ -210,7 +211,7 @@ function syncmod:prune-with-after-and-excludes-matching-none() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ @@ -233,7 +234,7 @@ function syncmod:prune-with-after-and-excludes-matching-all() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ @@ -256,7 +257,7 @@ function syncmod:prunes-a-directory-with-after() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ "prune": true(), "excludes": ".*", "after": $fixtures:mod-date } @@ -276,7 +277,7 @@ function syncmod:prunes-a-file-with-after() { let $_ := helper:setup-fs-extra($directory) return - file:sync( + util:file-sync( $fixtures:collection, $directory, map{ diff --git a/extensions/modules/file/pom.xml b/extensions/modules/file/pom.xml deleted file mode 100644 index 08da6131c37..00000000000 --- a/extensions/modules/file/pom.xml +++ /dev/null @@ -1,201 +0,0 @@ - - - - 4.0.0 - - - org.exist-db - exist-parent - 7.0.0-SNAPSHOT - ../../../exist-parent - - - exist-file - jar - - eXist-db File Module - eXist-db XQuery File Module - - - scm:git:https://github.com/exist-db/exist.git - scm:git:https://github.com/exist-db/exist.git - scm:git:https://github.com/exist-db/exist.git - HEAD - - - - - org.exist-db - exist-core - ${project.version} - - - - org.apache.logging.log4j - log4j-api - - - - com.evolvedbinary.j8fu - j8fu - test - - - com.evolvedbinary.multilock - multilock - - - - commons-io - commons-io - test - - - - net.sf.xmldb-org - xmldb-api - - - - jakarta.xml.bind - jakarta.xml.bind-api - test - - - - org.eclipse.angus - angus-activation - runtime - - - - org.apache.ant - ant - - - - - org.eclipse.jetty - jetty-deploy - test - - - - org.eclipse.jetty - jetty-jmx - test - - - - junit - junit - test - - - - ${project.groupId} - exist-jetty-config - ${project.version} - test - - - - org.apache.httpcomponents - httpcore - test - - - - org.apache.httpcomponents - fluent-hc - test - - - - commons-codec - commons-codec - test - - - - - - - - src/test/resources - false - - - src/test/resources-filtered - true - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - analyze - - analyze-only - - - true - - org.eclipse.angus:angus-activation:jar:${eclipse.angus-activation.version} - - - ${project.groupId}:exist-jetty-config:jar:${project.version} - org.eclipse.jetty:jetty-deploy:jar:${jetty.version} - org.eclipse.jetty:jetty-jmx:jar:${jetty.version} - - - com.sun.xml.bind:jaxb-impl:jar:${jaxb.impl.version} - - - - net.sf.xmldb-org:xmldb-api:jar - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - @{jacocoArgLine} - - ${project.basedir}/../../../exist-jetty-config/target/classes/org/exist/jetty - ${project.build.testOutputDirectory}/conf.xml - ${project.build.testOutputDirectory}/standalone-webapp - ${project.build.testOutputDirectory}/log4j2.xml - - - - - - diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Directory.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Directory.java deleted file mode 100644 index 9cde989a93d..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Directory.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Date; -import java.util.stream.Stream; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.dom.memtree.MemTreeBuilder; -import org.exist.util.FileUtils; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.BooleanValue; -import org.exist.xquery.value.DateTimeValue; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.NodeValue; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -/** - * eXist File Module Extension DirectoryList - * - * Enumerate a list of files and directories, including their size and modification time, found in - * a specified directory - * - * @author Dannes Wessels - * @author Andrzej Taramina - * @author Leif-Jöran Olsson - * @serial 2010-05-12 - * @version 1.2 - * - * @see org.exist.xquery.BasicFunction#BasicFunction(org.exist.xquery.XQueryContext, org.exist.xquery.FunctionSignature) - */ -public class Directory extends BasicFunction { - - private final static Logger logger = LogManager.getLogger(Directory.class); - - final static String NAMESPACE_URI = FileModule.NAMESPACE_URI; - final static String PREFIX = FileModule.PREFIX; - - public final static FunctionSignature[] signatures = { - new FunctionSignature( - new QName("list", NAMESPACE_URI, PREFIX), - "List all files and directories under the specified directory. " - + "This method is only available to the DBA role.", - new SequenceType[]{ - new FunctionParameterSequenceType("path", - Type.ITEM, Cardinality.EXACTLY_ONE, - "The directory path or URI in the file system."), - }, - new FunctionReturnSequenceType(Type.NODE, Cardinality.ZERO_OR_MORE, - "a node describing file and directory names and meta data.")) - }; - - public Directory(final XQueryContext context, final FunctionSignature signature) { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { - - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" - + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - - final String inputPath = args[0].getStringValue(); - final Path directoryPath = FileModuleHelper.getFile(inputPath, this); - - if (logger.isDebugEnabled()) { - logger.debug("Listing matching files in directory: {}", directoryPath.toAbsolutePath().toString()); - } - - if(!Files.isDirectory(directoryPath)) { - throw new XPathException(this, "'" + inputPath + "' does not point to a valid directory."); - } - - // Get list of files, null if baseDir does not point to a directory - context.pushDocumentContext(); - try(final Stream scannedFiles = Files.list(directoryPath)) { - - final MemTreeBuilder builder = context.getDocumentBuilder(); - - builder.startDocument(); - builder.startElement(new QName("list", null, null), null); - - scannedFiles.forEach(entry -> { - if (logger.isDebugEnabled()) { - logger.debug("Found: {}", entry.toAbsolutePath().toString()); - } - - String entryType = "unknown"; - if (Files.isRegularFile(entry)) { - entryType = "file"; - } else if (Files.isDirectory(entry)) { - entryType = "directory"; - } - - builder.startElement(new QName(entryType, NAMESPACE_URI, PREFIX), null); - - builder.addAttribute(new QName("name", null, null), FileUtils.fileName(entry)); - - try { - if (Files.isRegularFile(entry)) { - final Long sizeLong = Files.size(entry); - String sizeString = Long.toString(sizeLong); - String humanSize = getHumanSize(sizeLong, sizeString); - - builder.addAttribute(new QName("size", null, null), sizeString); - builder.addAttribute(new QName("human-size", null, null), humanSize); - } - - builder.addAttribute(new QName("modified", null, null), - new DateTimeValue(this, new Date(Files.getLastModifiedTime(entry).toMillis())).getStringValue()); - - builder.addAttribute(new QName("hidden", null, null), - new BooleanValue(this, Files.isHidden(entry)).getStringValue()); - - builder.addAttribute(new QName("canRead", null, null), - new BooleanValue(this, Files.isReadable(entry)).getStringValue()); - - builder.addAttribute(new QName("canWrite", null, null), - new BooleanValue(this, Files.isWritable(entry)).getStringValue()); - } catch (final IOException | XPathException ioe) { - LOG.warn(ioe); - } - builder.endElement(); - - }); - - builder.endElement(); - - return (NodeValue) builder.getDocument().getDocumentElement(); - } catch(final IOException ioe) { - throw new XPathException(this, ioe); - } finally { - context.popDocumentContext(); - } - } - - private String getHumanSize(final Long sizeLong, final String sizeString) { - String humanSize = "n/a"; - int sizeDigits = sizeString.length(); - - if (sizeDigits < 4) { - humanSize = Long.toString(Math.abs(sizeLong)); - - } else if (sizeDigits >= 4 && sizeDigits <= 6) { - if (sizeLong < 1024) { - // We don't want 0KB för e.g. 1006 Bytes. - humanSize = Long.toString(Math.abs(sizeLong)); - } else { - humanSize = Math.abs(sizeLong / 1024) + "KB"; - } - - } else if (sizeDigits >= 7 && sizeDigits <= 9) { - if (sizeLong < 1048576) { - humanSize = Math.abs(sizeLong / 1024) + "KB"; - } else { - humanSize = Math.abs(sizeLong / (1024 * 1024)) + "MB"; - } - - } else if (sizeDigits > 9) { - if (sizeLong < 1073741824) { - humanSize = Math.abs((sizeLong / (1024 * 1024))) + "MB"; - } else { - humanSize = Math.abs((sizeLong / (1024 * 1024 * 1024))) + "GB"; - } - } - return humanSize; - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryCreate.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryCreate.java deleted file mode 100644 index 2868b8633e1..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryCreate.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.io.IOException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.BooleanValue; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -/** - * @see java.nio.file.Files#createDirectory(Path, FileAttribute[]) - * @see java.nio.file.Files#createDirectories(Path, FileAttribute[]) - * - * @author Dannes Wessels - * - */ -public class DirectoryCreate extends BasicFunction { - - private final static Logger logger = LogManager.getLogger(DirectoryCreate.class); - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName("mkdir", FileModule.NAMESPACE_URI, FileModule.PREFIX), - "Create a directory. This method is only available to the DBA role.", - new SequenceType[]{ - new FunctionParameterSequenceType("path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the directory") - }, - new FunctionReturnSequenceType(Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "true if successful, false otherwise") - ), - new FunctionSignature( - new QName("mkdirs", FileModule.NAMESPACE_URI, FileModule.PREFIX), - "Create a directory including any necessary but nonexistent parent directories. " + - "This method is only available to the DBA role.", - new SequenceType[]{ - new FunctionParameterSequenceType("path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the directory") - }, - new FunctionReturnSequenceType(Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "true if successful, false otherwise") - ) - }; - - - public DirectoryCreate(final XQueryContext context, final FunctionSignature signature) { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { - - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - Sequence created = BooleanValue.FALSE; - - final String inputPath = args[0].itemAt(0).getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - try { - if (isCalledAs("mkdir") && Files.notExists(file)) { - Files.createDirectory(file); - created = BooleanValue.TRUE; - } else if (isCalledAs("mkdirs")) { - Files.createDirectories(file); - created = BooleanValue.TRUE; - } - } catch (final FileAlreadyExistsException e) { - created = BooleanValue.FALSE; - } catch(final IOException e) { - throw new XPathException(this, e); - } - - return created; - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryList.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryList.java deleted file mode 100644 index 10893583c83..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryList.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Date; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.apache.tools.ant.DirectoryScanner; -import org.exist.dom.QName; -import org.exist.dom.memtree.MemTreeBuilder; -import org.exist.util.FileUtils; -import org.exist.xquery.*; -import org.exist.xquery.value.DateTimeValue; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.NodeValue; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -import static org.exist.xquery.modules.file.FileErrorCode.DIRECTORY_NOT_FOUND; - -/** - * eXist File Module Extension DirectoryList - *

- * Enumerate a list of files, including their size and modification time, found - * in a specified directory, using a pattern - * - * @author Andrzej Taramina - * @author ljo - * @version 1.2 - * @serial 2009-08-09 - * @see org.exist.xquery.BasicFunction#BasicFunction(org.exist.xquery.XQueryContext, - * org.exist.xquery.FunctionSignature) - */ -public class DirectoryList extends BasicFunction { - - static final String NAMESPACE_URI = FileModule.NAMESPACE_URI; - static final String PREFIX = FileModule.PREFIX; - public static final FunctionSignature[] signatures = { - new FunctionSignature( - new QName("directory-list", NAMESPACE_URI, PREFIX), - "List all files, including their file size and modification time, " - + "found in or below a directory, $directory. Files are located in the server's " - + "file system, using filename patterns, $pattern. File pattern matching is based " - + "on code from Apache's Ant, thus following the same conventions. For example:\n\n" - + "'*.xml' matches any file ending with .xml in the current directory,\n- '**/*.xml' matches files " - + "in any directory below the specified directory. This method is only available to the DBA role.", - new SequenceType[]{ - new FunctionParameterSequenceType("path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The base directory path or URI in the file system where the files are located."), - new FunctionParameterSequenceType("pattern", Type.STRING, - Cardinality.ZERO_OR_MORE, "The file name pattern") - }, - new FunctionReturnSequenceType(Type.NODE, - Cardinality.ZERO_OR_ONE, "a node fragment that shows all matching " - + "filenames, including their file size and modification time, and " - + "the subdirectory they were found in") - ) - }; - static final QName FILE_ELEMENT = new QName("file", NAMESPACE_URI, PREFIX); - static final QName LIST_ELEMENT = new QName("list", NAMESPACE_URI, PREFIX); - - static final QName DIRECTORY_ATTRIBUTE = new QName("directory", null, null); - static final QName NAME_ATTRIBUTE = new QName("name", null, null); - static final QName SIZE_ATTRIBUTE = new QName("size", null, null); - static final QName HUMAN_SIZE_ATTRIBUTE = new QName("human-size", null, null); - static final QName MODIFIED_ATTRIBUTE = new QName("modified", null, null); - static final QName SUBDIR_ATTRIBUTE = new QName("subdir", null, null); - private static final Logger logger = LogManager.getLogger(DirectoryList.class); - - public DirectoryList(final XQueryContext context, final FunctionSignature signature) { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - final String inputPath = args[0].getStringValue(); - final Path baseDir = FileModuleHelper.getFile(inputPath, this); - - final Sequence patterns = args[1]; - - if (logger.isDebugEnabled()) { - logger.debug("Listing matching files in directory: {}", baseDir); - } - - context.pushDocumentContext(); - final MemTreeBuilder builder = context.getDocumentBuilder(); - - builder.startDocument(); - builder.startElement(LIST_ELEMENT, null); - builder.addAttribute(DIRECTORY_ATTRIBUTE, baseDir.toString()); - try { - final int patternsLen = patterns.getItemCount(); - final String[] includes = new String[patternsLen]; - for (int i = 0; i < patternsLen; i++) { - includes[i] = patterns.itemAt(0).getStringValue(); - } - - final DirectoryScanner directoryScanner = new DirectoryScanner(); - directoryScanner.setIncludes(includes); - directoryScanner.setBasedir(baseDir.toFile()); - directoryScanner.setCaseSensitive(true); - directoryScanner.scan(); - - for (final String includedFile : directoryScanner.getIncludedFiles()) { - final Path file = baseDir.resolve(includedFile); - - if (logger.isDebugEnabled()) { - logger.debug("Found: {}", file.toAbsolutePath()); - } - - final String relPath = file.toString().substring(baseDir.toString().length() + 1); - - builder.startElement(FILE_ELEMENT, null); - builder.addAttribute(NAME_ATTRIBUTE, FileUtils.fileName(file)); - - final long sizeLong = FileUtils.sizeQuietly(file); - builder.addAttribute(SIZE_ATTRIBUTE, Long.toString(sizeLong)); - builder.addAttribute(HUMAN_SIZE_ATTRIBUTE, getHumanSize(sizeLong)); - - builder.addAttribute(MODIFIED_ATTRIBUTE, - new DateTimeValue(this, - new Date(Files.getLastModifiedTime(file).toMillis())).getStringValue()); - - final int lastSeparatorPosition = relPath.lastIndexOf(java.io.File.separatorChar); - if (lastSeparatorPosition >= 0) { - final String relDir = relPath.substring(0, lastSeparatorPosition); - if (!relDir.isEmpty()) { - builder.addAttribute(SUBDIR_ATTRIBUTE, - relDir.replace(java.io.File.separatorChar, '/')); - } - } - - builder.endElement(); - } - - builder.endElement(); - - return (NodeValue) builder.getDocument().getDocumentElement(); - } catch (final IOException | IllegalStateException e) { - throw new XPathException(this, DIRECTORY_NOT_FOUND, e.getMessage()); - } finally { - context.popDocumentContext(); - } - } - - private String getHumanSize(final Long sizeLong) { - if (sizeLong < 1024) { - return Math.abs(sizeLong) + "B"; - } - if (sizeLong < 1048576) { - return Math.abs(sizeLong / 1024) + "KB"; - } - if (sizeLong < 1073741824) { - return Math.abs((sizeLong / (1024 * 1024))) + "MB"; - } - return Math.abs((sizeLong / (1024 * 1024 * 1024))) + "GB"; - } - -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileDelete.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileDelete.java deleted file mode 100644 index e16dcebbb5c..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileDelete.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.util.FileUtils; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.BooleanValue; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -/** - * @see org.exist.util.FileUtils#deleteQuietly(Path) - * - * @author Andrzej Taramina - */ -public class FileDelete extends BasicFunction { - - private final static Logger logger = LogManager.getLogger(FileDelete.class); - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( "delete", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Delete a file or directory. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file" ) - }, - new FunctionReturnSequenceType( Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "true if successful, false otherwise" ) ) - }; - - public FileDelete(final XQueryContext context, final FunctionSignature signature ) - { - super( context, signature ); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence ) throws XPathException { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - final String inputPath = args[0].getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - return BooleanValue.valueOf(FileUtils.deleteQuietly(file)); - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileErrorCode.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileErrorCode.java deleted file mode 100644 index b4277a3d7a7..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileErrorCode.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import org.exist.dom.QName; -import org.exist.xquery.ErrorCodes; - -class FileErrorCode extends ErrorCodes.ErrorCode { - public static final ErrorCodes.ErrorCode DIRECTORY_NOT_FOUND = new FileErrorCode("DIRECTORY_NOT_FOUND", - "The directory could not be found."); - - FileErrorCode(final String code, final String description) { - super(new QName(code, FileModule.NAMESPACE_URI, FileModule.PREFIX), description); - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileExists.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileExists.java deleted file mode 100644 index 7965d6027c2..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileExists.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.BooleanValue; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -/** - * @see java.nio.file.Files#exists(Path, LinkOption...) - * - * @author Andrzej Taramina - * - */ -public class FileExists extends BasicFunction { - - private final static Logger logger = LogManager.getLogger(FileExists.class); - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( "exists", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Tests if a file or directory exists. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file in the file system" ) - }, - new FunctionReturnSequenceType(Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "the boolean value true if the file exists, false otherwise" ) ) - }; - - public FileExists(final XQueryContext context, final FunctionSignature signature ) - { - super( context, signature ); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException - { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - final String inputPath = args[0].getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - return BooleanValue.valueOf(Files.exists(file)); - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsDirectory.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsDirectory.java deleted file mode 100644 index 256845935c0..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsDirectory.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.BooleanValue; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -/** - * @see java.nio.file.Files#isDirectory(Path, LinkOption...) - * - * @author Andrzej Taramina - * @author Loren Cahlander - */ -public class FileIsDirectory extends BasicFunction { - - private final static Logger logger = LogManager.getLogger(FileIsDirectory.class); - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( "is-directory", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Tests if a path is a directory. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file or directory" ) - }, - new FunctionReturnSequenceType( Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "true if the path is a directory" ) ) - }; - - public FileIsDirectory(final XQueryContext context, final FunctionSignature signature) - { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException - { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - final String inputPath = args[0].getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - return BooleanValue.valueOf(Files.isDirectory(file)); - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsReadable.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsReadable.java deleted file mode 100644 index 4461a4f6b51..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsReadable.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.nio.file.Files; -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.BooleanValue; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -/** - * @see java.nio.file.Files#isReadable(Path) - * - * @author Andrzej Taramina - * @author Loren Cahlander - */ -public class FileIsReadable extends BasicFunction { - - private static final Logger logger = LogManager.getLogger(FileIsReadable.class); - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( "is-readable", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Tests if a file is readable. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file" ) - }, - new FunctionReturnSequenceType( Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "true if file can be read" ) ) - }; - - public FileIsReadable(final XQueryContext context, final FunctionSignature signature) - { - super( context, signature ); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException - { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - final String inputPath = args[0].getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - return BooleanValue.valueOf(Files.isReadable(file)); - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsWriteable.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsWriteable.java deleted file mode 100644 index 787c3deb0b5..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsWriteable.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.nio.file.Files; -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.BooleanValue; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -/** - * @see java.nio.file.Files#isWritable(Path) - * - * @author Andrzej Taramina - * @author Loren Cahlander - */ -public class FileIsWriteable extends BasicFunction { - - private final static Logger logger = LogManager.getLogger(FileIsWriteable.class); - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( "is-writeable", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Tests if a file is writeable. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file" ) - }, - new FunctionReturnSequenceType( Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "true if the file has write permissions" ) ) - }; - - public FileIsWriteable(final XQueryContext context, final FunctionSignature signature) - { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException - { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - final String inputPath = args[0].getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - return BooleanValue.valueOf(Files.isWritable(file)); - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModule.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModule.java deleted file mode 100644 index ef7542a87a3..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModule.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.util.List; -import java.util.Map; -import org.exist.xquery.AbstractInternalModule; -import org.exist.xquery.FunctionDef; - -/** - * eXist File Module Extension - * - * An extension module for the eXist Native XML Database that allows various file-oriented - * activities. - * - * @author Andrzej Taramina - * @author ljo - * @serial 2008-03-06 - * @version 1.0 - * - * @see org.exist.xquery.AbstractInternalModule#AbstractInternalModule(org.exist.xquery.FunctionDef[], java.util.Map) - */ -public class FileModule extends AbstractInternalModule -{ - public final static String NAMESPACE_URI = "http://exist-db.org/xquery/file"; - - public final static String PREFIX = "file"; - public final static String INCLUSION_DATE = "2008-03-07"; - public final static String RELEASED_IN_VERSION = "eXist-1.4"; - - - private final static FunctionDef[] functions = { - new FunctionDef( Directory.signatures[0], Directory.class ), - new FunctionDef( DirectoryList.signatures[0], DirectoryList.class ), - new FunctionDef( FileRead.signatures[0], FileRead.class ), - new FunctionDef( FileRead.signatures[1], FileRead.class ), - new FunctionDef( FileReadBinary.signatures[0], FileReadBinary.class ), - new FunctionDef( FileReadUnicode.signatures[0], FileReadUnicode.class ), - new FunctionDef( FileReadUnicode.signatures[1], FileReadUnicode.class ), - new FunctionDef( SerializeToFile.signatures[0], SerializeToFile.class ), - new FunctionDef( SerializeToFile.signatures[1], SerializeToFile.class ), - new FunctionDef( SerializeToFile.signatures[2], SerializeToFile.class ), - new FunctionDef( SerializeToFile.signatures[3], SerializeToFile.class ), - new FunctionDef( FileExists.signatures[0], FileExists.class ), - new FunctionDef( FileIsReadable.signatures[0], FileIsReadable.class ), - new FunctionDef( FileIsWriteable.signatures[0], FileIsWriteable.class ), - new FunctionDef( FileIsDirectory.signatures[0], FileIsDirectory.class ), - new FunctionDef( FileDelete.signatures[0], FileDelete.class ), - new FunctionDef( FileMove.signatures[0], FileMove.class ), - new FunctionDef( DirectoryCreate.signatures[0], DirectoryCreate.class ), - new FunctionDef( DirectoryCreate.signatures[1], DirectoryCreate.class ), - new FunctionDef( Sync.signature, Sync.class) - }; - - - public FileModule(Map> parameters) - { - super( functions, parameters ); - } - - - public String getNamespaceURI() - { - return( NAMESPACE_URI ); - } - - - public String getDefaultPrefix() { - return( PREFIX ); - } - - - public String getDescription() - { - return( "A module for performing various operations on files and directories stored in the server file system." ); - } - - public String getReleaseVersion() { - return RELEASED_IN_VERSION; - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModuleHelper.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModuleHelper.java deleted file mode 100644 index df5f37c82f2..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModuleHelper.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.net.URI; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.exist.xquery.Expression; -import org.exist.xquery.XPathException; - -/** - * Helper class for FileModule - * - * @author Dannes Wessels - */ - - -public class FileModuleHelper { - - private FileModuleHelper() { - // no instance - } - - /** - * Convert path (URL, file path) to a File object. - * - * @param path Path written as OS specific path or as URL - * @return File object - * @throws XPathException Thrown when the URL cannot be used. - */ - public static Path getFile(String path, final Expression expression) throws XPathException { - if(path.startsWith("file:")){ - try { - return Paths.get(new URI(path)); - } catch (Exception ex) { // catch all (URISyntaxException) - throw new XPathException(expression, path + " is not a valid URI: '"+ ex.getMessage() +"'"); - } - } else { - return Paths.get(path); - } - } - -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileMove.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileMove.java deleted file mode 100644 index 01e6e538417..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileMove.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.io.IOException; -import java.nio.file.CopyOption; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.BooleanValue; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -/** - * @see java.nio.file.Files#move(Path, Path, CopyOption...) - * - * @author Dannes Wessels - * - */ -public class FileMove extends BasicFunction { - - private final static Logger logger = LogManager.getLogger(FileMove.class); - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( "move", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Move (rename) a file or directory. Exact operation is platform dependent. This " + - "method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "original", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file" ), - new FunctionParameterSequenceType( "destination", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file" ) - }, - new FunctionReturnSequenceType( Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "true if successful, false otherwise" ) ) - }; - - - public FileMove(final XQueryContext context, final FunctionSignature signature) - { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException - { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - Sequence moved = BooleanValue.FALSE; - - final String inputPath1 = args[0].getStringValue(); - final Path src = FileModuleHelper.getFile(inputPath1, this); - - final String inputPath2 = args[1].getStringValue(); - final Path dest = FileModuleHelper.getFile(inputPath2, this); - - try { - Files.move(src, dest); - return BooleanValue.TRUE; - } catch(final IOException ioe) { - LOG.error(ioe); - return BooleanValue.FALSE; - } - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileRead.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileRead.java deleted file mode 100644 index 79df63ef2a4..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileRead.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.StringValue; -import org.exist.xquery.value.Type; - -/** - * @author Pierrick Brihaye - * @author Dizzzz - * @author Andrzej Taramina - * @author Loren Cahlander - */ -public class FileRead extends BasicFunction { - - private final static Logger logger = LogManager.getLogger(FileRead.class); - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( "read", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Reads the content of file. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ) - }, - new FunctionReturnSequenceType( Type.STRING, - Cardinality.ZERO_OR_ONE, "the file contents" ) ), - - new FunctionSignature( - new QName( "read", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Reads the content of file. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ), - new FunctionParameterSequenceType( "encoding", Type.STRING, - Cardinality.EXACTLY_ONE, "The encoding type for the file" ) - }, - new FunctionReturnSequenceType( Type.STRING, - Cardinality.ZERO_OR_ONE, "the file contents" ) ) - }; - - public FileRead(final XQueryContext context, final FunctionSignature signature) - { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException - { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - final String inputPath = args[0].getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - final Charset encoding; - if(args.length == 2) { - encoding = Charset.forName(args[1].getStringValue()); - } else { - encoding = StandardCharsets.UTF_8; - } - - try { - return new StringValue(this, Files.readString(file, encoding)); - } catch(final IOException e ) { - throw new XPathException(this, e); - } - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadBinary.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadBinary.java deleted file mode 100644 index d3b2288f3a5..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadBinary.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.Base64BinaryValueType; -import org.exist.xquery.value.BinaryValueFromFile; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.Type; - -/** - * @author Andrzej Taramina - */ -public class FileReadBinary extends BasicFunction { - - private static final Logger logger = LogManager.getLogger(FileReadBinary.class); - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( "read-binary", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Reads the contents of a binary file. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ) - }, - new FunctionReturnSequenceType( Type.BASE64_BINARY, - Cardinality.ZERO_OR_ONE, "the file contents" ) ) - }; - - public FileReadBinary(final XQueryContext context, final FunctionSignature signature) - { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - final String inputPath = args[0].getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - return BinaryValueFromFile.getInstance(context, new Base64BinaryValueType(), file, this); - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadUnicode.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadUnicode.java deleted file mode 100644 index b9d9730f663..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadUnicode.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.io.IOException; -import java.io.StringWriter; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import org.exist.dom.QName; -import org.exist.xquery.BasicFunction; -import org.exist.xquery.Cardinality; -import org.exist.xquery.FunctionSignature; -import org.exist.xquery.XPathException; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.FunctionParameterSequenceType; -import org.exist.xquery.value.FunctionReturnSequenceType; -import org.exist.xquery.value.Sequence; -import org.exist.xquery.value.SequenceType; -import org.exist.xquery.value.StringValue; -import org.exist.xquery.value.Type; - -/** - * @author Pierrick Brihaye - * @author Dizzzz - * @author Andrzej Taramina - */ -public class FileReadUnicode extends BasicFunction { - - private final static Logger logger = LogManager.getLogger(FileReadUnicode.class); - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( "read-unicode", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Reads the contents of a file. Unicode BOM (Byte Order Marker) will be stripped " + - "off if found. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ) - }, - new FunctionReturnSequenceType( Type.STRING, - Cardinality.ZERO_OR_ONE, "the contents of the file" ) ), - - new FunctionSignature( - new QName( "read-unicode", FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Reads the contents of a file. Unicode BOM (Byte Order Marker) will be stripped " + - "off if found. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ), - new FunctionParameterSequenceType( "encoding", Type.STRING, - Cardinality.EXACTLY_ONE, "The file is read with the encoding specified." ) - }, - new FunctionReturnSequenceType( Type.STRING, - Cardinality.ZERO_OR_ONE, "the contents of the file" ) ) - }; - - public FileReadUnicode(final XQueryContext context, final FunctionSignature signature) { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { - if (!context.getSubject().hasDbaRole()) { - XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - final String inputPath = args[0].getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - final Charset encoding; - if(args.length == 2) { - encoding = Charset.forName(args[1].getStringValue()); - } else { - encoding = StandardCharsets.UTF_8; - } - - try(final UnicodeReader reader = new UnicodeReader(Files.newInputStream(file), encoding.name()); - final StringWriter sw = new StringWriter()) { - - char[] buf = new char[1024]; - int len; - while(( len = reader.read( buf ) ) > 0) { - sw.write( buf, 0, len) ; - } - - return new StringValue(this, sw.toString()); - } catch(final IOException e) { - throw new XPathException(this, e); - } - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/SerializeToFile.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/SerializeToFile.java deleted file mode 100644 index 1a14bc02ba2..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/SerializeToFile.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.exist.dom.QName; -import org.exist.storage.serializers.Serializer; -import org.exist.xquery.*; -import org.exist.xquery.util.SerializerUtils; -import org.exist.xquery.value.*; -import org.xml.sax.SAXException; - -import javax.xml.transform.OutputKeys; -import java.io.*; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Properties; - -public class SerializeToFile extends BasicFunction { - private final static Logger logger = LogManager.getLogger(SerializeToFile.class); - - private final static String FN_SERIALIZE_LN = "serialize"; - private final static String FN_SERIALIZE_BINARY_LN = "serialize-binary"; - - public final static FunctionSignature signatures[] = { - new FunctionSignature( - new QName( FN_SERIALIZE_LN, FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Writes the node set into a file on the file system. $parameters contains a " + - "sequence of zero or more serialization parameters specified as key=value pairs. The " + - "serialization options are the same as those recognized by \"declare option exist:serialize\". " + - "The function does NOT automatically inherit the serialization options of the XQuery it is " + - "called from. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "node-set", Type.NODE, - Cardinality.ZERO_OR_MORE, "The contents to write to the file system." ), - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file" ), - new FunctionParameterSequenceType( "parameters", Type.ITEM, - Cardinality.ZERO_OR_MORE, "The serialization parameters: either a sequence of key=value pairs or an output:serialization-parameters " + - "element as defined by the standard fn:serialize function." ) - }, - new FunctionReturnSequenceType( Type.BOOLEAN, - Cardinality.ZERO_OR_ONE, "true on success - false if the specified file can not be " - + "created or is not writable. The empty sequence is returned if the argument sequence is empty." ) - ), - - new FunctionSignature( - new QName( FN_SERIALIZE_LN, FileModule.NAMESPACE_URI, FileModule.PREFIX ), - "Writes the node set into a file on the file system, optionally appending to it. " + - "$parameters contains a sequence of zero or more serialization parameters specified as " + - "key=value pairs. The serialization options are the same as those recognized by " + - "\"declare option exist:serialize\". " + - "The function does NOT automatically inherit the serialization options of the XQuery it is " + - "called from. This method is only available to the DBA role.", - new SequenceType[] { - new FunctionParameterSequenceType( "node-set", Type.NODE, - Cardinality.ZERO_OR_MORE, "The contents to write to the file system." ), - new FunctionParameterSequenceType( "path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file" ), - new FunctionParameterSequenceType( "parameters", Type.ITEM, - Cardinality.ZERO_OR_MORE, "The serialization parameters: either a sequence of key=value pairs or an output:serialization-parameters " + - "element as defined by the standard fn:serialize function." ), - new FunctionParameterSequenceType( "append", Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "Should content be appended?") - }, - new FunctionReturnSequenceType( Type.BOOLEAN, - Cardinality.ZERO_OR_ONE, "true on success - false if the specified file can " - + "not be created or is not writable. The empty sequence is returned if the argument sequence is empty." ) - ), - - new FunctionSignature( - new QName(FN_SERIALIZE_BINARY_LN, FileModule.NAMESPACE_URI, FileModule.PREFIX), - "Writes binary data into a file on the file system. This method is only available to the DBA role.", - new SequenceType[]{ - new FunctionParameterSequenceType("binarydata", Type.BASE64_BINARY, - Cardinality.EXACTLY_ONE, "The contents to write to the file system."), - new FunctionParameterSequenceType("path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file") - }, - new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true on success - false if the specified file can not be created or is not writable") - ), - - new FunctionSignature( - new QName(FN_SERIALIZE_BINARY_LN, FileModule.NAMESPACE_URI, FileModule.PREFIX), - "Writes binary data into a file on the file system, optionally appending the content. This method is only available to the DBA role.", - new SequenceType[]{ - new FunctionParameterSequenceType("binarydata", Type.BASE64_BINARY, - Cardinality.EXACTLY_ONE, "The contents to write to the file system."), - new FunctionParameterSequenceType("path", Type.ITEM, - Cardinality.EXACTLY_ONE, "The full path or URI to the file"), - new FunctionParameterSequenceType("append", Type.BOOLEAN, - Cardinality.EXACTLY_ONE, "Should content be appended?") - }, - new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, - "true on success - false if the specified file can not be created or is not writable") - ) - }; - - - public SerializeToFile(final XQueryContext context, final FunctionSignature signature) { - super(context, signature); - } - - @Override - public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { - - if(args[0].isEmpty()) { - return Sequence.EMPTY_SEQUENCE; - } - - if(!context.getSubject().hasDbaRole()) { - final XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); - logger.error("Invalid user", xPathException); - throw xPathException; - } - - - //check the file output path - final String inputPath = args[1].getStringValue(); - final Path file = FileModuleHelper.getFile(inputPath, this); - - if(Files.isDirectory(file)) { - logger.debug("Cannot serialize file. Output file is a directory: {}", file.toAbsolutePath().toString()); - return BooleanValue.FALSE; - } - - if(Files.exists(file) && !Files.isWritable(file)) { - logger.debug("Cannot serialize file. Cannot write to file {}", file.toAbsolutePath().toString()); - return BooleanValue.FALSE; - } - - if(isCalledAs(FN_SERIALIZE_LN)) { - //parse serialization options from third argument to function - final Properties outputProperties = parseXMLSerializationOptions( args[2] ); - final boolean doAppend = (args.length > 3) && "true".equals(args[3].itemAt(0).getStringValue()); - - //do the serialization - serializeXML(args[0].iterate(), outputProperties, file, doAppend); - - } else if(isCalledAs(FN_SERIALIZE_BINARY_LN)) { - final boolean doAppend = (args.length > 2) && "true".equals(args[2].itemAt(0).getStringValue()); - serializeBinary((BinaryValue)args[0].itemAt(0), file, doAppend); - - } else { - throw new XPathException(this, "Unknown function name"); - } - - return BooleanValue.TRUE; - } - - - private Properties parseXMLSerializationOptions(final Sequence sSerializeParams) throws XPathException { - //parse serialization options - final Properties outputProperties = new Properties(); - - // defaults - outputProperties.setProperty( OutputKeys.INDENT, "yes" ); - outputProperties.setProperty( OutputKeys.OMIT_XML_DECLARATION, "yes" ); - outputProperties.setProperty( OutputKeys.ENCODING, "UTF-8" ); - - if (sSerializeParams.hasOne() && Type.subTypeOf(sSerializeParams.getItemType(), Type.NODE)) { - SerializerUtils.getSerializationOptions(this, (NodeValue) sSerializeParams.itemAt(0), outputProperties); - } else { - SequenceIterator siSerializeParams = sSerializeParams.iterate(); - while(siSerializeParams.hasNext()) { - final String serializeParam = siSerializeParams.nextItem().getStringValue(); - final String opt[] = Option.parseKeyValuePair(serializeParam); - if(opt != null && opt.length == 2) { - outputProperties.setProperty( opt[0], opt[1] ); - } - } - } - return outputProperties; - } - - - private void serializeXML(final SequenceIterator siNode, final Properties outputProperties, final Path file, final boolean doAppend) throws XPathException { - final Serializer serializer = context.getBroker().borrowSerializer(); - - StandardOpenOption ops[] = doAppend ? new StandardOpenOption[]{StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND} - : new StandardOpenOption[]{StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING}; - - - try (final OutputStream os = new BufferedOutputStream(Files.newOutputStream(file, ops)); - final Writer writer = new OutputStreamWriter(os, Charset.forName(outputProperties.getProperty(OutputKeys.ENCODING)))) { - - serializer.setProperties(outputProperties); - - while (siNode.hasNext()) { - final NodeValue nv = (NodeValue) siNode.nextItem(); - serializer.serialize(nv, writer); - } - } catch(UnsupportedOperationException | IllegalArgumentException e){ - throw new XPathException(this, "Error wile writing to file: " + e.getMessage(), e); - - } catch(final IOException | SAXException e) { - throw new XPathException(this, "Cannot serialize file. A problem occurred while serializing the node set: " + e.getMessage(), e); - } finally { - context.getBroker().returnSerializer(serializer); - } - } - - private void serializeBinary(final BinaryValue binary, final Path file, final boolean doAppend) throws XPathException { - - StandardOpenOption ops[] = doAppend ? new StandardOpenOption[]{StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND} - : new StandardOpenOption[]{StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING}; - - try(final OutputStream os = new BufferedOutputStream(Files.newOutputStream(file, ops))) { - binary.streamBinaryTo(os); - } catch(final IOException ioe) { - throw new XPathException(this, "Cannot serialize file. A problem occurred while serializing the binary data: " + ioe.getMessage(), ioe); - } - } -} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/UnicodeReader.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/UnicodeReader.java deleted file mode 100644 index c73927113f6..00000000000 --- a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/UnicodeReader.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PushbackInputStream; -import java.io.Reader; - - - -/** - * Generic unicode textreader, which will use BOM mark - * to identify the encoding to be used. If BOM is not found - * then use a given default or system encoding. This is a bug fix - * workaround for a known issue with InputStreamReader not detecting and - * ignoring the UTF-* BOM (EF BB BF). - * - * http://www.unicode.org/unicode/faq/utf_bom.html - * BOMs: - * 00 00 FE FF = UTF-32, big-endian - * FF FE 00 00 = UTF-32, little-endian - * EF BB BF = UTF-8, - * FE FF = UTF-16, big-endian - * FF FE = UTF-16, little-endian - * - * Win2k Notepad: - * Unicode format = UTF-16LE - * - * Based on code by Thomas Weidenfeller and Aki Nieminen - * - * @author Andrzej Taramina - * @serial 2008-03-06 - * @version 1.1 - */ - -public class UnicodeReader extends Reader -{ - PushbackInputStream internalIn; - InputStreamReader internalIn2 = null; - String defaultEnc; - - private static final int BOM_SIZE = 4; - - - /** - * - * @param in inputstream to be read - */ - - public UnicodeReader( InputStream in ) - { - internalIn = new PushbackInputStream( in, BOM_SIZE ); - this.defaultEnc = null; - } - - - /** - * - * @param in inputstream to be read - * @param defaultEnc default encoding if stream does not have - * BOM marker. Give NULL to use system-level default. - */ - - public UnicodeReader( InputStream in, String defaultEnc ) - { - internalIn = new PushbackInputStream( in, BOM_SIZE ); - this.defaultEnc = defaultEnc; - } - - - public String getDefaultEncoding() - { - return( defaultEnc ); - } - - - /** - * Get stream encoding or NULL if stream is uninitialized. - * Call init() or read() method to initialize it. - * - * @return the encoding - */ - public String getEncoding() - { - String ret = null; - - if( internalIn2 != null ) { - ret = internalIn2.getEncoding(); - } - - return( ret ); - } - - - /** - * Read-ahead four bytes and check for BOM marks. Extra bytes are - * unread back to the stream, only BOM bytes are skipped. - * - * @throws IOException if an I/O error occurs - */ - protected void init() throws IOException - { - if( internalIn2 == null ) { - - String encoding; - byte bom[] = new byte[BOM_SIZE]; - int n; - int unread; - n = internalIn.read( bom, 0, bom.length ); - - if( (bom[0] == (byte)0x00) && (bom[1] == (byte)0x00) && (bom[2] == (byte)0xFE) && (bom[3] == (byte)0xFF) ) { - encoding = "UTF-32BE"; - unread = n - 4; - } else if( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) && (bom[2] == (byte)0x00) && (bom[3] == (byte)0x00) ) { - encoding = "UTF-32LE"; - unread = n - 4; - } else if( (bom[0] == (byte)0xEF) && (bom[1] == (byte)0xBB) && (bom[2] == (byte)0xBF) ) { - encoding = "UTF-8"; - unread = n - 3; - } else if( (bom[0] == (byte)0xFE) && (bom[1] == (byte)0xFF) ) { - encoding = "UTF-16BE"; - unread = n - 2; - } else if( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) ) { - encoding = "UTF-16LE"; - unread = n - 2; - } else { - // Unicode BOM mark not found, unread all bytes - encoding = defaultEnc; - unread = n; - } - - //System.out.println("read=" + n + ", unread=" + unread); - - if( unread > 0 ) { - internalIn.unread( bom, (n - unread), unread ); - } - - // Use given encoding - if( encoding == null ) { - internalIn2 = new InputStreamReader( internalIn ); - } else { - internalIn2 = new InputStreamReader( internalIn, encoding ); - } - } - } - - - public void close() throws IOException - { - init(); - internalIn2.close(); - } - - public int read( char[] cbuf, int off, int len ) throws IOException - { - init(); - return( internalIn2.read( cbuf, off, len ) ); - } - -} \ No newline at end of file diff --git a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/AbstractBinariesTest.java b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/AbstractBinariesTest.java deleted file mode 100644 index 54692e89bec..00000000000 --- a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/AbstractBinariesTest.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import com.evolvedbinary.j8fu.function.Consumer2E; -import org.exist.xmldb.XmldbURI; -import org.junit.*; -import org.junit.rules.TemporaryFolder; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Random; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.exist.test.TestConstants.TEST_COLLECTION_URI; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Tests for accessing binaries using XQuery via various APIs. - * - * @see Binary streaming is broken - * - * @author Adam Retter - */ -public abstract class AbstractBinariesTest { - - protected static final XmldbURI TEST_COLLECTION = TEST_COLLECTION_URI.append("BinariesTest"); - protected static final String BIN1_FILENAME = "1.bin"; - protected static final byte[] BIN1_CONTENT = "1234567890".getBytes(UTF_8); - - @ClassRule - public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Before - public void setup() throws Exception { - storeBinaryFile(TEST_COLLECTION.append(BIN1_FILENAME), BIN1_CONTENT); - } - - @After - public void cleanup() throws Exception { - removeCollection(TEST_COLLECTION); - } - - /** - * {@see https://github.com/eXist-db/exist/issues/790#error-case-3} - */ - @Test - public void readBinary() throws Exception { - final byte[] data = randomData(1024 * 1024 * 10); // 10KB - final Path tmpFile = createTemporaryFile(data); - - final String query = "import module namespace file = \"http://exist-db.org/xquery/file\";\n" + - "file:read-binary('" + tmpFile.toAbsolutePath() + "')"; - - final QueryResultAccessor resultsAccessor = executeXQuery(query); - - resultsAccessor.accept(results -> { - assertEquals(1, size(results)); - - final U item = item(results, 0); - assertTrue(isBinaryType(item)); - assertArrayEquals(data, getBytes(item)); - }); - } - - /** - * {@see https://github.com/eXist-db/exist/issues/790#error-case-4} - */ - @Test - public void readAndWriteBinary() throws Exception { - final byte[] data = randomData(1024 * 1024); // 1MB - final Path tmpInFile = createTemporaryFile(data); - - final Path tmpOutFile = temporaryFolder.newFile().toPath(); - - final String query = "import module namespace file = \"http://exist-db.org/xquery/file\";\n" + - "let $bin := file:read-binary('" + tmpInFile.toAbsolutePath() + "')\n" + - "return\n" + - " file:serialize-binary($bin, '" + tmpOutFile.toAbsolutePath() + "')"; - - final QueryResultAccessor resultsAccessor = executeXQuery(query); - - resultsAccessor.accept(results -> { - assertEquals(1, size(results)); - - final U item = item(results, 0); - assertTrue(isBooleanType(item)); - assertEquals(true, getBoolean(item)); - }); - - assertArrayEquals(Files.readAllBytes(tmpInFile), Files.readAllBytes(tmpOutFile)); - } - - protected byte[] randomData(final int size) { - final byte data[] = new byte[size]; - new Random().nextBytes(data); - return data; - } - - protected Path createTemporaryFile(final byte[] data) throws IOException { - final Path f = temporaryFolder.newFile().toPath(); - Files.write(f, data); - return f; - } - - @FunctionalInterface interface QueryResultAccessor extends Consumer2E, AssertionError, E> { - } - - protected abstract void storeBinaryFile(final XmldbURI filePath, final byte[] content) throws Exception; - protected abstract void removeCollection(final XmldbURI collectionUri) throws Exception; - protected abstract QueryResultAccessor executeXQuery(final String query) throws Exception; - protected abstract long size(T results) throws E; - protected abstract U item(T results, int index) throws E; - protected abstract boolean isBinaryType(U item) throws E; - protected abstract boolean isBooleanType(U item) throws E; - protected abstract byte[] getBytes(U item) throws E; - protected abstract boolean getBoolean(U item) throws E; -} diff --git a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/EmbeddedBinariesTest.java b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/EmbeddedBinariesTest.java deleted file mode 100644 index 593431a5fe2..00000000000 --- a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/EmbeddedBinariesTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import org.exist.collections.Collection; -import org.exist.source.Source; -import org.exist.source.StringSource; -import org.exist.storage.BrokerPool; -import org.exist.storage.DBBroker; -import org.exist.storage.XQueryPool; -import org.exist.storage.lock.Lock; -import org.exist.storage.lock.ManagedCollectionLock; -import org.exist.storage.txn.Txn; -import org.exist.test.ExistEmbeddedServer; -import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; -import org.exist.util.MimeType; -import org.exist.util.StringInputSource; -import org.exist.xmldb.XmldbURI; -import org.exist.xquery.CompiledXQuery; -import org.exist.xquery.XQuery; -import org.exist.xquery.XQueryContext; -import org.exist.xquery.value.*; -import org.junit.ClassRule; - -import java.io.IOException; -import java.util.Optional; - -/** - * @author Adam Retter - */ -public class EmbeddedBinariesTest extends AbstractBinariesTest { - - @ClassRule - public static ExistEmbeddedServer existEmbeddedServer = new ExistEmbeddedServer(true, true); - - @Override - protected void storeBinaryFile(final XmldbURI filePath, final byte[] content) throws Exception { - final BrokerPool brokerPool = existEmbeddedServer.getBrokerPool(); - try(final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject())); - final Txn transaction = brokerPool.getTransactionManager().beginTransaction()) { - - try(final ManagedCollectionLock collectionLock = brokerPool.getLockManager().acquireCollectionWriteLock(filePath.removeLastSegment())) { - final Collection collection = broker.getOrCreateCollection(transaction, filePath.removeLastSegment()); - - broker.storeDocument(transaction, filePath.lastSegment(), new StringInputSource(content), MimeType.BINARY_TYPE, collection); - - broker.saveCollection(transaction, collection); - } - - transaction.commit(); - } - } - - @Override - protected void removeCollection(final XmldbURI collectionUri) throws Exception { - final BrokerPool brokerPool = existEmbeddedServer.getBrokerPool(); - try(final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject())); - final Txn transaction = brokerPool.getTransactionManager().beginTransaction(); - final Collection collection = broker.openCollection(collectionUri, Lock.LockMode.WRITE_LOCK)) { - if(collection != null) { - broker.removeCollection(transaction, collection); - } - - transaction.commit(); - } - } - - @Override - protected QueryResultAccessor executeXQuery(final String query) throws Exception { - final Source source = new StringSource(query); - final BrokerPool brokerPool = existEmbeddedServer.getBrokerPool(); - final XQueryPool pool = brokerPool.getXQueryPool(); - final XQuery xquery = brokerPool.getXQueryService(); - - try(final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject()))) { - final CompiledXQuery existingCompiled = pool.borrowCompiledXQuery(broker, source); - - final XQueryContext context; - final CompiledXQuery compiled; - if (existingCompiled == null) { - context = new XQueryContext(brokerPool); - compiled = xquery.compile(context, source); - } else { - context = existingCompiled.getContext(); - context.prepareForReuse(); - compiled = existingCompiled; - } - - final Sequence results = xquery.execute(broker, compiled, null); - - return consumer2E -> { - try { -// context.runCleanupTasks(); //TODO(AR) shows the ordering issue with binary values (see comment below) - - consumer2E.accept(results); - } finally { - //TODO(AR) performing #runCleanupTasks causes the stream to be closed, so if we do so before we are finished with the results, serialization fails. - context.runCleanupTasks(); - pool.returnCompiledXQuery(source, compiled); - } - }; - } - } - - @Override - protected long size(final Sequence results) { - return results.getItemCount(); - } - - @Override - protected Item item(final Sequence results, final int index) { - return results.itemAt(index); - } - - @Override - protected boolean isBinaryType(final Item item) { - return Type.BASE64_BINARY == item.getType() || Type.HEX_BINARY == item.getType(); - } - - @Override - protected boolean isBooleanType(final Item item) throws IOException { - return Type.BOOLEAN == item.getType(); - } - - @Override - protected byte[] getBytes(final Item item) throws IOException { - if (item instanceof Base64BinaryDocument doc) { - try (final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { - doc.streamBinaryTo(baos); - return baos.toByteArray(); - } - } else { - final BinaryValueFromFile file = (BinaryValueFromFile) item; - try (final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { - file.streamBinaryTo(baos); - return baos.toByteArray(); - } - } - } - - @Override - protected boolean getBoolean(final Item item) throws IOException { - return ((BooleanValue)item).getValue(); - } -} diff --git a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/RestBinariesTest.java b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/RestBinariesTest.java deleted file mode 100644 index de93012eda0..00000000000 --- a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/RestBinariesTest.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.client.fluent.Executor; -import org.apache.http.client.fluent.Request; -import org.apache.http.entity.ContentType; -import org.exist.http.jaxb.Query; -import org.exist.http.jaxb.Result; -import org.exist.test.ExistWebServer; -import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; -import org.exist.xmldb.XmldbURI; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; - -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Marshaller; -import jakarta.xml.bind.Unmarshaller; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.apache.http.HttpStatus.SC_CREATED; -import static org.apache.http.HttpStatus.SC_OK; -import static org.exist.TestUtils.ADMIN_DB_PWD; -import static org.exist.TestUtils.ADMIN_DB_USER; -import static org.junit.Assert.assertArrayEquals; - -/** - * @author Adam Retter - */ -public class RestBinariesTest extends AbstractBinariesTest { - - @ClassRule - public static final ExistWebServer existWebServer = new ExistWebServer(true, false, true, true); - - private static Executor executor = null; - - @BeforeClass - public static void setupExecutor() { - executor = Executor.newInstance() - .auth(new HttpHost("localhost", existWebServer.getPort()), ADMIN_DB_USER, ADMIN_DB_PWD) - .authPreemptive(new HttpHost("localhost", existWebServer.getPort())); - } - - /** - * {@see https://github.com/eXist-db/exist/issues/790#error-case-5} - * - * response:stream is used to return Base64 encoded binary. - */ - @Test - public void readAndStreamBinarySax() throws IOException, JAXBException { - final byte[] data = randomData(1024 * 1024); // 1MB - final Path tmpInFile = createTemporaryFile(data); - - final String query = "import module namespace file = \"http://exist-db.org/xquery/file\";\n" + - "import module namespace response = \"http://exist-db.org/xquery/response\";\n" + - "let $bin := file:read-binary('" + tmpInFile.toAbsolutePath() + "')\n" + - "return response:stream($bin, 'media-type=application/octet-stream')"; - - final HttpResponse response = postXquery(query); - - final HttpEntity entity = response.getEntity(); - try(final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { - entity.writeTo(baos); - - assertArrayEquals(Files.readAllBytes(tmpInFile), Base64.decodeBase64(baos.toByteArray())); - } - } - - /** - * {@see https://github.com/eXist-db/exist/issues/790#error-case-5} - * - * response:stream-binary is used to return raw binary. - */ - @Test - public void readAndStreamBinaryRaw() throws IOException, JAXBException { - final byte[] data = randomData(1024 * 1024); // 1MB - final Path tmpInFile = createTemporaryFile(data); - - final String query = "import module namespace file = \"http://exist-db.org/xquery/file\";\n" + - "import module namespace response = \"http://exist-db.org/xquery/response\";\n" + - "let $bin := file:read-binary('" + tmpInFile.toAbsolutePath() + "')\n" + - "return response:stream-binary($bin, 'media-type=application/octet-stream', ())"; - - final HttpResponse response = postXquery(query); - - final HttpEntity entity = response.getEntity(); - try(final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { - entity.writeTo(baos); - - assertArrayEquals(Files.readAllBytes(tmpInFile), baos.toByteArray()); - } - } - - - @Override - protected void storeBinaryFile(final XmldbURI filePath, final byte[] content) throws Exception { - final HttpResponse response = executor.execute(Request.Put(getRestUrl() + filePath.toString()) - .setHeader("Content-Type", "application/octet-stream") - .bodyByteArray(content) - ).returnResponse(); - - if(response.getStatusLine().getStatusCode() != SC_CREATED) { - throw new Exception("Unable to store binary file: " + filePath); - } - } - - private String getRestUrl() { - return "http://localhost:" + existWebServer.getPort() + "/rest"; - } - - @Override - protected void removeCollection(final XmldbURI collectionUri) throws Exception { - final HttpResponse response = executor.execute(Request.Delete(getRestUrl() + collectionUri.toString())) - .returnResponse(); - - if(response.getStatusLine().getStatusCode() != SC_OK) { - throw new Exception("Unable to delete collection: " + collectionUri); - } - } - - @Override - protected QueryResultAccessor executeXQuery(final String xquery) throws Exception { - final HttpResponse response = postXquery(xquery); - final HttpEntity entity = response.getEntity(); - try(final InputStream is = entity.getContent()) { - final JAXBContext jaxbContext = JAXBContext.newInstance("org.exist.http.jaxb"); - final Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); - final Result result = (Result)unmarshaller.unmarshal(is); - - return consumer -> consumer.accept(result); - } - } - - private HttpResponse postXquery(final String xquery) throws JAXBException, IOException { - final Query query = new Query(); - query.setText(xquery); - - final JAXBContext jaxbContext = JAXBContext.newInstance("org.exist.http.jaxb"); - final Marshaller marshaller = jaxbContext.createMarshaller(); - - final HttpResponse response; - try(final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { - marshaller.marshal(query, baos); - response = executor.execute(Request.Post(getRestUrl() + "/db/") - .bodyByteArray(baos.toByteArray(), ContentType.APPLICATION_XML) - ).returnResponse(); - } - - if(response.getStatusLine().getStatusCode() != SC_OK) { - throw new IOException("Unable to query, HTTP response code: " + response.getStatusLine().getStatusCode()); - } - - return response; - } - - @Override - protected long size(final Result result) throws Exception { - return result.getCount(); - } - - @Override - protected Result.Value item(final Result results, final int index) throws Exception { - return results.getValue().get(index); - } - - @Override - protected boolean isBinaryType(final Result.Value item) throws Exception { - final String type = item.getType(); - return "xs:base64Binary".equals(type) || "xs:hexBinary".equals(type); - } - - @Override - protected boolean isBooleanType(Result.Value item) throws Exception { - return "xs:boolean".equals(item.getType()); - } - - @Override - protected byte[] getBytes(final Result.Value item) throws Exception { - return switch (item.getType()) { - case "xs:base64Binary" -> Base64.decodeBase64(item.getContent().getFirst().toString()); - case "xs:hexBinary" -> Hex.decodeHex(item.getContent().getFirst().toString()); - default -> throw new UnsupportedOperationException(); - }; - } - - @Override - protected boolean getBoolean(final Result.Value item) throws Exception { - return Boolean.parseBoolean(item.getContent().getFirst().toString()); - } -} diff --git a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java deleted file mode 100644 index 08358285365..00000000000 --- a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * eXist-db Open Source Native XML Database - * Copyright (C) 2001 The eXist-db Authors - * - * info@exist-db.org - * http://www.exist-db.org - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -package org.exist.xquery.modules.file; - -import org.exist.test.ExistWebServer; -import org.exist.xmldb.XmldbURI; -import org.junit.ClassRule; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.xmldb.api.DatabaseManager; -import org.xmldb.api.base.*; -import org.xmldb.api.modules.BinaryResource; -import org.xmldb.api.modules.CollectionManagementService; -import org.xmldb.api.modules.XQueryService; - -import java.util.ArrayDeque; -import java.util.Arrays; -import java.util.Deque; - -import static org.exist.TestUtils.ADMIN_DB_PWD; -import static org.exist.TestUtils.ADMIN_DB_USER; -import static org.xmldb.api.base.ResourceType.BINARY_RESOURCE; - -/** - * @author Adam Retter - */ -@RunWith(Parameterized.class) -public class XmldbBinariesTest extends AbstractBinariesTest { - - @ClassRule - public static final ExistWebServer existWebServer = new ExistWebServer(true, false, true, true); - private static final String PORT_PLACEHOLDER = "${PORT}"; - - @Parameterized.Parameters(name = "{0}") - public static java.util.Collection data() { - return Arrays.asList(new Object[][] { - { "local", "xmldb:exist://" }, - { "remote", "xmldb:exist://localhost:" + PORT_PLACEHOLDER + "/xmlrpc" } - }); - } - - @Parameterized.Parameter - public String apiName; - - @Parameterized.Parameter(value = 1) - public String baseUri; - - private final String getBaseUri() { - return baseUri.replace(PORT_PLACEHOLDER, Integer.toString(existWebServer.getPort())); - } - - @Override - protected void storeBinaryFile(final XmldbURI filePath, byte[] content) throws Exception { - Collection colRoot = null; - try { - colRoot = DatabaseManager.getCollection(getBaseUri() + "/db", ADMIN_DB_USER, ADMIN_DB_PWD); - - final XmldbURI collectionNames[] = filePath.removeLastSegment().getPathSegments(); - - final Deque cols = new ArrayDeque<>(); - try { - Collection current = colRoot; - for (int i = 1; i < collectionNames.length; i++) { - final Collection child = getOrCreateCollection(current, collectionNames[i].toString()); - cols.push(child); - current = child; - } - - final String fileName = filePath.lastSegment().toString(); - final Resource resource = current.createResource(fileName, BinaryResource.class); - resource.setContent(content); - current.storeResource(resource); - - } finally { - while(!cols.isEmpty()) { - try { - cols.pop().close(); - } catch(XMLDBException e) { - - } - } - } - } finally { - if(colRoot != null) { - colRoot.close(); - } - } - } - - private Collection getOrCreateCollection(final Collection parent, final String childName) throws XMLDBException { - Collection child = parent.getChildCollection(childName); - if(child == null) { - final CollectionManagementService cms = parent.getService(CollectionManagementService.class); - child = cms.createCollection(childName); - } - return child; - } - - @Override - protected void removeCollection(final XmldbURI collectionUri) throws Exception { - Collection colRoot = null; - try { - colRoot = DatabaseManager.getCollection(getBaseUri() + "/db", ADMIN_DB_USER, ADMIN_DB_PWD); - - final Collection colTest = colRoot.getChildCollection("test"); - try { - final CollectionManagementService cms = colTest.getService(CollectionManagementService.class); - - final String testCollectionName = collectionUri.lastSegment().toString(); - cms.removeCollection(testCollectionName); - } finally { - if(colTest != null) { - colTest.close(); - } - } - } finally { - if(colRoot != null) { - colRoot.close(); - } - } - } - - @Override - protected QueryResultAccessor executeXQuery(final String query) throws Exception { - return consumer -> { - Collection colRoot = null; - try { - colRoot = DatabaseManager.getCollection(getBaseUri() + "/db", ADMIN_DB_USER, ADMIN_DB_PWD); - final XQueryService xqueryService = colRoot.getService(XQueryService.class); - - final CompiledExpression compiledExpression = xqueryService.compile(query); - final ResourceSet results = xqueryService.execute(compiledExpression); - - - try { - // compiledExpression.reset(); // shows the ordering issue with binary values (see comment below) - - consumer.accept(results); - } finally { - //the following calls cause the streams of any binary result values to be closed, so if we did so before we are finished with the results, serialization would fail. - results.clear(); - compiledExpression.reset(); - } - } finally { - colRoot.close(); - } - }; - } - - @Override - protected long size(final ResourceSet results) throws XMLDBException { - return results.getSize(); - } - - @Override - protected Resource item(final ResourceSet results, final int index) throws XMLDBException { - return results.getResource(index); - } - - @Override - protected boolean isBinaryType(final Resource item) throws XMLDBException { - return BINARY_RESOURCE.equals(item.getResourceType()); - } - - @Override - protected boolean isBooleanType(final Resource item) throws XMLDBException { - final String value = item.getContent().toString(); - return "true".equals(value) || "false".equals(value); - } - - @Override - protected byte[] getBytes(final Resource item) throws XMLDBException { - return (byte[])item.getContent(); - } - - @Override - protected boolean getBoolean(final Resource item) throws XMLDBException { - return Boolean.parseBoolean(item.getContent().toString()); - } -} diff --git a/extensions/modules/file/src/test/resources-filtered/conf.xml b/extensions/modules/file/src/test/resources-filtered/conf.xml deleted file mode 100644 index 11c020c728e..00000000000 --- a/extensions/modules/file/src/test/resources-filtered/conf.xml +++ /dev/null @@ -1,781 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/extensions/modules/file/src/test/resources/log4j2.xml b/extensions/modules/file/src/test/resources/log4j2.xml deleted file mode 100644 index b355862720a..00000000000 --- a/extensions/modules/file/src/test/resources/log4j2.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - diff --git a/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml b/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml deleted file mode 100644 index 76b81502392..00000000000 --- a/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/web.xml b/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/web.xml deleted file mode 100644 index 4722b24716c..00000000000 --- a/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/web.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - - eXist-db – Open Source Native XML Database - eXist-db XML Database - - - org.exist.xmlrpc.RpcServlet - org.exist.xmlrpc.RpcServlet - - enabledForExtensions - true - - - - - EXistServlet - org.exist.http.servlets.EXistServlet - - configuration - conf.xml - - - basedir - WEB-INF/ - - - start - true - - 2 - - - - XQueryURLRewrite - org.exist.http.urlrewrite.XQueryURLRewrite - - config - WEB-INF/controller-config.xml - - - - - XSLTServlet - org.exist.http.servlets.XSLTServlet - - - - XQueryURLRewrite - /* - - diff --git a/extensions/modules/file/src/test/xquery/modules/file/directory-list.xqm b/extensions/modules/file/src/test/xquery/modules/file/directory-list.xqm deleted file mode 100644 index 629eeea50d4..00000000000 --- a/extensions/modules/file/src/test/xquery/modules/file/directory-list.xqm +++ /dev/null @@ -1,47 +0,0 @@ -(: - : eXist-db Open Source Native XML Database - : Copyright (C) 2001 The eXist-db Authors - : - : info@exist-db.org - : http://www.exist-db.org - : - : This library is free software; you can redistribute it and/or - : modify it under the terms of the GNU Lesser General Public - : License as published by the Free Software Foundation; either - : version 2.1 of the License, or (at your option) any later version. - : - : This library is distributed in the hope that it will be useful, - : but WITHOUT ANY WARRANTY; without even the implied warranty of - : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - : Lesser General Public License for more details. - : - : You should have received a copy of the GNU Lesser General Public - : License along with this library; if not, write to the Free Software - : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - :) -xquery version "3.1"; - -module namespace dirlist="http://exist-db.org/testsuite/modules/file/dirlist"; - - -import module namespace file="http://exist-db.org/xquery/file"; - - -declare namespace test="http://exist-db.org/xquery/xqsuite"; - - -declare - %test:assertError("file:DIRECTORY_NOT_FOUND") -function dirlist:non-existent-error-code() { - file:directory-list("/non/existent", ()) -}; - -declare - %test:assertXPath("contains($result,'basedir /non/existent does not exist.') or contains($result,'basedir \non\existent does not exist.')") -function dirlist:non-existent-error-description() { - try { - file:directory-list("/non/existent", ()) - } catch file:DIRECTORY_NOT_FOUND { - $err:description - } -}; diff --git a/extensions/modules/file/src/test/xquery/modules/file/read-binary.xqm b/extensions/modules/file/src/test/xquery/modules/file/read-binary.xqm deleted file mode 100644 index b00dd6c15f4..00000000000 --- a/extensions/modules/file/src/test/xquery/modules/file/read-binary.xqm +++ /dev/null @@ -1,57 +0,0 @@ -(: - : eXist-db Open Source Native XML Database - : Copyright (C) 2001 The eXist-db Authors - : - : info@exist-db.org - : http://www.exist-db.org - : - : This library is free software; you can redistribute it and/or - : modify it under the terms of the GNU Lesser General Public - : License as published by the Free Software Foundation; either - : version 2.1 of the License, or (at your option) any later version. - : - : This library is distributed in the hope that it will be useful, - : but WITHOUT ANY WARRANTY; without even the implied warranty of - : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - : Lesser General Public License for more details. - : - : You should have received a copy of the GNU Lesser General Public - : License along with this library; if not, write to the Free Software - : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - :) -xquery version "3.1"; - -module namespace readbinary="http://exist-db.org/testsuite/modules/file/read-binary"; - - -import module namespace file="http://exist-db.org/xquery/file"; -import module namespace util="http://exist-db.org/xquery/util"; -import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; - -declare namespace test="http://exist-db.org/xquery/xqsuite"; - - -declare variable $readbinary:suite := "read-binary"; - -declare - %test:setUp -function readbinary:setup() as empty-sequence() { -}; - -declare - %test:tearDown -function readbinary:tear-down() as empty-sequence() { - helper:clear-suite-fs($readbinary:suite) -}; - -declare - %test:assertEquals("SERVER_SECRET=123!") -function readbinary:without-serialization() { - let $directory := helper:get-test-directory($readbinary:suite) - let $_ := helper:setup-fs-extra($directory) - - return - concat($directory, "/.env") - => file:read-binary() - => util:binary-to-string() -}; diff --git a/extensions/modules/file/src/test/xquery/modules/file/serialize.xqm b/extensions/modules/file/src/test/xquery/modules/file/serialize.xqm deleted file mode 100644 index 2610614faf4..00000000000 --- a/extensions/modules/file/src/test/xquery/modules/file/serialize.xqm +++ /dev/null @@ -1,148 +0,0 @@ -(: - : eXist-db Open Source Native XML Database - : Copyright (C) 2001 The eXist-db Authors - : - : info@exist-db.org - : http://www.exist-db.org - : - : This library is free software; you can redistribute it and/or - : modify it under the terms of the GNU Lesser General Public - : License as published by the Free Software Foundation; either - : version 2.1 of the License, or (at your option) any later version. - : - : This library is distributed in the hope that it will be useful, - : but WITHOUT ANY WARRANTY; without even the implied warranty of - : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - : Lesser General Public License for more details. - : - : You should have received a copy of the GNU Lesser General Public - : License along with this library; if not, write to the Free Software - : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - :) -xquery version "3.1"; - -module namespace serialize="http://exist-db.org/testsuite/modules/file/serialize"; - -import module namespace file="http://exist-db.org/xquery/file"; -import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; -import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; - - -declare namespace test="http://exist-db.org/xquery/xqsuite"; - - -declare variable $serialize:suite := "serialize"; -declare variable $serialize:text := data; -declare variable $serialize:xml := - - - text - -; - - -declare - %test:tearDown -function serialize:tear-down() as empty-sequence() { - helper:clear-suite-fs($serialize:suite) -}; - -declare - %test:assertEquals("datamoredata") -function serialize:append() { - let $directory := helper:get-test-directory($serialize:suite) - let $_ := file:mkdirs($directory) - - let $path := $directory || "/append-test.txt" - - let $_ := file:serialize-binary(xs:base64Binary("data"), $path) - let $_ := file:serialize-binary(xs:base64Binary("moredata"), $path, true()) - - return file:read-binary($path) => xs:string() -}; - -declare - %test:assertEquals("moredata") -function serialize:overwrite() { - let $directory := helper:get-test-directory($serialize:suite) - let $_ := file:mkdirs($directory) - - let $path := $directory || "/overwrite-test.txt" - let $_ := file:serialize-binary(xs:base64Binary("data"), $path) - let $_ := file:serialize-binary(xs:base64Binary("moredata"), $path, false()) - - return file:read-binary($path) => xs:string() -}; - -declare - %test:assertEquals("data") -function serialize:serialize3() { - let $directory := helper:get-test-directory($serialize:suite) - let $_ := file:mkdirs($directory) - - let $path := $directory || "/serialize-3-test.txt" - let $_ := file:serialize($serialize:text, $path, ()) - let $_ := file:serialize($serialize:text, $path, ()) - - return file:read($path) -}; - -declare - %test:assertTrue -function serialize:xml-defaults() { - let $directory := helper:get-test-directory($serialize:suite) - let $_ := file:mkdirs($directory) - - let $path := $directory || "/xml-defaults-test.xml" - let $_ := file:serialize($serialize:xml, $path, ()) - - return file:read($path) eq - "" || $fixtures:NL || - " " || $fixtures:NL || - " " || $fixtures:NL || - " text" || $fixtures:NL || - " " || $fixtures:NL || - "" -}; - -declare - %test:assertTrue -function serialize:xml-final-newline() { - let $directory := helper:get-test-directory($serialize:suite) - let $_ := file:mkdirs($directory) - - let $path := $directory || "/xml-final-newline.xml" - let $_ := file:serialize($serialize:xml, $path, - - - ) - - return file:read($path) eq - "" || $fixtures:NL || - " " || $fixtures:NL || - " " || $fixtures:NL || - " text" || $fixtures:NL || - " " || $fixtures:NL || - "" || $fixtures:NL -}; - -declare - %test:assertTrue -function serialize:xml-minified() { - let $directory := helper:get-test-directory($serialize:suite) - let $_ := file:mkdirs($directory) - - let $path := $directory || "/xml-minified.xml" - let $_ := file:serialize($serialize:xml, $path, - - - ) - - return file:read($path) eq - "" || $fixtures:NL || - " text" || $fixtures:NL || - " " -}; - diff --git a/extensions/modules/image/pom.xml b/extensions/modules/image/pom.xml index c0e0709c31f..f6919e277f9 100644 --- a/extensions/modules/image/pom.xml +++ b/extensions/modules/image/pom.xml @@ -75,7 +75,7 @@ org.exist-db - exist-file + exist-expath ${project.version} test @@ -112,8 +112,8 @@ - - org.exist-db:exist-file:jar:${project.version} + + org.exist-db:exist-expath:jar:${project.version} diff --git a/extensions/modules/image/src/test/resources-filtered/conf.xml b/extensions/modules/image/src/test/resources-filtered/conf.xml index 9df613700e8..49cab6955c6 100644 --- a/extensions/modules/image/src/test/resources-filtered/conf.xml +++ b/extensions/modules/image/src/test/resources-filtered/conf.xml @@ -747,7 +747,7 @@ - + diff --git a/extensions/modules/pom.xml b/extensions/modules/pom.xml index 0f8bf723555..fe5d12dc45b 100644 --- a/extensions/modules/pom.xml +++ b/extensions/modules/pom.xml @@ -54,7 +54,6 @@ exi expathrepo expathrepo/expathrepo-trigger-test - file image jndi mail From 31bd3e9701df33fdba9c1e47060752b5f7dcd972 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Thu, 12 Mar 2026 00:23:20 -0400 Subject: [PATCH 002/326] [feature] Improve EXPath File Module for W3C XQTS compliance Add missing function overloads: - file:read-text/read-text-lines 3-arg $fallback form - file:create-temp-dir/create-temp-file 2-arg form Fix error codes per EXPath File 4.0 spec: - file:copy/move raise file:no-dir when target parent missing - file:create-dir raises file:exists when path component is a file - file:read-binary rejects negative $length with file:out-of-range - file:write-binary validates $offset against file size Fix readBinary hang: replace BinaryValueFromInputStream (which uses CachingFilterInputStream/FilterInputStreamCacheMonitor infrastructure that prevents clean BrokerPool shutdown) with BinaryValueFromBinaryString. Reads file into byte[], base64-encodes, wraps in lightweight value type with no open handles and no-op close(). Tradeoff: ~2.4x memory for file content, acceptable for typical file module use cases. Resolve relative paths against XQuery static base URI when set as a file: URI, falling back to JVM working directory. Detect XML-illegal characters in read-text/read-text-lines: raise file:io-error by default, or replace with U+FFFD when $fallback=true. QT4 XQTS expath-file: 183/190 (96.3%), 0 hangs, 0 errors. --- .../expath/exist/file/ExpathFileModule.java | 9 +- .../exist/file/ExpathFileModuleHelper.java | 42 ++++- .../org/expath/exist/file/FileAppend.java | 2 +- .../java/org/expath/exist/file/FileIO.java | 155 ++++++++++++++---- .../expath/exist/file/FileManipulation.java | 57 ++++++- .../java/org/expath/exist/file/FilePaths.java | 4 +- .../org/expath/exist/file/FileProperties.java | 2 +- .../exist/file/FileSystemProperties.java | 13 ++ .../java/org/expath/exist/file/FileWrite.java | 9 +- 9 files changed, 246 insertions(+), 47 deletions(-) diff --git a/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java index bc0fa4ba42a..71515b9dcdc 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java @@ -50,6 +50,7 @@ public class ExpathFileModule extends AbstractInternalModule { new FunctionDef(FileProperties.signatures[6], FileProperties.class), // FileIO: read-text(1), read-text(2), read-text-lines(1), read-text-lines(2), + // read-text(3-fallback), read-text-lines(3-fallback), // read-binary(1), read-binary(2), read-binary(3) new FunctionDef(FileIO.signatures[0], FileIO.class), new FunctionDef(FileIO.signatures[1], FileIO.class), @@ -58,6 +59,8 @@ public class ExpathFileModule extends AbstractInternalModule { new FunctionDef(FileIO.signatures[4], FileIO.class), new FunctionDef(FileIO.signatures[5], FileIO.class), new FunctionDef(FileIO.signatures[6], FileIO.class), + new FunctionDef(FileIO.signatures[7], FileIO.class), + new FunctionDef(FileIO.signatures[8], FileIO.class), // FileWrite: write(2), write(3), write-text(2), write-text(3), // write-text-lines(2), write-text-lines(3), write-binary(2), write-binary(3) @@ -81,7 +84,9 @@ public class ExpathFileModule extends AbstractInternalModule { new FunctionDef(FileAppend.signatures[6], FileAppend.class), // FileManipulation: copy, move, delete(1), delete(2), create-dir, - // create-temp-dir, create-temp-file, list(1), list(2), list(3), + // create-temp-dir(2), create-temp-dir(3), + // create-temp-file(2), create-temp-file(3), + // list(1), list(2), list(3), // children, descendants, list-roots new FunctionDef(FileManipulation.signatures[0], FileManipulation.class), new FunctionDef(FileManipulation.signatures[1], FileManipulation.class), @@ -96,6 +101,8 @@ public class ExpathFileModule extends AbstractInternalModule { new FunctionDef(FileManipulation.signatures[10], FileManipulation.class), new FunctionDef(FileManipulation.signatures[11], FileManipulation.class), new FunctionDef(FileManipulation.signatures[12], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[13], FileManipulation.class), + new FunctionDef(FileManipulation.signatures[14], FileManipulation.class), // FilePaths: name, parent, path-to-native, path-to-uri, resolve-path(1), resolve-path(2) new FunctionDef(FilePaths.signatures[0], FilePaths.class), diff --git a/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModuleHelper.java b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModuleHelper.java index b6b48182471..ea59d882c31 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModuleHelper.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModuleHelper.java @@ -58,6 +58,7 @@ public static void checkDbaRole(final XQueryContext context, final Expression ex /** * Resolve a path string (file: URI or native path) to a {@link Path}. + * Relative paths are resolved against the JVM working directory. * * @param path the path string or file: URI * @param expression the calling expression (for error reporting) @@ -65,12 +66,49 @@ public static void checkDbaRole(final XQueryContext context, final Expression ex * @throws XPathException if the path is invalid */ public static Path getPath(final String path, final Expression expression) throws XPathException { + return getPath(path, expression, null); + } + + /** + * Resolve a path string (file: URI or native path) to a {@link Path}. + * Relative paths are resolved against the XQuery static base URI if it is a + * file: URI, otherwise against the JVM working directory. + * + * @param path the path string or file: URI + * @param expression the calling expression (for error reporting) + * @param context the XQuery context (may be null) + * @return the resolved Path + * @throws XPathException if the path is invalid + */ + public static Path getPath(final String path, final Expression expression, final XQueryContext context) throws XPathException { try { if (path.startsWith("file:")) { return Paths.get(new URI(path)); - } else { - return Paths.get(path); } + + final Path p = Paths.get(path); + if (p.isAbsolute()) { + return p; + } + + // Resolve relative paths against static base URI if available + if (context != null) { + try { + final String baseUri = context.getBaseURI().getStringValue(); + if (baseUri != null && baseUri.startsWith("file:")) { + final Path basePath = Paths.get(new URI(baseUri)); + // Base URI may point to a file; resolve against its parent directory + final Path baseDir = java.nio.file.Files.isDirectory(basePath) ? basePath : basePath.getParent(); + if (baseDir != null) { + return baseDir.resolve(p); + } + } + } catch (final Exception ignored) { + // Fall through to default resolution + } + } + + return p; } catch (final InvalidPathException e) { throw new XPathException(expression, ExpathFileErrorCode.INVALID_PATH, "Invalid path: " + path + " - " + e.getMessage()); diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileAppend.java b/extensions/expath/src/main/java/org/expath/exist/file/FileAppend.java index 88de70ab611..78d75f2b4eb 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/FileAppend.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileAppend.java @@ -138,7 +138,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro ExpathFileModuleHelper.checkDbaRole(context, this); final String pathStr = args[0].getStringValue(); - final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this, context); checkParentDir(path); diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileIO.java b/extensions/expath/src/main/java/org/expath/exist/file/FileIO.java index a5c4e35be1b..39a2eb3a297 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/FileIO.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileIO.java @@ -36,7 +36,7 @@ import org.exist.xquery.XPathException; import org.exist.xquery.XQueryContext; import org.exist.xquery.value.Base64BinaryValueType; -import org.exist.xquery.value.BinaryValueFromInputStream; +import org.exist.xquery.value.BinaryValueFromBinaryString; import org.exist.xquery.value.FunctionParameterSequenceType; import org.exist.xquery.value.FunctionReturnSequenceType; import org.exist.xquery.value.Sequence; @@ -88,6 +88,24 @@ public class FileIO extends BasicFunction { new SequenceType[]{FILE_PARAM, ENCODING_PARAM}, new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "the lines of the file.") ), + // file:read-text($file as xs:string, $encoding as xs:string?, $fallback as xs:boolean) as xs:string + new FunctionSignature( + new QName("read-text", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Reads the contents of a file as text. If $fallback is true, invalid characters are replaced.", + new SequenceType[]{FILE_PARAM, ENCODING_PARAM, + new FunctionParameterSequenceType("fallback", Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "If true, replace invalid characters with the Unicode replacement character.")}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the file contents as string.") + ), + // file:read-text-lines($file as xs:string, $encoding as xs:string?, $fallback as xs:boolean) as xs:string* + new FunctionSignature( + new QName("read-text-lines", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Reads the contents of a file as a sequence of lines. If $fallback is true, invalid characters are replaced.", + new SequenceType[]{FILE_PARAM, ENCODING_PARAM, + new FunctionParameterSequenceType("fallback", Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "If true, replace invalid characters with the Unicode replacement character.")}, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "the lines of the file.") + ), // file:read-binary($file as xs:string) as xs:base64Binary new FunctionSignature( new QName("read-binary", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), @@ -130,7 +148,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro ExpathFileModuleHelper.checkDbaRole(context, this); final String pathStr = args[0].getStringValue(); - final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this, context); if (!Files.exists(path)) { throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, @@ -154,11 +172,16 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro private Sequence readText(final Path path, final Sequence[] args) throws XPathException { final Charset encoding = getEncoding(args, 1); + final boolean fallback = args.length > 2 && !args[2].isEmpty() + && args[2].itemAt(0).toJavaObject(Boolean.class); try { - final String content = Files.readString(path, encoding); + final String content = readFileText(path, encoding, fallback); // Normalize newlines per spec: CR or CRLF -> LF final String normalized = content.replace("\r\n", "\n").replace("\r", "\n"); return new StringValue(this, normalized); + } catch (final java.nio.charset.MalformedInputException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, + "Invalid characters in file for encoding " + encoding.name()); } catch (final IOException e) { throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); } @@ -166,8 +189,10 @@ private Sequence readText(final Path path, final Sequence[] args) throws XPathEx private Sequence readTextLines(final Path path, final Sequence[] args) throws XPathException { final Charset encoding = getEncoding(args, 1); + final boolean fallback = args.length > 2 && !args[2].isEmpty() + && args[2].itemAt(0).toJavaObject(Boolean.class); try { - final String content = Files.readString(path, encoding); + final String content = readFileText(path, encoding, fallback); // Split at newline boundaries per spec final String[] lines = content.split("\r\n|\r|\n", -1); final ValueSequence result = new ValueSequence(lines.length); @@ -177,6 +202,9 @@ private Sequence readTextLines(final Path path, final Sequence[] args) throws XP result.add(new StringValue(this, lines[i])); } return result; + } catch (final java.nio.charset.MalformedInputException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, + "Invalid characters in file for encoding " + encoding.name()); } catch (final IOException e) { throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); } @@ -184,41 +212,106 @@ private Sequence readTextLines(final Path path, final Sequence[] args) throws XP private Sequence readBinary(final Path path, final Sequence[] args) throws XPathException { final long offset = args.length > 1 && !args[1].isEmpty() ? args[1].itemAt(0).toJavaObject(Long.class) : 0; - final long length = args.length > 2 && !args[2].isEmpty() ? args[2].itemAt(0).toJavaObject(Long.class) : -1; + final boolean hasLength = args.length > 2 && !args[2].isEmpty(); + final long length = hasLength ? args[2].itemAt(0).toJavaObject(Long.class) : -1; try { final long fileSize = Files.size(path); - if (offset < 0 || offset > fileSize) { - throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, - "Offset " + offset + " is out of range for file of size " + fileSize); - } - if (length < -1) { - throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, - "Length must not be negative: " + length); - } - if (length >= 0 && offset + length > fileSize) { - throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, - "Offset + length exceeds file size: " + (offset + length) + " > " + fileSize); - } + validateBinaryRange(offset, length, hasLength, fileSize); + + final byte[] data = readBinaryData(path, offset, hasLength, length, fileSize); + final String base64 = java.util.Base64.getEncoder().encodeToString(data); + return new BinaryValueFromBinaryString(this, new Base64BinaryValueType(), base64); + } catch (final IOException e) { + throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); + } + } - if (offset == 0 && length < 0) { - // Read entire file - final InputStream is = Files.newInputStream(path); - return BinaryValueFromInputStream.getInstance(context, new Base64BinaryValueType(), is, this); + private void validateBinaryRange(final long offset, final long length, final boolean hasLength, final long fileSize) throws XPathException { + if (offset < 0 || offset > fileSize) { + throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, + "Offset " + offset + " is out of range for file of size " + fileSize); + } + if (hasLength && length < 0) { + throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, + "Length must not be negative: " + length); + } + if (hasLength && offset + length > fileSize) { + throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, + "Offset + length exceeds file size: " + (offset + length) + " > " + fileSize); + } + } + + private byte[] readBinaryData(final Path path, final long offset, final boolean hasLength, final long length, final long fileSize) throws IOException { + if (offset == 0 && !hasLength) { + return Files.readAllBytes(path); + } + try (final RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) { + raf.seek(offset); + final int readLen = hasLength ? (int) length : (int) (fileSize - offset); + final byte[] data = new byte[readLen]; + raf.readFully(data); + return data; + } + } + + /** + * Reads a file as text with the given encoding. + * If fallback is true, malformed byte sequences and XML-illegal characters + * are replaced with U+FFFD. Otherwise, an IOException is thrown if the file + * contains malformed bytes or XML-illegal characters. + */ + private String readFileText(final Path path, final Charset encoding, final boolean fallback) throws IOException { + final String content; + if (fallback) { + final java.nio.charset.CharsetDecoder decoder = encoding.newDecoder() + .onMalformedInput(java.nio.charset.CodingErrorAction.REPLACE) + .onUnmappableCharacter(java.nio.charset.CodingErrorAction.REPLACE) + .replaceWith("\uFFFD"); + final byte[] bytes = Files.readAllBytes(path); + content = decoder.decode(java.nio.ByteBuffer.wrap(bytes)).toString(); + // Replace XML-illegal characters with U+FFFD + return replaceXmlIllegalChars(content); + } else { + content = Files.readString(path, encoding); + // Check for XML-illegal characters + checkXmlIllegalChars(content); + return content; + } + } + + /** + * Check if a string contains characters illegal in XML 1.0 and throw IOException if so. + * XML 1.0 allows: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] + */ + private void checkXmlIllegalChars(final String text) throws IOException { + for (int i = 0; i < text.length(); i++) { + final char c = text.charAt(i); + if (c < 0x20 && c != 0x9 && c != 0xA && c != 0xD) { + throw new IOException("File contains XML-illegal character U+" + + String.format("%04X", (int) c) + " at position " + i); + } + if (c >= 0xFFFE) { + throw new IOException("File contains XML-illegal character U+" + + String.format("%04X", (int) c) + " at position " + i); } + } + } - // Partial read - try (final RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) { - raf.seek(offset); - final int readLen = length >= 0 ? (int) length : (int) (fileSize - offset); - final byte[] data = new byte[readLen]; - raf.readFully(data); - final InputStream bis = new java.io.ByteArrayInputStream(data); - return BinaryValueFromInputStream.getInstance(context, new Base64BinaryValueType(), bis, this); + /** + * Replace characters illegal in XML 1.0 with U+FFFD. + */ + private String replaceXmlIllegalChars(final String text) { + final StringBuilder sb = new StringBuilder(text.length()); + for (int i = 0; i < text.length(); i++) { + final char c = text.charAt(i); + if ((c < 0x20 && c != 0x9 && c != 0xA && c != 0xD) || c >= 0xFFFE) { + sb.append('\uFFFD'); + } else { + sb.append(c); } - } catch (final IOException e) { - throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage()); } + return sb.toString(); } private Charset getEncoding(final Sequence[] args, final int index) throws XPathException { diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java b/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java index 23f5821a8de..3ec848837da 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java @@ -97,6 +97,16 @@ public class FileManipulation extends BasicFunction { }, new FunctionReturnSequenceType(Type.ITEM, Cardinality.EMPTY_SEQUENCE, "empty sequence.") ), + // file:create-temp-dir($prefix, $suffix) + new FunctionSignature( + new QName("create-temp-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Creates a temporary directory in the system default temp directory.", + new SequenceType[]{ + new FunctionParameterSequenceType("prefix", Type.STRING, Cardinality.ZERO_OR_ONE, "Prefix for the directory name."), + new FunctionParameterSequenceType("suffix", Type.STRING, Cardinality.ZERO_OR_ONE, "Suffix for the directory name.") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "The path of the created temporary directory.") + ), // file:create-temp-dir($prefix, $suffix, $dir) new FunctionSignature( new QName("create-temp-dir", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), @@ -108,6 +118,16 @@ public class FileManipulation extends BasicFunction { }, new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "The path of the created temporary directory.") ), + // file:create-temp-file($prefix, $suffix) + new FunctionSignature( + new QName("create-temp-file", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), + "Creates a temporary file in the system default temp directory.", + new SequenceType[]{ + new FunctionParameterSequenceType("prefix", Type.STRING, Cardinality.ZERO_OR_ONE, "Prefix for the file name."), + new FunctionParameterSequenceType("suffix", Type.STRING, Cardinality.ZERO_OR_ONE, "Suffix for the file name.") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "The path of the created temporary file.") + ), // file:create-temp-file($prefix, $suffix, $dir) new FunctionSignature( new QName("create-temp-file", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX), @@ -185,7 +205,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro } final String pathStr = args[0].getStringValue(); - final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this, context); if (isCalledAs("copy")) { return copy(path, args); @@ -215,7 +235,15 @@ private Sequence copy(final Path source, final Sequence[] args) throws XPathExce throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, "Source does not exist: " + source.toAbsolutePath()); } - final Path target = ExpathFileModuleHelper.getPath(args[1].getStringValue(), this); + final Path target = ExpathFileModuleHelper.getPath(args[1].getStringValue(), this, context); + + // Check target parent directory exists + final Path targetParent = target.toAbsolutePath().getParent(); + if (targetParent != null && !Files.isDirectory(targetParent)) { + throw new XPathException(this, ExpathFileErrorCode.NO_DIR, + "Target parent directory does not exist: " + targetParent); + } + try { if (Files.isDirectory(source)) { copyDirectory(source, target); @@ -252,7 +280,15 @@ private Sequence move(final Path source, final Sequence[] args) throws XPathExce throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, "Source does not exist: " + source.toAbsolutePath()); } - final Path target = ExpathFileModuleHelper.getPath(args[1].getStringValue(), this); + final Path target = ExpathFileModuleHelper.getPath(args[1].getStringValue(), this, context); + + // Check target parent directory exists + final Path targetParent = target.toAbsolutePath().getParent(); + if (targetParent != null && !Files.isDirectory(targetParent)) { + throw new XPathException(this, ExpathFileErrorCode.NO_DIR, + "Target parent directory does not exist: " + targetParent); + } + try { // If target is an existing directory, move into it final Path actualTarget = Files.isDirectory(target) ? target.resolve(source.getFileName()) : target; @@ -305,9 +341,14 @@ private Sequence delete(final Path path, final Sequence[] args) throws XPathExce } private Sequence createDir(final Path path) throws XPathException { - if (Files.exists(path) && !Files.isDirectory(path)) { - throw new XPathException(this, ExpathFileErrorCode.EXISTS, - "Path exists and is not a directory: " + path.toAbsolutePath()); + // Check if the path itself or any ancestor is an existing non-directory file + Path check = path.toAbsolutePath().normalize(); + while (check != null) { + if (Files.exists(check) && !Files.isDirectory(check)) { + throw new XPathException(this, ExpathFileErrorCode.EXISTS, + "Path exists and is not a directory: " + check); + } + check = check.getParent(); } try { Files.createDirectories(path); @@ -321,7 +362,7 @@ private Sequence createTempDir(final Sequence[] args) throws XPathException { final String prefix = args.length > 0 && !args[0].isEmpty() ? args[0].getStringValue() : ""; final String suffix = args.length > 1 && !args[1].isEmpty() ? args[1].getStringValue() : ""; final Path dir = args.length > 2 && !args[2].isEmpty() - ? ExpathFileModuleHelper.getPath(args[2].getStringValue(), this) + ? ExpathFileModuleHelper.getPath(args[2].getStringValue(), this, context) : Paths.get(System.getProperty("java.io.tmpdir")); if (!Files.isDirectory(dir)) { @@ -347,7 +388,7 @@ private Sequence createTempFile(final Sequence[] args) throws XPathException { final String prefix = args.length > 0 && !args[0].isEmpty() ? args[0].getStringValue() : ""; final String suffix = args.length > 1 && !args[1].isEmpty() ? args[1].getStringValue() : ""; final Path dir = args.length > 2 && !args[2].isEmpty() - ? ExpathFileModuleHelper.getPath(args[2].getStringValue(), this) + ? ExpathFileModuleHelper.getPath(args[2].getStringValue(), this, context) : Paths.get(System.getProperty("java.io.tmpdir")); if (!Files.isDirectory(dir)) { diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FilePaths.java b/extensions/expath/src/main/java/org/expath/exist/file/FilePaths.java index ee8c260bc96..1d3baa9ded5 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/FilePaths.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/FilePaths.java @@ -101,7 +101,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro ExpathFileModuleHelper.checkDbaRole(context, this); final String pathStr = args[0].getStringValue(); - final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this, context); if (isCalledAs("name")) { final Path fileName = path.getFileName(); @@ -128,7 +128,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro return new AnyURIValue(this, abs.toUri().toString()); } else if (isCalledAs("resolve-path")) { if (args.length > 1 && !args[1].isEmpty()) { - final Path base = ExpathFileModuleHelper.getPath(args[1].getStringValue(), this); + final Path base = ExpathFileModuleHelper.getPath(args[1].getStringValue(), this, context); return new StringValue(this, base.resolve(path).toAbsolutePath().normalize().toString()); } return new StringValue(this, path.toAbsolutePath().normalize().toString()); diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileProperties.java b/extensions/expath/src/main/java/org/expath/exist/file/FileProperties.java index d7027f06977..6b794bd3a94 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/FileProperties.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileProperties.java @@ -126,7 +126,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro ExpathFileModuleHelper.checkDbaRole(context, this); final String pathStr = args[0].getStringValue(); - final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this, context); if (isCalledAs("exists")) { return BooleanValue.valueOf(Files.exists(path)); diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileSystemProperties.java b/extensions/expath/src/main/java/org/expath/exist/file/FileSystemProperties.java index 0af85e24717..735735cd638 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/FileSystemProperties.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileSystemProperties.java @@ -122,6 +122,19 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro } return Sequence.EMPTY_SEQUENCE; } else if (isCalledAs("current-dir")) { + // If a file: base URI is set (e.g., sandpit), use its directory as the working directory + try { + final String baseURI = context.getBaseURI().getStringValue(); + if (baseURI != null && !baseURI.isEmpty() && baseURI.startsWith("file:")) { + final Path basePath = Paths.get(new URI(baseURI)); + final Path dir = java.nio.file.Files.isDirectory(basePath) ? basePath : basePath.getParent(); + if (dir != null) { + return new StringValue(this, dir.toString() + File.separator); + } + } + } catch (final Exception ignored) { + // Fall through to JVM CWD + } return new StringValue(this, System.getProperty("user.dir") + File.separator); } diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileWrite.java b/extensions/expath/src/main/java/org/expath/exist/file/FileWrite.java index d89eeb103a8..d1fe234ebae 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/FileWrite.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileWrite.java @@ -148,7 +148,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro ExpathFileModuleHelper.checkDbaRole(context, this); final String pathStr = args[0].getStringValue(); - final Path path = ExpathFileModuleHelper.getPath(pathStr, this); + final Path path = ExpathFileModuleHelper.getPath(pathStr, this, context); checkParentDir(path); @@ -251,6 +251,13 @@ private Sequence writeBinary(final Path path, final Sequence[] args) throws XPat throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, "Offset must not be negative: " + offset); } + if (Files.exists(path)) { + final long fileSize = Files.size(path); + if (offset > fileSize) { + throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE, + "Offset " + offset + " exceeds file size " + fileSize); + } + } try (final RandomAccessFile raf = new RandomAccessFile(path.toFile(), "rw"); final InputStream is = binaryValue.getInputStream()) { raf.seek(offset); From b05fa817a74d867eeb5cb58e85439230e65e5e6e Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Fri, 20 Mar 2026 03:20:57 -0400 Subject: [PATCH 003/326] [refactor] Fix Codacy PMD violations in EXPath File Module Remove unused ENTRY_PARAM field, replace raw RuntimeException with IllegalStateException, fix parameter reassignment in noOtherNCNameAttribute(), and suppress NPathComplexity on list(). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../main/java/org/expath/exist/ZipFileFunctions.java | 2 -- .../java/org/expath/exist/file/FileManipulation.java | 1 + .../org/expath/tools/model/exist/EXistElement.java | 10 ++++------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/extensions/expath/src/main/java/org/expath/exist/ZipFileFunctions.java b/extensions/expath/src/main/java/org/expath/exist/ZipFileFunctions.java index f4093a61b64..f91f6e81165 100644 --- a/extensions/expath/src/main/java/org/expath/exist/ZipFileFunctions.java +++ b/extensions/expath/src/main/java/org/expath/exist/ZipFileFunctions.java @@ -53,8 +53,6 @@ public class ZipFileFunctions extends BasicFunction { private static final Logger logger = LogManager.getLogger(ZipFileFunctions.class); private final static FunctionParameterSequenceType HREF_PARAM = new FunctionParameterSequenceType("href", Type.ANY_URI, Cardinality.EXACTLY_ONE, "The URI for locating the Zip file"); - private final static FunctionParameterSequenceType ENTRY_PARAM = new FunctionParameterSequenceType("entry", Type.ELEMENT, Cardinality.EXACTLY_ONE, "A zip:entry element describing the contents of the file"); - private final static String FILE_ENTRIES = "entries"; private final static String ZIP_FILE = "zip-file"; private final static String UPDATE_ENTRIES = "update"; diff --git a/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java b/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java index 3ec848837da..d671ce40a13 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/FileManipulation.java @@ -404,6 +404,7 @@ private Sequence createTempFile(final Sequence[] args) throws XPathException { } } + @SuppressWarnings("PMD.NPathComplexity") private Sequence list(final Path dir, final Sequence[] args) throws XPathException { if (!Files.exists(dir)) { throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND, diff --git a/extensions/expath/src/main/java/org/expath/tools/model/exist/EXistElement.java b/extensions/expath/src/main/java/org/expath/tools/model/exist/EXistElement.java index a78a5eb87d4..5a6a464f867 100644 --- a/extensions/expath/src/main/java/org/expath/tools/model/exist/EXistElement.java +++ b/extensions/expath/src/main/java/org/expath/tools/model/exist/EXistElement.java @@ -115,7 +115,7 @@ public Sequence getContent() { } return new EXistSequence(valueSequence, context); } catch(final XPathException xpe) { - throw new RuntimeException(xpe.getMessage(), xpe); + throw new IllegalStateException("Failed to build content sequence", xpe); } } @@ -153,13 +153,11 @@ public Iterable children(final String ns) { } @Override - public void noOtherNCNameAttribute(final String[] names, String[] forbidden_ns) throws ToolsException { - if ( forbidden_ns == null ) { - forbidden_ns = new String[] { }; - } + public void noOtherNCNameAttribute(final String[] names, final String[] forbidden_ns) throws ToolsException { + final String[] effectiveForbiddenNs = forbidden_ns == null ? new String[]{} : forbidden_ns; final String[] sorted_names = sortCopy(names); - final String[] sorted_ns = sortCopy(forbidden_ns); + final String[] sorted_ns = sortCopy(effectiveForbiddenNs); final NamedNodeMap attributes = element.getNode().getAttributes(); From 2245fe15411d1bc73cab2c16fa733fbb0f95adfb Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Fri, 20 Mar 2026 20:14:16 -0400 Subject: [PATCH 004/326] [feature] Coexist with original file module instead of replacing it Change the EXPath File module to coexist alongside eXist's original file module rather than replacing it. This is a non-breaking, additive change suitable for a feature release. - Change default prefix from "file" to "exfile" in ExpathFileModule.java (namespace URI unchanged: http://expath.org/ns/file) - Restore original file module (extensions/modules/file/) from develop - Register BOTH modules in conf.xml: - http://exist-db.org/xquery/file (original, unchanged) - http://expath.org/ns/file (EXPath File 4.0, new) - Keep file:sync in original FileModule; util:file-sync also available - Update all EXPath test files to use exfile: prefix - Restore exist-distribution pom.xml dependency on exist-file - Restore image module's original dependency on exist-file Both module test suites pass: - EXPath File: 108 tests, 0 failures - Original File: 63 tests, 0 failures Co-Authored-By: Claude Opus 4.6 (1M context) --- .../resources/org/exist/xmldb/allowAnyUri.xml | 2 +- exist-distribution/pom.xml | 6 + exist-distribution/src/main/config/conf.xml | 1 + .../expath/exist/file/ExpathFileModule.java | 2 +- .../src/test/resources-filtered/conf.xml | 1 + .../expath/src/test/resources/util/helper.xqm | 28 +- .../org/expath/exist/file/file-tests.xql | 250 +++--- .../xquery/org/expath/exist/file/sync.xqm | 8 +- .../xquery/org/expath/exist/file/syncmod.xqm | 2 +- extensions/modules/file/pom.xml | 201 +++++ .../exist/xquery/modules/file/Directory.java | 205 +++++ .../xquery/modules/file/DirectoryCreate.java | 115 +++ .../xquery/modules/file/DirectoryList.java | 191 +++++ .../exist/xquery/modules/file/FileDelete.java | 82 ++ .../xquery/modules/file/FileErrorCode.java | 34 + .../exist/xquery/modules/file/FileExists.java | 85 ++ .../xquery/modules/file/FileIsDirectory.java | 85 ++ .../xquery/modules/file/FileIsReadable.java | 84 ++ .../xquery/modules/file/FileIsWriteable.java | 84 ++ .../exist/xquery/modules/file/FileModule.java | 100 +++ .../xquery/modules/file/FileModuleHelper.java | 63 ++ .../exist/xquery/modules/file/FileMove.java | 101 +++ .../exist/xquery/modules/file/FileRead.java | 110 +++ .../xquery/modules/file/FileReadBinary.java | 80 ++ .../xquery/modules/file/FileReadUnicode.java | 118 +++ .../xquery/modules/file/SerializeToFile.java | 236 ++++++ .../org/exist/xquery/modules/file/Sync.java | 593 +++++++++++++ .../xquery/modules/file/UnicodeReader.java | 181 ++++ .../org/exist/xquery/modules/file/repo.xsl | 61 ++ .../modules/file/AbstractBinariesTest.java | 140 ++++ .../modules/file/EmbeddedBinariesTest.java | 164 ++++ .../xquery/modules/file/RestBinariesTest.java | 219 +++++ .../modules/file/XmldbBinariesTest.java | 199 +++++ .../java/xquery/modules/file/FileTests.java | 33 + .../file/src/test/resources-filtered/conf.xml | 781 ++++++++++++++++++ .../file/src/test/resources/log4j2.xml | 36 + .../WEB-INF/controller-config.xml | 33 + .../standalone-webapp/WEB-INF/web.xml | 80 ++ .../file/src/test/resources/util/fixtures.xqm | 127 +++ .../file/src/test/resources/util/helper.xqm | 219 +++++ .../xquery/modules/file/directory-list.xqm | 47 ++ .../test/xquery/modules/file/read-binary.xqm | 57 ++ .../test/xquery/modules/file/serialize.xqm | 148 ++++ .../xquery/modules/file/sync-serialize.xqm | 249 ++++++ .../src/test/xquery/modules/file/sync.xqm | 423 ++++++++++ .../src/test/xquery/modules/file/syncmod.xqm | 293 +++++++ extensions/modules/image/pom.xml | 6 +- .../src/test/resources-filtered/conf.xml | 2 +- extensions/modules/pom.xml | 1 + 49 files changed, 6216 insertions(+), 150 deletions(-) create mode 100644 extensions/modules/file/pom.xml create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Directory.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryCreate.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryList.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileDelete.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileErrorCode.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileExists.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsDirectory.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsReadable.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsWriteable.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModule.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModuleHelper.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileMove.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileRead.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadBinary.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadUnicode.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/SerializeToFile.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java create mode 100644 extensions/modules/file/src/main/java/org/exist/xquery/modules/file/UnicodeReader.java create mode 100644 extensions/modules/file/src/main/resources/org/exist/xquery/modules/file/repo.xsl create mode 100644 extensions/modules/file/src/test/java/org/exist/xquery/modules/file/AbstractBinariesTest.java create mode 100644 extensions/modules/file/src/test/java/org/exist/xquery/modules/file/EmbeddedBinariesTest.java create mode 100644 extensions/modules/file/src/test/java/org/exist/xquery/modules/file/RestBinariesTest.java create mode 100644 extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java create mode 100644 extensions/modules/file/src/test/java/xquery/modules/file/FileTests.java create mode 100644 extensions/modules/file/src/test/resources-filtered/conf.xml create mode 100644 extensions/modules/file/src/test/resources/log4j2.xml create mode 100644 extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml create mode 100644 extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/web.xml create mode 100644 extensions/modules/file/src/test/resources/util/fixtures.xqm create mode 100644 extensions/modules/file/src/test/resources/util/helper.xqm create mode 100644 extensions/modules/file/src/test/xquery/modules/file/directory-list.xqm create mode 100644 extensions/modules/file/src/test/xquery/modules/file/read-binary.xqm create mode 100644 extensions/modules/file/src/test/xquery/modules/file/serialize.xqm create mode 100644 extensions/modules/file/src/test/xquery/modules/file/sync-serialize.xqm create mode 100644 extensions/modules/file/src/test/xquery/modules/file/sync.xqm create mode 100644 extensions/modules/file/src/test/xquery/modules/file/syncmod.xqm diff --git a/exist-core/src/test/resources/org/exist/xmldb/allowAnyUri.xml b/exist-core/src/test/resources/org/exist/xmldb/allowAnyUri.xml index 589e37370c4..46f2ccd8e99 100644 --- a/exist-core/src/test/resources/org/exist/xmldb/allowAnyUri.xml +++ b/exist-core/src/test/resources/org/exist/xmldb/allowAnyUri.xml @@ -1022,7 +1022,7 @@ - + diff --git a/exist-distribution/pom.xml b/exist-distribution/pom.xml index 6fa28396d37..a1373fec076 100644 --- a/exist-distribution/pom.xml +++ b/exist-distribution/pom.xml @@ -198,6 +198,12 @@ ${project.version} runtime + + ${project.groupId} + exist-file + ${project.version} + runtime + ${project.groupId} exist-image diff --git a/exist-distribution/src/main/config/conf.xml b/exist-distribution/src/main/config/conf.xml index 708b45097f5..67261afc561 100644 --- a/exist-distribution/src/main/config/conf.xml +++ b/exist-distribution/src/main/config/conf.xml @@ -1048,6 +1048,7 @@ + diff --git a/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java index 71515b9dcdc..c7bbbcb8277 100644 --- a/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java +++ b/extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java @@ -35,7 +35,7 @@ public class ExpathFileModule extends AbstractInternalModule { public static final String NAMESPACE_URI = "http://expath.org/ns/file"; - public static final String PREFIX = "file"; + public static final String PREFIX = "exfile"; public static final String INCLUSION_DATE = "2025-05-01"; public static final String RELEASED_IN_VERSION = "7.0.0"; diff --git a/extensions/expath/src/test/resources-filtered/conf.xml b/extensions/expath/src/test/resources-filtered/conf.xml index ffcfb5d71b0..97aeb503126 100644 --- a/extensions/expath/src/test/resources-filtered/conf.xml +++ b/extensions/expath/src/test/resources-filtered/conf.xml @@ -745,6 +745,7 @@ + diff --git a/extensions/expath/src/test/resources/util/helper.xqm b/extensions/expath/src/test/resources/util/helper.xqm index c34307815bb..bbb4b494e99 100644 --- a/extensions/expath/src/test/resources/util/helper.xqm +++ b/extensions/expath/src/test/resources/util/helper.xqm @@ -23,7 +23,7 @@ xquery version "3.1"; module namespace helper="http://exist-db.org/xquery/test/util/helper"; import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "fixtures.xqm"; -import module namespace file="http://expath.org/ns/file"; +import module namespace exfile="http://expath.org/ns/file"; import module namespace xmldb="http://exist-db.org/xquery/xmldb"; import module namespace util="http://exist-db.org/xquery/util"; @@ -76,15 +76,15 @@ declare function helper:clear-suite-fs ($suite as xs:string) as empty-sequence() $suite )) return - if (file:exists($dir)) then - let $_ := file:delete($dir, true()) + if (exfile:exists($dir)) then + let $_ := exfile:delete($dir, true()) return () else () }; declare function helper:clear-fs ($directory as xs:string) as empty-sequence() { - if (file:exists($directory)) then - let $_ := file:delete($directory, true()) + if (exfile:exists($directory)) then + let $_ := exfile:delete($directory, true()) return () else () }; @@ -106,13 +106,13 @@ declare function helper:glue-path ($parts as xs:string+) as xs:string { : @returns given directory to allow use in pipeline (chain of arrow operators) :) declare function helper:setup-fs-extra ($directory as xs:string) as xs:string { - let $_ := file:create-dir($directory) - let $_ := file:create-dir(helper:glue-path(($directory, "test"))) + let $_ := exfile:create-dir($directory) + let $_ := exfile:create-dir(helper:glue-path(($directory, "test"))) let $_ := ( - file:write-binary( + exfile:write-binary( helper:glue-path(($directory, ".env")), util:string-to-binary("SERVER_SECRET=123!")), - file:write-binary( + exfile:write-binary( helper:glue-path(($directory, "test", "three.s")), util:string-to-binary("...")) ) @@ -132,12 +132,12 @@ declare function helper:get-updated-from-sync-result ($result as element(utilns: }; declare function helper:list-files-and-directories ($directory as xs:string) as xs:string* { - (: EXPath file:list returns relative path strings, directories end with separator :) - for $entry in file:list($directory) + (: EXPath exfile:list returns relative path strings, directories end with separator :) + for $entry in exfile:list($directory) return (: strip trailing separator from directory names :) - if (ends-with($entry, file:dir-separator())) - then substring($entry, 1, string-length($entry) - string-length(file:dir-separator())) + if (ends-with($entry, exfile:dir-separator())) + then substring($entry, 1, string-length($entry) - string-length(exfile:dir-separator())) else $entry }; @@ -214,7 +214,7 @@ declare function helper:maybe-remove-item-at-index($sequence as xs:anyAtomicType declare function helper:assert-file-contents($expected as xs:string, $path-parts as xs:string+) as xs:boolean { let $path := helper:glue-path($path-parts) - let $actual := file:read-text($path) + let $actual := exfile:read-text($path) return if ( diff --git a/extensions/expath/src/test/xquery/org/expath/exist/file/file-tests.xql b/extensions/expath/src/test/xquery/org/expath/exist/file/file-tests.xql index 12b6233a274..7079255ba83 100644 --- a/extensions/expath/src/test/xquery/org/expath/exist/file/file-tests.xql +++ b/extensions/expath/src/test/xquery/org/expath/exist/file/file-tests.xql @@ -25,11 +25,11 @@ module namespace ft="http://exist-db.org/testsuite/expath-file"; import module namespace test="http://exist-db.org/xquery/xqsuite" at "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql"; -import module namespace file="http://expath.org/ns/file"; +import module namespace exfile="http://expath.org/ns/file"; (: ======== Helper variables ======== :) -declare variable $ft:temp-dir := file:temp-dir(); +declare variable $ft:temp-dir := exfile:temp-dir(); declare variable $ft:test-dir := $ft:temp-dir || "expath-file-test/"; (: ======== Setup / Teardown ======== :) @@ -37,14 +37,14 @@ declare variable $ft:test-dir := $ft:temp-dir || "expath-file-test/"; declare %test:setUp function ft:setup() { - file:create-dir($ft:test-dir) + exfile:create-dir($ft:test-dir) }; declare %test:tearDown function ft:teardown() { - if (file:exists($ft:test-dir)) then - file:delete($ft:test-dir, true()) + if (exfile:exists($ft:test-dir)) then + exfile:delete($ft:test-dir, true()) else () }; @@ -54,44 +54,44 @@ function ft:teardown() { declare %test:assertExists function ft:dir-separator() { - file:dir-separator() + exfile:dir-separator() }; declare %test:assertExists function ft:line-separator() { - file:line-separator() + exfile:line-separator() }; declare %test:assertExists function ft:path-separator() { - file:path-separator() + exfile:path-separator() }; declare %test:assertExists function ft:temp-dir() { - file:temp-dir() + exfile:temp-dir() }; declare %test:assertExists function ft:current-dir() { - file:current-dir() + exfile:current-dir() }; (: temp-dir and current-dir should end with separator :) declare %test:assertTrue function ft:temp-dir-ends-with-separator() { - ends-with(file:temp-dir(), file:dir-separator()) + ends-with(exfile:temp-dir(), exfile:dir-separator()) }; declare %test:assertTrue function ft:current-dir-ends-with-separator() { - ends-with(file:current-dir(), file:dir-separator()) + ends-with(exfile:current-dir(), exfile:dir-separator()) }; (: ======== File Properties ======== :) @@ -99,73 +99,73 @@ function ft:current-dir-ends-with-separator() { declare %test:assertTrue function ft:exists-temp-dir() { - file:exists(file:temp-dir()) + exfile:exists(exfile:temp-dir()) }; declare %test:assertFalse function ft:exists-nonexistent() { - file:exists($ft:test-dir || "nonexistent-file.txt") + exfile:exists($ft:test-dir || "nonexistent-file.txt") }; declare %test:assertTrue function ft:is-dir-temp() { - file:is-dir(file:temp-dir()) + exfile:is-dir(exfile:temp-dir()) }; declare %test:assertFalse function ft:is-dir-nonexistent() { - file:is-dir($ft:test-dir || "nonexistent") + exfile:is-dir($ft:test-dir || "nonexistent") }; declare %test:assertTrue function ft:is-file-after-write() { let $path := $ft:test-dir || "is-file-test.txt" - let $_ := file:write-text($path, "hello") - return file:is-file($path) + let $_ := exfile:write-text($path, "hello") + return exfile:is-file($path) }; declare %test:assertFalse function ft:is-file-on-dir() { - file:is-file($ft:test-dir) + exfile:is-file($ft:test-dir) }; declare %test:assertTrue function ft:is-absolute-absolute-path() { - file:is-absolute(file:temp-dir()) + exfile:is-absolute(exfile:temp-dir()) }; declare %test:assertFalse function ft:is-absolute-relative-path() { - file:is-absolute("relative/path") + exfile:is-absolute("relative/path") }; declare %test:assertExists function ft:last-modified() { let $path := $ft:test-dir || "last-mod-test.txt" - let $_ := file:write-text($path, "test") - return file:last-modified($path) + let $_ := exfile:write-text($path, "test") + return exfile:last-modified($path) }; declare %test:assertEquals(5) function ft:size-file() { let $path := $ft:test-dir || "size-test.txt" - let $_ := file:write-text($path, "hello") - return file:size($path) + let $_ := exfile:write-text($path, "hello") + return exfile:size($path) }; declare %test:assertEquals(0) function ft:size-dir() { - file:size($ft:test-dir) + exfile:size($ft:test-dir) }; (: ======== Read / Write Text ======== :) @@ -174,32 +174,32 @@ declare %test:assertEquals("hello world") function ft:write-read-text() { let $path := $ft:test-dir || "write-read.txt" - let $_ := file:write-text($path, "hello world") - return file:read-text($path) + let $_ := exfile:write-text($path, "hello world") + return exfile:read-text($path) }; declare %test:assertEquals("héllo wörld") function ft:write-read-text-utf8() { let $path := $ft:test-dir || "write-read-utf8.txt" - let $_ := file:write-text($path, "héllo wörld") - return file:read-text($path) + let $_ := exfile:write-text($path, "héllo wörld") + return exfile:read-text($path) }; declare %test:assertEquals("line1", "line2", "line3") function ft:write-read-text-lines() { let $path := $ft:test-dir || "lines-test.txt" - let $_ := file:write-text-lines($path, ("line1", "line2", "line3")) - return file:read-text-lines($path) + let $_ := exfile:write-text-lines($path, ("line1", "line2", "line3")) + return exfile:read-text-lines($path) }; declare %test:assertEquals(0) function ft:write-text-lines-empty() { let $path := $ft:test-dir || "empty-lines.txt" - let $_ := file:write-text-lines($path, ()) - return count(file:read-text-lines($path)) + let $_ := exfile:write-text-lines($path, ()) + return count(exfile:read-text-lines($path)) }; (: Verify newline normalization: CR and CRLF -> LF :) @@ -208,8 +208,8 @@ declare function ft:read-text-lines-normalization() { let $path := $ft:test-dir || "crlf-test.txt" (: Write raw bytes with CRLF line endings :) - let $_ := file:write-text($path, "a b c") - return file:read-text-lines($path) + let $_ := exfile:write-text($path, "a b c") + return exfile:read-text-lines($path) }; (: ======== Read / Write Binary ======== :) @@ -219,8 +219,8 @@ declare function ft:write-read-binary() { let $path := $ft:test-dir || "binary-test.bin" let $data := xs:base64Binary("SGVsbG8gV29ybGQ=") (: "Hello World" :) - let $_ := file:write-binary($path, $data) - return file:read-binary($path) + let $_ := exfile:write-binary($path, $data) + return exfile:read-binary($path) }; (: ======== Append ======== :) @@ -229,18 +229,18 @@ declare %test:assertEquals("helloworld") function ft:append-text() { let $path := $ft:test-dir || "append-test.txt" - let $_ := file:write-text($path, "hello") - let $_ := file:append-text($path, "world") - return file:read-text($path) + let $_ := exfile:write-text($path, "hello") + let $_ := exfile:append-text($path, "world") + return exfile:read-text($path) }; declare %test:assertEquals("line1", "line2", "line3", "line4") function ft:append-text-lines() { let $path := $ft:test-dir || "append-lines.txt" - let $_ := file:write-text-lines($path, ("line1", "line2")) - let $_ := file:append-text-lines($path, ("line3", "line4")) - return file:read-text-lines($path) + let $_ := exfile:write-text-lines($path, ("line1", "line2")) + let $_ := exfile:append-text-lines($path, ("line3", "line4")) + return exfile:read-text-lines($path) }; (: ======== Directory Operations ======== :) @@ -249,24 +249,24 @@ declare %test:assertTrue function ft:create-dir() { let $dir := $ft:test-dir || "subdir/" - let $_ := file:create-dir($dir) - return file:is-dir($dir) + let $_ := exfile:create-dir($dir) + return exfile:is-dir($dir) }; declare %test:assertTrue function ft:create-dir-nested() { let $dir := $ft:test-dir || "a/b/c/" - let $_ := file:create-dir($dir) - return file:is-dir($dir) + let $_ := exfile:create-dir($dir) + return exfile:is-dir($dir) }; declare %test:assertExists function ft:create-temp-dir() { - let $dir := file:create-temp-dir("test-", "-dir", $ft:test-dir) + let $dir := exfile:create-temp-dir("test-", "-dir", $ft:test-dir) return - if (file:is-dir($dir)) then + if (exfile:is-dir($dir)) then $dir else () @@ -275,9 +275,9 @@ function ft:create-temp-dir() { declare %test:assertExists function ft:create-temp-file() { - let $f := file:create-temp-file("test-", ".tmp", $ft:test-dir) + let $f := exfile:create-temp-file("test-", ".tmp", $ft:test-dir) return - if (file:is-file($f)) then + if (exfile:is-file($f)) then $f else () @@ -288,42 +288,42 @@ function ft:create-temp-file() { declare %test:assertExists function ft:list-dir() { - let $_ := file:write-text($ft:test-dir || "list-a.txt", "a") - let $_ := file:write-text($ft:test-dir || "list-b.txt", "b") - return file:list($ft:test-dir) + let $_ := exfile:write-text($ft:test-dir || "list-a.txt", "a") + let $_ := exfile:write-text($ft:test-dir || "list-b.txt", "b") + return exfile:list($ft:test-dir) }; declare %test:assertTrue function ft:list-contains-file() { - let $_ := file:write-text($ft:test-dir || "list-find.txt", "find me") - return "list-find.txt" = file:list($ft:test-dir) + let $_ := exfile:write-text($ft:test-dir || "list-find.txt", "find me") + return "list-find.txt" = exfile:list($ft:test-dir) }; declare %test:assertTrue function ft:list-dir-trailing-separator() { - let $_ := file:create-dir($ft:test-dir || "list-subdir") - let $entries := file:list($ft:test-dir) + let $_ := exfile:create-dir($ft:test-dir || "list-subdir") + let $entries := exfile:list($ft:test-dir) return some $e in $entries satisfies - starts-with($e, "list-subdir") and ends-with($e, file:dir-separator()) + starts-with($e, "list-subdir") and ends-with($e, exfile:dir-separator()) }; declare %test:assertTrue function ft:list-recursive() { - let $_ := file:create-dir($ft:test-dir || "rec-dir") - let $_ := file:write-text($ft:test-dir || "rec-dir/nested.txt", "nested") - let $entries := file:list($ft:test-dir, true()) + let $_ := exfile:create-dir($ft:test-dir || "rec-dir") + let $_ := exfile:write-text($ft:test-dir || "rec-dir/nested.txt", "nested") + let $entries := exfile:list($ft:test-dir, true()) return some $e in $entries satisfies contains($e, "nested.txt") }; declare %test:assertTrue function ft:list-pattern() { - let $_ := file:write-text($ft:test-dir || "pat-a.txt", "a") - let $_ := file:write-text($ft:test-dir || "pat-b.xml", "b") - let $entries := file:list($ft:test-dir, false(), "*.txt") + let $_ := exfile:write-text($ft:test-dir || "pat-a.txt", "a") + let $_ := exfile:write-text($ft:test-dir || "pat-b.xml", "b") + let $entries := exfile:list($ft:test-dir, false(), "*.txt") return (some $e in $entries satisfies $e = "pat-a.txt") and not(some $e in $entries satisfies $e = "pat-b.xml") @@ -332,30 +332,30 @@ function ft:list-pattern() { declare %test:assertExists function ft:children() { - let $_ := file:write-text($ft:test-dir || "child.txt", "x") - return file:children($ft:test-dir) + let $_ := exfile:write-text($ft:test-dir || "child.txt", "x") + return exfile:children($ft:test-dir) }; declare %test:assertTrue function ft:children-absolute-paths() { - let $_ := file:write-text($ft:test-dir || "child-abs.txt", "x") - let $children := file:children($ft:test-dir) - return every $c in $children satisfies file:is-absolute($c) + let $_ := exfile:write-text($ft:test-dir || "child-abs.txt", "x") + let $children := exfile:children($ft:test-dir) + return every $c in $children satisfies exfile:is-absolute($c) }; declare %test:assertExists function ft:descendants() { - let $_ := file:create-dir($ft:test-dir || "desc-dir") - let $_ := file:write-text($ft:test-dir || "desc-dir/deep.txt", "deep") - return file:descendants($ft:test-dir) + let $_ := exfile:create-dir($ft:test-dir || "desc-dir") + let $_ := exfile:write-text($ft:test-dir || "desc-dir/deep.txt", "deep") + return exfile:descendants($ft:test-dir) }; declare %test:assertExists function ft:list-roots() { - file:list-roots() + exfile:list-roots() }; (: ======== Copy / Move / Delete ======== :) @@ -365,9 +365,9 @@ declare function ft:copy-file() { let $src := $ft:test-dir || "copy-src.txt" let $dst := $ft:test-dir || "copy-dst.txt" - let $_ := file:write-text($src, "copy content") - let $_ := file:copy($src, $dst) - return file:read-text($dst) + let $_ := exfile:write-text($src, "copy content") + let $_ := exfile:copy($src, $dst) + return exfile:read-text($dst) }; declare @@ -375,29 +375,29 @@ declare function ft:move-file() { let $src := $ft:test-dir || "move-src.txt" let $dst := $ft:test-dir || "move-dst.txt" - let $_ := file:write-text($src, "move content") - let $_ := file:move($src, $dst) + let $_ := exfile:write-text($src, "move content") + let $_ := exfile:move($src, $dst) return - file:exists($dst) and not(file:exists($src)) + exfile:exists($dst) and not(exfile:exists($src)) }; declare %test:assertFalse function ft:delete-file() { let $path := $ft:test-dir || "delete-me.txt" - let $_ := file:write-text($path, "bye") - let $_ := file:delete($path) - return file:exists($path) + let $_ := exfile:write-text($path, "bye") + let $_ := exfile:delete($path) + return exfile:exists($path) }; declare %test:assertFalse function ft:delete-dir-recursive() { let $dir := $ft:test-dir || "delete-dir/" - let $_ := file:create-dir($dir) - let $_ := file:write-text($dir || "inner.txt", "inner") - let $_ := file:delete($dir, true()) - return file:exists($dir) + let $_ := exfile:create-dir($dir) + let $_ := exfile:write-text($dir || "inner.txt", "inner") + let $_ := exfile:delete($dir, true()) + return exfile:exists($dir) }; (: ======== Path Functions ======== :) @@ -405,57 +405,57 @@ function ft:delete-dir-recursive() { declare %test:assertEquals("file.txt") function ft:name() { - file:name("/some/path/file.txt") + exfile:name("/some/path/file.txt") }; declare %test:assertEquals("") function ft:name-root() { - file:name("/") + exfile:name("/") }; declare %test:assertExists function ft:parent() { - file:parent("/some/path/file.txt") + exfile:parent("/some/path/file.txt") }; declare %test:assertEmpty function ft:parent-root() { - file:parent("/") + exfile:parent("/") }; declare %test:assertTrue function ft:parent-ends-with-separator() { - ends-with(file:parent("/some/path/file.txt"), file:dir-separator()) + ends-with(exfile:parent("/some/path/file.txt"), exfile:dir-separator()) }; declare %test:assertExists function ft:path-to-native() { (: temp-dir definitely exists :) - file:path-to-native(file:temp-dir()) + exfile:path-to-native(exfile:temp-dir()) }; declare %test:assertTrue function ft:path-to-uri-starts-with-file() { - starts-with(string(file:path-to-uri("/tmp")), "file:/") + starts-with(string(exfile:path-to-uri("/tmp")), "file:/") }; declare %test:assertTrue function ft:resolve-path-absolute() { - let $resolved := file:resolve-path("relative") - return file:is-absolute($resolved) + let $resolved := exfile:resolve-path("relative") + return exfile:is-absolute($resolved) }; declare %test:assertTrue function ft:resolve-path-with-base() { - let $resolved := file:resolve-path("child.txt", "/some/base/") + let $resolved := exfile:resolve-path("child.txt", "/some/base/") return contains($resolved, "base") and contains($resolved, "child.txt") }; @@ -465,8 +465,8 @@ declare %test:assertTrue function ft:write-xml() { let $path := $ft:test-dir || "write-xml.xml" - let $_ := file:write($path, text) - let $content := file:read-text($path) + let $_ := exfile:write($path, text) + let $content := exfile:read-text($path) return contains($content, "") and contains($content, "text") }; @@ -474,74 +474,74 @@ declare %test:assertTrue function ft:append-xml() { let $path := $ft:test-dir || "append-xml.xml" - let $_ := file:write($path, ) - let $_ := file:append($path, ) - let $content := file:read-text($path) + let $_ := exfile:write($path, ) + let $_ := exfile:append($path, ) + let $content := exfile:read-text($path) return contains($content, "first") and contains($content, "second") }; (: ======== Error Conditions ======== :) declare - %test:assertError("file:not-found") + %test:assertError("exfile:not-found") function ft:read-text-not-found() { - file:read-text($ft:test-dir || "does-not-exist.txt") + exfile:read-text($ft:test-dir || "does-not-exist.txt") }; declare - %test:assertError("file:is-dir") + %test:assertError("exfile:is-dir") function ft:read-text-is-dir() { - file:read-text($ft:test-dir) + exfile:read-text($ft:test-dir) }; declare - %test:assertError("file:not-found") + %test:assertError("exfile:not-found") function ft:last-modified-not-found() { - file:last-modified($ft:test-dir || "does-not-exist.txt") + exfile:last-modified($ft:test-dir || "does-not-exist.txt") }; declare - %test:assertError("file:not-found") + %test:assertError("exfile:not-found") function ft:size-not-found() { - file:size($ft:test-dir || "does-not-exist.txt") + exfile:size($ft:test-dir || "does-not-exist.txt") }; declare - %test:assertError("file:not-found") + %test:assertError("exfile:not-found") function ft:delete-not-found() { - file:delete($ft:test-dir || "does-not-exist.txt") + exfile:delete($ft:test-dir || "does-not-exist.txt") }; declare - %test:assertError("file:not-found") + %test:assertError("exfile:not-found") function ft:children-not-found() { - file:children($ft:test-dir || "does-not-exist/") + exfile:children($ft:test-dir || "does-not-exist/") }; declare - %test:assertError("file:no-dir") + %test:assertError("exfile:no-dir") function ft:children-not-dir() { let $path := $ft:test-dir || "not-a-dir.txt" - let $_ := file:write-text($path, "x") - return file:children($path) + let $_ := exfile:write-text($path, "x") + return exfile:children($path) }; declare - %test:assertError("file:no-dir") + %test:assertError("exfile:no-dir") function ft:write-text-no-parent-dir() { - file:write-text($ft:test-dir || "no-such-parent/file.txt", "hello") + exfile:write-text($ft:test-dir || "no-such-parent/file.txt", "hello") }; declare - %test:assertError("file:unknown-encoding") + %test:assertError("exfile:unknown-encoding") function ft:read-text-bad-encoding() { let $path := $ft:test-dir || "encoding-test.txt" - let $_ := file:write-text($path, "hello") - return file:read-text($path, "not-a-real-encoding") + let $_ := exfile:write-text($path, "hello") + return exfile:read-text($path, "not-a-real-encoding") }; declare - %test:assertError("file:not-found") + %test:assertError("exfile:not-found") function ft:path-to-native-not-found() { - file:path-to-native("/this/path/surely/does/not/exist/anywhere") + exfile:path-to-native("/this/path/surely/does/not/exist/anywhere") }; diff --git a/extensions/expath/src/test/xquery/org/expath/exist/file/sync.xqm b/extensions/expath/src/test/xquery/org/expath/exist/file/sync.xqm index f9ee5dd1c74..d6f9d860ace 100644 --- a/extensions/expath/src/test/xquery/org/expath/exist/file/sync.xqm +++ b/extensions/expath/src/test/xquery/org/expath/exist/file/sync.xqm @@ -24,7 +24,7 @@ xquery version "3.1"; module namespace sync="http://exist-db.org/xquery/test/util/sync"; import module namespace util="http://exist-db.org/xquery/util"; -import module namespace file="http://expath.org/ns/file"; +import module namespace exfile="http://expath.org/ns/file"; import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; @@ -304,15 +304,15 @@ function sync:prune-with-after-and-excludes() { let $directory := helper:get-test-directory($sync:suite) let $_ := helper:setup-fs-extra($directory) let $_ := ( - file:write-binary( + exfile:write-binary( $directory || "/excluded.xq", util:string-to-binary("1") ), - file:write-binary( + exfile:write-binary( $directory || "/pruned.xql", util:string-to-binary("1") ), - file:write-binary( + exfile:write-binary( $directory || "/readme.md", util:string-to-binary("oh oh") ) diff --git a/extensions/expath/src/test/xquery/org/expath/exist/file/syncmod.xqm b/extensions/expath/src/test/xquery/org/expath/exist/file/syncmod.xqm index ee1859dcf5d..f37282be8da 100644 --- a/extensions/expath/src/test/xquery/org/expath/exist/file/syncmod.xqm +++ b/extensions/expath/src/test/xquery/org/expath/exist/file/syncmod.xqm @@ -24,7 +24,7 @@ xquery version "3.1"; module namespace syncmod="http://exist-db.org/xquery/test/util/syncmod"; import module namespace util="http://exist-db.org/xquery/util"; -import module namespace file="http://expath.org/ns/file"; +import module namespace exfile="http://expath.org/ns/file"; import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; diff --git a/extensions/modules/file/pom.xml b/extensions/modules/file/pom.xml new file mode 100644 index 00000000000..08da6131c37 --- /dev/null +++ b/extensions/modules/file/pom.xml @@ -0,0 +1,201 @@ + + + + 4.0.0 + + + org.exist-db + exist-parent + 7.0.0-SNAPSHOT + ../../../exist-parent + + + exist-file + jar + + eXist-db File Module + eXist-db XQuery File Module + + + scm:git:https://github.com/exist-db/exist.git + scm:git:https://github.com/exist-db/exist.git + scm:git:https://github.com/exist-db/exist.git + HEAD + + + + + org.exist-db + exist-core + ${project.version} + + + + org.apache.logging.log4j + log4j-api + + + + com.evolvedbinary.j8fu + j8fu + test + + + com.evolvedbinary.multilock + multilock + + + + commons-io + commons-io + test + + + + net.sf.xmldb-org + xmldb-api + + + + jakarta.xml.bind + jakarta.xml.bind-api + test + + + + org.eclipse.angus + angus-activation + runtime + + + + org.apache.ant + ant + + + + + org.eclipse.jetty + jetty-deploy + test + + + + org.eclipse.jetty + jetty-jmx + test + + + + junit + junit + test + + + + ${project.groupId} + exist-jetty-config + ${project.version} + test + + + + org.apache.httpcomponents + httpcore + test + + + + org.apache.httpcomponents + fluent-hc + test + + + + commons-codec + commons-codec + test + + + + + + + + src/test/resources + false + + + src/test/resources-filtered + true + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + + analyze-only + + + true + + org.eclipse.angus:angus-activation:jar:${eclipse.angus-activation.version} + + + ${project.groupId}:exist-jetty-config:jar:${project.version} + org.eclipse.jetty:jetty-deploy:jar:${jetty.version} + org.eclipse.jetty:jetty-jmx:jar:${jetty.version} + + + com.sun.xml.bind:jaxb-impl:jar:${jaxb.impl.version} + + + + net.sf.xmldb-org:xmldb-api:jar + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + @{jacocoArgLine} + + ${project.basedir}/../../../exist-jetty-config/target/classes/org/exist/jetty + ${project.build.testOutputDirectory}/conf.xml + ${project.build.testOutputDirectory}/standalone-webapp + ${project.build.testOutputDirectory}/log4j2.xml + + + + + + diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Directory.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Directory.java new file mode 100644 index 00000000000..9cde989a93d --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Directory.java @@ -0,0 +1,205 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Date; +import java.util.stream.Stream; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.dom.memtree.MemTreeBuilder; +import org.exist.util.FileUtils; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.DateTimeValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.NodeValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * eXist File Module Extension DirectoryList + * + * Enumerate a list of files and directories, including their size and modification time, found in + * a specified directory + * + * @author Dannes Wessels + * @author Andrzej Taramina + * @author Leif-Jöran Olsson + * @serial 2010-05-12 + * @version 1.2 + * + * @see org.exist.xquery.BasicFunction#BasicFunction(org.exist.xquery.XQueryContext, org.exist.xquery.FunctionSignature) + */ +public class Directory extends BasicFunction { + + private final static Logger logger = LogManager.getLogger(Directory.class); + + final static String NAMESPACE_URI = FileModule.NAMESPACE_URI; + final static String PREFIX = FileModule.PREFIX; + + public final static FunctionSignature[] signatures = { + new FunctionSignature( + new QName("list", NAMESPACE_URI, PREFIX), + "List all files and directories under the specified directory. " + + "This method is only available to the DBA role.", + new SequenceType[]{ + new FunctionParameterSequenceType("path", + Type.ITEM, Cardinality.EXACTLY_ONE, + "The directory path or URI in the file system."), + }, + new FunctionReturnSequenceType(Type.NODE, Cardinality.ZERO_OR_MORE, + "a node describing file and directory names and meta data.")) + }; + + public Directory(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + + final String inputPath = args[0].getStringValue(); + final Path directoryPath = FileModuleHelper.getFile(inputPath, this); + + if (logger.isDebugEnabled()) { + logger.debug("Listing matching files in directory: {}", directoryPath.toAbsolutePath().toString()); + } + + if(!Files.isDirectory(directoryPath)) { + throw new XPathException(this, "'" + inputPath + "' does not point to a valid directory."); + } + + // Get list of files, null if baseDir does not point to a directory + context.pushDocumentContext(); + try(final Stream scannedFiles = Files.list(directoryPath)) { + + final MemTreeBuilder builder = context.getDocumentBuilder(); + + builder.startDocument(); + builder.startElement(new QName("list", null, null), null); + + scannedFiles.forEach(entry -> { + if (logger.isDebugEnabled()) { + logger.debug("Found: {}", entry.toAbsolutePath().toString()); + } + + String entryType = "unknown"; + if (Files.isRegularFile(entry)) { + entryType = "file"; + } else if (Files.isDirectory(entry)) { + entryType = "directory"; + } + + builder.startElement(new QName(entryType, NAMESPACE_URI, PREFIX), null); + + builder.addAttribute(new QName("name", null, null), FileUtils.fileName(entry)); + + try { + if (Files.isRegularFile(entry)) { + final Long sizeLong = Files.size(entry); + String sizeString = Long.toString(sizeLong); + String humanSize = getHumanSize(sizeLong, sizeString); + + builder.addAttribute(new QName("size", null, null), sizeString); + builder.addAttribute(new QName("human-size", null, null), humanSize); + } + + builder.addAttribute(new QName("modified", null, null), + new DateTimeValue(this, new Date(Files.getLastModifiedTime(entry).toMillis())).getStringValue()); + + builder.addAttribute(new QName("hidden", null, null), + new BooleanValue(this, Files.isHidden(entry)).getStringValue()); + + builder.addAttribute(new QName("canRead", null, null), + new BooleanValue(this, Files.isReadable(entry)).getStringValue()); + + builder.addAttribute(new QName("canWrite", null, null), + new BooleanValue(this, Files.isWritable(entry)).getStringValue()); + } catch (final IOException | XPathException ioe) { + LOG.warn(ioe); + } + builder.endElement(); + + }); + + builder.endElement(); + + return (NodeValue) builder.getDocument().getDocumentElement(); + } catch(final IOException ioe) { + throw new XPathException(this, ioe); + } finally { + context.popDocumentContext(); + } + } + + private String getHumanSize(final Long sizeLong, final String sizeString) { + String humanSize = "n/a"; + int sizeDigits = sizeString.length(); + + if (sizeDigits < 4) { + humanSize = Long.toString(Math.abs(sizeLong)); + + } else if (sizeDigits >= 4 && sizeDigits <= 6) { + if (sizeLong < 1024) { + // We don't want 0KB för e.g. 1006 Bytes. + humanSize = Long.toString(Math.abs(sizeLong)); + } else { + humanSize = Math.abs(sizeLong / 1024) + "KB"; + } + + } else if (sizeDigits >= 7 && sizeDigits <= 9) { + if (sizeLong < 1048576) { + humanSize = Math.abs(sizeLong / 1024) + "KB"; + } else { + humanSize = Math.abs(sizeLong / (1024 * 1024)) + "MB"; + } + + } else if (sizeDigits > 9) { + if (sizeLong < 1073741824) { + humanSize = Math.abs((sizeLong / (1024 * 1024))) + "MB"; + } else { + humanSize = Math.abs((sizeLong / (1024 * 1024 * 1024))) + "GB"; + } + } + return humanSize; + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryCreate.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryCreate.java new file mode 100644 index 00000000000..2868b8633e1 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryCreate.java @@ -0,0 +1,115 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * @see java.nio.file.Files#createDirectory(Path, FileAttribute[]) + * @see java.nio.file.Files#createDirectories(Path, FileAttribute[]) + * + * @author Dannes Wessels + * + */ +public class DirectoryCreate extends BasicFunction { + + private final static Logger logger = LogManager.getLogger(DirectoryCreate.class); + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName("mkdir", FileModule.NAMESPACE_URI, FileModule.PREFIX), + "Create a directory. This method is only available to the DBA role.", + new SequenceType[]{ + new FunctionParameterSequenceType("path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the directory") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "true if successful, false otherwise") + ), + new FunctionSignature( + new QName("mkdirs", FileModule.NAMESPACE_URI, FileModule.PREFIX), + "Create a directory including any necessary but nonexistent parent directories. " + + "This method is only available to the DBA role.", + new SequenceType[]{ + new FunctionParameterSequenceType("path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the directory") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "true if successful, false otherwise") + ) + }; + + + public DirectoryCreate(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + Sequence created = BooleanValue.FALSE; + + final String inputPath = args[0].itemAt(0).getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + try { + if (isCalledAs("mkdir") && Files.notExists(file)) { + Files.createDirectory(file); + created = BooleanValue.TRUE; + } else if (isCalledAs("mkdirs")) { + Files.createDirectories(file); + created = BooleanValue.TRUE; + } + } catch (final FileAlreadyExistsException e) { + created = BooleanValue.FALSE; + } catch(final IOException e) { + throw new XPathException(this, e); + } + + return created; + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryList.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryList.java new file mode 100644 index 00000000000..10893583c83 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/DirectoryList.java @@ -0,0 +1,191 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Date; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.apache.tools.ant.DirectoryScanner; +import org.exist.dom.QName; +import org.exist.dom.memtree.MemTreeBuilder; +import org.exist.util.FileUtils; +import org.exist.xquery.*; +import org.exist.xquery.value.DateTimeValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.NodeValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +import static org.exist.xquery.modules.file.FileErrorCode.DIRECTORY_NOT_FOUND; + +/** + * eXist File Module Extension DirectoryList + *

+ * Enumerate a list of files, including their size and modification time, found + * in a specified directory, using a pattern + * + * @author Andrzej Taramina + * @author ljo + * @version 1.2 + * @serial 2009-08-09 + * @see org.exist.xquery.BasicFunction#BasicFunction(org.exist.xquery.XQueryContext, + * org.exist.xquery.FunctionSignature) + */ +public class DirectoryList extends BasicFunction { + + static final String NAMESPACE_URI = FileModule.NAMESPACE_URI; + static final String PREFIX = FileModule.PREFIX; + public static final FunctionSignature[] signatures = { + new FunctionSignature( + new QName("directory-list", NAMESPACE_URI, PREFIX), + "List all files, including their file size and modification time, " + + "found in or below a directory, $directory. Files are located in the server's " + + "file system, using filename patterns, $pattern. File pattern matching is based " + + "on code from Apache's Ant, thus following the same conventions. For example:\n\n" + + "'*.xml' matches any file ending with .xml in the current directory,\n- '**/*.xml' matches files " + + "in any directory below the specified directory. This method is only available to the DBA role.", + new SequenceType[]{ + new FunctionParameterSequenceType("path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The base directory path or URI in the file system where the files are located."), + new FunctionParameterSequenceType("pattern", Type.STRING, + Cardinality.ZERO_OR_MORE, "The file name pattern") + }, + new FunctionReturnSequenceType(Type.NODE, + Cardinality.ZERO_OR_ONE, "a node fragment that shows all matching " + + "filenames, including their file size and modification time, and " + + "the subdirectory they were found in") + ) + }; + static final QName FILE_ELEMENT = new QName("file", NAMESPACE_URI, PREFIX); + static final QName LIST_ELEMENT = new QName("list", NAMESPACE_URI, PREFIX); + + static final QName DIRECTORY_ATTRIBUTE = new QName("directory", null, null); + static final QName NAME_ATTRIBUTE = new QName("name", null, null); + static final QName SIZE_ATTRIBUTE = new QName("size", null, null); + static final QName HUMAN_SIZE_ATTRIBUTE = new QName("human-size", null, null); + static final QName MODIFIED_ATTRIBUTE = new QName("modified", null, null); + static final QName SUBDIR_ATTRIBUTE = new QName("subdir", null, null); + private static final Logger logger = LogManager.getLogger(DirectoryList.class); + + public DirectoryList(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + final String inputPath = args[0].getStringValue(); + final Path baseDir = FileModuleHelper.getFile(inputPath, this); + + final Sequence patterns = args[1]; + + if (logger.isDebugEnabled()) { + logger.debug("Listing matching files in directory: {}", baseDir); + } + + context.pushDocumentContext(); + final MemTreeBuilder builder = context.getDocumentBuilder(); + + builder.startDocument(); + builder.startElement(LIST_ELEMENT, null); + builder.addAttribute(DIRECTORY_ATTRIBUTE, baseDir.toString()); + try { + final int patternsLen = patterns.getItemCount(); + final String[] includes = new String[patternsLen]; + for (int i = 0; i < patternsLen; i++) { + includes[i] = patterns.itemAt(0).getStringValue(); + } + + final DirectoryScanner directoryScanner = new DirectoryScanner(); + directoryScanner.setIncludes(includes); + directoryScanner.setBasedir(baseDir.toFile()); + directoryScanner.setCaseSensitive(true); + directoryScanner.scan(); + + for (final String includedFile : directoryScanner.getIncludedFiles()) { + final Path file = baseDir.resolve(includedFile); + + if (logger.isDebugEnabled()) { + logger.debug("Found: {}", file.toAbsolutePath()); + } + + final String relPath = file.toString().substring(baseDir.toString().length() + 1); + + builder.startElement(FILE_ELEMENT, null); + builder.addAttribute(NAME_ATTRIBUTE, FileUtils.fileName(file)); + + final long sizeLong = FileUtils.sizeQuietly(file); + builder.addAttribute(SIZE_ATTRIBUTE, Long.toString(sizeLong)); + builder.addAttribute(HUMAN_SIZE_ATTRIBUTE, getHumanSize(sizeLong)); + + builder.addAttribute(MODIFIED_ATTRIBUTE, + new DateTimeValue(this, + new Date(Files.getLastModifiedTime(file).toMillis())).getStringValue()); + + final int lastSeparatorPosition = relPath.lastIndexOf(java.io.File.separatorChar); + if (lastSeparatorPosition >= 0) { + final String relDir = relPath.substring(0, lastSeparatorPosition); + if (!relDir.isEmpty()) { + builder.addAttribute(SUBDIR_ATTRIBUTE, + relDir.replace(java.io.File.separatorChar, '/')); + } + } + + builder.endElement(); + } + + builder.endElement(); + + return (NodeValue) builder.getDocument().getDocumentElement(); + } catch (final IOException | IllegalStateException e) { + throw new XPathException(this, DIRECTORY_NOT_FOUND, e.getMessage()); + } finally { + context.popDocumentContext(); + } + } + + private String getHumanSize(final Long sizeLong) { + if (sizeLong < 1024) { + return Math.abs(sizeLong) + "B"; + } + if (sizeLong < 1048576) { + return Math.abs(sizeLong / 1024) + "KB"; + } + if (sizeLong < 1073741824) { + return Math.abs((sizeLong / (1024 * 1024))) + "MB"; + } + return Math.abs((sizeLong / (1024 * 1024 * 1024))) + "GB"; + } + +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileDelete.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileDelete.java new file mode 100644 index 00000000000..e16dcebbb5c --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileDelete.java @@ -0,0 +1,82 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.util.FileUtils; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * @see org.exist.util.FileUtils#deleteQuietly(Path) + * + * @author Andrzej Taramina + */ +public class FileDelete extends BasicFunction { + + private final static Logger logger = LogManager.getLogger(FileDelete.class); + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( "delete", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Delete a file or directory. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file" ) + }, + new FunctionReturnSequenceType( Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "true if successful, false otherwise" ) ) + }; + + public FileDelete(final XQueryContext context, final FunctionSignature signature ) + { + super( context, signature ); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence ) throws XPathException { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + final String inputPath = args[0].getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + return BooleanValue.valueOf(FileUtils.deleteQuietly(file)); + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileErrorCode.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileErrorCode.java new file mode 100644 index 00000000000..b4277a3d7a7 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileErrorCode.java @@ -0,0 +1,34 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import org.exist.dom.QName; +import org.exist.xquery.ErrorCodes; + +class FileErrorCode extends ErrorCodes.ErrorCode { + public static final ErrorCodes.ErrorCode DIRECTORY_NOT_FOUND = new FileErrorCode("DIRECTORY_NOT_FOUND", + "The directory could not be found."); + + FileErrorCode(final String code, final String description) { + super(new QName(code, FileModule.NAMESPACE_URI, FileModule.PREFIX), description); + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileExists.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileExists.java new file mode 100644 index 00000000000..7965d6027c2 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileExists.java @@ -0,0 +1,85 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * @see java.nio.file.Files#exists(Path, LinkOption...) + * + * @author Andrzej Taramina + * + */ +public class FileExists extends BasicFunction { + + private final static Logger logger = LogManager.getLogger(FileExists.class); + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( "exists", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Tests if a file or directory exists. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file in the file system" ) + }, + new FunctionReturnSequenceType(Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "the boolean value true if the file exists, false otherwise" ) ) + }; + + public FileExists(final XQueryContext context, final FunctionSignature signature ) + { + super( context, signature ); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException + { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + final String inputPath = args[0].getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + return BooleanValue.valueOf(Files.exists(file)); + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsDirectory.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsDirectory.java new file mode 100644 index 00000000000..256845935c0 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsDirectory.java @@ -0,0 +1,85 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * @see java.nio.file.Files#isDirectory(Path, LinkOption...) + * + * @author Andrzej Taramina + * @author Loren Cahlander + */ +public class FileIsDirectory extends BasicFunction { + + private final static Logger logger = LogManager.getLogger(FileIsDirectory.class); + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( "is-directory", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Tests if a path is a directory. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file or directory" ) + }, + new FunctionReturnSequenceType( Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "true if the path is a directory" ) ) + }; + + public FileIsDirectory(final XQueryContext context, final FunctionSignature signature) + { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException + { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + final String inputPath = args[0].getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + return BooleanValue.valueOf(Files.isDirectory(file)); + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsReadable.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsReadable.java new file mode 100644 index 00000000000..4461a4f6b51 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsReadable.java @@ -0,0 +1,84 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * @see java.nio.file.Files#isReadable(Path) + * + * @author Andrzej Taramina + * @author Loren Cahlander + */ +public class FileIsReadable extends BasicFunction { + + private static final Logger logger = LogManager.getLogger(FileIsReadable.class); + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( "is-readable", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Tests if a file is readable. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file" ) + }, + new FunctionReturnSequenceType( Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "true if file can be read" ) ) + }; + + public FileIsReadable(final XQueryContext context, final FunctionSignature signature) + { + super( context, signature ); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException + { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + final String inputPath = args[0].getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + return BooleanValue.valueOf(Files.isReadable(file)); + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsWriteable.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsWriteable.java new file mode 100644 index 00000000000..787c3deb0b5 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileIsWriteable.java @@ -0,0 +1,84 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * @see java.nio.file.Files#isWritable(Path) + * + * @author Andrzej Taramina + * @author Loren Cahlander + */ +public class FileIsWriteable extends BasicFunction { + + private final static Logger logger = LogManager.getLogger(FileIsWriteable.class); + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( "is-writeable", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Tests if a file is writeable. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file" ) + }, + new FunctionReturnSequenceType( Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "true if the file has write permissions" ) ) + }; + + public FileIsWriteable(final XQueryContext context, final FunctionSignature signature) + { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException + { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + final String inputPath = args[0].getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + return BooleanValue.valueOf(Files.isWritable(file)); + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModule.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModule.java new file mode 100644 index 00000000000..ef7542a87a3 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModule.java @@ -0,0 +1,100 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.util.List; +import java.util.Map; +import org.exist.xquery.AbstractInternalModule; +import org.exist.xquery.FunctionDef; + +/** + * eXist File Module Extension + * + * An extension module for the eXist Native XML Database that allows various file-oriented + * activities. + * + * @author Andrzej Taramina + * @author ljo + * @serial 2008-03-06 + * @version 1.0 + * + * @see org.exist.xquery.AbstractInternalModule#AbstractInternalModule(org.exist.xquery.FunctionDef[], java.util.Map) + */ +public class FileModule extends AbstractInternalModule +{ + public final static String NAMESPACE_URI = "http://exist-db.org/xquery/file"; + + public final static String PREFIX = "file"; + public final static String INCLUSION_DATE = "2008-03-07"; + public final static String RELEASED_IN_VERSION = "eXist-1.4"; + + + private final static FunctionDef[] functions = { + new FunctionDef( Directory.signatures[0], Directory.class ), + new FunctionDef( DirectoryList.signatures[0], DirectoryList.class ), + new FunctionDef( FileRead.signatures[0], FileRead.class ), + new FunctionDef( FileRead.signatures[1], FileRead.class ), + new FunctionDef( FileReadBinary.signatures[0], FileReadBinary.class ), + new FunctionDef( FileReadUnicode.signatures[0], FileReadUnicode.class ), + new FunctionDef( FileReadUnicode.signatures[1], FileReadUnicode.class ), + new FunctionDef( SerializeToFile.signatures[0], SerializeToFile.class ), + new FunctionDef( SerializeToFile.signatures[1], SerializeToFile.class ), + new FunctionDef( SerializeToFile.signatures[2], SerializeToFile.class ), + new FunctionDef( SerializeToFile.signatures[3], SerializeToFile.class ), + new FunctionDef( FileExists.signatures[0], FileExists.class ), + new FunctionDef( FileIsReadable.signatures[0], FileIsReadable.class ), + new FunctionDef( FileIsWriteable.signatures[0], FileIsWriteable.class ), + new FunctionDef( FileIsDirectory.signatures[0], FileIsDirectory.class ), + new FunctionDef( FileDelete.signatures[0], FileDelete.class ), + new FunctionDef( FileMove.signatures[0], FileMove.class ), + new FunctionDef( DirectoryCreate.signatures[0], DirectoryCreate.class ), + new FunctionDef( DirectoryCreate.signatures[1], DirectoryCreate.class ), + new FunctionDef( Sync.signature, Sync.class) + }; + + + public FileModule(Map> parameters) + { + super( functions, parameters ); + } + + + public String getNamespaceURI() + { + return( NAMESPACE_URI ); + } + + + public String getDefaultPrefix() { + return( PREFIX ); + } + + + public String getDescription() + { + return( "A module for performing various operations on files and directories stored in the server file system." ); + } + + public String getReleaseVersion() { + return RELEASED_IN_VERSION; + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModuleHelper.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModuleHelper.java new file mode 100644 index 00000000000..df5f37c82f2 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileModuleHelper.java @@ -0,0 +1,63 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.exist.xquery.Expression; +import org.exist.xquery.XPathException; + +/** + * Helper class for FileModule + * + * @author Dannes Wessels + */ + + +public class FileModuleHelper { + + private FileModuleHelper() { + // no instance + } + + /** + * Convert path (URL, file path) to a File object. + * + * @param path Path written as OS specific path or as URL + * @return File object + * @throws XPathException Thrown when the URL cannot be used. + */ + public static Path getFile(String path, final Expression expression) throws XPathException { + if(path.startsWith("file:")){ + try { + return Paths.get(new URI(path)); + } catch (Exception ex) { // catch all (URISyntaxException) + throw new XPathException(expression, path + " is not a valid URI: '"+ ex.getMessage() +"'"); + } + } else { + return Paths.get(path); + } + } + +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileMove.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileMove.java new file mode 100644 index 00000000000..01e6e538417 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileMove.java @@ -0,0 +1,101 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.io.IOException; +import java.nio.file.CopyOption; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * @see java.nio.file.Files#move(Path, Path, CopyOption...) + * + * @author Dannes Wessels + * + */ +public class FileMove extends BasicFunction { + + private final static Logger logger = LogManager.getLogger(FileMove.class); + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( "move", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Move (rename) a file or directory. Exact operation is platform dependent. This " + + "method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "original", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file" ), + new FunctionParameterSequenceType( "destination", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file" ) + }, + new FunctionReturnSequenceType( Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "true if successful, false otherwise" ) ) + }; + + + public FileMove(final XQueryContext context, final FunctionSignature signature) + { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException + { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + Sequence moved = BooleanValue.FALSE; + + final String inputPath1 = args[0].getStringValue(); + final Path src = FileModuleHelper.getFile(inputPath1, this); + + final String inputPath2 = args[1].getStringValue(); + final Path dest = FileModuleHelper.getFile(inputPath2, this); + + try { + Files.move(src, dest); + return BooleanValue.TRUE; + } catch(final IOException ioe) { + LOG.error(ioe); + return BooleanValue.FALSE; + } + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileRead.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileRead.java new file mode 100644 index 00000000000..79df63ef2a4 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileRead.java @@ -0,0 +1,110 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; + +/** + * @author Pierrick Brihaye + * @author Dizzzz + * @author Andrzej Taramina + * @author Loren Cahlander + */ +public class FileRead extends BasicFunction { + + private final static Logger logger = LogManager.getLogger(FileRead.class); + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( "read", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Reads the content of file. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ) + }, + new FunctionReturnSequenceType( Type.STRING, + Cardinality.ZERO_OR_ONE, "the file contents" ) ), + + new FunctionSignature( + new QName( "read", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Reads the content of file. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ), + new FunctionParameterSequenceType( "encoding", Type.STRING, + Cardinality.EXACTLY_ONE, "The encoding type for the file" ) + }, + new FunctionReturnSequenceType( Type.STRING, + Cardinality.ZERO_OR_ONE, "the file contents" ) ) + }; + + public FileRead(final XQueryContext context, final FunctionSignature signature) + { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException + { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + final String inputPath = args[0].getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + final Charset encoding; + if(args.length == 2) { + encoding = Charset.forName(args[1].getStringValue()); + } else { + encoding = StandardCharsets.UTF_8; + } + + try { + return new StringValue(this, Files.readString(file, encoding)); + } catch(final IOException e ) { + throw new XPathException(this, e); + } + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadBinary.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadBinary.java new file mode 100644 index 00000000000..d3b2288f3a5 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadBinary.java @@ -0,0 +1,80 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.Base64BinaryValueType; +import org.exist.xquery.value.BinaryValueFromFile; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * @author Andrzej Taramina + */ +public class FileReadBinary extends BasicFunction { + + private static final Logger logger = LogManager.getLogger(FileReadBinary.class); + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( "read-binary", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Reads the contents of a binary file. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ) + }, + new FunctionReturnSequenceType( Type.BASE64_BINARY, + Cardinality.ZERO_OR_ONE, "the file contents" ) ) + }; + + public FileReadBinary(final XQueryContext context, final FunctionSignature signature) + { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + final String inputPath = args[0].getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + return BinaryValueFromFile.getInstance(context, new Base64BinaryValueType(), file, this); + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadUnicode.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadUnicode.java new file mode 100644 index 00000000000..b9d9730f663 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/FileReadUnicode.java @@ -0,0 +1,118 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.io.IOException; +import java.io.StringWriter; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; + +/** + * @author Pierrick Brihaye + * @author Dizzzz + * @author Andrzej Taramina + */ +public class FileReadUnicode extends BasicFunction { + + private final static Logger logger = LogManager.getLogger(FileReadUnicode.class); + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( "read-unicode", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Reads the contents of a file. Unicode BOM (Byte Order Marker) will be stripped " + + "off if found. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ) + }, + new FunctionReturnSequenceType( Type.STRING, + Cardinality.ZERO_OR_ONE, "the contents of the file" ) ), + + new FunctionSignature( + new QName( "read-unicode", FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Reads the contents of a file. Unicode BOM (Byte Order Marker) will be stripped " + + "off if found. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The directory path or URI in the file system." ), + new FunctionParameterSequenceType( "encoding", Type.STRING, + Cardinality.EXACTLY_ONE, "The file is read with the encoding specified." ) + }, + new FunctionReturnSequenceType( Type.STRING, + Cardinality.ZERO_OR_ONE, "the contents of the file" ) ) + }; + + public FileReadUnicode(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (!context.getSubject().hasDbaRole()) { + XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + final String inputPath = args[0].getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + final Charset encoding; + if(args.length == 2) { + encoding = Charset.forName(args[1].getStringValue()); + } else { + encoding = StandardCharsets.UTF_8; + } + + try(final UnicodeReader reader = new UnicodeReader(Files.newInputStream(file), encoding.name()); + final StringWriter sw = new StringWriter()) { + + char[] buf = new char[1024]; + int len; + while(( len = reader.read( buf ) ) > 0) { + sw.write( buf, 0, len) ; + } + + return new StringValue(this, sw.toString()); + } catch(final IOException e) { + throw new XPathException(this, e); + } + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/SerializeToFile.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/SerializeToFile.java new file mode 100644 index 00000000000..1a14bc02ba2 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/SerializeToFile.java @@ -0,0 +1,236 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.exist.dom.QName; +import org.exist.storage.serializers.Serializer; +import org.exist.xquery.*; +import org.exist.xquery.util.SerializerUtils; +import org.exist.xquery.value.*; +import org.xml.sax.SAXException; + +import javax.xml.transform.OutputKeys; +import java.io.*; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Properties; + +public class SerializeToFile extends BasicFunction { + private final static Logger logger = LogManager.getLogger(SerializeToFile.class); + + private final static String FN_SERIALIZE_LN = "serialize"; + private final static String FN_SERIALIZE_BINARY_LN = "serialize-binary"; + + public final static FunctionSignature signatures[] = { + new FunctionSignature( + new QName( FN_SERIALIZE_LN, FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Writes the node set into a file on the file system. $parameters contains a " + + "sequence of zero or more serialization parameters specified as key=value pairs. The " + + "serialization options are the same as those recognized by \"declare option exist:serialize\". " + + "The function does NOT automatically inherit the serialization options of the XQuery it is " + + "called from. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "node-set", Type.NODE, + Cardinality.ZERO_OR_MORE, "The contents to write to the file system." ), + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file" ), + new FunctionParameterSequenceType( "parameters", Type.ITEM, + Cardinality.ZERO_OR_MORE, "The serialization parameters: either a sequence of key=value pairs or an output:serialization-parameters " + + "element as defined by the standard fn:serialize function." ) + }, + new FunctionReturnSequenceType( Type.BOOLEAN, + Cardinality.ZERO_OR_ONE, "true on success - false if the specified file can not be " + + "created or is not writable. The empty sequence is returned if the argument sequence is empty." ) + ), + + new FunctionSignature( + new QName( FN_SERIALIZE_LN, FileModule.NAMESPACE_URI, FileModule.PREFIX ), + "Writes the node set into a file on the file system, optionally appending to it. " + + "$parameters contains a sequence of zero or more serialization parameters specified as " + + "key=value pairs. The serialization options are the same as those recognized by " + + "\"declare option exist:serialize\". " + + "The function does NOT automatically inherit the serialization options of the XQuery it is " + + "called from. This method is only available to the DBA role.", + new SequenceType[] { + new FunctionParameterSequenceType( "node-set", Type.NODE, + Cardinality.ZERO_OR_MORE, "The contents to write to the file system." ), + new FunctionParameterSequenceType( "path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file" ), + new FunctionParameterSequenceType( "parameters", Type.ITEM, + Cardinality.ZERO_OR_MORE, "The serialization parameters: either a sequence of key=value pairs or an output:serialization-parameters " + + "element as defined by the standard fn:serialize function." ), + new FunctionParameterSequenceType( "append", Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "Should content be appended?") + }, + new FunctionReturnSequenceType( Type.BOOLEAN, + Cardinality.ZERO_OR_ONE, "true on success - false if the specified file can " + + "not be created or is not writable. The empty sequence is returned if the argument sequence is empty." ) + ), + + new FunctionSignature( + new QName(FN_SERIALIZE_BINARY_LN, FileModule.NAMESPACE_URI, FileModule.PREFIX), + "Writes binary data into a file on the file system. This method is only available to the DBA role.", + new SequenceType[]{ + new FunctionParameterSequenceType("binarydata", Type.BASE64_BINARY, + Cardinality.EXACTLY_ONE, "The contents to write to the file system."), + new FunctionParameterSequenceType("path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true on success - false if the specified file can not be created or is not writable") + ), + + new FunctionSignature( + new QName(FN_SERIALIZE_BINARY_LN, FileModule.NAMESPACE_URI, FileModule.PREFIX), + "Writes binary data into a file on the file system, optionally appending the content. This method is only available to the DBA role.", + new SequenceType[]{ + new FunctionParameterSequenceType("binarydata", Type.BASE64_BINARY, + Cardinality.EXACTLY_ONE, "The contents to write to the file system."), + new FunctionParameterSequenceType("path", Type.ITEM, + Cardinality.EXACTLY_ONE, "The full path or URI to the file"), + new FunctionParameterSequenceType("append", Type.BOOLEAN, + Cardinality.EXACTLY_ONE, "Should content be appended?") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "true on success - false if the specified file can not be created or is not writable") + ) + }; + + + public SerializeToFile(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + + if(args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + if(!context.getSubject().hasDbaRole()) { + final XPathException xPathException = new XPathException(this, "Permission denied, calling user '" + context.getSubject().getName() + "' must be a DBA to call this function."); + logger.error("Invalid user", xPathException); + throw xPathException; + } + + + //check the file output path + final String inputPath = args[1].getStringValue(); + final Path file = FileModuleHelper.getFile(inputPath, this); + + if(Files.isDirectory(file)) { + logger.debug("Cannot serialize file. Output file is a directory: {}", file.toAbsolutePath().toString()); + return BooleanValue.FALSE; + } + + if(Files.exists(file) && !Files.isWritable(file)) { + logger.debug("Cannot serialize file. Cannot write to file {}", file.toAbsolutePath().toString()); + return BooleanValue.FALSE; + } + + if(isCalledAs(FN_SERIALIZE_LN)) { + //parse serialization options from third argument to function + final Properties outputProperties = parseXMLSerializationOptions( args[2] ); + final boolean doAppend = (args.length > 3) && "true".equals(args[3].itemAt(0).getStringValue()); + + //do the serialization + serializeXML(args[0].iterate(), outputProperties, file, doAppend); + + } else if(isCalledAs(FN_SERIALIZE_BINARY_LN)) { + final boolean doAppend = (args.length > 2) && "true".equals(args[2].itemAt(0).getStringValue()); + serializeBinary((BinaryValue)args[0].itemAt(0), file, doAppend); + + } else { + throw new XPathException(this, "Unknown function name"); + } + + return BooleanValue.TRUE; + } + + + private Properties parseXMLSerializationOptions(final Sequence sSerializeParams) throws XPathException { + //parse serialization options + final Properties outputProperties = new Properties(); + + // defaults + outputProperties.setProperty( OutputKeys.INDENT, "yes" ); + outputProperties.setProperty( OutputKeys.OMIT_XML_DECLARATION, "yes" ); + outputProperties.setProperty( OutputKeys.ENCODING, "UTF-8" ); + + if (sSerializeParams.hasOne() && Type.subTypeOf(sSerializeParams.getItemType(), Type.NODE)) { + SerializerUtils.getSerializationOptions(this, (NodeValue) sSerializeParams.itemAt(0), outputProperties); + } else { + SequenceIterator siSerializeParams = sSerializeParams.iterate(); + while(siSerializeParams.hasNext()) { + final String serializeParam = siSerializeParams.nextItem().getStringValue(); + final String opt[] = Option.parseKeyValuePair(serializeParam); + if(opt != null && opt.length == 2) { + outputProperties.setProperty( opt[0], opt[1] ); + } + } + } + return outputProperties; + } + + + private void serializeXML(final SequenceIterator siNode, final Properties outputProperties, final Path file, final boolean doAppend) throws XPathException { + final Serializer serializer = context.getBroker().borrowSerializer(); + + StandardOpenOption ops[] = doAppend ? new StandardOpenOption[]{StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND} + : new StandardOpenOption[]{StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING}; + + + try (final OutputStream os = new BufferedOutputStream(Files.newOutputStream(file, ops)); + final Writer writer = new OutputStreamWriter(os, Charset.forName(outputProperties.getProperty(OutputKeys.ENCODING)))) { + + serializer.setProperties(outputProperties); + + while (siNode.hasNext()) { + final NodeValue nv = (NodeValue) siNode.nextItem(); + serializer.serialize(nv, writer); + } + } catch(UnsupportedOperationException | IllegalArgumentException e){ + throw new XPathException(this, "Error wile writing to file: " + e.getMessage(), e); + + } catch(final IOException | SAXException e) { + throw new XPathException(this, "Cannot serialize file. A problem occurred while serializing the node set: " + e.getMessage(), e); + } finally { + context.getBroker().returnSerializer(serializer); + } + } + + private void serializeBinary(final BinaryValue binary, final Path file, final boolean doAppend) throws XPathException { + + StandardOpenOption ops[] = doAppend ? new StandardOpenOption[]{StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.APPEND} + : new StandardOpenOption[]{StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING}; + + try(final OutputStream os = new BufferedOutputStream(Files.newOutputStream(file, ops))) { + binary.streamBinaryTo(os); + } catch(final IOException ioe) { + throw new XPathException(this, "Cannot serialize file. A problem occurred while serializing the binary data: " + ioe.getMessage(), ioe); + } + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java new file mode 100644 index 00000000000..23d205c45b7 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java @@ -0,0 +1,593 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.net.URISyntaxException; +import java.util.*; +import java.util.stream.Stream; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.*; +import javax.xml.transform.sax.SAXResult; +import javax.xml.transform.sax.SAXTransformerFactory; +import javax.xml.transform.sax.TransformerHandler; +import javax.xml.transform.stream.StreamSource; + +import org.apache.tools.ant.DirectoryScanner; +import org.exist.collections.Collection; +import org.exist.dom.persistent.BinaryDocument; +import org.exist.dom.persistent.DocumentImpl; +import org.exist.dom.QName; +import org.exist.dom.memtree.MemTreeBuilder; +import org.exist.security.PermissionDeniedException; +import org.exist.storage.lock.Lock.LockMode; +import org.exist.storage.lock.ManagedLock; +import org.exist.storage.serializers.EXistOutputKeys; +import org.exist.storage.serializers.Serializer; +import org.exist.util.FileUtils; +import org.exist.util.LockException; +import org.exist.util.serializer.SAXSerializer; +import org.exist.util.serializer.SerializerPool; +import org.exist.xmldb.XmldbURI; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.util.SerializerUtils; +import org.exist.xquery.value.*; +import org.exist.xslt.TransformerFactoryAllocator; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; +import uk.ac.ic.doc.slurp.multilock.MultiLock; + +public class Sync extends BasicFunction { + + public static final String PRUNE_OPT = "prune"; + public static final String AFTER_OPT = "after"; + public static final String EXCLUDES_OPT = "excludes"; + + public static final QName FILE_SYNC_ELEMENT = new QName("sync", FileModule.NAMESPACE_URI); + public static final QName FILE_UPDATE_ELEMENT = new QName("update", FileModule.NAMESPACE_URI); + public static final QName FILE_DELETE_ELEMENT = new QName("delete", FileModule.NAMESPACE_URI); + public static final QName FILE_ERROR_ELEMENT = new QName("error", FileModule.NAMESPACE_URI); + + // TODO(JL) Figure out which namespace all attributes should be in (possible breaking change) + // https://github.com/eXist-db/exist/issues/4207 + public static final QName FILE_COLLECTION_ATTRIBUTE = new QName("collection", FileModule.NAMESPACE_URI); + public static final QName FILE_DIR_ATTRIBUTE = new QName("dir", FileModule.NAMESPACE_URI); + + public static final QName FILE_ATTRIBUTE = new QName("file", XMLConstants.NULL_NS_URI); + public static final QName NAME_ATTRIBUTE = new QName("name", XMLConstants.NULL_NS_URI); + public static final QName COLLECTION_ATTRIBUTE = new QName("collection", XMLConstants.NULL_NS_URI); + public static final QName TYPE_ATTRIBUTE = new QName("type", XMLConstants.NULL_NS_URI); + public static final QName MODIFIED_ATTRIBUTE = new QName("modified", XMLConstants.NULL_NS_URI); + + public static final FunctionSignature signature = + new FunctionSignature( + new QName("sync", FileModule.NAMESPACE_URI, FileModule.PREFIX), + "Synchronize a collection with a directory hierarchy." + + "This method is only available to the DBA role. ", + new SequenceType[]{ + new FunctionParameterSequenceType("collection", Type.STRING, Cardinality.EXACTLY_ONE, + "Absolute path to the collection to synchronize to disk."), + new FunctionParameterSequenceType("targetPath", Type.ITEM, Cardinality.EXACTLY_ONE, + "The path or URI to the target directory. Relative paths resolve against EXIST_HOME."), + new FunctionParameterSequenceType("dateTimeOrOptionsMap", Type.ITEM, Cardinality.ZERO_OR_ONE, + "Options as map(*). The available settings are:" + + "\"" + PRUNE_OPT + "\": delete any file/dir that does not correspond to a doc/collection in the DB. " + + "\"" + AFTER_OPT + "\": only resources modified after this date will be taken into account." + + "\"" + EXCLUDES_OPT + "\": files on the file system matching any of these patterns will be left untouched." + + "(deprecated) If the third parameter is of type xs:dateTime, it is the same as setting the \"" + AFTER_OPT + "\" option.") + }, + new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.EXACTLY_ONE, "A report (file:sync) which files and directories were updated (file:update) or deleted (file:delete).") + ); + + private static final Properties DEFAULT_PROPERTIES = new Properties(); + + static { + DEFAULT_PROPERTIES.put(OutputKeys.INDENT, "yes"); + DEFAULT_PROPERTIES.put(OutputKeys.OMIT_XML_DECLARATION, "no"); + DEFAULT_PROPERTIES.put(EXistOutputKeys.EXPAND_XINCLUDES, "no"); + DEFAULT_PROPERTIES.put(OutputKeys.ENCODING, "UTF-8"); + } + + private Properties outputProperties = new Properties(); + + public Sync(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + + if (!context.getSubject().hasDbaRole()) { + throw new XPathException(this, "Function file:sync is only available to the DBA role"); + } + + final String collectionPath = args[0].getStringValue(); + final String target = args[1].getStringValue(); + final Map options = getOptions(args[2]); + + return startSync(target, collectionPath, options); + } + + private Map getOptions(final Sequence parameter) throws XPathException { + final Map options = new HashMap<>(); + options.put(AFTER_OPT, Sequence.EMPTY_SEQUENCE); + options.put(PRUNE_OPT, new BooleanValue(this, false)); + options.put(EXCLUDES_OPT, Sequence.EMPTY_SEQUENCE); + + if (parameter.isEmpty()) { + outputProperties = DEFAULT_PROPERTIES; + return options; + } + + final Item item = parameter.itemAt(0); + + if (item.getType() == Type.MAP_ITEM) { + final AbstractMapType optionsMap = (AbstractMapType) item; + + outputProperties = SerializerUtils.getSerializationOptions(this, optionsMap); + + // override defaults set in SerializerUtils + for(String p : DEFAULT_PROPERTIES.stringPropertyNames()) { + if (optionsMap.get(new StringValue(this, p)).isEmpty()) { + outputProperties.setProperty(p, DEFAULT_PROPERTIES.getProperty(p)); + } + } + + final Sequence seq = optionsMap.get(new StringValue(this, EXCLUDES_OPT)); + if (!seq.isEmpty() && seq.getItemType() != Type.STRING) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Invalid value for option \"excludes\", expected xs:string* got " + + Type.getTypeName(seq.getItemType())); + } + options.put(EXCLUDES_OPT, seq); + + checkOption(optionsMap, PRUNE_OPT, Type.BOOLEAN, options); + checkOption(optionsMap, AFTER_OPT, Type.DATE_TIME, options); + } else if (parameter.itemAt(0).getType() == Type.DATE_TIME) { + options.put(AFTER_OPT, parameter); + } else { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Invalid 3rd parameter, allowed parameter types are xs:dateTime or map(*) got " + Type.getTypeName(item.getType())); + } + return options; + } + + private void checkOption( + final AbstractMapType optionsMap, + final String name, + final int expectedType, + final Map options + ) throws XPathException { + final Sequence p = optionsMap.get(new StringValue(this, name)); + + if (p.isEmpty()) { + return; // nothing to do, continue + } + + if (p.hasMany() || !Type.subTypeOf(p.getItemType(),expectedType)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Invalid value type for option \"" + name + "\", expected " + + Type.getTypeName(expectedType) + " got " + + Type.getTypeName(p.itemAt(0).getType())); + } + + options.put(name, p); + } + + private Sequence startSync( + final String target, + final String collectionPath, + final Map options + ) throws XPathException { + final Date startDate = options.get(AFTER_OPT).hasOne() ? ((DateTimeValue) options.get(AFTER_OPT)).getDate() : null; + + final boolean prune = ((BooleanValue) options.get(PRUNE_OPT)).getValue(); + + final List excludes = new ArrayList<>(Collections.emptyList()); + for (final SequenceIterator si = options.get(EXCLUDES_OPT).iterate(); si.hasNext(); ) { + excludes.add(si.nextItem().getStringValue()); + } + + final Path p = FileModuleHelper.getFile(target, this); + context.pushDocumentContext(); + final MemTreeBuilder output = context.getDocumentBuilder(); + final Path targetDir; + try { + if (p.isAbsolute()) { + targetDir = p; + } else { + final Optional home = context.getBroker().getConfiguration().getExistHome(); + targetDir = FileUtils.resolve(home, target); + } + + output.startDocument(); + output.startElement(FILE_SYNC_ELEMENT, null); + output.addAttribute(FILE_COLLECTION_ATTRIBUTE, collectionPath); + output.addAttribute(FILE_DIR_ATTRIBUTE, targetDir.toAbsolutePath().toString()); + + final String rootTargetAbsPath = targetDir.toAbsolutePath().toString(); + final String separator = rootTargetAbsPath.endsWith(File.separator) ? "" : File.separator; + syncCollection(XmldbURI.create(collectionPath), rootTargetAbsPath + separator, targetDir, startDate, prune, excludes, output); + + output.endElement(); + output.endDocument(); + } catch (final PermissionDeniedException | LockException e) { + throw new XPathException(this, e); + } finally { + context.popDocumentContext(); + } + return output.getDocument(); + } + + private void syncCollection( + final XmldbURI collectionPath, + final String rootTargetAbsPath, + final Path targetDir, + final Date startDate, + final boolean prune, + final List excludes, + final MemTreeBuilder output + ) throws PermissionDeniedException, LockException { + final Path targetDirectory; + try { + targetDirectory = Files.createDirectories(targetDir); + } catch (final IOException ioe) { + reportError(output, "Failed to create output directory: " + targetDir.toAbsolutePath() + + " for collection " + collectionPath); + return; + } + + if (!Files.isWritable(targetDirectory)) { + reportError(output, "Failed to write to output directory: " + targetDirectory.toAbsolutePath()); + return; + } + + final List subCollections = handleCollection(collectionPath, rootTargetAbsPath, targetDirectory, startDate, prune, excludes, output); + + for (final XmldbURI childURI : subCollections) { + final Path childDir = targetDirectory.resolve(childURI.lastSegment().toString()); + syncCollection(collectionPath.append(childURI), rootTargetAbsPath, childDir, startDate, prune, excludes, output); + } + } + + private List handleCollection( + final XmldbURI collectionPath, + final String rootTargetAbsPath, + final Path targetDirectory, + final Date startDate, + final boolean prune, + final List excludes, + final MemTreeBuilder output + ) throws PermissionDeniedException, LockException { + try (final Collection collection = context.getBroker().openCollection(collectionPath, LockMode.READ_LOCK)) { + if (collection == null) { + reportError(output, "Collection not found: " + collectionPath); + return Collections.emptyList(); + } + + if (prune) { + pruneCollectionEntries(collection, rootTargetAbsPath, targetDirectory, excludes, output); + } + + for (final Iterator i = collection.iterator(context.getBroker()); i.hasNext(); ) { + final DocumentImpl doc = i.next(); + final Path targetFile = targetDirectory.resolve(doc.getFileURI().toASCIIString()); + saveFile(targetFile, doc, startDate, output); + } + + final List subCollections = new ArrayList<>(collection.getChildCollectionCount(context.getBroker())); + for (final Iterator i = collection.collectionIterator(context.getBroker()); i.hasNext(); ) { + subCollections.add(i.next()); + } + return subCollections; + } + } + + private void pruneCollectionEntries( + final Collection collection, + final String rootTargetAbsPath, + final Path targetDir, + final List excludes, + final MemTreeBuilder output) { + try (final Stream fileStream = Files.walk(targetDir, 1)) { + fileStream.forEach(path -> { + try { + // guard against deletion of output folder + if (rootTargetAbsPath.startsWith(path.toString())) { + return; + } + + if (isExcludedPath(rootTargetAbsPath, path, excludes)) { + return; + } + + final String fileName = path.getFileName().toString(); + final XmldbURI dbname = XmldbURI.xmldbUriFor(fileName); + final String currentCollection = collection.getURI().getCollectionPath(); + + if (collection.hasDocument(context.getBroker(), dbname) + || collection.hasChildCollection(context.getBroker(), dbname) + || currentCollection.endsWith("/" + fileName)) { + return; + } + + // handle non-empty directories + if (Files.isDirectory(path)) { + deleteWithExcludes(rootTargetAbsPath, path, excludes, output); + } else { + Files.deleteIfExists(path); + // reporting + output.startElement(FILE_DELETE_ELEMENT, null); + output.addAttribute(FILE_ATTRIBUTE, path.toAbsolutePath().toString()); + output.addAttribute(NAME_ATTRIBUTE, fileName); + output.endElement(); + } + + } catch (final IOException | URISyntaxException + | PermissionDeniedException | LockException e) { + reportError(output, e.getMessage()); + } + }); + } catch (final IOException e) { + reportError(output, e.getMessage()); + } + } + + private void saveFile(final Path targetFile, final DocumentImpl doc, final Date startDate, final MemTreeBuilder output) throws LockException { + // the resource has not changed in the selected period + if (startDate != null && doc.getLastModified() <= startDate.getTime()) { + return; + } + try (final ManagedLock lock = context.getBroker().getBrokerPool().getLockManager().acquireDocumentReadLock(doc.getURI())) { + // the file on the disk appears to be up-to-date + if (Files.exists(targetFile) && Files.getLastModifiedTime(targetFile).compareTo(FileTime.fromMillis(doc.getLastModified())) >= 0) { + return; + } + + output.startElement(FILE_UPDATE_ELEMENT, null); + output.addAttribute(FILE_ATTRIBUTE, targetFile.toAbsolutePath().toString()); + output.addAttribute(NAME_ATTRIBUTE, doc.getFileURI().toString()); + output.addAttribute(COLLECTION_ATTRIBUTE, doc.getCollection().getURI().toString()); + output.addAttribute(MODIFIED_ATTRIBUTE, new DateTimeValue(this, new Date(doc.getLastModified())).getStringValue()); + + if (doc.getResourceType() == DocumentImpl.BINARY_FILE) { + output.addAttribute(TYPE_ATTRIBUTE, "binary"); + output.endElement(); + saveBinary(targetFile, (BinaryDocument) doc, output); + } else { + output.addAttribute(TYPE_ATTRIBUTE, "xml"); + output.endElement(); + saveXML(targetFile, doc, output); + } + } catch (final XPathException e) { + reportError(output, e.getMessage()); + } catch (final IOException e) { + reportError(output, "IO error while saving file: " + targetFile.toAbsolutePath()); + } + } + + private void saveXML(final Path targetFile, final DocumentImpl doc, final MemTreeBuilder output) throws IOException { + final SAXSerializer sax = (SAXSerializer) SerializerPool.getInstance().borrowObject(SAXSerializer.class); + try { + final boolean isRepoXML = Files.exists(targetFile) && "repo.xml".equals(FileUtils.fileName(targetFile)); + + if (isRepoXML) { + processRepoDesc(targetFile, doc, sax, output); + } else { + final Serializer serializer = context.getBroker().borrowSerializer(); + try (final Writer writer = new OutputStreamWriter(new BufferedOutputStream(Files.newOutputStream(targetFile)), StandardCharsets.UTF_8)) { + sax.setOutput(writer, outputProperties); + serializer.setProperties(outputProperties); + + serializer.setSAXHandlers(sax, sax); + serializer.toSAX(doc); + } finally { + context.getBroker().returnSerializer(serializer); + } + } + } catch (final SAXException e) { + reportError(output, "SAX exception while saving file " + targetFile.toAbsolutePath() + ": " + e.getMessage()); + } finally { + SerializerPool.getInstance().returnObject(sax); + } + } + + /** + * Merge repo.xml modified by user with original file. This is necessary because we have to + * remove sensitive information during upload (default password) and need to restore it + * when the package is synchronized back to disk. + */ + private void processRepoDesc(final Path targetFile, final DocumentImpl doc, final SAXSerializer sax, final MemTreeBuilder output) { + try { + final DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + final Document original = builder.parse(targetFile.toFile()); + + final Serializer serializer = context.getBroker().borrowSerializer(); + + try (final Writer writer = new OutputStreamWriter(new BufferedOutputStream(Files.newOutputStream(targetFile)), StandardCharsets.UTF_8)) { + sax.setOutput(writer, outputProperties); + + final StreamSource styleSource = new StreamSource(Sync.class.getResourceAsStream("repo.xsl")); + + final SAXTransformerFactory factory = TransformerFactoryAllocator.getTransformerFactory(context.getBroker().getBrokerPool()); + final TransformerHandler handler = factory.newTransformerHandler(styleSource); + handler.getTransformer().setParameter("original", original.getDocumentElement()); + handler.setResult(new SAXResult(sax)); + + serializer.reset(); + serializer.setProperties(outputProperties); + serializer.setSAXHandlers(handler, handler); + + serializer.toSAX(doc); + } finally { + context.getBroker().returnSerializer(serializer); + } + } catch (final ParserConfigurationException e) { + reportError(output, "Parser exception while saving file " + targetFile.toAbsolutePath() + ": " + e.getMessage()); + } catch (final SAXException e) { + reportError(output, "SAX exception while saving file " + targetFile.toAbsolutePath() + ": " + e.getMessage()); + } catch (final IOException e) { + reportError(output, "IO exception while saving file " + targetFile.toAbsolutePath() + ": " + e.getMessage()); + } catch (final TransformerException e) { + reportError(output, "Transformation exception while saving file " + targetFile.toAbsolutePath() + ": " + e.getMessage()); + } + } + + private void saveBinary(final Path targetFile, final BinaryDocument binary, final MemTreeBuilder output) { + try (final InputStream is = context.getBroker().getBinaryResource(binary)) { + Files.copy(is, targetFile, StandardCopyOption.REPLACE_EXISTING); + } catch (final Exception e) { + reportError(output, e.getMessage()); + } + } + + private void reportError(final MemTreeBuilder output, final String msg) { + output.startElement(FILE_ERROR_ELEMENT, null); + output.characters(msg); + output.endElement(); + } + + /** + * We need to convert to a relative path in relation to rootTargetAbsPath, + * as all the exclusion patterns are relative to rootTargetAbsPath. + * + * @param rootTargetAbsPath the root target (abs)path + * @param path (abs)path to check for being excluded. Should be subdir of rootTargetAbsPath + * @param excludes exclude patterns (in the convention of DirectoryScanner.match) + * @return true if the (rel)path in question is matched by some of the exclusion patterns + */ + private static boolean isExcludedPath(final String rootTargetAbsPath, final Path path, final List excludes) { + if (excludes.isEmpty()) { + return false; + } + // root folder cannot be excluded + // path will then also be one character shorter than rootTargetApsPath + // and throw when attempting to construct the relative path + if (rootTargetAbsPath.startsWith(path.toString())) { + return false; + } + + final String absPath = path.toAbsolutePath().toString(); + final String relPath = absPath.substring(rootTargetAbsPath.length()); + final String normalizedPath = relPath.startsWith(File.separator) + ? relPath.substring(File.separator.length()) + : relPath; + + return matchAny(excludes, normalizedPath); + } + + /** + * Check if any of the patterns matches the path. + */ + public static boolean matchAny(final Iterable patterns, final String path) { + for (final String pattern : patterns) { + if (DirectoryScanner.match(pattern, path)) { + return true; + } + } + return false; + } + + private static void deleteWithExcludes(final String root, final Path path, final List excludes, final MemTreeBuilder output) throws IOException { + if (Files.isDirectory(path)) { + Files.walkFileTree(path, new DeleteDirWithExcludesVisitor(root, excludes, output)); + } else { + Files.deleteIfExists(path); + } + } + + private static class DeleteDirWithExcludesVisitor extends SimpleFileVisitor { + + private final List excludes; + private final String root; + private final MemTreeBuilder output; + private boolean hasExcludedChildren = false; + + public DeleteDirWithExcludesVisitor(final String root, final List excludes, final MemTreeBuilder output) { + this.output = output; + this.excludes = excludes; + this.root = root; + } + + @Override + public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) { + if (isExcludedPath(root, dir, excludes)) { + hasExcludedChildren = true; + return FileVisitResult.SKIP_SUBTREE; + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + if (isExcludedPath(root, file, excludes)) { + hasExcludedChildren = true; + return FileVisitResult.CONTINUE; + } + Files.deleteIfExists(file); + + output.startElement(FILE_DELETE_ELEMENT, null); + output.addAttribute(FILE_ATTRIBUTE, file.toAbsolutePath().toString()); + output.addAttribute(NAME_ATTRIBUTE, file.getFileName().toString()); + output.endElement(); + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException { + if (exc != null) { + throw exc; + } + // deletion would fail due to non-empty directory + if (hasExcludedChildren) { + return FileVisitResult.CONTINUE; + } + Files.deleteIfExists(dir); + + output.startElement(FILE_DELETE_ELEMENT, null); + output.addAttribute(FILE_ATTRIBUTE, dir.toAbsolutePath().toString()); + output.addAttribute(NAME_ATTRIBUTE, dir.getFileName().toString()); + output.endElement(); + + return FileVisitResult.CONTINUE; + } + } +} diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/UnicodeReader.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/UnicodeReader.java new file mode 100644 index 00000000000..c73927113f6 --- /dev/null +++ b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/UnicodeReader.java @@ -0,0 +1,181 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PushbackInputStream; +import java.io.Reader; + + + +/** + * Generic unicode textreader, which will use BOM mark + * to identify the encoding to be used. If BOM is not found + * then use a given default or system encoding. This is a bug fix + * workaround for a known issue with InputStreamReader not detecting and + * ignoring the UTF-* BOM (EF BB BF). + * + * http://www.unicode.org/unicode/faq/utf_bom.html + * BOMs: + * 00 00 FE FF = UTF-32, big-endian + * FF FE 00 00 = UTF-32, little-endian + * EF BB BF = UTF-8, + * FE FF = UTF-16, big-endian + * FF FE = UTF-16, little-endian + * + * Win2k Notepad: + * Unicode format = UTF-16LE + * + * Based on code by Thomas Weidenfeller and Aki Nieminen + * + * @author Andrzej Taramina + * @serial 2008-03-06 + * @version 1.1 + */ + +public class UnicodeReader extends Reader +{ + PushbackInputStream internalIn; + InputStreamReader internalIn2 = null; + String defaultEnc; + + private static final int BOM_SIZE = 4; + + + /** + * + * @param in inputstream to be read + */ + + public UnicodeReader( InputStream in ) + { + internalIn = new PushbackInputStream( in, BOM_SIZE ); + this.defaultEnc = null; + } + + + /** + * + * @param in inputstream to be read + * @param defaultEnc default encoding if stream does not have + * BOM marker. Give NULL to use system-level default. + */ + + public UnicodeReader( InputStream in, String defaultEnc ) + { + internalIn = new PushbackInputStream( in, BOM_SIZE ); + this.defaultEnc = defaultEnc; + } + + + public String getDefaultEncoding() + { + return( defaultEnc ); + } + + + /** + * Get stream encoding or NULL if stream is uninitialized. + * Call init() or read() method to initialize it. + * + * @return the encoding + */ + public String getEncoding() + { + String ret = null; + + if( internalIn2 != null ) { + ret = internalIn2.getEncoding(); + } + + return( ret ); + } + + + /** + * Read-ahead four bytes and check for BOM marks. Extra bytes are + * unread back to the stream, only BOM bytes are skipped. + * + * @throws IOException if an I/O error occurs + */ + protected void init() throws IOException + { + if( internalIn2 == null ) { + + String encoding; + byte bom[] = new byte[BOM_SIZE]; + int n; + int unread; + n = internalIn.read( bom, 0, bom.length ); + + if( (bom[0] == (byte)0x00) && (bom[1] == (byte)0x00) && (bom[2] == (byte)0xFE) && (bom[3] == (byte)0xFF) ) { + encoding = "UTF-32BE"; + unread = n - 4; + } else if( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) && (bom[2] == (byte)0x00) && (bom[3] == (byte)0x00) ) { + encoding = "UTF-32LE"; + unread = n - 4; + } else if( (bom[0] == (byte)0xEF) && (bom[1] == (byte)0xBB) && (bom[2] == (byte)0xBF) ) { + encoding = "UTF-8"; + unread = n - 3; + } else if( (bom[0] == (byte)0xFE) && (bom[1] == (byte)0xFF) ) { + encoding = "UTF-16BE"; + unread = n - 2; + } else if( (bom[0] == (byte)0xFF) && (bom[1] == (byte)0xFE) ) { + encoding = "UTF-16LE"; + unread = n - 2; + } else { + // Unicode BOM mark not found, unread all bytes + encoding = defaultEnc; + unread = n; + } + + //System.out.println("read=" + n + ", unread=" + unread); + + if( unread > 0 ) { + internalIn.unread( bom, (n - unread), unread ); + } + + // Use given encoding + if( encoding == null ) { + internalIn2 = new InputStreamReader( internalIn ); + } else { + internalIn2 = new InputStreamReader( internalIn, encoding ); + } + } + } + + + public void close() throws IOException + { + init(); + internalIn2.close(); + } + + public int read( char[] cbuf, int off, int len ) throws IOException + { + init(); + return( internalIn2.read( cbuf, off, len ) ); + } + +} \ No newline at end of file diff --git a/extensions/modules/file/src/main/resources/org/exist/xquery/modules/file/repo.xsl b/extensions/modules/file/src/main/resources/org/exist/xquery/modules/file/repo.xsl new file mode 100644 index 00000000000..73b08785f5c --- /dev/null +++ b/extensions/modules/file/src/main/resources/org/exist/xquery/modules/file/repo.xsl @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/AbstractBinariesTest.java b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/AbstractBinariesTest.java new file mode 100644 index 00000000000..54692e89bec --- /dev/null +++ b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/AbstractBinariesTest.java @@ -0,0 +1,140 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import com.evolvedbinary.j8fu.function.Consumer2E; +import org.exist.xmldb.XmldbURI; +import org.junit.*; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Random; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.exist.test.TestConstants.TEST_COLLECTION_URI; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests for accessing binaries using XQuery via various APIs. + * + * @see Binary streaming is broken + * + * @author Adam Retter + */ +public abstract class AbstractBinariesTest { + + protected static final XmldbURI TEST_COLLECTION = TEST_COLLECTION_URI.append("BinariesTest"); + protected static final String BIN1_FILENAME = "1.bin"; + protected static final byte[] BIN1_CONTENT = "1234567890".getBytes(UTF_8); + + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Before + public void setup() throws Exception { + storeBinaryFile(TEST_COLLECTION.append(BIN1_FILENAME), BIN1_CONTENT); + } + + @After + public void cleanup() throws Exception { + removeCollection(TEST_COLLECTION); + } + + /** + * {@see https://github.com/eXist-db/exist/issues/790#error-case-3} + */ + @Test + public void readBinary() throws Exception { + final byte[] data = randomData(1024 * 1024 * 10); // 10KB + final Path tmpFile = createTemporaryFile(data); + + final String query = "import module namespace file = \"http://exist-db.org/xquery/file\";\n" + + "file:read-binary('" + tmpFile.toAbsolutePath() + "')"; + + final QueryResultAccessor resultsAccessor = executeXQuery(query); + + resultsAccessor.accept(results -> { + assertEquals(1, size(results)); + + final U item = item(results, 0); + assertTrue(isBinaryType(item)); + assertArrayEquals(data, getBytes(item)); + }); + } + + /** + * {@see https://github.com/eXist-db/exist/issues/790#error-case-4} + */ + @Test + public void readAndWriteBinary() throws Exception { + final byte[] data = randomData(1024 * 1024); // 1MB + final Path tmpInFile = createTemporaryFile(data); + + final Path tmpOutFile = temporaryFolder.newFile().toPath(); + + final String query = "import module namespace file = \"http://exist-db.org/xquery/file\";\n" + + "let $bin := file:read-binary('" + tmpInFile.toAbsolutePath() + "')\n" + + "return\n" + + " file:serialize-binary($bin, '" + tmpOutFile.toAbsolutePath() + "')"; + + final QueryResultAccessor resultsAccessor = executeXQuery(query); + + resultsAccessor.accept(results -> { + assertEquals(1, size(results)); + + final U item = item(results, 0); + assertTrue(isBooleanType(item)); + assertEquals(true, getBoolean(item)); + }); + + assertArrayEquals(Files.readAllBytes(tmpInFile), Files.readAllBytes(tmpOutFile)); + } + + protected byte[] randomData(final int size) { + final byte data[] = new byte[size]; + new Random().nextBytes(data); + return data; + } + + protected Path createTemporaryFile(final byte[] data) throws IOException { + final Path f = temporaryFolder.newFile().toPath(); + Files.write(f, data); + return f; + } + + @FunctionalInterface interface QueryResultAccessor extends Consumer2E, AssertionError, E> { + } + + protected abstract void storeBinaryFile(final XmldbURI filePath, final byte[] content) throws Exception; + protected abstract void removeCollection(final XmldbURI collectionUri) throws Exception; + protected abstract QueryResultAccessor executeXQuery(final String query) throws Exception; + protected abstract long size(T results) throws E; + protected abstract U item(T results, int index) throws E; + protected abstract boolean isBinaryType(U item) throws E; + protected abstract boolean isBooleanType(U item) throws E; + protected abstract byte[] getBytes(U item) throws E; + protected abstract boolean getBoolean(U item) throws E; +} diff --git a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/EmbeddedBinariesTest.java b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/EmbeddedBinariesTest.java new file mode 100644 index 00000000000..593431a5fe2 --- /dev/null +++ b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/EmbeddedBinariesTest.java @@ -0,0 +1,164 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import org.exist.collections.Collection; +import org.exist.source.Source; +import org.exist.source.StringSource; +import org.exist.storage.BrokerPool; +import org.exist.storage.DBBroker; +import org.exist.storage.XQueryPool; +import org.exist.storage.lock.Lock; +import org.exist.storage.lock.ManagedCollectionLock; +import org.exist.storage.txn.Txn; +import org.exist.test.ExistEmbeddedServer; +import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; +import org.exist.util.MimeType; +import org.exist.util.StringInputSource; +import org.exist.xmldb.XmldbURI; +import org.exist.xquery.CompiledXQuery; +import org.exist.xquery.XQuery; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.*; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.Optional; + +/** + * @author Adam Retter + */ +public class EmbeddedBinariesTest extends AbstractBinariesTest { + + @ClassRule + public static ExistEmbeddedServer existEmbeddedServer = new ExistEmbeddedServer(true, true); + + @Override + protected void storeBinaryFile(final XmldbURI filePath, final byte[] content) throws Exception { + final BrokerPool brokerPool = existEmbeddedServer.getBrokerPool(); + try(final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject())); + final Txn transaction = brokerPool.getTransactionManager().beginTransaction()) { + + try(final ManagedCollectionLock collectionLock = brokerPool.getLockManager().acquireCollectionWriteLock(filePath.removeLastSegment())) { + final Collection collection = broker.getOrCreateCollection(transaction, filePath.removeLastSegment()); + + broker.storeDocument(transaction, filePath.lastSegment(), new StringInputSource(content), MimeType.BINARY_TYPE, collection); + + broker.saveCollection(transaction, collection); + } + + transaction.commit(); + } + } + + @Override + protected void removeCollection(final XmldbURI collectionUri) throws Exception { + final BrokerPool brokerPool = existEmbeddedServer.getBrokerPool(); + try(final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject())); + final Txn transaction = brokerPool.getTransactionManager().beginTransaction(); + final Collection collection = broker.openCollection(collectionUri, Lock.LockMode.WRITE_LOCK)) { + if(collection != null) { + broker.removeCollection(transaction, collection); + } + + transaction.commit(); + } + } + + @Override + protected QueryResultAccessor executeXQuery(final String query) throws Exception { + final Source source = new StringSource(query); + final BrokerPool brokerPool = existEmbeddedServer.getBrokerPool(); + final XQueryPool pool = brokerPool.getXQueryPool(); + final XQuery xquery = brokerPool.getXQueryService(); + + try(final DBBroker broker = brokerPool.get(Optional.of(brokerPool.getSecurityManager().getSystemSubject()))) { + final CompiledXQuery existingCompiled = pool.borrowCompiledXQuery(broker, source); + + final XQueryContext context; + final CompiledXQuery compiled; + if (existingCompiled == null) { + context = new XQueryContext(brokerPool); + compiled = xquery.compile(context, source); + } else { + context = existingCompiled.getContext(); + context.prepareForReuse(); + compiled = existingCompiled; + } + + final Sequence results = xquery.execute(broker, compiled, null); + + return consumer2E -> { + try { +// context.runCleanupTasks(); //TODO(AR) shows the ordering issue with binary values (see comment below) + + consumer2E.accept(results); + } finally { + //TODO(AR) performing #runCleanupTasks causes the stream to be closed, so if we do so before we are finished with the results, serialization fails. + context.runCleanupTasks(); + pool.returnCompiledXQuery(source, compiled); + } + }; + } + } + + @Override + protected long size(final Sequence results) { + return results.getItemCount(); + } + + @Override + protected Item item(final Sequence results, final int index) { + return results.itemAt(index); + } + + @Override + protected boolean isBinaryType(final Item item) { + return Type.BASE64_BINARY == item.getType() || Type.HEX_BINARY == item.getType(); + } + + @Override + protected boolean isBooleanType(final Item item) throws IOException { + return Type.BOOLEAN == item.getType(); + } + + @Override + protected byte[] getBytes(final Item item) throws IOException { + if (item instanceof Base64BinaryDocument doc) { + try (final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { + doc.streamBinaryTo(baos); + return baos.toByteArray(); + } + } else { + final BinaryValueFromFile file = (BinaryValueFromFile) item; + try (final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { + file.streamBinaryTo(baos); + return baos.toByteArray(); + } + } + } + + @Override + protected boolean getBoolean(final Item item) throws IOException { + return ((BooleanValue)item).getValue(); + } +} diff --git a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/RestBinariesTest.java b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/RestBinariesTest.java new file mode 100644 index 00000000000..de93012eda0 --- /dev/null +++ b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/RestBinariesTest.java @@ -0,0 +1,219 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.Hex; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.client.fluent.Executor; +import org.apache.http.client.fluent.Request; +import org.apache.http.entity.ContentType; +import org.exist.http.jaxb.Query; +import org.exist.http.jaxb.Result; +import org.exist.test.ExistWebServer; +import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; +import org.exist.xmldb.XmldbURI; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; +import jakarta.xml.bind.Unmarshaller; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.apache.http.HttpStatus.SC_CREATED; +import static org.apache.http.HttpStatus.SC_OK; +import static org.exist.TestUtils.ADMIN_DB_PWD; +import static org.exist.TestUtils.ADMIN_DB_USER; +import static org.junit.Assert.assertArrayEquals; + +/** + * @author Adam Retter + */ +public class RestBinariesTest extends AbstractBinariesTest { + + @ClassRule + public static final ExistWebServer existWebServer = new ExistWebServer(true, false, true, true); + + private static Executor executor = null; + + @BeforeClass + public static void setupExecutor() { + executor = Executor.newInstance() + .auth(new HttpHost("localhost", existWebServer.getPort()), ADMIN_DB_USER, ADMIN_DB_PWD) + .authPreemptive(new HttpHost("localhost", existWebServer.getPort())); + } + + /** + * {@see https://github.com/eXist-db/exist/issues/790#error-case-5} + * + * response:stream is used to return Base64 encoded binary. + */ + @Test + public void readAndStreamBinarySax() throws IOException, JAXBException { + final byte[] data = randomData(1024 * 1024); // 1MB + final Path tmpInFile = createTemporaryFile(data); + + final String query = "import module namespace file = \"http://exist-db.org/xquery/file\";\n" + + "import module namespace response = \"http://exist-db.org/xquery/response\";\n" + + "let $bin := file:read-binary('" + tmpInFile.toAbsolutePath() + "')\n" + + "return response:stream($bin, 'media-type=application/octet-stream')"; + + final HttpResponse response = postXquery(query); + + final HttpEntity entity = response.getEntity(); + try(final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { + entity.writeTo(baos); + + assertArrayEquals(Files.readAllBytes(tmpInFile), Base64.decodeBase64(baos.toByteArray())); + } + } + + /** + * {@see https://github.com/eXist-db/exist/issues/790#error-case-5} + * + * response:stream-binary is used to return raw binary. + */ + @Test + public void readAndStreamBinaryRaw() throws IOException, JAXBException { + final byte[] data = randomData(1024 * 1024); // 1MB + final Path tmpInFile = createTemporaryFile(data); + + final String query = "import module namespace file = \"http://exist-db.org/xquery/file\";\n" + + "import module namespace response = \"http://exist-db.org/xquery/response\";\n" + + "let $bin := file:read-binary('" + tmpInFile.toAbsolutePath() + "')\n" + + "return response:stream-binary($bin, 'media-type=application/octet-stream', ())"; + + final HttpResponse response = postXquery(query); + + final HttpEntity entity = response.getEntity(); + try(final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { + entity.writeTo(baos); + + assertArrayEquals(Files.readAllBytes(tmpInFile), baos.toByteArray()); + } + } + + + @Override + protected void storeBinaryFile(final XmldbURI filePath, final byte[] content) throws Exception { + final HttpResponse response = executor.execute(Request.Put(getRestUrl() + filePath.toString()) + .setHeader("Content-Type", "application/octet-stream") + .bodyByteArray(content) + ).returnResponse(); + + if(response.getStatusLine().getStatusCode() != SC_CREATED) { + throw new Exception("Unable to store binary file: " + filePath); + } + } + + private String getRestUrl() { + return "http://localhost:" + existWebServer.getPort() + "/rest"; + } + + @Override + protected void removeCollection(final XmldbURI collectionUri) throws Exception { + final HttpResponse response = executor.execute(Request.Delete(getRestUrl() + collectionUri.toString())) + .returnResponse(); + + if(response.getStatusLine().getStatusCode() != SC_OK) { + throw new Exception("Unable to delete collection: " + collectionUri); + } + } + + @Override + protected QueryResultAccessor executeXQuery(final String xquery) throws Exception { + final HttpResponse response = postXquery(xquery); + final HttpEntity entity = response.getEntity(); + try(final InputStream is = entity.getContent()) { + final JAXBContext jaxbContext = JAXBContext.newInstance("org.exist.http.jaxb"); + final Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); + final Result result = (Result)unmarshaller.unmarshal(is); + + return consumer -> consumer.accept(result); + } + } + + private HttpResponse postXquery(final String xquery) throws JAXBException, IOException { + final Query query = new Query(); + query.setText(xquery); + + final JAXBContext jaxbContext = JAXBContext.newInstance("org.exist.http.jaxb"); + final Marshaller marshaller = jaxbContext.createMarshaller(); + + final HttpResponse response; + try(final UnsynchronizedByteArrayOutputStream baos = UnsynchronizedByteArrayOutputStream.builder().get()) { + marshaller.marshal(query, baos); + response = executor.execute(Request.Post(getRestUrl() + "/db/") + .bodyByteArray(baos.toByteArray(), ContentType.APPLICATION_XML) + ).returnResponse(); + } + + if(response.getStatusLine().getStatusCode() != SC_OK) { + throw new IOException("Unable to query, HTTP response code: " + response.getStatusLine().getStatusCode()); + } + + return response; + } + + @Override + protected long size(final Result result) throws Exception { + return result.getCount(); + } + + @Override + protected Result.Value item(final Result results, final int index) throws Exception { + return results.getValue().get(index); + } + + @Override + protected boolean isBinaryType(final Result.Value item) throws Exception { + final String type = item.getType(); + return "xs:base64Binary".equals(type) || "xs:hexBinary".equals(type); + } + + @Override + protected boolean isBooleanType(Result.Value item) throws Exception { + return "xs:boolean".equals(item.getType()); + } + + @Override + protected byte[] getBytes(final Result.Value item) throws Exception { + return switch (item.getType()) { + case "xs:base64Binary" -> Base64.decodeBase64(item.getContent().getFirst().toString()); + case "xs:hexBinary" -> Hex.decodeHex(item.getContent().getFirst().toString()); + default -> throw new UnsupportedOperationException(); + }; + } + + @Override + protected boolean getBoolean(final Result.Value item) throws Exception { + return Boolean.parseBoolean(item.getContent().getFirst().toString()); + } +} diff --git a/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java new file mode 100644 index 00000000000..08358285365 --- /dev/null +++ b/extensions/modules/file/src/test/java/org/exist/xquery/modules/file/XmldbBinariesTest.java @@ -0,0 +1,199 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.modules.file; + +import org.exist.test.ExistWebServer; +import org.exist.xmldb.XmldbURI; +import org.junit.ClassRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.xmldb.api.DatabaseManager; +import org.xmldb.api.base.*; +import org.xmldb.api.modules.BinaryResource; +import org.xmldb.api.modules.CollectionManagementService; +import org.xmldb.api.modules.XQueryService; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; + +import static org.exist.TestUtils.ADMIN_DB_PWD; +import static org.exist.TestUtils.ADMIN_DB_USER; +import static org.xmldb.api.base.ResourceType.BINARY_RESOURCE; + +/** + * @author Adam Retter + */ +@RunWith(Parameterized.class) +public class XmldbBinariesTest extends AbstractBinariesTest { + + @ClassRule + public static final ExistWebServer existWebServer = new ExistWebServer(true, false, true, true); + private static final String PORT_PLACEHOLDER = "${PORT}"; + + @Parameterized.Parameters(name = "{0}") + public static java.util.Collection data() { + return Arrays.asList(new Object[][] { + { "local", "xmldb:exist://" }, + { "remote", "xmldb:exist://localhost:" + PORT_PLACEHOLDER + "/xmlrpc" } + }); + } + + @Parameterized.Parameter + public String apiName; + + @Parameterized.Parameter(value = 1) + public String baseUri; + + private final String getBaseUri() { + return baseUri.replace(PORT_PLACEHOLDER, Integer.toString(existWebServer.getPort())); + } + + @Override + protected void storeBinaryFile(final XmldbURI filePath, byte[] content) throws Exception { + Collection colRoot = null; + try { + colRoot = DatabaseManager.getCollection(getBaseUri() + "/db", ADMIN_DB_USER, ADMIN_DB_PWD); + + final XmldbURI collectionNames[] = filePath.removeLastSegment().getPathSegments(); + + final Deque cols = new ArrayDeque<>(); + try { + Collection current = colRoot; + for (int i = 1; i < collectionNames.length; i++) { + final Collection child = getOrCreateCollection(current, collectionNames[i].toString()); + cols.push(child); + current = child; + } + + final String fileName = filePath.lastSegment().toString(); + final Resource resource = current.createResource(fileName, BinaryResource.class); + resource.setContent(content); + current.storeResource(resource); + + } finally { + while(!cols.isEmpty()) { + try { + cols.pop().close(); + } catch(XMLDBException e) { + + } + } + } + } finally { + if(colRoot != null) { + colRoot.close(); + } + } + } + + private Collection getOrCreateCollection(final Collection parent, final String childName) throws XMLDBException { + Collection child = parent.getChildCollection(childName); + if(child == null) { + final CollectionManagementService cms = parent.getService(CollectionManagementService.class); + child = cms.createCollection(childName); + } + return child; + } + + @Override + protected void removeCollection(final XmldbURI collectionUri) throws Exception { + Collection colRoot = null; + try { + colRoot = DatabaseManager.getCollection(getBaseUri() + "/db", ADMIN_DB_USER, ADMIN_DB_PWD); + + final Collection colTest = colRoot.getChildCollection("test"); + try { + final CollectionManagementService cms = colTest.getService(CollectionManagementService.class); + + final String testCollectionName = collectionUri.lastSegment().toString(); + cms.removeCollection(testCollectionName); + } finally { + if(colTest != null) { + colTest.close(); + } + } + } finally { + if(colRoot != null) { + colRoot.close(); + } + } + } + + @Override + protected QueryResultAccessor executeXQuery(final String query) throws Exception { + return consumer -> { + Collection colRoot = null; + try { + colRoot = DatabaseManager.getCollection(getBaseUri() + "/db", ADMIN_DB_USER, ADMIN_DB_PWD); + final XQueryService xqueryService = colRoot.getService(XQueryService.class); + + final CompiledExpression compiledExpression = xqueryService.compile(query); + final ResourceSet results = xqueryService.execute(compiledExpression); + + + try { + // compiledExpression.reset(); // shows the ordering issue with binary values (see comment below) + + consumer.accept(results); + } finally { + //the following calls cause the streams of any binary result values to be closed, so if we did so before we are finished with the results, serialization would fail. + results.clear(); + compiledExpression.reset(); + } + } finally { + colRoot.close(); + } + }; + } + + @Override + protected long size(final ResourceSet results) throws XMLDBException { + return results.getSize(); + } + + @Override + protected Resource item(final ResourceSet results, final int index) throws XMLDBException { + return results.getResource(index); + } + + @Override + protected boolean isBinaryType(final Resource item) throws XMLDBException { + return BINARY_RESOURCE.equals(item.getResourceType()); + } + + @Override + protected boolean isBooleanType(final Resource item) throws XMLDBException { + final String value = item.getContent().toString(); + return "true".equals(value) || "false".equals(value); + } + + @Override + protected byte[] getBytes(final Resource item) throws XMLDBException { + return (byte[])item.getContent(); + } + + @Override + protected boolean getBoolean(final Resource item) throws XMLDBException { + return Boolean.parseBoolean(item.getContent().toString()); + } +} diff --git a/extensions/modules/file/src/test/java/xquery/modules/file/FileTests.java b/extensions/modules/file/src/test/java/xquery/modules/file/FileTests.java new file mode 100644 index 00000000000..f3abd2ce42c --- /dev/null +++ b/extensions/modules/file/src/test/java/xquery/modules/file/FileTests.java @@ -0,0 +1,33 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package xquery.modules.file; + +import org.exist.test.runner.XSuite; +import org.junit.runner.RunWith; + +@RunWith(XSuite.class) +@XSuite.XSuiteFiles({ + "src/test/xquery/modules/file" +}) +public class FileTests { +} + diff --git a/extensions/modules/file/src/test/resources-filtered/conf.xml b/extensions/modules/file/src/test/resources-filtered/conf.xml new file mode 100644 index 00000000000..11c020c728e --- /dev/null +++ b/extensions/modules/file/src/test/resources-filtered/conf.xml @@ -0,0 +1,781 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/extensions/modules/file/src/test/resources/log4j2.xml b/extensions/modules/file/src/test/resources/log4j2.xml new file mode 100644 index 00000000000..b355862720a --- /dev/null +++ b/extensions/modules/file/src/test/resources/log4j2.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + diff --git a/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml b/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml new file mode 100644 index 00000000000..76b81502392 --- /dev/null +++ b/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/web.xml b/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/web.xml new file mode 100644 index 00000000000..4722b24716c --- /dev/null +++ b/extensions/modules/file/src/test/resources/standalone-webapp/WEB-INF/web.xml @@ -0,0 +1,80 @@ + + + + + eXist-db – Open Source Native XML Database + eXist-db XML Database + + + org.exist.xmlrpc.RpcServlet + org.exist.xmlrpc.RpcServlet + + enabledForExtensions + true + + + + + EXistServlet + org.exist.http.servlets.EXistServlet + + configuration + conf.xml + + + basedir + WEB-INF/ + + + start + true + + 2 + + + + XQueryURLRewrite + org.exist.http.urlrewrite.XQueryURLRewrite + + config + WEB-INF/controller-config.xml + + + + + XSLTServlet + org.exist.http.servlets.XSLTServlet + + + + XQueryURLRewrite + /* + + diff --git a/extensions/modules/file/src/test/resources/util/fixtures.xqm b/extensions/modules/file/src/test/resources/util/fixtures.xqm new file mode 100644 index 00000000000..22b42d54215 --- /dev/null +++ b/extensions/modules/file/src/test/resources/util/fixtures.xqm @@ -0,0 +1,127 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures"; + +(: simple xml :) + +declare variable $fixtures:XML := document {}; + +declare variable $fixtures:SIMPLE_XML_INDENTED := +"" || $fixtures:NL || +" " || $fixtures:NL || +"" +; + +declare variable $fixtures:SIMPLE_XML_UNINDENTED := ""; + +(: more complex xml :) + +declare variable $fixtures:COMPLEX_XML := + + This is a very long line. Certainly longer than eighty characters. It is here to see what happens to lines longer than a certain limit. Let's see. + + + + + MIXED CONTENT + + The next word is highlighted. + +

+ +; + +(: FIXME(JL) cannot use StringConstructor here because that will cause all comparisons to fail on Windows :) +(: see https://github.com/eXist-db/exist/issues/4301 :) +declare variable $fixtures:COMPLEX_XML_INDENTED := +"" || $fixtures:NL || +" This is a very long line. Certainly longer than eighty characters. It is here to see what happens to lines longer than a certain limit. Let's see." || $fixtures:NL || +" " || $fixtures:NL || +" " || $fixtures:NL || +" " || $fixtures:NL || +$fixtures:NL || +" MIXED CONTENT" || $fixtures:NL || +$fixtures:NL || +" " || $fixtures:NL || +" The next word is highlighted." || $fixtures:NL || +" " || $fixtures:NL || +"

" || $fixtures:NL || +"" +; + +declare variable $fixtures:COMPLEX_XML_UNINDENTED := +"" || +"This is a very long line. Certainly longer than eighty characters. It is here to see what happens to lines longer than a certain limit. Let's see." || +"" || +"" || +"" || $fixtures:NL || +$fixtures:NL || +" MIXED CONTENT" || $fixtures:NL || +$fixtures:NL || +" " || $fixtures:NL || +" The next word is highlighted." || $fixtures:NL || +" " || +"

" || +"" +; + + +declare variable $fixtures:TXT := +``[12 12 +This is just a Text +]`` +; + +declare variable $fixtures:XQY := "xquery version ""3.1""; 0 to 9"; +declare variable $fixtures:BIN := "To bin or not to bin..."; + +(: other constants :) + +declare variable $fixtures:XML_DECLARATION := ''; +declare variable $fixtures:NL := " "; + +(: modification dates :) + +declare variable $fixtures:now := current-dateTime(); +declare variable $fixtures:mod-date := $fixtures:now; +declare variable $fixtures:mod-date-2 := $fixtures:now + xs:dayTimeDuration('PT2H'); + +(: collections :) + +declare variable $fixtures:collection-name := "file-module-test"; +declare variable $fixtures:child-collection-name := "data"; +declare variable $fixtures:collection := "/db/" || $fixtures:collection-name; +declare variable $fixtures:child-collection := $fixtures:collection || "/" || $fixtures:child-collection-name; + +(: file sync results :) + +declare variable $fixtures:ALL-UPDATED := ("test-text.txt", "test-query.xq", "bin", "test-data.xml"); + +declare variable $fixtures:ROOT-FS := ("bin", "test-text.txt", "test-query.xq", "data"); + +declare variable $fixtures:EXTRA-DATA := ("test", ".env"); + +declare variable $fixtures:ROOT-FS-EXTRA := ("test", "bin", ".env", "test-text.txt", "test-query.xq", "data"); diff --git a/extensions/modules/file/src/test/resources/util/helper.xqm b/extensions/modules/file/src/test/resources/util/helper.xqm new file mode 100644 index 00000000000..adddc4a214d --- /dev/null +++ b/extensions/modules/file/src/test/resources/util/helper.xqm @@ -0,0 +1,219 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace helper="http://exist-db.org/xquery/test/util/helper"; +import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "fixtures.xqm"; +import module namespace file="http://exist-db.org/xquery/file"; +import module namespace xmldb="http://exist-db.org/xquery/xmldb"; +import module namespace util="http://exist-db.org/xquery/util"; + +declare variable $helper:error := xs:QName("helper:assert-sync-error"); + +declare variable $helper:path-separator := util:system-property("file.separator"); + +(: +/db + /file-module-test + /data + test-data.xml + test-text.txt + test-query.xq + bin +:) +declare function helper:setup-db() as empty-sequence() { + let $_ := ( + xmldb:create-collection("/db", $fixtures:collection-name), + helper:create-db-resource($fixtures:collection, "test-text.txt", $fixtures:TXT), + helper:create-db-resource($fixtures:collection, "test-query.xq", $fixtures:XQY), + helper:create-db-resource($fixtures:collection, "bin", $fixtures:BIN), + + xmldb:create-collection($fixtures:collection, $fixtures:child-collection-name), + helper:create-db-resource($fixtures:child-collection, "test-data.xml", $fixtures:XML) + ) + return () +}; + +declare function helper:clear-db() { + xmldb:remove($fixtures:collection) +}; + +declare function helper:create-db-resource($collection as xs:string, $resource as xs:string, $content as item()) as empty-sequence() { + let $_ := xmldb:store($collection, $resource, $content) + return () +}; + +declare function helper:modify-db-resource($collection as xs:string, $resource as xs:string) as empty-sequence() { + let $_ := xmldb:touch($collection, $resource, $fixtures:mod-date-2) + return () +}; + +declare function helper:clear-suite-fs ($suite as xs:string) as empty-sequence() { + let $_ := + helper:glue-path(( + util:system-property("java.io.tmpdir"), + $suite + )) + => file:delete() + return () +}; + +declare function helper:clear-fs ($directory as xs:string) as empty-sequence() { + let $_ := file:delete($directory) + return () +}; + +declare function helper:get-test-directory ($suite as xs:string) as xs:string { + helper:glue-path(( + util:system-property("java.io.tmpdir"), + $suite, + util:uuid() + )) +}; + +declare function helper:glue-path ($parts as xs:string+) as xs:string { + string-join($parts, $helper:path-separator) +}; + +(: + : clear FS state and simulate additional data on the file system in a specific directory + : @returns given directory to allow use in pipeline (chain of arrow operators) + :) +declare function helper:setup-fs-extra ($directory as xs:string) as xs:string { + let $action1 := file:mkdirs($directory) + let $action2 := file:mkdirs(helper:glue-path(($directory, "test"))) + let $action3 := ( + (: cannot use fixtures here because this will lead to consumed input streams! :) + file:serialize-binary( + util:string-to-binary("SERVER_SECRET=123!"), + helper:glue-path(($directory, ".env"))), + file:serialize-binary( + util:string-to-binary("..."), + helper:glue-path(($directory, "test", "three.s"))) + ) + return $directory +}; + +declare function helper:get-deleted-from-sync-result ($result as element(file:sync)) as xs:string* { + $result//file:delete/@name/string() +}; + +declare function helper:get-dir-from-sync-result ($result as element(file:sync)) as xs:string* { + $result/@file:dir/string() +}; + +declare function helper:get-updated-from-sync-result ($result as element(file:sync)) as xs:string* { + $result//file:update/@name/string() +}; + +declare function helper:list-files-and-directories ($directory as xs:string) as xs:string* { + file:list($directory)//(file:file|file:directory)/@name/string() +}; + +declare function helper:sync-with-options ($directory as xs:string, $options as item()?) as element(file:sync) { + file:sync($fixtures:collection, $directory, $options)/* +}; + +declare function helper:assert-sync-result ( + $result as document-node(element(file:sync)), + $expected as map(xs:string, xs:string*) +) as xs:boolean { + helper:assert-permutation-of( + $expected?updated, + helper:get-updated-from-sync-result($result/*), + "updated" + ) + and + helper:assert-permutation-of( + $expected?deleted, + helper:get-deleted-from-sync-result($result/*), + "deleted" + ) + and + helper:assert-permutation-of( + $expected?fs, + helper:get-dir-from-sync-result($result/*) + => helper:list-files-and-directories(), + "filesystem" + ) +}; + +declare function helper:assert-permutation-of( + $expected as xs:anyAtomicType*, + $actual as xs:anyAtomicType*, + $label as xs:string +) as xs:boolean { + let $test := fold-left( + $expected, + [true(), $actual], + helper:permutation-reducer#2 + ) + + return + if (empty($expected) and not(empty($actual))) + then error($helper:error, + "Assertion failed (" || $label || "): expected empty sequence" || + " but got (" || string-join($actual, ", ") || ")") + else if (not($test?1 or exists($test?2))) + then error($helper:error, + "Assertion failed (" || $label || "): expected permutation of " || + "(" || string-join($expected, ", ") || ")" || + " but got (" || string-join($actual, ", ") || ")") + else true() +}; + +declare function helper:permutation-reducer ($result, $next) as array(*) { + let $first-index := index-of($result?2, $next)[1] + return [ + $result?1 and $first-index > 0, + helper:maybe-remove-item-at-index($result?2, $first-index) + ] +}; + +declare function helper:maybe-remove-item-at-index($sequence as xs:anyAtomicType*, $index as xs:integer?) as xs:anyAtomicType* { + if ($index = 1) + then subsequence($sequence, 2) + else if ($index > 1) + then ( + subsequence($sequence, 1, $index - 1), + subsequence($sequence, $index + 1) + ) + else $sequence (: do nothing - will be handled later :) +}; + +declare function helper:assert-file-contents($expected as xs:string, $path-parts as xs:string+) as xs:boolean { + let $path := helper:glue-path($path-parts) + let $actual := file:read($path) + + return + if ( + exists($actual) and + count($actual) = 1 and + $actual eq $expected) + then true() + else error( + $helper:error, + "File Content Assertion failed: expected file " || $path || " " || + "to contain string " || "<[" || $expected || "]>" || + " but was <[" || $actual || "]>" + ) +}; diff --git a/extensions/modules/file/src/test/xquery/modules/file/directory-list.xqm b/extensions/modules/file/src/test/xquery/modules/file/directory-list.xqm new file mode 100644 index 00000000000..629eeea50d4 --- /dev/null +++ b/extensions/modules/file/src/test/xquery/modules/file/directory-list.xqm @@ -0,0 +1,47 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace dirlist="http://exist-db.org/testsuite/modules/file/dirlist"; + + +import module namespace file="http://exist-db.org/xquery/file"; + + +declare namespace test="http://exist-db.org/xquery/xqsuite"; + + +declare + %test:assertError("file:DIRECTORY_NOT_FOUND") +function dirlist:non-existent-error-code() { + file:directory-list("/non/existent", ()) +}; + +declare + %test:assertXPath("contains($result,'basedir /non/existent does not exist.') or contains($result,'basedir \non\existent does not exist.')") +function dirlist:non-existent-error-description() { + try { + file:directory-list("/non/existent", ()) + } catch file:DIRECTORY_NOT_FOUND { + $err:description + } +}; diff --git a/extensions/modules/file/src/test/xquery/modules/file/read-binary.xqm b/extensions/modules/file/src/test/xquery/modules/file/read-binary.xqm new file mode 100644 index 00000000000..b00dd6c15f4 --- /dev/null +++ b/extensions/modules/file/src/test/xquery/modules/file/read-binary.xqm @@ -0,0 +1,57 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace readbinary="http://exist-db.org/testsuite/modules/file/read-binary"; + + +import module namespace file="http://exist-db.org/xquery/file"; +import module namespace util="http://exist-db.org/xquery/util"; +import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; + +declare namespace test="http://exist-db.org/xquery/xqsuite"; + + +declare variable $readbinary:suite := "read-binary"; + +declare + %test:setUp +function readbinary:setup() as empty-sequence() { +}; + +declare + %test:tearDown +function readbinary:tear-down() as empty-sequence() { + helper:clear-suite-fs($readbinary:suite) +}; + +declare + %test:assertEquals("SERVER_SECRET=123!") +function readbinary:without-serialization() { + let $directory := helper:get-test-directory($readbinary:suite) + let $_ := helper:setup-fs-extra($directory) + + return + concat($directory, "/.env") + => file:read-binary() + => util:binary-to-string() +}; diff --git a/extensions/modules/file/src/test/xquery/modules/file/serialize.xqm b/extensions/modules/file/src/test/xquery/modules/file/serialize.xqm new file mode 100644 index 00000000000..2610614faf4 --- /dev/null +++ b/extensions/modules/file/src/test/xquery/modules/file/serialize.xqm @@ -0,0 +1,148 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace serialize="http://exist-db.org/testsuite/modules/file/serialize"; + +import module namespace file="http://exist-db.org/xquery/file"; +import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; +import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; + + +declare namespace test="http://exist-db.org/xquery/xqsuite"; + + +declare variable $serialize:suite := "serialize"; +declare variable $serialize:text := data; +declare variable $serialize:xml := + + + text + +; + + +declare + %test:tearDown +function serialize:tear-down() as empty-sequence() { + helper:clear-suite-fs($serialize:suite) +}; + +declare + %test:assertEquals("datamoredata") +function serialize:append() { + let $directory := helper:get-test-directory($serialize:suite) + let $_ := file:mkdirs($directory) + + let $path := $directory || "/append-test.txt" + + let $_ := file:serialize-binary(xs:base64Binary("data"), $path) + let $_ := file:serialize-binary(xs:base64Binary("moredata"), $path, true()) + + return file:read-binary($path) => xs:string() +}; + +declare + %test:assertEquals("moredata") +function serialize:overwrite() { + let $directory := helper:get-test-directory($serialize:suite) + let $_ := file:mkdirs($directory) + + let $path := $directory || "/overwrite-test.txt" + let $_ := file:serialize-binary(xs:base64Binary("data"), $path) + let $_ := file:serialize-binary(xs:base64Binary("moredata"), $path, false()) + + return file:read-binary($path) => xs:string() +}; + +declare + %test:assertEquals("data") +function serialize:serialize3() { + let $directory := helper:get-test-directory($serialize:suite) + let $_ := file:mkdirs($directory) + + let $path := $directory || "/serialize-3-test.txt" + let $_ := file:serialize($serialize:text, $path, ()) + let $_ := file:serialize($serialize:text, $path, ()) + + return file:read($path) +}; + +declare + %test:assertTrue +function serialize:xml-defaults() { + let $directory := helper:get-test-directory($serialize:suite) + let $_ := file:mkdirs($directory) + + let $path := $directory || "/xml-defaults-test.xml" + let $_ := file:serialize($serialize:xml, $path, ()) + + return file:read($path) eq + "" || $fixtures:NL || + " " || $fixtures:NL || + " " || $fixtures:NL || + " text" || $fixtures:NL || + " " || $fixtures:NL || + "" +}; + +declare + %test:assertTrue +function serialize:xml-final-newline() { + let $directory := helper:get-test-directory($serialize:suite) + let $_ := file:mkdirs($directory) + + let $path := $directory || "/xml-final-newline.xml" + let $_ := file:serialize($serialize:xml, $path, + + + ) + + return file:read($path) eq + "" || $fixtures:NL || + " " || $fixtures:NL || + " " || $fixtures:NL || + " text" || $fixtures:NL || + " " || $fixtures:NL || + "" || $fixtures:NL +}; + +declare + %test:assertTrue +function serialize:xml-minified() { + let $directory := helper:get-test-directory($serialize:suite) + let $_ := file:mkdirs($directory) + + let $path := $directory || "/xml-minified.xml" + let $_ := file:serialize($serialize:xml, $path, + + + ) + + return file:read($path) eq + "" || $fixtures:NL || + " text" || $fixtures:NL || + " " +}; + diff --git a/extensions/modules/file/src/test/xquery/modules/file/sync-serialize.xqm b/extensions/modules/file/src/test/xquery/modules/file/sync-serialize.xqm new file mode 100644 index 00000000000..c26c201a77a --- /dev/null +++ b/extensions/modules/file/src/test/xquery/modules/file/sync-serialize.xqm @@ -0,0 +1,249 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +(: + : test serialization defaults and setting different serialization options on + : file:sync#3 + :) +module namespace syse="http://exist-db.org/xquery/test/file/sync-serialize"; + + +import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; +import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; + + +declare namespace test="http://exist-db.org/xquery/xqsuite"; + +declare variable $syse:suite := "syse"; + +declare variable $syse:simple-file-name := "simple-data.xml"; +declare variable $syse:complex-file-name := "complex-data.xml"; + +declare + %test:setUp +function syse:setup() as empty-sequence() { + let $_ := ( + xmldb:create-collection("/db", $fixtures:collection-name), + helper:create-db-resource($fixtures:collection, $syse:simple-file-name, $fixtures:XML), + helper:create-db-resource($fixtures:collection, $syse:complex-file-name, $fixtures:COMPLEX_XML) + ) + return () +}; + +declare + %test:tearDown +function syse:tear-down() { + helper:clear-db(), + helper:clear-suite-fs($syse:suite) +}; + +declare + %test:assertEquals("true", "true") +function syse:defaults() { + let $directory := helper:get-test-directory($syse:suite) + let $sync := file:sync( + $fixtures:collection, + $directory, + () + ) + + return ( + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:SIMPLE_XML_INDENTED, + ($directory, $syse:simple-file-name) + ), + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:COMPLEX_XML_INDENTED, + ($directory, $syse:complex-file-name) + ) + ) +}; + +declare + %test:assertEquals("true", "true") +function syse:indent-no() { + let $directory := helper:get-test-directory($syse:suite) + let $sync := file:sync( + $fixtures:collection, + $directory, + map{"indent": false()} + ) + + return ( + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:SIMPLE_XML_UNINDENTED, + ($directory, $syse:simple-file-name) + ), + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:COMPLEX_XML_UNINDENTED, + ($directory, $syse:complex-file-name) + ) + ) +}; + +declare + %test:assertEquals("true", "true") +function syse:indent-yes() { + let $directory := helper:get-test-directory($syse:suite) + let $sync := file:sync( + $fixtures:collection, + $directory, + map{"indent": true()} + ) + + return ( + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:SIMPLE_XML_INDENTED, + ($directory, $syse:simple-file-name) + ), + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:COMPLEX_XML_INDENTED, + ($directory, $syse:complex-file-name) + ) + ) +}; + +declare + %test:assertEquals("true", "true") +function syse:omit-xml-declaration-no() { + let $directory := helper:get-test-directory($syse:suite) + let $sync := file:sync( + $fixtures:collection, + $directory, + map{"omit-xml-declaration": false()} + ) + + return ( + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:SIMPLE_XML_INDENTED, + ($directory, $syse:simple-file-name) + ), + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:COMPLEX_XML_INDENTED, + ($directory, $syse:complex-file-name) + ) + ) +}; + +declare + %test:assertEquals("true", "true") +function syse:omit-xml-declaration-yes() { + let $directory := helper:get-test-directory($syse:suite) + let $sync := file:sync( + $fixtures:collection, + $directory, + map{"omit-xml-declaration": true()} + ) + + return ( + helper:assert-file-contents( + $fixtures:SIMPLE_XML_INDENTED, + ($directory, $syse:simple-file-name) + ), + helper:assert-file-contents( + $fixtures:COMPLEX_XML_INDENTED, + ($directory, $syse:complex-file-name) + ) + ) +}; + +declare + %test:assertEquals("true", "true") +function syse:unindented-no-declaration() { + let $directory := helper:get-test-directory($syse:suite) + let $sync := file:sync( + $fixtures:collection, + $directory, + map{ + "omit-xml-declaration": true(), + "indent": false() + } + ) + + return ( + helper:assert-file-contents( + $fixtures:SIMPLE_XML_UNINDENTED, + ($directory, $syse:simple-file-name) + ), + helper:assert-file-contents( + $fixtures:COMPLEX_XML_UNINDENTED, + ($directory, $syse:complex-file-name) + ) + ) +}; + +declare + %test:assertEquals("true", "true") +function syse:insert-final-newline-yes() { + let $directory := helper:get-test-directory($syse:suite) + let $sync := file:sync( + $fixtures:collection, + $directory, + map{ "exist:insert-final-newline": true() } + ) + + return ( + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:SIMPLE_XML_INDENTED || $fixtures:NL, + ($directory, $syse:simple-file-name) + ), + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:COMPLEX_XML_INDENTED || $fixtures:NL, + ($directory, $syse:complex-file-name) + ) + ) +}; + +declare + %test:assertEquals("true", "true") +function syse:insert-final-newline-no() { + let $directory := helper:get-test-directory($syse:suite) + let $sync := file:sync( + $fixtures:collection, + $directory, + map{ "exist:insert-final-newline": false() } + ) + + return ( + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:SIMPLE_XML_INDENTED, + ($directory, $syse:simple-file-name) + ), + helper:assert-file-contents( + $fixtures:XML_DECLARATION || $fixtures:NL || + $fixtures:COMPLEX_XML_INDENTED, + ($directory, $syse:complex-file-name) + ) + ) +}; diff --git a/extensions/modules/file/src/test/xquery/modules/file/sync.xqm b/extensions/modules/file/src/test/xquery/modules/file/sync.xqm new file mode 100644 index 00000000000..645ee8bb024 --- /dev/null +++ b/extensions/modules/file/src/test/xquery/modules/file/sync.xqm @@ -0,0 +1,423 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace sync="http://exist-db.org/xquery/test/file/sync"; + + +import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; +import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; + +declare namespace test="http://exist-db.org/xquery/xqsuite"; + +declare variable $sync:suite := "sync"; + +declare + %test:setUp +function sync:setup() as empty-sequence() { + helper:setup-db() +}; + +declare + %test:tearDown +function sync:tear-down() { + helper:clear-db(), + helper:clear-suite-fs($sync:suite) +}; + +declare + %test:assertTrue +function sync:simple() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + () + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertTrue +function sync:empty-options-map() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + map{} + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertError +function sync:deprecated-options() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + $fixtures:mod-date + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertError("err:XPTY0004") +function sync:bad-options-1() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + xs:date("2012-12-21") + ) +}; + +declare + %test:assertError("err:XPTY0004") +function sync:bad-options-2() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + "2012-12-21T10:12:21" + ) +}; + +declare + %test:assertError("err:XPTY0004") +function sync:bad-options-3() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + "lizard" + ) +}; + +declare + %test:assertError("err:XPTY0004") +function sync:bad-options-4() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + "" + ) +}; + +(: + : TODO(JL) should also report %test:assertError("err:XPTY0004") + : it is wrapped in org.exist.xquery.XPathException and therefore not recognized + :) +declare + %test:assertError +function sync:bad-options-5() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + (1, map{}, "") + ) +}; + +declare + %test:assertError("err:XPTY0004") +function sync:bad-options-6() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + map{ "prune": "true" } + ) +}; + +declare + %test:assertError("err:XPTY0004") +function sync:bad-options-7() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + map{ "prune": "no" } + ) +}; + +declare + %test:assertError("err:XPTY0004") +function sync:bad-options-8() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + map{ "after": 1234325 } + ) +}; + +declare + %test:assertError("err:XPTY0004") +function sync:bad-options-9() { + file:sync( + $fixtures:collection, + helper:get-test-directory($sync:suite), + map{ "excludes": [] } + ) +}; + +declare + %test:assertTrue +function sync:do-not-prune() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "prune": false() } + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (), + "fs": $fixtures:ROOT-FS-EXTRA + }) +}; + +declare + %test:assertTrue +function sync:prune() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "prune": true() } + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": ("test", "three.s", ".env"), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertTrue +function sync:prune-with-excludes-matching-none() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "prune": true(), "excludes": "*.txt" } + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": ("test", "three.s", ".env"), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertTrue +function sync:after() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "after": $fixtures:mod-date } + ) + => helper:assert-sync-result(map { + "updated": (), + "deleted": (), + "fs": ($fixtures:EXTRA-DATA, "data") (: TODO: data should not be here! :) + }) +}; + +declare + %test:assertTrue +function sync:after-mod-date-2() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "after": $fixtures:mod-date-2 } + ) + => helper:assert-sync-result(map { + "updated": (), + "deleted": (), + "fs": ($fixtures:EXTRA-DATA, "data") (: TODO: data should not be here! :) + }) +}; + +declare + %test:assertTrue +function sync:after-with-excludes() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "after": $fixtures:mod-date, "excludes": ".env" } + ) + => helper:assert-sync-result(map { + "updated": (), + "deleted": (), + "fs": ($fixtures:EXTRA-DATA, "data") (: TODO: data should not be here! :) + }) +}; + +declare + %test:assertTrue +function sync:prune-with-after-and-excludes() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + let $_ := ( + file:serialize-binary( + util:string-to-binary("1"), + $directory || "/excluded.xq" + ), + file:serialize-binary( + util:string-to-binary("1"), + $directory || "/pruned.xql" + ), + file:serialize-binary( + util:string-to-binary("oh oh"), + $directory || "/readme.md" + ) + ) + + return + file:sync( + $fixtures:collection, + $directory, + map{ + "after": $fixtures:mod-date, + "excludes": "*.xq", + "prune": true() + } + ) + => helper:assert-sync-result(map { + "updated": (), + "deleted": ("three.s", "test", ".env", "pruned.xql", "readme.md"), + "fs": ("excluded.xq", "data") (: TODO: data should not be here! :) + }) +}; + +declare + %test:assertTrue +function sync:prunes-a-directory() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "prune": true(), "excludes": ".*" } + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": ("test", "three.s"), + "fs": ($fixtures:ROOT-FS, ".env") + }) +}; + +declare + %test:assertTrue +function sync:prunes-a-file() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "prune": true(), "excludes": "test" || $helper:path-separator || "*" } + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (".env"), + "fs": ($fixtures:ROOT-FS, "test") + }) +}; + +declare + %test:assertTrue +function sync:prunes-with-multiple-excludes() { + let $directory := helper:get-test-directory($sync:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ + "prune": true(), + "excludes": (".env", "**" || $helper:path-separator || "three.?") + } + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (), + "fs": ($fixtures:ROOT-FS, ".env", "test") + }) +}; + +declare + %test:assertTrue +function sync:twice() { + let $directory := helper:get-test-directory($sync:suite) + (: + : ensure that files on disk are always recognized as newer by waiting one second until + : syncing to disk, see https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8177809 + :) + let $_ := util:wait(1000) + let $_ := file:sync( + $fixtures:collection, + $directory, + () + ) + + return + file:sync( + $fixtures:collection, + $directory, + () + ) + => helper:assert-sync-result(map { + "updated": (), + "deleted": (), + "fs": $fixtures:ROOT-FS + }) +}; diff --git a/extensions/modules/file/src/test/xquery/modules/file/syncmod.xqm b/extensions/modules/file/src/test/xquery/modules/file/syncmod.xqm new file mode 100644 index 00000000000..178010889f0 --- /dev/null +++ b/extensions/modules/file/src/test/xquery/modules/file/syncmod.xqm @@ -0,0 +1,293 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +module namespace syncmod="http://exist-db.org/xquery/test/file/syncmod"; + + +import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm"; +import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm"; + +declare namespace test="http://exist-db.org/xquery/xqsuite"; + +declare variable $syncmod:suite := "sync-modified"; + +(: + : Same setup as for basic sync tests in sync.xqm + : In addition this time two files are modified an hour after ($fixtures:mod-date) + :) +declare + %test:setUp +function syncmod:setup() as empty-sequence() { + helper:setup-db(), + helper:modify-db-resource($fixtures:child-collection, "test-data.xml"), + helper:modify-db-resource($fixtures:collection, "test-text.txt") +}; + +declare + %test:tearDown +function syncmod:tearDown() { + helper:clear-db(), + helper:clear-suite-fs($syncmod:suite) +}; + +declare + %test:assertTrue +function syncmod:simple() { + file:sync( + $fixtures:collection, + helper:get-test-directory($syncmod:suite), + () + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertTrue +function syncmod:empty-options() { + file:sync( + $fixtures:collection, + helper:get-test-directory($syncmod:suite), + map{} + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertError +function syncmod:deprecated-options() { + file:sync( + $fixtures:collection, + helper:get-test-directory($syncmod:suite), + $fixtures:mod-date + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertTrue +function syncmod:do-not-prune() { + let $directory := helper:get-test-directory($syncmod:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "prune": false() } + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": (), + "fs": $fixtures:ROOT-FS-EXTRA + }) +}; + +declare + %test:assertTrue +function syncmod:prune() { + let $directory := helper:get-test-directory($syncmod:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "prune": true() } + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": ("test", "three.s", ".env"), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertTrue +function syncmod:prune-with-excludes-matching-none() { + let $directory := helper:get-test-directory($syncmod:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "prune": true(), "excludes": "*.txt" } + ) + => helper:assert-sync-result(map { + "updated": $fixtures:ALL-UPDATED, + "deleted": ("test", "three.s", ".env"), + "fs": $fixtures:ROOT-FS + }) +}; + +declare + %test:assertTrue +function syncmod:after() { + let $directory := helper:get-test-directory($syncmod:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "after": $fixtures:mod-date } + ) + => helper:assert-sync-result(map { + "updated": ("test-text.txt", "test-data.xml"), + "deleted": (), + "fs": ("test", ".env", "test-text.txt", "data") + }) +}; + +(: collections seem to be synced regardless of their content :) +declare + %test:assertTrue +function syncmod:after-mod-date-2() { + file:sync( + $fixtures:collection, + helper:get-test-directory($syncmod:suite), + map{ "after": $fixtures:mod-date-2 } + ) + => helper:assert-sync-result(map { + "updated": (), + "deleted": (), + "fs": ("data") (: TODO: data should not be here! :) + }) +}; + +declare + %test:pending("this would only work if exclude patterns would exclude DB resources from syncing") + %test:assertTrue +function syncmod:exclude-changed-files() { + file:sync( + $fixtures:collection, + helper:get-test-directory($syncmod:suite), + map{ "excludes":("*.txt", "data/*"), "after": $fixtures:mod-date } + ) + => helper:assert-sync-result(map { + "updated": (), + "deleted": (), + "fs": () + }) +}; + +declare + %test:assertTrue +function syncmod:prune-with-after-and-excludes-matching-none() { + let $directory := helper:get-test-directory($syncmod:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ + "after": $fixtures:mod-date, + "excludes": "QQQ", + "prune": true() + } + ) + => helper:assert-sync-result(map { + "updated": ("test-text.txt", "test-data.xml"), + "deleted": (".env", "test"), + "fs": ("test-text.txt", "data") + }) +}; + +declare + %test:assertTrue +function syncmod:prune-with-after-and-excludes-matching-all() { + let $directory := helper:get-test-directory($syncmod:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ + "after": $fixtures:mod-date, + "excludes": "**", + "prune": true() + } + ) + => helper:assert-sync-result(map { + "updated": ("test-text.txt"), (: TODO , "test-data.xml" is missing here :) + "deleted": (), + "fs": (".env", "test", "test-text.txt", "data") + }) +}; + +declare + %test:assertTrue +function syncmod:prunes-a-directory-with-after() { + let $directory := helper:get-test-directory($syncmod:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ "prune": true(), "excludes": ".*", "after": $fixtures:mod-date } + ) + => helper:assert-sync-result(map { + "updated": ("test-text.txt", "test-data.xml"), + "deleted": ("test", "three.s"), + "fs": (".env", "test-text.txt", "data") + }) +}; + +declare + %test:pending + %test:assertTrue +function syncmod:prunes-a-file-with-after() { + let $directory := helper:get-test-directory($syncmod:suite) + let $_ := helper:setup-fs-extra($directory) + + return + file:sync( + $fixtures:collection, + $directory, + map{ + "after": $fixtures:mod-date, + "excludes": "test/*", + "prune": true() + } + ) + => helper:assert-sync-result(map { + "updated": ("test-text.txt", "test-data.xml"), + "deleted": (".env"), + "fs": ("test", "test-text.txt", "data") + }) +}; diff --git a/extensions/modules/image/pom.xml b/extensions/modules/image/pom.xml index f6919e277f9..c0e0709c31f 100644 --- a/extensions/modules/image/pom.xml +++ b/extensions/modules/image/pom.xml @@ -75,7 +75,7 @@ org.exist-db - exist-expath + exist-file ${project.version} test @@ -112,8 +112,8 @@ - - org.exist-db:exist-expath:jar:${project.version} + + org.exist-db:exist-file:jar:${project.version} diff --git a/extensions/modules/image/src/test/resources-filtered/conf.xml b/extensions/modules/image/src/test/resources-filtered/conf.xml index 49cab6955c6..9df613700e8 100644 --- a/extensions/modules/image/src/test/resources-filtered/conf.xml +++ b/extensions/modules/image/src/test/resources-filtered/conf.xml @@ -747,7 +747,7 @@ - + diff --git a/extensions/modules/pom.xml b/extensions/modules/pom.xml index fe5d12dc45b..3eb56caa2e9 100644 --- a/extensions/modules/pom.xml +++ b/extensions/modules/pom.xml @@ -52,6 +52,7 @@ cqlparser example exi + file expathrepo expathrepo/expathrepo-trigger-test image From ce2d704ec59179a224091a34974d35e0f9310be5 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Sat, 4 Apr 2026 09:51:59 -0400 Subject: [PATCH 005/326] [feature] Add XQuery 4.0 syntax to ANTLR 2 grammar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds XQ4 syntax support to the ANTLR 2 parser and tree walker: - Pipeline operator (->): chainable expression transformation - Focus functions (fn { }): anonymous functions with implicit context - Keyword arguments: named parameter passing at call sites - String templates (`` `{expr}` ``): interpolated string literals - Otherwise operator: fallback for empty sequences - Braced if: if (cond) { expr } without else clause - Try/finally: cleanup expressions that always execute - For member: iterate over array members in FLWOR - While clause: conditional FLWOR iteration - Default parameter values in function declarations - Mapping arrow (=>) and method call (=?>) - Ternary conditional (if..then..else as expression) - QName literals (#name): symbolic name references - Hex/binary integer literals (0xNN, 0bNN) - Numeric underscore separators (1_000_000) - Choice/enum cast types - Version gating: XQuery 4.0 features require version declaration Grammar sections added in labeled blocks per feature area within the XQuery 4.0 Parser Extensions section. Spec: QT4 XQuery 4.0 §3 (Expressions), §4 (Modules and Prologs) XQTS: QT4 parser-dependent test sets (1898/2163, 87.7%) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../antlr/org/exist/xquery/parser/XQuery.g | 865 +++++- .../org/exist/xquery/parser/XQueryTree.g | 2370 ++++++++++++----- .../xquery/ReservedKeywordsAsNCNamesTest.java | 115 + 3 files changed, 2601 insertions(+), 749 deletions(-) create mode 100644 exist-core/src/test/java/org/exist/xquery/ReservedKeywordsAsNCNamesTest.java diff --git a/exist-core/src/main/antlr/org/exist/xquery/parser/XQuery.g b/exist-core/src/main/antlr/org/exist/xquery/parser/XQuery.g index d852d700444..a670bf3cb69 100644 --- a/exist-core/src/main/antlr/org/exist/xquery/parser/XQuery.g +++ b/exist-core/src/main/antlr/org/exist/xquery/parser/XQuery.g @@ -83,6 +83,7 @@ options { protected Deque> globalStack = new ArrayDeque<>(); protected Deque elementStack = new ArrayDeque<>(); protected XQueryLexer lexer; + protected boolean xq4Enabled = false; public XQueryParser(XQueryLexer lexer) { this((TokenStream)lexer); @@ -90,6 +91,8 @@ options { setASTNodeClass("org.exist.xquery.parser.XQueryAST"); } + public boolean isXQ4() { return xq4Enabled; } + public boolean foundErrors() { return foundError; } @@ -192,6 +195,28 @@ imaginaryTokenDefinitions PREVIOUS_ITEM NEXT_ITEM WINDOW_VARS + FOCUS_FUNCTION + KEYWORD_ARG + FOR_MEMBER + STRING_TEMPLATE + FOR_KEY + FOR_VALUE + FOR_KEY_VALUE + VALUE_VAR + SWITCH_BOOLEAN + MAPPING_ARROW + FILTER_AM + QNAME_LITERAL + PARAM_DEFAULT + CHOICE_TYPE + ENUM_TYPE + TERNARY + SEQ_DESTRUCTURE + ARRAY_DESTRUCTURE + MAP_DESTRUCTURE + DESTRUCTURE_VAR_TYPE + RECORD_TEST + RECORD_FIELD ; // === XPointer === @@ -272,7 +297,7 @@ prolog throws XPathException ( "declare" "variable" ) => varDeclUp { inSetters = false; } | - ( "declare" "context" "item" ) + ( "declare" "context" ("item" | "value") ) => contextItemDeclUp { inSetters = false; } | ( "declare" MOD ) @@ -292,7 +317,12 @@ importDecl throws XPathException versionDecl throws XPathException : "xquery" "version" v:STRING_LITERAL ( "encoding"! enc:STRING_LITERAL )? - { #versionDecl = #(#[VERSION_DECL, v.getText()], enc); } + { + #versionDecl = #(#[VERSION_DECL, v.getText()], enc); + if ("4.0".equals(v.getText())) { + xq4Enabled = true; + } + } ; setter @@ -441,7 +471,7 @@ contextItemDeclUp! throws XPathException contextItemDecl [XQueryAST decl] throws XPathException : - "context"! "item"! ( typeDeclaration )? + "context"! ( "item"! | "value"! ) ( typeDeclaration )? ( COLON! EQ! e1:expr | @@ -464,10 +494,22 @@ annotation String name= null; } : - MOD! name=eqName! (LPAREN! literal (COMMA! literal)* RPAREN!)? + MOD! name=eqName! (LPAREN! annotationLiteral (COMMA! annotationLiteral)* RPAREN!)? { #annotation= #(#[ANNOT_DECL, name], #annotation); } ; +// XQ4: annotation parameters support literals, true(), false(), and negated numeric literals +// Note: true()/false() must be matched via NCNAME + semantic predicate, NOT as "true"/"false" keywords. +// Using quoted keyword syntax would register them in testLiterals, breaking true()/false() function +// calls throughout the grammar (ANTLR 2 converts all NCNAMEs matching keywords to LITERAL_xxx tokens). +annotationLiteral +: + literal + | ( { LT(1).getText().equals("true") || LT(1).getText().equals("false") }? b:NCNAME LPAREN! RPAREN! + { #annotationLiteral = #[STRING_LITERAL, #b.getText()]; #b = null; } ) + | MINUS! n:numericLiteral { #n.setText("-" + #n.getText()); #annotationLiteral = #n; } + ; + eqName returns [String name] { name= null; } : @@ -550,7 +592,10 @@ param throws XPathException { String varName= null; } : DOLLAR! varName=eqName ( t:typeDeclaration )? - { #param= #(#[VARIABLE_BINDING, varName], #t); } + ( ( { xq4Enabled }? COLON EQ ) => COLON! EQ! pd:exprSingle! + { #pd = #(#[PARAM_DEFAULT, "param-default"], #pd); } + )? + { #param= #(#[VARIABLE_BINDING, varName], #t, #pd); } ; uriList throws XPathException @@ -588,10 +633,16 @@ itemType throws XPathException | ( "function" LPAREN ) => functionTest | + ( "fn" LPAREN ) => fnShorthandFunctionTest + | ( "map" LPAREN ) => mapType | ( "array" LPAREN ) => arrayType | + ( "record" LPAREN ) => recordType + | + ( "enum" LPAREN ) => enumType + | ( LPAREN ) => parenthesizedItemType | ( . LPAREN ) => kindTest @@ -600,13 +651,51 @@ itemType throws XPathException ; parenthesizedItemType throws XPathException +{ int count = 0; } : - LPAREN! itemType RPAREN! + LPAREN! itemType { count++; } ( UNION! itemType { count++; } )* RPAREN! + { + if (count > 1) { + #parenthesizedItemType = #(#[CHOICE_TYPE, "choice-type"], #parenthesizedItemType); + } + } + ; + +enumType throws XPathException +{ List enumValues = new ArrayList(); } +: + e:"enum"! LPAREN! + s1:STRING_LITERAL! { enumValues.add(s1.getText()); } + ( COMMA! s2:STRING_LITERAL! { enumValues.add(s2.getText()); } )* + RPAREN! + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < enumValues.size(); i++) { + if (i > 0) sb.append(","); + sb.append(enumValues.get(i)); + } + #enumType = #(#[ENUM_TYPE, sb.toString()]); + #enumType.copyLexInfo(#e); + } ; singleType throws XPathException +{ int count = 0; } : - atomicType ( QUESTION )? + ( + ( "enum" LPAREN ) => enumType ( QUESTION )? + | + ( LPAREN ) => + LPAREN! atomicType { count++; } ( UNION! atomicType { count++; } )* RPAREN! + { + if (count > 1) { + #singleType = #(#[CHOICE_TYPE, "choice-type"], #singleType); + } + } + ( QUESTION )? + | + atomicType ( QUESTION )? + ) ; atomicType throws XPathException @@ -634,10 +723,38 @@ anyFunctionTest throws XPathException typedFunctionTest throws XPathException : - "function"! LPAREN! (sequenceType (COMMA! sequenceType)*)? RPAREN! "as" sequenceType + "function"! LPAREN! (fnShorthandParam (COMMA! fnShorthandParam)*)? RPAREN! "as" sequenceType { #typedFunctionTest = #(#[FUNCTION_TEST, "anyFunction"], #typedFunctionTest); } ; +// XQ4: fn(...) as shorthand for function(...) in type positions +fnShorthandFunctionTest throws XPathException +: + ( "fn" LPAREN STAR RPAREN) => fnShorthandAnyFunctionTest + | + fnShorthandTypedFunctionTest + ; + +fnShorthandAnyFunctionTest throws XPathException +: + "fn"! LPAREN! s2:STAR RPAREN! + { #fnShorthandAnyFunctionTest = #(#[FUNCTION_TEST, "anyFunction"], #s2); } + ; + +fnShorthandTypedFunctionTest throws XPathException +: + "fn"! LPAREN! (fnShorthandParam (COMMA! fnShorthandParam)*)? RPAREN! "as" sequenceType + { #fnShorthandTypedFunctionTest = #(#[FUNCTION_TEST, "anyFunction"], #fnShorthandTypedFunctionTest); } + ; + +// XQ4: fn() type parameters can optionally have names: fn($name as type, ...) +fnShorthandParam throws XPathException +: + ( DOLLAR ) => DOLLAR! eqName! "as"! sequenceType + | + sequenceType + ; + mapType throws XPathException : ( "map" LPAREN STAR ) => anyMapTypeTest @@ -686,6 +803,50 @@ arrayTypeTest throws XPathException } ; +recordType throws XPathException +: + ( "record" LPAREN STAR ) => anyRecordTypeTest + | + ( "record" LPAREN RPAREN ) => emptyRecordTypeTest + | + recordTypeTest + ; + +anyRecordTypeTest throws XPathException +: + m:"record"! LPAREN! s:STAR RPAREN! + { + #anyRecordTypeTest = #(#[RECORD_TEST, "record"], #s); + #anyRecordTypeTest.copyLexInfo(#m); + } + ; + +emptyRecordTypeTest throws XPathException +: + m:"record"! LPAREN! RPAREN! + { + #emptyRecordTypeTest = #(#[RECORD_TEST, "record"]); + #emptyRecordTypeTest.copyLexInfo(#m); + } + ; + +recordTypeTest throws XPathException +: + m:"record"! LPAREN! recordFieldDecl ( COMMA! ( STAR | recordFieldDecl ) )* RPAREN! + { + #recordTypeTest = #(#[RECORD_TEST, "record"], #recordTypeTest); + } + ; + +recordFieldDecl throws XPathException +{ String fieldName = null; } +: + fieldName=ncnameOrKeyword! ( QUESTION )? ( "as"! sequenceType )? + { + #recordFieldDecl = #(#[RECORD_FIELD, fieldName], #recordFieldDecl); + } + ; + // === Expressions === queryBody throws XPathException: expr ; @@ -702,7 +863,7 @@ expr throws XPathException exprSingle throws XPathException : - ( ( "for" | "let" ) ("tumbling" | "sliding" | DOLLAR ) ) => flworExpr + ( ( "for" | "let" ) ("tumbling" | "sliding" | "member" | "key" | "value" | DOLLAR) ) => flworExpr | ( "try" LCURLY ) => tryCatchExpr | ( ( "some" | "every" ) DOLLAR ) => quantifiedExpr | ( "if" LPAREN ) => ifExpr @@ -752,11 +913,14 @@ renameExpr throws XPathException "rename" exprSingle "as"! exprSingle ; -// === try/catch === +// === try/catch/finally === tryCatchExpr throws XPathException : "try"^ LCURLY! tryTargetExpr RCURLY! - (catchClause)+ + ( + (catchClause)+ ( { xq4Enabled }? finallyClause )? + | { xq4Enabled }? finallyClause + ) ; tryTargetExpr throws XPathException @@ -769,6 +933,11 @@ catchClause throws XPathException "catch"^ catchErrorList (catchVars)? LCURLY! expr RCURLY! ; +finallyClause throws XPathException +: + "finally"^ LCURLY! (expr)? RCURLY! + ; + catchErrorList throws XPathException : nameTest (UNION! nameTest)* @@ -809,14 +978,14 @@ flworExpr throws XPathException initialClause throws XPathException : - ( ( "for" DOLLAR ) => forClause + ( ( "for" ( "member" | "key" | "value" | DOLLAR ) ) => forClause | ( "for" ( "tumbling" | "sliding" ) ) => windowClause | letClause ) ; intermediateClause throws XPathException : - ( initialClause | whereClause | groupByClause | orderByClause | countClause ) + ( initialClause | whereClause | whileClause | groupByClause | orderByClause | countClause ) ; whereClause throws XPathException @@ -824,6 +993,11 @@ whereClause throws XPathException "where"^ exprSingle ; +whileClause throws XPathException +: + { xq4Enabled }? "while"^ exprSingle + ; + countClause throws XPathException { String varName; } : @@ -833,7 +1007,77 @@ countClause throws XPathException forClause throws XPathException : - "for"^ inVarBinding ( COMMA! inVarBinding )* + "for"^ forBinding ( COMMA! forBinding )* + ; + +forBinding throws XPathException +: + ( { xq4Enabled }? "member" ) => memberVarBinding + | ( { xq4Enabled }? "key" ) => keyVarBinding + | ( { xq4Enabled }? "value" ) => valueVarBinding + | inVarBinding + ; + +memberVarBinding throws XPathException +{ String varName; } +: + "member"! DOLLAR! varName=v:varName! ( typeDeclaration )? + ( positionalVar )? + "in"! exprSingle + { + #memberVarBinding= #(#[VARIABLE_BINDING, varName], #memberVarBinding); + #memberVarBinding.copyLexInfo(#v); + #memberVarBinding= #(#[FOR_MEMBER, null], #memberVarBinding); + } + ; + +keyVarBinding throws XPathException +{ String varName; } +: + "key"! DOLLAR! varName=v:varName! ( typeDeclaration )? + ( + ( "value" DOLLAR ) => keyValueVarPart + )? + ( positionalVar )? + "in"! exprSingle + { + #keyVarBinding= #(#[VARIABLE_BINDING, varName], #keyVarBinding); + #keyVarBinding.copyLexInfo(#v); + // Check if we have a value variable (keyValueVarPart was matched) + boolean hasValueVar = false; + AST child = #keyVarBinding.getFirstChild(); + while (child != null) { + if (child.getType() == VALUE_VAR) { hasValueVar = true; break; } + child = child.getNextSibling(); + } + if (hasValueVar) { + #keyVarBinding= #(#[FOR_KEY_VALUE, null], #keyVarBinding); + } else { + #keyVarBinding= #(#[FOR_KEY, null], #keyVarBinding); + } + } + ; + +keyValueVarPart throws XPathException +{ String valueVarName; } +: + "value"! DOLLAR! valueVarName=varName! ( typeDeclaration )? + { + #keyValueVarPart = #(#[VALUE_VAR, valueVarName], #keyValueVarPart); + } + ; + +valueVarBinding throws XPathException +{ String varName; } +: + "value"! DOLLAR! varName=v:varName! ( typeDeclaration )? + ( positionalVar )? + "in"! exprSingle + { + #valueVarBinding= #(#[VARIABLE_BINDING, varName], #valueVarBinding); + #valueVarBinding.copyLexInfo(#v); + #valueVarBinding= #(#[FOR_VALUE, null], #valueVarBinding); + } ; letClause throws XPathException @@ -904,6 +1148,16 @@ windowVars throws XPathException letVarBinding throws XPathException { String varName; } : + // XQ4: sequence destructuring - let $($x, $y) := expr + ( DOLLAR LPAREN ) => letDestructureSeq + | + // XQ4: array destructuring - let $[$x, $y] := expr + ( DOLLAR LPPAREN ) => letDestructureArray + | + // XQ4: map destructuring - let ${$x, $y} := expr + ( DOLLAR LCURLY ) => letDestructureMap + | + // Standard let binding DOLLAR! varName=v:varName! ( typeDeclaration )? COLON! EQ! exprSingle { @@ -912,6 +1166,67 @@ letVarBinding throws XPathException } ; +// XQ4: Per-variable type annotations: "x+,y" means $x has a DESTRUCTURE_VAR_TYPE child, $y does not +letDestructureSeq throws XPathException +{ String vn; + StringBuilder sb = new StringBuilder(); } +: + d:DOLLAR! LPAREN! + DOLLAR! vn=varName! { sb.append(vn); } + ( destructureVarType { sb.append("+"); } )? + ( COMMA! DOLLAR! vn=varName! { sb.append(",").append(vn); } + ( destructureVarType { sb.append("+"); } )? )* + RPAREN! ( typeDeclaration )? + COLON! EQ! exprSingle + { + #letDestructureSeq = #(#[SEQ_DESTRUCTURE, sb.toString()], #letDestructureSeq); + #letDestructureSeq.copyLexInfo(#d); + } + ; + +letDestructureArray throws XPathException +{ String vn; + StringBuilder sb = new StringBuilder(); } +: + d:DOLLAR! LPPAREN! + DOLLAR! vn=varName! { sb.append(vn); } + ( destructureVarType { sb.append("+"); } )? + ( COMMA! DOLLAR! vn=varName! { sb.append(",").append(vn); } + ( destructureVarType { sb.append("+"); } )? )* + RPPAREN! ( typeDeclaration )? + COLON! EQ! exprSingle + { + #letDestructureArray = #(#[ARRAY_DESTRUCTURE, sb.toString()], #letDestructureArray); + #letDestructureArray.copyLexInfo(#d); + } + ; + +letDestructureMap throws XPathException +{ String vn; + StringBuilder sb = new StringBuilder(); } +: + d:DOLLAR! LCURLY! + DOLLAR! vn=varName! { sb.append(vn); } + ( destructureVarType { sb.append("+"); } )? + ( COMMA! DOLLAR! vn=varName! { sb.append(",").append(vn); } + ( destructureVarType { sb.append("+"); } )? )* + RCURLY! ( typeDeclaration )? + COLON! EQ! exprSingle + { + #letDestructureMap = #(#[MAP_DESTRUCTURE, sb.toString()], #letDestructureMap); + #letDestructureMap.copyLexInfo(#d); + } + ; + +// Helper: wraps typeDeclaration in DESTRUCTURE_VAR_TYPE imaginary token +destructureVarType throws XPathException +: + td:typeDeclaration + { + #destructureVarType = #(#[DESTRUCTURE_VAR_TYPE, "vartype"], #td); + } + ; + orderByClause throws XPathException : ( "order"! "by"! | "stable"! "order"! "by"! ) orderSpecList @@ -973,9 +1288,26 @@ quantifiedInVarBinding throws XPathException switchExpr throws XPathException : - "switch"^ LPAREN! expr RPAREN! - ( switchCaseClause )+ - "default" "return"! exprSingle + "switch"^ LPAREN! + ( + // XQ4 omitted comparand - boolean mode: switch () { case boolExpr return ... } + ( RPAREN ) => + RPAREN! switchBooleanMarker + | + expr RPAREN! + ) + ( + // XQ4 braced syntax: switch (...) { case ... default ... } + ( LCURLY "case" ) => + LCURLY! ( switchCaseClause )+ "default" "return"! exprSingle RCURLY! + | + ( switchCaseClause )+ "default" "return"! exprSingle + ) + ; + +switchBooleanMarker +: + { #switchBooleanMarker = #(#[SWITCH_BOOLEAN, "switch-boolean"]); } ; switchCaseClause throws XPathException @@ -988,8 +1320,13 @@ typeswitchExpr throws XPathException { String varName; } : "typeswitch"^ LPAREN! expr RPAREN! - ( caseClause )+ - "default" ( defaultVar )? "return"! exprSingle + ( + // XQ4 braced syntax: typeswitch (...) { case ... default ... } + ( LCURLY "case" ) => + LCURLY! ( caseClause )+ "default" ( defaultVar )? "return"! exprSingle RCURLY! + | + ( caseClause )+ "default" ( defaultVar )? "return"! exprSingle + ) ; caseClause throws XPathException @@ -1024,12 +1361,28 @@ defaultVar throws XPathException ; ifExpr throws XPathException +{ + org.exist.xquery.parser.XQueryAST emptyNode = null; +} : - "if"^ LPAREN! expr RPAREN! t:"then"! thenExpr:exprSingle e:"else"! elseExpr:exprSingle - { - #thenExpr.copyLexInfo(#t); - #elseExpr.copyLexInfo(#e); - } + "if"^ LPAREN! expr RPAREN! + ( + // Traditional: if (cond) then expr else expr + ( "then" ) => + t:"then"! thenExpr:exprSingle e:"else"! elseExpr:exprSingle + { + #thenExpr.copyLexInfo(#t); + #elseExpr.copyLexInfo(#e); + } + | + // XQ4 Braced: if (cond) { expr } (no else clause; returns empty sequence if false) + LCURLY! bracedThenExpr:expr RCURLY! + { + // Synthesize empty sequence as implicit else branch + emptyNode = (org.exist.xquery.parser.XQueryAST) #(#[PARENTHESIZED, "()"]); + #ifExpr.addChild(emptyNode); + } + ) ; // === Logical === @@ -1037,6 +1390,12 @@ ifExpr throws XPathException orExpr throws XPathException : andExpr ( "or"^ andExpr )* + ( + { xq4Enabled }? DOUBLE_QUESTION! exprSingle DOUBLE_BANG! exprSingle + { + #orExpr = #(#[TERNARY, "ternary"], #orExpr); + } + )? ; andExpr throws XPathException @@ -1061,23 +1420,33 @@ castableExpr throws XPathException castExpr throws XPathException : - arrowExpr ( "cast"^ "as"! singleType )? + pipelineExpr ( "cast"^ "as"! singleType )? + ; + +pipelineExpr throws XPathException +: + arrowExpr ( { xq4Enabled }? PIPELINE_OP^ arrowExpr )* ; comparisonExpr throws XPathException : - r1:stringConcatExpr ( - ( BEFORE ) => BEFORE^ stringConcatExpr + r1:otherwiseExpr ( + ( BEFORE ) => BEFORE^ otherwiseExpr | - ( AFTER ) => AFTER^ stringConcatExpr - | ( ( "eq"^ | "ne"^ | "lt"^ | "le"^ | "gt"^ | "ge"^ ) stringConcatExpr ) + ( AFTER ) => AFTER^ otherwiseExpr + | ( ( "eq"^ | "ne"^ | "lt"^ | "le"^ | "gt"^ | "ge"^ ) otherwiseExpr ) | ( GT EQ ) => GT^ EQ^ r2:rangeExpr { #comparisonExpr = #(#[GTEQ, ">="], #r1, #r2); } - | ( ( EQ^ | NEQ^ | GT^ | LT^ | LTEQ^ ) stringConcatExpr ) - | ( ( "is"^ | "isnot"^ ) stringConcatExpr ) + | ( ( EQ^ | NEQ^ | GT^ | LT^ | LTEQ^ ) otherwiseExpr ) + | ( ( "is"^ | "isnot"^ ) otherwiseExpr ) )? ; +otherwiseExpr throws XPathException +: + stringConcatExpr ( { xq4Enabled }? "otherwise"^ stringConcatExpr )* + ; + stringConcatExpr throws XPathException { boolean isConcat = false; } : @@ -1222,13 +1591,15 @@ stepExpr throws XPathException | ( ( "element" | "attribute" | "text" | "document" | "comment" | "namespace-node" | "processing-instruction" | "namespace" | "ordered" | - "unordered" | "map" | "array" ) LCURLY ) => + "unordered" | "map" | "array" | "fn" | "function" ) LCURLY ) => postfixExpr | ( ( "element" | "attribute" | "processing-instruction" | "namespace" ) eqName LCURLY ) => postfixExpr | + ( "fn" LPAREN ) => postfixExpr + | ( MOD | DOLLAR | ( eqName ( LPAREN | HASH ) ) | SELF | LPAREN | literal | XML_COMMENT | LT | - XML_PI | QUESTION | LPPAREN | STRING_CONSTRUCTOR_START ) + XML_PI | QUESTION | LPPAREN | STRING_CONSTRUCTOR_START | STRING_TEMPLATE_START | LCURLY | HASH ) => postfixExpr | axisStep @@ -1272,6 +1643,7 @@ forwardAxisSpecifier : "child" | "self" | "attribute" | "descendant" | "descendant-or-self" | "following-sibling" | "following" + | "following-or-self" | "following-sibling-or-self" ; reverseAxis : reverseAxisSpecifier COLON! COLON! ; @@ -1279,6 +1651,7 @@ reverseAxis : reverseAxisSpecifier COLON! COLON! ; reverseAxisSpecifier : "parent" | "ancestor" | "ancestor-or-self" | "preceding-sibling" | "preceding" + | "preceding-or-self" | "preceding-sibling-or-self" ; nodeTest throws XPathException @@ -1326,18 +1699,42 @@ postfixExpr throws XPathException | (LPAREN) => dynamicFunCall | + // XQ4: ?[ must come before ? lookup to disambiguate + (QUESTION LPPAREN) => filterExprAM + | (QUESTION) => lookup )* ; arrowExpr throws XPathException : - unaryExpr ( ARROW_OP^ arrowFunctionSpecifier argumentList )* + unaryExpr ( + ARROW_OP^ arrowFunctionSpecifier argumentList + | + { xq4Enabled }? MAPPING_ARROW_OP^ arrowFunctionSpecifier argumentList + | + { xq4Enabled }? METHOD_CALL_OP^ NCNAME argumentList + )* ; arrowFunctionSpecifier throws XPathException { String name= null; } : + // XQ4: inline/focus function expression + ( MOD | ( ("function" | "fn") (LPAREN | LCURLY) ) ) => inlineOrFocusFunctionExpr + | + // XQ4: named function reference (eqName '#' arity) + ( eqName HASH ) => namedFunctionRef + | + // XQ4: map constructor as function + ( "map" LCURLY ) => mapConstructor + | + // XQ4: bare map constructor as function + ( LCURLY ) => bareMapConstructor + | + // XQ4: array constructor as function + ( LPPAREN | ("array" LCURLY) ) => arrayConstructor + | name=n:eqName { #arrowFunctionSpecifier= #[EQNAME, name]; @@ -1349,8 +1746,17 @@ arrowFunctionSpecifier throws XPathException varRef ; +filterExprAM throws XPathException +: + q:QUESTION! LPPAREN! expr RPPAREN! + { + #filterExprAM = #(#[FILTER_AM, "filter-am"], #filterExprAM); + #filterExprAM.copyLexInfo(#q); + } + ; + lookup throws XPathException -{ String name= null; } +{ String name= null; String varName= null; } : q:QUESTION! ( @@ -1360,18 +1766,59 @@ lookup throws XPathException #lookup.copyLexInfo(#q); } | + // XQ4: decimal and double literals as key selectors (?1.2, ?1.2e0) + dbl:DOUBLE_LITERAL + { + #lookup = #(#[LOOKUP, "?"], #dbl); + #lookup.copyLexInfo(#q); + } + | + dec:DECIMAL_LITERAL + { + #lookup = #(#[LOOKUP, "?"], #dec); + #lookup.copyLexInfo(#q); + } + | pos:INTEGER_LITERAL { #lookup = #(#[LOOKUP, "?"], #pos); #lookup.copyLexInfo(#q); } | + // XQ4: string literal as key selector (?"first value") + str:STRING_LITERAL + { + #lookup = #(#[LOOKUP, "?"], #str); + #lookup.copyLexInfo(#q); + } + | paren:parenthesizedExpr { #lookup = #(#[LOOKUP, "?"], #paren); #lookup.copyLexInfo(#q); } | + // XQ4: variable reference as key selector (?$var) + DOLLAR! varName=v:varName + { + #lookup = #(#[LOOKUP, "?"], #[VARIABLE_REF, varName]); + #lookup.copyLexInfo(#q); + } + | + // XQ4: context item as key selector (?.) + dot:SELF + { + #lookup = #(#[LOOKUP, "?"], #dot); + #lookup.copyLexInfo(#q); + } + | + // XQ4: QName literal as key selector (?#name) + qnl:qnameLiteral + { + #lookup = #(#[LOOKUP, "?"], #qnl); + #lookup.copyLexInfo(#q); + } + | STAR { #lookup = #(#[LOOKUP, "?*"]); @@ -1423,9 +1870,18 @@ primaryExpr throws XPathException | ( "map" LCURLY ) => mapConstructor | + ( LCURLY RCURLY ) => bareMapConstructor + | + ( LCURLY exprSingle COLON ) => bareMapConstructor + | directConstructor | - ( MOD | "function" LPAREN | eqName HASH ) => functionItemExpr + ( { xq4Enabled }? ( "fn" | "function" ) LCURLY ) => focusFunctionExpr + | + // XQ4: QName literal (#local, #prefix:local, #Q{uri}local) + ( { xq4Enabled }? HASH ) => qnameLiteral + | + ( MOD | ( "fn" | "function" ) LPAREN | eqName HASH ) => functionItemExpr | ( eqName LPAREN ) => functionCall | @@ -1433,6 +1889,8 @@ primaryExpr throws XPathException | ( STRING_CONSTRUCTOR_START ) => stringConstructor | + ( { xq4Enabled }? STRING_TEMPLATE_START ) => stringTemplate + | contextItemExpr | parenthesizedExpr @@ -1459,10 +1917,32 @@ stringConstructorContent throws XPathException stringConstructorInterpolation throws XPathException : STRING_CONSTRUCTOR_INTERPOLATION_START^ - { lexer.inStringConstructor = false; } + { lexer.inStringConstructor = false; lexer.stringConstructorInterpolationDepth++; } ( expr )? STRING_CONSTRUCTOR_INTERPOLATION_END! - { lexer.inStringConstructor = true; } + { lexer.stringConstructorInterpolationDepth--; lexer.inStringConstructor = true; } + ; + +stringTemplate throws XPathException +: + st:STRING_TEMPLATE_START! + { lexer.inStringTemplate = true; } + ( STRING_TEMPLATE_CONTENT | stringTemplateInterpolation )* + STRING_TEMPLATE_END! + { lexer.inStringTemplate = false; } + { + #stringTemplate = #(#[STRING_TEMPLATE, null], #stringTemplate); + #stringTemplate.copyLexInfo(#st); + } + ; + +stringTemplateInterpolation throws XPathException +: + lc:LCURLY! + { lexer.inStringTemplate = false; lexer.stringTemplateDepth++; } + ( expr )? + RCURLY! + { lexer.stringTemplateDepth--; lexer.inStringTemplate = true; } ; mapConstructor throws XPathException @@ -1474,6 +1954,15 @@ mapConstructor throws XPathException } ; +bareMapConstructor throws XPathException +: + lc:LCURLY! ( mapAssignment ( COMMA! mapAssignment )* )? RCURLY! + { + #bareMapConstructor = #(#[MAP, "map"], #bareMapConstructor); + #bareMapConstructor.copyLexInfo(#lc); + } + ; + mapAssignment throws XPathException : (exprSingle COLON! EQ!) => exprSingle COLON^ eq:EQ^ exprSingle @@ -1525,6 +2014,16 @@ literal STRING_LITERAL^ | numericLiteral ; +qnameLiteral throws XPathException +{ String name = null; } +: + h:HASH! name=eqName + { + #qnameLiteral = #(#[QNAME_LITERAL, name]); + #qnameLiteral.copyLexInfo(#h); + } + ; + numericLiteral : DOUBLE_LITERAL^ | DECIMAL_LITERAL^ | INTEGER_LITERAL^ @@ -1539,7 +2038,7 @@ parenthesizedExpr throws XPathException functionItemExpr throws XPathException : - ( MOD | "function" ) => inlineFunctionExpr + ( MOD | "function" | "fn" ) => inlineOrFocusFunctionExpr | namedFunctionRef ; @@ -1553,24 +2052,36 @@ namedFunctionRef throws XPathException } ; -inlineFunctionExpr throws XPathException +inlineOrFocusFunctionExpr throws XPathException : - ann:annotations! "function"! lp:LPAREN! ( paramList )? - RPAREN! ( returnType )? - functionBody + ann:annotations! ( "function"! | "fn"! ) + ( + (LPAREN) => lp:LPAREN! ( paramList )? + RPAREN! ( returnType )? + functionBody + { + #inlineOrFocusFunctionExpr = #(#[INLINE_FUNCTION_DECL, null], #ann, #inlineOrFocusFunctionExpr); + #inlineOrFocusFunctionExpr.copyLexInfo(#lp); + } + | + lc:LCURLY! ( expr )? RCURLY! + { + #inlineOrFocusFunctionExpr = #(#[FOCUS_FUNCTION, null], #inlineOrFocusFunctionExpr); + #inlineOrFocusFunctionExpr.copyLexInfo(#lc); + } + ) + exception catch [RecognitionException e] { - #inlineFunctionExpr = #(#[INLINE_FUNCTION_DECL, null], null, #inlineFunctionExpr); - #inlineFunctionExpr.copyLexInfo(#lp); + throw new XPathException(e.getLine(), e.getColumn(), ErrorCodes.XPST0003, "Syntax error within inline function: " + e.getMessage()); } - exception catch [RecognitionException e] + ; + +focusFunctionExpr throws XPathException +: + ( "fn"! | "function"! ) lc:LCURLY! ( expr )? RCURLY! { - if (#lp == null) { - throw new XPathException(e.getLine(), e.getColumn(), ErrorCodes.XPST0003, "Syntax error within inline function: " + e.getMessage()); - } else { - #lp.setLine(e.getLine()); - #lp.setColumn(e.getColumn()); - throw new XPathException(#lp, ErrorCodes.XPST0003, "Syntax error within user defined function: " + e.getMessage()); - } + #focusFunctionExpr = #(#[FOCUS_FUNCTION, null], #focusFunctionExpr); + #focusFunctionExpr.copyLexInfo(#lc); } ; @@ -1595,8 +2106,34 @@ argumentList throws XPathException argument throws XPathException : - (QUESTION! ( NCNAME | INTEGER_LITERAL | LPAREN | STAR )) => lookup + (QUESTION ( ncnameOrKeyword | INTEGER_LITERAL | DECIMAL_LITERAL | DOUBLE_LITERAL | STRING_LITERAL | LPAREN | DOLLAR | SELF | HASH | STAR )) => unaryLookup | argumentPlaceholder + | ( { xq4Enabled }? ncnameOrKeyword COLON ( EQ | ncnameOrKeyword COLON EQ ) ) => keywordArgument + | exprSingle + ; + +// XQ4: keyword arguments - name := value, or prefix:name := value +keywordArgument throws XPathException +{ String kwName = null; String prefix = null; String local = null; } +: + // Prefixed keyword: prefix:name := value + ( ( ncnameOrKeyword COLON ncnameOrKeyword COLON EQ ) => + prefix=ncnameOrKeyword! COLON! local=ncnameOrKeyword! COLON! EQ! keywordArgumentValue + { kwName = prefix + ":" + local; } + | + // Simple keyword: name := value + kwName=ncnameOrKeyword! COLON! EQ! keywordArgumentValue + ) + { + #keywordArgument = #(#[KEYWORD_ARG, kwName], #keywordArgument); + } + ; + +// XQ4: keyword argument value can be an expression or argument placeholder (?) +// Use lookahead to distinguish bare ? (placeholder) from ?key (unary lookup) +keywordArgumentValue throws XPathException +: + ( QUESTION ( RPAREN | COMMA ) ) => argumentPlaceholder | exprSingle ; @@ -1606,7 +2143,7 @@ contextItemExpr : SELF ; kindTest : - textTest | anyKindTest | elementTest | attributeTest | + textTest | anyKindTest | gnodeTest | elementTest | attributeTest | commentTest | namespaceNodeTest | piTest | documentTest ; @@ -1620,6 +2157,13 @@ anyKindTest "node"^ LPAREN! RPAREN! ; +// XQ4: gnode() is a synonym for node() +gnodeTest +: + "gnode"! LPAREN! RPAREN! + { #gnodeTest = #[LITERAL_node, "node"]; } + ; + elementTest : "element"^ LPAREN! @@ -2074,8 +2618,23 @@ ncnameOrKeyword returns [String name] name=reservedKeywords ; +/** + * Top-level dispatcher for reserved keywords usable as NCNames. + * Split into feature-area sub-rules to reduce merge conflicts on the + * next integration branch. Each feature branch owns its sub-rule; + * merging adds a single alternative here instead of interleaving 80+ lines. + */ reservedKeywords returns [String name] { name= null; } +: + name=coreReservedKeywords + | + name=xq4Keywords + ; + +// ---- Core reserved keywords (XQuery 3.1 + eXist-db extensions) ---- +coreReservedKeywords returns [String name] +{ name= null; } : "element" { name = "element"; } | @@ -2125,6 +2684,14 @@ reservedKeywords returns [String name] | "preceding" { name = "preceding"; } | + "following-or-self" { name = "following-or-self"; } + | + "preceding-or-self" { name = "preceding-or-self"; } + | + "following-sibling-or-self" { name = "following-sibling-or-self"; } + | + "preceding-sibling-or-self" { name = "preceding-sibling-or-self"; } + | "item" { name= "item"; } | "empty" { name= "empty"; } @@ -2137,8 +2704,8 @@ reservedKeywords returns [String name] | "namespace-node" { name= "namespace-node"; } | - "namespace" { name= "namespace"; } - | + "namespace" { name= "namespace"; } + | "if" { name= "if"; } | "then" { name= "then"; } @@ -2177,8 +2744,8 @@ reservedKeywords returns [String name] | "by" { name = "by"; } | - "group" { name = "group"; } - | + "group" { name = "group"; } + | "some" { name = "some"; } | "every" { name = "every"; } @@ -2289,7 +2856,7 @@ reservedKeywords returns [String name] | "tumbling" { name = "tumbling"; } | - "sliding" { name = "sliding"; } + "sliding" { name = "sliding"; } | "window" { name = "window"; } | @@ -2304,6 +2871,47 @@ reservedKeywords returns [String name] "next" { name = "next"; } | "when" { name = "when"; } + | + "ascending" { name = "ascending"; } + | + "descending" { name = "descending"; } + | + "greatest" { name = "greatest"; } + | + "least" { name = "least"; } + | + "satisfies" { name = "satisfies"; } + | + "schema-attribute" { name = "schema-attribute"; } + | + "castable" { name = "castable"; } + | + "idiv" { name = "idiv"; } + | + "processing-instruction" { name = "processing-instruction"; } + | + "allowing" { name = "allowing"; } + ; + +// ---- XQuery 4.0 keywords (feature/xquery-4.0-parser) ---- +xq4Keywords returns [String name] +{ name= null; } +: + "fn" { name = "fn"; } + | + "member" { name = "member"; } + | + "otherwise" { name = "otherwise"; } + | + "key" { name = "key"; } + | + "while" { name = "while"; } + | + "finally" { name = "finally"; } + | + "record" { name = "record"; } + | + "gnode" { name = "gnode"; } ; @@ -2324,6 +2932,9 @@ options { protected boolean wsExplicit= false; protected boolean parseStringLiterals= true; protected boolean inStringConstructor = false; + protected boolean inStringTemplate = false; + protected int stringTemplateDepth = 0; + protected int stringConstructorInterpolationDepth = 0; protected boolean inElementContent= false; protected boolean inAttributeContent= false; protected boolean inFunctionBody= false; @@ -2352,11 +2963,35 @@ options { newline(); } } + + /** + * Disambiguate (# as pragma vs ( + #QName literal. + * Scans past (# and the QName. Returns true (pragma) if the QName + * is followed by whitespace or #). Returns false (QName literal) + * if followed by , or ). + */ + private boolean isPragmaContext() throws CharStreamException { + // LA(1)='(' LA(2)='#' -- start scanning from LA(3) + int i = 3; + // Skip the QName (letters, digits, -, ., _, :) + while (Character.isLetterOrDigit(LA(i)) || LA(i) == '-' || LA(i) == '.' || LA(i) == '_' || LA(i) == ':') { + i++; + } + char afterQName = LA(i); + // If followed by , or ) it's a QName literal argument + if (afterQName == ',' || afterQName == ')') { + return false; + } + // Otherwise it's a pragma (whitespace, #), or other pragma content) + return true; + } } protected SLASH options { paraphrase="single slash '/'"; }: '/' ; protected DSLASH options { paraphrase="double slash '//'"; }: '/' '/' ; protected BANG : '!' ; +protected DOUBLE_BANG options { paraphrase="double bang '!!'"; }: '!' '!' ; +protected DOUBLE_QUESTION options { paraphrase="double question '??'"; }: '?' '?' ; protected MOD : '%' ; protected COLON : ':' ; protected COMMA : ',' ; @@ -2374,7 +3009,10 @@ protected SELF options { paraphrase="."; }: '.' ; protected PARENT options { paraphrase=".."; }: ".." ; protected UNION options { paraphrase="union"; }: '|' ; protected CONCAT options { paraphrase="||"; }: '|' '|'; +protected METHOD_CALL_OP options { paraphrase="method call operator"; }: '=' '?' '>'; +protected MAPPING_ARROW_OP options { paraphrase="mapping arrow operator"; }: '=' '!' '>'; protected ARROW_OP options { paraphrase="arrow operator"; }: '=' '>'; +protected PIPELINE_OP options { paraphrase="pipeline operator"; }: '-' '>'; protected AT options { paraphrase="@ char"; }: '@' ; protected DOLLAR options { paraphrase="dollar sign '$'"; }: '$' ; protected EQ options { paraphrase="="; }: '=' ; @@ -2408,12 +3046,17 @@ protected LETTER protected DIGITS : - ( DIGIT )+ + ( DIGIT )+ ( '_' ( DIGIT )+ )* ; protected HEX_DIGITS : - ( '0'..'9' | 'a'..'f' | 'A'..'F' )+ + ( '0'..'9' | 'a'..'f' | 'A'..'F' )+ ( '_' ( '0'..'9' | 'a'..'f' | 'A'..'F' )+ )* + ; + +protected BINARY_DIGITS +: + ( '0' | '1' )+ ( '_' ( '0' | '1' )+ )* ; protected NCNAME @@ -2470,16 +3113,26 @@ protected INTEGER_LITERAL { !(inElementContent || inAttributeContent) }? DIGITS ; +protected HEX_INTEGER_LITERAL +: + { !(inElementContent || inAttributeContent) }? '0' ('x' | 'X') HEX_DIGITS + ; + +protected BINARY_INTEGER_LITERAL +: + { !(inElementContent || inAttributeContent) }? '0' ('b' | 'B') BINARY_DIGITS + ; + protected DOUBLE_LITERAL : { !(inElementContent || inAttributeContent) }? - ( ( '.' DIGITS ) | ( DIGITS ( '.' ( DIGIT )* )? ) ) ( 'e' | 'E' ) ( '+' | '-' )? DIGITS + ( ( '.' DIGITS ) | ( DIGITS ( '.' ( DIGITS )? )? ) ) ( 'e' | 'E' ) ( '+' | '-' )? DIGITS ; protected DECIMAL_LITERAL : { !(inElementContent || inAttributeContent) }? - ( '.' DIGITS ) | ( DIGITS ( '.' ( DIGIT )* )? ) + ( '.' DIGITS ) | ( DIGITS ( '.' ( DIGITS )? )? ) ; protected PREDEFINED_ENTITY_REF @@ -2520,7 +3173,6 @@ options { : ( ( '\n' ) => '\n' { newline(); } | - ( '&' ) => ( PREDEFINED_ENTITY_REF | CHAR_REF ) | ( ( ']' '`' ) ~ ( '`' ) ) => ( ']' '`' ) | ( ']' ~ ( '`' ) ) => ']' | ( '`' ~ ( '{') ) => '`' | @@ -2528,6 +3180,21 @@ options { )+ ; +protected STRING_TEMPLATE_START options { paraphrase="start of string template"; }: '`'; +protected STRING_TEMPLATE_END options { paraphrase="end of string template"; }: '`'; + +protected STRING_TEMPLATE_CONTENT +options { + testLiterals = false; + paraphrase = "string template content"; +} +: + ( + '\n' { newline(); } | + ~ ( '\n' | '{' | '}' | '`') + )+ + ; + protected BRACED_URI_LITERAL options { paraphrase="braced uri literal"; @@ -2641,6 +3308,46 @@ options { testLiterals = false; } : + { inStringTemplate }? + ( '`' '`' ) => '`' '`' { + $setType(STRING_TEMPLATE_CONTENT); + } + | + { inStringTemplate }? + ( '{' '{' ) => '{' '{' { + $setType(STRING_TEMPLATE_CONTENT); + } + | + { inStringTemplate }? + ( '}' '}' ) => '}' '}' { + $setType(STRING_TEMPLATE_CONTENT); + } + | + { inStringTemplate }? + STRING_TEMPLATE_END { + $setType(STRING_TEMPLATE_END); + } + | + { inStringTemplate }? + LCURLY { + $setType(LCURLY); + } + | + { inStringTemplate }? + STRING_TEMPLATE_CONTENT { + $setType(STRING_TEMPLATE_CONTENT); + } + | + { !inStringConstructor && !inStringTemplate && !inElementContent && !inAttributeContent }? + ( '`' '`' '[' ) => STRING_CONSTRUCTOR_START { + $setType(STRING_CONSTRUCTOR_START); + } + | + { !inStringConstructor && !inStringTemplate && !inElementContent && !inAttributeContent }? + STRING_TEMPLATE_START { + $setType(STRING_TEMPLATE_START); + } + | { !inStringConstructor }? STRING_CONSTRUCTOR_START { $setType(STRING_CONSTRUCTOR_START); @@ -2656,7 +3363,7 @@ options { $setType(STRING_CONSTRUCTOR_INTERPOLATION_START); } | - { !inStringConstructor }? + { !inStringConstructor && stringTemplateDepth == 0 && stringConstructorInterpolationDepth > 0 }? STRING_CONSTRUCTOR_INTERPOLATION_END { $setType(STRING_CONSTRUCTOR_INTERPOLATION_END); } @@ -2777,7 +3484,7 @@ options { ( NAME_START_CHAR ) => ncname:NCNAME { $setType(ncname.getType()); } | - { parseStringLiterals && !inElementContent && !inStringConstructor }? + { parseStringLiterals && !inElementContent && !inStringConstructor && !inStringTemplate }? STRING_LITERAL { $setType(STRING_LITERAL); } | BRACED_URI_LITERAL { $setType(BRACED_URI_LITERAL); } @@ -2801,7 +3508,15 @@ options { ( '.' ) => SELF { $setType(SELF); } | - ( INTEGER_LITERAL ( '.' ( INTEGER_LITERAL )? )? ( 'e' | 'E' ) ) + // XQ4: hex integer literals (0xFF, 0xCAFE_BABE) + ( '0' ('x' | 'X') ) + => HEX_INTEGER_LITERAL { $setType(INTEGER_LITERAL); } + | + // XQ4: binary integer literals (0b1010, 0b1111_0000) + ( '0' ('b' | 'B') ) + => BINARY_INTEGER_LITERAL { $setType(INTEGER_LITERAL); } + | + ( INTEGER_LITERAL ( '.' ( DIGITS )? )? ( 'e' | 'E' ) ) => DOUBLE_LITERAL { $setType(DOUBLE_LITERAL); } | @@ -2816,6 +3531,8 @@ options { { !(inAttributeContent || inElementContent) }? DSLASH { $setType(DSLASH); } | + ( DOUBLE_BANG ) => DOUBLE_BANG { $setType(DOUBLE_BANG); } + | BANG { $setType(BANG); } | COLON { $setType(COLON); } @@ -2828,10 +3545,17 @@ options { | STAR { $setType(STAR); } | + // XQ4: Unicode multiplication sign (U+00D7) as alternative to * + '\u00D7' { $setType(STAR); } + | + ( DOUBLE_QUESTION ) => DOUBLE_QUESTION { $setType(DOUBLE_QUESTION); } + | QUESTION { $setType(QUESTION); } | PLUS { $setType(PLUS); } | + ( PIPELINE_OP ) => PIPELINE_OP { $setType(PIPELINE_OP); } + | MINUS { $setType(MINUS); } | LPPAREN { $setType(LPPAREN); } @@ -2846,6 +3570,10 @@ options { | DOLLAR { $setType(DOLLAR); } | + ( METHOD_CALL_OP ) => METHOD_CALL_OP { $setType(METHOD_CALL_OP); } + | + ( MAPPING_ARROW_OP ) => MAPPING_ARROW_OP { $setType(MAPPING_ARROW_OP); } + | ARROW_OP { $setType(ARROW_OP); } | EQ { $setType(EQ); } @@ -2863,6 +3591,7 @@ options { | XML_CDATA_END { $setType(XML_CDATA_END); } | + { LA(1) == '(' && LA(2) == '#' && isPragmaContext() }? PRAGMA_START { $setType(PRAGMA_START); diff --git a/exist-core/src/main/antlr/org/exist/xquery/parser/XQueryTree.g b/exist-core/src/main/antlr/org/exist/xquery/parser/XQueryTree.g index 20308296806..45ca178fd5e 100644 --- a/exist-core/src/main/antlr/org/exist/xquery/parser/XQueryTree.g +++ b/exist-core/src/main/antlr/org/exist/xquery/parser/XQueryTree.g @@ -139,6 +139,11 @@ options { List windowConditions = null; WindowExpr.WindowType windowType = null; boolean allowEmpty = false; + QName valueVarName = null; + SequenceType valueSequenceType = null; + // XQ4 destructuring + List destructureVarNames = null; + List destructureVarTypes = null; } /** @@ -267,14 +272,20 @@ throws PermissionDeniedException, EXistException, XPathException v:VERSION_DECL { final String version = v.getText(); - if (version.equals("3.1")) { + if (version.equals("4.0")) { + context.setXQueryVersion(40); + staticContext.setXQueryVersion(40); + } else if (version.equals("3.1")) { context.setXQueryVersion(31); + staticContext.setXQueryVersion(31); } else if (version.equals("3.0")) { context.setXQueryVersion(30); + staticContext.setXQueryVersion(30); } else if (version.equals("1.0")) { context.setXQueryVersion(10); + staticContext.setXQueryVersion(10); } else { - throw new XPathException(v, ErrorCodes.XQST0031, "Wrong XQuery version: require 1.0, 3.0 or 3.1"); + throw new XPathException(v, ErrorCodes.XQST0031, "Wrong XQuery version: require 1.0, 3.0, 3.1 or 4.0"); } } ( enc:STRING_LITERAL )? @@ -828,7 +839,13 @@ throws PermissionDeniedException, EXistException, XPathException { QName qn= null; try { - qn = QName.parse(staticContext, name.getText(), staticContext.getDefaultFunctionNamespace()); + // XQ4 (PR2200): unprefixed function declarations go into "no namespace" + // instead of the default function namespace (fn:) + if (name.getText() != null && !name.getText().contains(":") && staticContext.getXQueryVersion() >= 40) { + qn = new QName(name.getText(), ""); + } else { + qn = QName.parse(staticContext, name.getText(), staticContext.getDefaultFunctionNamespace()); + } } catch (final IllegalQNameException iqe) { throw new XPathException(name.getLine(), name.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + name.getText()); } @@ -930,11 +947,42 @@ throws PermissionDeniedException, EXistException, XPathException ) ; +focusFunctionDecl [PathExpr path] +returns [Expression step] +throws PermissionDeniedException, EXistException, XPathException +{ step = null; }: + #( + ff:FOCUS_FUNCTION + { + PathExpr body = new PathExpr(context); + body.setASTNode(focusFunctionDecl_AST_in); + + // Create a function with a single implicit parameter + FunctionSignature signature = new FunctionSignature(InlineFunction.INLINE_FUNCTION_QNAME); + UserDefinedFunction func = new UserDefinedFunction(context, signature); + func.setASTNode(ff); + + // Add the implicit focus parameter: $(.focus) as item()* + FunctionParameterSequenceType focusParam = new FunctionParameterSequenceType( + FocusFunction.FOCUS_PARAM_NAME, Type.ITEM, Cardinality.ZERO_OR_MORE, + "implicit focus parameter"); + signature.setArgumentTypes(new SequenceType[] { focusParam }); + signature.setReturnType(new SequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE)); + func.addVariable(FocusFunction.FOCUS_PARAM_NAME); + } + ( expr [body] )? + { + func.setFunctionBody(body); + step = new FocusFunction(context, func); + } + ) + ; + /** * Parse params in function declaration. */ paramList [List vars] -throws XPathException +throws PermissionDeniedException, EXistException, XPathException : param [vars] ( param [vars] )* ; @@ -943,7 +991,7 @@ throws XPathException * Single function param. */ param [List vars] -throws XPathException +throws PermissionDeniedException, EXistException, XPathException : #( varname:VARIABLE_BINDING @@ -959,6 +1007,18 @@ throws XPathException sequenceType [var] ) )? + ( + #( + PARAM_DEFAULT + { + PathExpr defaultExpr = new PathExpr(context); + } + expr [defaultExpr] + { + var.setDefaultValue(defaultExpr.simplify()); + } + ) + )? ) ; @@ -1132,6 +1192,38 @@ throws XPathException ) ) | + #( + RECORD_TEST { type.setPrimaryType(Type.RECORD); } + ( + STAR + { type.setRecordExtensible(true); } + | + ( + ( + #( + rf:RECORD_FIELD + { + final String fieldName = rf.getText(); + boolean optional = false; + SequenceType fieldType = null; + } + ( QUESTION { optional = true; } )? + ( + { fieldType = new SequenceType(); } + sequenceType [fieldType] + )? + { + type.addRecordField(new SequenceType.RecordField( + fieldName, optional, fieldType)); + } + ) + | + STAR { type.setRecordExtensible(true); } + )* + ) + )? + ) + | #( "item" { type.setPrimaryType(Type.ITEM); } ) @@ -1262,6 +1354,37 @@ throws XPathException #( "schema-element" EQNAME ) )? ) + | + #( + CHOICE_TYPE + { + List alternatives = new ArrayList(); + } + ( + { + SequenceType altType = new SequenceType(); + } + sequenceType [altType] + { + alternatives.add(altType); + } + )+ + { + for (final SequenceType alt : alternatives) { + type.addChoiceAlternative(alt); + } + type.setPrimaryType(Type.ITEM); + } + ) + | + #( + en:ENUM_TYPE + { + String enumText = en.getText(); + String[] enumVals = enumText.split(",", -1); + type.setEnumValues(enumVals); + } + ) ) ( STAR { type.setCardinality(Cardinality.ZERO_OR_MORE); } @@ -1293,6 +1416,14 @@ throws PermissionDeniedException, EXistException, XPathException | step=arrowOp [path] | + step=mappingArrowOp [path] + | + step=pipelineOp [path] + | + step=methodCallOp [path] // XQ4 method call operator =?> + | + step=otherwiseExpr [path] + | step=typeCastExpr [path] | // sequence constructor: @@ -1363,301 +1494,1047 @@ throws PermissionDeniedException, EXistException, XPathException } ) | - // conditional: + step=exprFlowControl [path] + | + // treat as: #( - astIf:"if" + "treat" { - PathExpr testExpr= new PathExpr(context); - PathExpr thenExpr= new PathExpr(context); - PathExpr elseExpr= new PathExpr(context); + PathExpr expr = new PathExpr(context); + expr.setASTNode(expr_AST_in); + SequenceType type= new SequenceType(); } - step=expr [testExpr] - step=astThen:expr [thenExpr] - step=astElse:expr [elseExpr] + step=expr [expr] + sequenceType [type] { - thenExpr.setASTNode(astThen); - elseExpr.setASTNode(astElse); - ConditionalExpression cond = - new ConditionalExpression(context, testExpr, thenExpr, - new DebuggableExpression(elseExpr)); - cond.setASTNode(astIf); - path.add(cond); - step = cond; + step = new TreatAsExpression(context, expr, type); + step.setASTNode(expr_AST_in); + path.add(step); } ) | - // quantified expression: some + // switch #( - "some" + switchAST:"switch" { - List clauses= new ArrayList(); - PathExpr satisfiesExpr = new PathExpr(context); - satisfiesExpr.setASTNode(expr_AST_in); + PathExpr operand = new PathExpr(context); + operand.setASTNode(expr_AST_in); + boolean booleanMode = false; } ( - #( - someVarName:VARIABLE_BINDING - { - ForLetClause clause= new ForLetClause(); - PathExpr inputSequence = new PathExpr(context); - inputSequence.setASTNode(expr_AST_in); - } - ( - #( - "as" - { SequenceType type= new SequenceType(); } - sequenceType[type] - ) - { clause.sequenceType = type; } - )? - step=expr[inputSequence] - { - try { - clause.varName = QName.parse(staticContext, someVarName.getText(), null); - } catch (final IllegalQNameException iqe) { - throw new XPathException(someVarName.getLine(), someVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + someVarName.getText()); - } - clause.inputSequence= inputSequence; - clauses.add(clause); - } - ) - )* - step=expr[satisfiesExpr] - { - Expression action = satisfiesExpr; - for (int i= clauses.size() - 1; i >= 0; i--) { - ForLetClause clause= (ForLetClause) clauses.get(i); - BindingExpression expr = new QuantifiedExpression(context, QuantifiedExpression.SOME); - expr.setASTNode(expr_AST_in); - expr.setVariable(clause.varName); - expr.setSequenceType(clause.sequenceType); - expr.setInputSequence(clause.inputSequence); - expr.setReturnExpression(action); - satisfiesExpr= null; - action= expr; - } - path.add(action); - step = action; - } - ) - | - // quantified expression: every - #( - "every" + SWITCH_BOOLEAN + { booleanMode = true; } + | + step=expr [operand] + ) { - List clauses= new ArrayList(); - PathExpr satisfiesExpr = new PathExpr(context); - satisfiesExpr.setASTNode(expr_AST_in); + SwitchExpression switchExpr = new SwitchExpression(context, operand); + switchExpr.setBooleanMode(booleanMode); + switchExpr.setASTNode(switchAST); + path.add(switchExpr); } ( - #( - everyVarName:VARIABLE_BINDING - { - ForLetClause clause= new ForLetClause(); - PathExpr inputSequence = new PathExpr(context); - inputSequence.setASTNode(expr_AST_in); - } - ( - #( - "as" - { SequenceType type= new SequenceType(); } - sequenceType[type] - ) - { clause.sequenceType = type; } - )? - step=expr[inputSequence] - { - try { - clause.varName = QName.parse(staticContext, everyVarName.getText(), null); - } catch (final IllegalQNameException iqe) { - throw new XPathException(everyVarName.getLine(), everyVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + everyVarName.getText()); - } - clause.inputSequence= inputSequence; - clauses.add(clause); - } - ) - )* - step=expr[satisfiesExpr] - { - Expression action = satisfiesExpr; - for (int i= clauses.size() - 1; i >= 0; i--) { - ForLetClause clause= (ForLetClause) clauses.get(i); - BindingExpression expr = new QuantifiedExpression(context, QuantifiedExpression.EVERY); - expr.setASTNode(expr_AST_in); - expr.setVariable(clause.varName); - expr.setSequenceType(clause.sequenceType); - expr.setInputSequence(clause.inputSequence); - expr.setReturnExpression(action); - satisfiesExpr= null; - action= expr; + { + List caseOperands = new ArrayList(2); + PathExpr returnExpr = new PathExpr(context); + returnExpr.setASTNode(expr_AST_in); } - path.add(action); - step = action; - } + (( + { + PathExpr caseOperand = new PathExpr(context); + caseOperand.setASTNode(expr_AST_in); + } + "case" + expr [caseOperand] + { caseOperands.add(caseOperand); } + )+ + #( + "return" + step= expr [returnExpr] + { switchExpr.addCase(caseOperands, returnExpr); } + )) + )+ + ( + "default" + { + PathExpr returnExpr = new PathExpr(context); + returnExpr.setASTNode(expr_AST_in); + } + step=expr [returnExpr] + { + switchExpr.setDefault(returnExpr); + } + ) + { step = switchExpr; } ) | - //try/catch expression + // typeswitch #( - astTry:"try" + "typeswitch" { - PathExpr tryTargetExpr = new PathExpr(context); - tryTargetExpr.setASTNode(expr_AST_in); + PathExpr operand = new PathExpr(context); + operand.setASTNode(expr_AST_in); } - step=expr [tryTargetExpr] + step=expr [operand] { - TryCatchExpression cond = new TryCatchExpression(context, tryTargetExpr); - cond.setASTNode(astTry); - path.add(cond); + TypeswitchExpression tswitch = new TypeswitchExpression(context, operand); + tswitch.setASTNode(expr_AST_in); + path.add(tswitch); } ( { - final List catchErrorList = new ArrayList<>(2); - final List catchVars = new ArrayList<>(3); - final PathExpr catchExpr = new PathExpr(context); - catchExpr.setASTNode(expr_AST_in); + PathExpr returnExpr = new PathExpr(context); + returnExpr.setASTNode(expr_AST_in); + QName qn = null; + List types = new ArrayList(2); + SequenceType type = new SequenceType(); } #( - astCatch:"catch" - (catchErrorList [catchErrorList]) + "case" ( - { - QName qncode = null; - QName qndesc = null; - QName qnval = null; - } - code:CATCH_ERROR_CODE + var:VARIABLE_BINDING { try { - qncode = QName.parse(staticContext, code.getText()); - catchVars.add(qncode); + qn = QName.parse(staticContext, var.getText()); } catch (final IllegalQNameException iqe) { - throw new XPathException(code.getLine(), code.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + code.getText()); + throw new XPathException(var.getLine(), var.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + var.getText()); } } - ( - desc:CATCH_ERROR_DESC - { - try { - qndesc = QName.parse(staticContext, desc.getText()); - catchVars.add(qndesc); - } catch (final IllegalQNameException iqe) { - throw new XPathException(desc.getLine(), desc.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + desc.getText()); - } - } - - ( - val:CATCH_ERROR_VAL - { - try { - qnval = QName.parse(staticContext, val.getText()); - catchVars.add(qnval); - } catch (final IllegalQNameException iqe) { - throw new XPathException(val.getLine(), val.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + val.getText()); - } - } - - )? - )? )? - step= expr [catchExpr] - { - catchExpr.setASTNode(astCatch); - cond.addCatchClause(catchErrorList, catchVars, catchExpr); - } + ( + sequenceType[type] + { + types.add(type); + type = new SequenceType(); + } + )+ + // Need return as root in following to disambiguate + // e.g. ( case a xs:integer ( * 3 3 ) ) + // which gives xs:integer* and no operator left for 3 3 ... + // Now ( case a xs:integer ( return ( + 3 3 ) ) ) /ljo + #( + "return" + step= expr [returnExpr] + { + SequenceType[] atype = new SequenceType[types.size()]; + atype = types.toArray(atype); + tswitch.addCase(atype, qn, returnExpr); + } + ) ) + )+ + ( + "default" + { + PathExpr returnExpr = new PathExpr(context); + returnExpr.setASTNode(expr_AST_in); + QName qn = null; + } + ( + dvar:VARIABLE_BINDING + { + try { + qn = QName.parse(staticContext, dvar.getText()); + } catch (final IllegalQNameException iqe) { + throw new XPathException(dvar.getLine(), dvar.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + dvar.getText()); + } + } + )? + step=expr [returnExpr] + { + tswitch.setDefault(qn, returnExpr); + } + ) + { step = tswitch; } + ) + | + // logical operator: or + #( + "or" + { + PathExpr left= new PathExpr(context); + left.setASTNode(expr_AST_in); + } + step=expr [left] + { + PathExpr right= new PathExpr(context); + right.setASTNode(expr_AST_in); + } + step=expr [right] + ) + { + OpOr or= new OpOr(context); + or.addPath(left); + or.addPath(right); + path.addPath(or); + step = or; + } + | + // logical operator: and + #( + "and" + { + PathExpr left= new PathExpr(context); + left.setASTNode(expr_AST_in); + + PathExpr right= new PathExpr(context); + right.setASTNode(expr_AST_in); + } + step=expr [left] + step=expr [right] + ) + { + OpAnd and= new OpAnd(context); + and.addPath(left); + and.addPath(right); + path.addPath(and); + step = and; + } + | + // union expressions: | and union + #( + UNION + { + PathExpr left= new PathExpr(context); + left.setASTNode(expr_AST_in); + PathExpr right= new PathExpr(context); + right.setASTNode(expr_AST_in); + } + step=expr [left] + step=expr [right] + ) + { + Union union= new Union(context, left, right); + path.add(union); + step = union; + } + | + // intersections: + #( "intersect" { - step = cond; + PathExpr left = new PathExpr(context); + left.setASTNode(expr_AST_in); + + PathExpr right = new PathExpr(context); + right.setASTNode(expr_AST_in); + } + step=expr [left] + step=expr [right] + ) + { + Intersect intersect = new Intersect(context, left, right); + path.add(intersect); + step = intersect; + } + | + #( "except" + { + PathExpr left = new PathExpr(context); + left.setASTNode(expr_AST_in); + + PathExpr right = new PathExpr(context); + right.setASTNode(expr_AST_in); } + step=expr [left] + step=expr [right] ) + { + Except intersect = new Except(context, left, right); + path.add(intersect); + step = intersect; + } | - // FLWOR expressions: let and for + // absolute path expression starting with a / #( - r:"return" + ABSOLUTE_SLASH { - List clauses= new ArrayList(); - Expression action= new PathExpr(context); - action.setASTNode(r); - PathExpr whereExpr= null; - List orderBy= null; + RootNode root= new RootNode(context); + path.add(root); + } + ( step=expr [path] )? + ) + | + // absolute path expression starting with // + #( + ABSOLUTE_DSLASH + { + RootNode root= new RootNode(context); + path.add(root); } ( - #( - f:"for" - ( - #( - varName:VARIABLE_BINDING - { - ForLetClause clause= new ForLetClause(); - clause.ast = varName; - PathExpr inputSequence= new PathExpr(context); - inputSequence.setASTNode(expr_AST_in);inputSequence.setASTNode(expr_AST_in); - final DistinctVariableNames distinctVariableNames = new DistinctVariableNames(); - } - ( - #( - "as" - { clause.sequenceType= new SequenceType(); } - sequenceType [clause.sequenceType] - ) - )? - ( - "empty" - { clause.allowEmpty = true; } - )? - ( - posVar:POSITIONAL_VAR - { - try { - clause.posVar = distinctVariableNames.check(ErrorCodes.XQST0089, posVar, QName.parse(staticContext, posVar.getText(), null)); - } catch (final IllegalQNameException iqe) { - throw new XPathException(posVar.getLine(), posVar.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + posVar.getText()); - } - } - )? - step=expr [inputSequence] - { - try { - clause.varName = distinctVariableNames.check(ErrorCodes.XQST0089, varName, QName.parse(staticContext, varName.getText(), null)); - } catch (final IllegalQNameException iqe) { - throw new XPathException(varName.getLine(), varName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + varName.getText()); - } - clause.inputSequence= inputSequence; - clauses.add(clause); - } - ) - )+ - ) - | - #( - l:"let" - ( - #( - letVarName:VARIABLE_BINDING - { - ForLetClause clause= new ForLetClause(); - clause.ast = letVarName; - clause.type = FLWORClause.ClauseType.LET; - PathExpr inputSequence= new PathExpr(context); - inputSequence.setASTNode(expr_AST_in); - } + step=expr [path] + { + if (step instanceof LocationStep) { + LocationStep s= (LocationStep) step; + if (s.getAxis() == Constants.ATTRIBUTE_AXIS || + (s.getTest().getType() == Type.ATTRIBUTE && s.getAxis() == Constants.CHILD_AXIS)) + // combines descendant-or-self::node()/attribute:* + s.setAxis(Constants.DESCENDANT_ATTRIBUTE_AXIS); + else { + s.setAxis(Constants.DESCENDANT_SELF_AXIS); + s.setAbbreviated(true); + } + } else + step.setPrimaryAxis(Constants.DESCENDANT_SELF_AXIS); + } + )? + ) + | + // range expression: to + #( + "to" + { + PathExpr start= new PathExpr(context); + start.setASTNode(expr_AST_in); + + PathExpr end= new PathExpr(context); + end.setASTNode(expr_AST_in); + + List args= new ArrayList(2); + args.add(start); + args.add(end); + } + step=expr [start] + step=expr [end] + { + RangeExpression range= new RangeExpression(context); + range.setASTNode(expr_AST_in); + range.setArguments(args); + path.addPath(range); + step = range; + } + ) + | + step=generalComp [path] + | + step=valueComp [path] + | + step=nodeComp [path] + | + step=primaryExpr [path] + | + step=pathExpr [path] + | + step=extensionExpr [path] + | + step=numericExpr [path] + | + step=updateExpr [path] + ; + +/** + * Flow control expressions extracted from expr to avoid + * Java method size limit (64KB bytecode). + * Handles: conditional, ternary, quantified (some/every), + * try/catch/finally, FLWOR, instance of. + */ +exprFlowControl [PathExpr path] +returns [Expression step] +throws PermissionDeniedException, EXistException, XPathException +{ step = null; } +: + // conditional: + #( + astIf:"if" + { + PathExpr testExpr= new PathExpr(context); + PathExpr thenExpr= new PathExpr(context); + PathExpr elseExpr= new PathExpr(context); + } + step=expr [testExpr] + step=astThen:expr [thenExpr] + step=astElse:expr [elseExpr] + { + thenExpr.setASTNode(astThen); + elseExpr.setASTNode(astElse); + ConditionalExpression cond = + new ConditionalExpression(context, testExpr, thenExpr, + new DebuggableExpression(elseExpr)); + cond.setASTNode(astIf); + path.add(cond); + step = cond; + } + ) + | + // ternary conditional: condition ?? then !! else + #( + astTernary:TERNARY + { + PathExpr ternTestExpr = new PathExpr(context); + PathExpr ternThenExpr = new PathExpr(context); + PathExpr ternElseExpr = new PathExpr(context); + } + step=expr [ternTestExpr] + step=expr [ternThenExpr] + step=expr [ternElseExpr] + { + ConditionalExpression ternCond = + new ConditionalExpression(context, ternTestExpr, ternThenExpr, + new DebuggableExpression(ternElseExpr)); + ternCond.setASTNode(astTernary); + path.add(ternCond); + step = ternCond; + } + ) + | + // quantified expression: some + #( + "some" + { + List clauses= new ArrayList(); + PathExpr satisfiesExpr = new PathExpr(context); + satisfiesExpr.setASTNode(exprFlowControl_AST_in); + } + ( + #( + someVarName:VARIABLE_BINDING + { + ForLetClause clause= new ForLetClause(); + PathExpr inputSequence = new PathExpr(context); + inputSequence.setASTNode(exprFlowControl_AST_in); + } + ( + #( + "as" + { SequenceType type= new SequenceType(); } + sequenceType[type] + ) + { clause.sequenceType = type; } + )? + step=expr[inputSequence] + { + try { + clause.varName = QName.parse(staticContext, someVarName.getText(), null); + } catch (final IllegalQNameException iqe) { + throw new XPathException(someVarName.getLine(), someVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + someVarName.getText()); + } + clause.inputSequence= inputSequence; + clauses.add(clause); + } + ) + )* + step=expr[satisfiesExpr] + { + Expression action = satisfiesExpr; + for (int i= clauses.size() - 1; i >= 0; i--) { + ForLetClause clause= (ForLetClause) clauses.get(i); + BindingExpression expr = new QuantifiedExpression(context, QuantifiedExpression.SOME); + expr.setASTNode(exprFlowControl_AST_in); + expr.setVariable(clause.varName); + expr.setSequenceType(clause.sequenceType); + expr.setInputSequence(clause.inputSequence); + expr.setReturnExpression(action); + satisfiesExpr= null; + action= expr; + } + path.add(action); + step = action; + } + ) + | + // quantified expression: every + #( + "every" + { + List clauses= new ArrayList(); + PathExpr satisfiesExpr = new PathExpr(context); + satisfiesExpr.setASTNode(exprFlowControl_AST_in); + } + ( + #( + everyVarName:VARIABLE_BINDING + { + ForLetClause clause= new ForLetClause(); + PathExpr inputSequence = new PathExpr(context); + inputSequence.setASTNode(exprFlowControl_AST_in); + } + ( + #( + "as" + { SequenceType type= new SequenceType(); } + sequenceType[type] + ) + { clause.sequenceType = type; } + )? + step=expr[inputSequence] + { + try { + clause.varName = QName.parse(staticContext, everyVarName.getText(), null); + } catch (final IllegalQNameException iqe) { + throw new XPathException(everyVarName.getLine(), everyVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + everyVarName.getText()); + } + clause.inputSequence= inputSequence; + clauses.add(clause); + } + ) + )* + step=expr[satisfiesExpr] + { + Expression action = satisfiesExpr; + for (int i= clauses.size() - 1; i >= 0; i--) { + ForLetClause clause= (ForLetClause) clauses.get(i); + BindingExpression expr = new QuantifiedExpression(context, QuantifiedExpression.EVERY); + expr.setASTNode(exprFlowControl_AST_in); + expr.setVariable(clause.varName); + expr.setSequenceType(clause.sequenceType); + expr.setInputSequence(clause.inputSequence); + expr.setReturnExpression(action); + satisfiesExpr= null; + action= expr; + } + path.add(action); + step = action; + } + ) + | + //try/catch expression + #( + astTry:"try" + { + PathExpr tryTargetExpr = new PathExpr(context); + tryTargetExpr.setASTNode(exprFlowControl_AST_in); + } + step=expr [tryTargetExpr] + { + TryCatchExpression cond = new TryCatchExpression(context, tryTargetExpr); + cond.setASTNode(astTry); + path.add(cond); + } + ( + { + final List catchErrorList = new ArrayList<>(2); + final List catchVars = new ArrayList<>(3); + final PathExpr catchExpr = new PathExpr(context); + catchExpr.setASTNode(exprFlowControl_AST_in); + } + #( + astCatch:"catch" + (catchErrorList [catchErrorList]) + ( + { + QName qncode = null; + QName qndesc = null; + QName qnval = null; + } + code:CATCH_ERROR_CODE + { + try { + qncode = QName.parse(staticContext, code.getText()); + catchVars.add(qncode); + } catch (final IllegalQNameException iqe) { + throw new XPathException(code.getLine(), code.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + code.getText()); + } + } + ( + desc:CATCH_ERROR_DESC + { + try { + qndesc = QName.parse(staticContext, desc.getText()); + catchVars.add(qndesc); + } catch (final IllegalQNameException iqe) { + throw new XPathException(desc.getLine(), desc.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + desc.getText()); + } + } + + ( + val:CATCH_ERROR_VAL + { + try { + qnval = QName.parse(staticContext, val.getText()); + catchVars.add(qnval); + } catch (final IllegalQNameException iqe) { + throw new XPathException(val.getLine(), val.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + val.getText()); + } + } + + )? + )? + )? + step= expr [catchExpr] + { + catchExpr.setASTNode(astCatch); + cond.addCatchClause(catchErrorList, catchVars, catchExpr); + } + ) + )* + ( + #( + astFinally:"finally" + { + final PathExpr finallyExpr = new PathExpr(context); + finallyExpr.setASTNode(astFinally); + } + (step=expr [finallyExpr])? + { + finallyExpr.setASTNode(astFinally); + cond.setFinallyExpr(finallyExpr); + } + ) + )? + + { + step = cond; + } + ) + | + // FLWOR expressions: let and for + #( + r:"return" + { + List clauses= new ArrayList(); + Expression action= new PathExpr(context); + action.setASTNode(r); + PathExpr whereExpr= null; + List orderBy= null; + } + ( + #( + f:"for" + ( + #( + varName:VARIABLE_BINDING + { + ForLetClause clause= new ForLetClause(); + clause.ast = varName; + PathExpr inputSequence= new PathExpr(context); + inputSequence.setASTNode(exprFlowControl_AST_in);inputSequence.setASTNode(exprFlowControl_AST_in); + final DistinctVariableNames distinctVariableNames = new DistinctVariableNames(); + } + ( + #( + "as" + { clause.sequenceType= new SequenceType(); } + sequenceType [clause.sequenceType] + ) + )? + ( + "empty" + { clause.allowEmpty = true; } + )? + ( + posVar:POSITIONAL_VAR + { + try { + clause.posVar = distinctVariableNames.check(ErrorCodes.XQST0089, posVar, QName.parse(staticContext, posVar.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(posVar.getLine(), posVar.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + posVar.getText()); + } + } + )? + step=expr [inputSequence] + { + try { + clause.varName = distinctVariableNames.check(ErrorCodes.XQST0089, varName, QName.parse(staticContext, varName.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(varName.getLine(), varName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + varName.getText()); + } + clause.inputSequence= inputSequence; + clauses.add(clause); + } + ) + | + #( + FOR_MEMBER + #( + memberVarName:VARIABLE_BINDING + { + ForLetClause clause= new ForLetClause(); + clause.ast = memberVarName; + clause.type = FLWORClause.ClauseType.FOR_MEMBER; + PathExpr inputSequence= new PathExpr(context); + inputSequence.setASTNode(exprFlowControl_AST_in); + final DistinctVariableNames memberDistinctVars = new DistinctVariableNames(); + } + ( + #( + "as" + { clause.sequenceType= new SequenceType(); } + sequenceType [clause.sequenceType] + ) + )? + ( + memberPosVar:POSITIONAL_VAR + { + try { + clause.posVar = memberDistinctVars.check(ErrorCodes.XQST0089, memberPosVar, QName.parse(staticContext, memberPosVar.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(memberPosVar.getLine(), memberPosVar.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + memberPosVar.getText()); + } + } + )? + step=expr [inputSequence] + { + try { + clause.varName = memberDistinctVars.check(ErrorCodes.XQST0089, memberVarName, QName.parse(staticContext, memberVarName.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(memberVarName.getLine(), memberVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + memberVarName.getText()); + } + clause.inputSequence= inputSequence; + clauses.add(clause); + } + ) + ) + | + #( + FOR_KEY + #( + keyVarName:VARIABLE_BINDING + { + ForLetClause clause= new ForLetClause(); + clause.ast = keyVarName; + clause.type = FLWORClause.ClauseType.FOR_KEY; + PathExpr inputSequence= new PathExpr(context); + inputSequence.setASTNode(exprFlowControl_AST_in); + final DistinctVariableNames keyDistinctVars = new DistinctVariableNames(); + } + ( + #( + "as" + { clause.sequenceType= new SequenceType(); } + sequenceType [clause.sequenceType] + ) + )? + ( + keyPosVar:POSITIONAL_VAR + { + try { + clause.posVar = keyDistinctVars.check(ErrorCodes.XQST0089, keyPosVar, QName.parse(staticContext, keyPosVar.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(keyPosVar.getLine(), keyPosVar.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + keyPosVar.getText()); + } + } + )? + step=expr [inputSequence] + { + try { + clause.varName = keyDistinctVars.check(ErrorCodes.XQST0089, keyVarName, QName.parse(staticContext, keyVarName.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(keyVarName.getLine(), keyVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + keyVarName.getText()); + } + clause.inputSequence= inputSequence; + clauses.add(clause); + } + ) + ) + | + #( + FOR_VALUE + #( + valueVarName:VARIABLE_BINDING + { + ForLetClause clause= new ForLetClause(); + clause.ast = valueVarName; + clause.type = FLWORClause.ClauseType.FOR_VALUE; + PathExpr inputSequence= new PathExpr(context); + inputSequence.setASTNode(exprFlowControl_AST_in); + final DistinctVariableNames valueDistinctVars = new DistinctVariableNames(); + } + ( + #( + "as" + { clause.sequenceType= new SequenceType(); } + sequenceType [clause.sequenceType] + ) + )? + ( + valuePosVar:POSITIONAL_VAR + { + try { + clause.posVar = valueDistinctVars.check(ErrorCodes.XQST0089, valuePosVar, QName.parse(staticContext, valuePosVar.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(valuePosVar.getLine(), valuePosVar.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + valuePosVar.getText()); + } + } + )? + step=expr [inputSequence] + { + try { + clause.varName = valueDistinctVars.check(ErrorCodes.XQST0089, valueVarName, QName.parse(staticContext, valueVarName.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(valueVarName.getLine(), valueVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + valueVarName.getText()); + } + clause.inputSequence= inputSequence; + clauses.add(clause); + } + ) + ) + | + #( + FOR_KEY_VALUE + #( + kvKeyVarName:VARIABLE_BINDING + { + ForLetClause clause= new ForLetClause(); + clause.ast = kvKeyVarName; + clause.type = FLWORClause.ClauseType.FOR_KEY_VALUE; + PathExpr inputSequence= new PathExpr(context); + inputSequence.setASTNode(exprFlowControl_AST_in); + final DistinctVariableNames kvDistinctVars = new DistinctVariableNames(); + } + ( + #( + "as" + { clause.sequenceType= new SequenceType(); } + sequenceType [clause.sequenceType] + ) + )? + ( + #( + kvValueVar:VALUE_VAR + { + try { + clause.valueVarName = kvDistinctVars.check(ErrorCodes.XQST0089, kvValueVar, QName.parse(staticContext, kvValueVar.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(kvValueVar.getLine(), kvValueVar.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + kvValueVar.getText()); + } + } + ( + #( + "as" + { clause.valueSequenceType = new SequenceType(); } + sequenceType [clause.valueSequenceType] + ) + )? + ) + )? + ( + kvPosVar:POSITIONAL_VAR + { + try { + clause.posVar = kvDistinctVars.check(ErrorCodes.XQST0089, kvPosVar, QName.parse(staticContext, kvPosVar.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(kvPosVar.getLine(), kvPosVar.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + kvPosVar.getText()); + } + } + )? + step=expr [inputSequence] + { + try { + clause.varName = kvDistinctVars.check(ErrorCodes.XQST0089, kvKeyVarName, QName.parse(staticContext, kvKeyVarName.getText(), null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(kvKeyVarName.getLine(), kvKeyVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + kvKeyVarName.getText()); + } + clause.inputSequence= inputSequence; + clauses.add(clause); + } + ) + ) + )+ + ) + | + #( + l:"let" + ( + #( + letVarName:VARIABLE_BINDING + { + ForLetClause clause= new ForLetClause(); + clause.ast = letVarName; + clause.type = FLWORClause.ClauseType.LET; + PathExpr inputSequence= new PathExpr(context); + inputSequence.setASTNode(exprFlowControl_AST_in); + } + ( + #( + "as" + { clause.sequenceType= new SequenceType(); } + sequenceType [clause.sequenceType] + ) + )? + step=expr [inputSequence] + { + try { + clause.varName = QName.parse(staticContext, letVarName.getText(), null); + } catch (final IllegalQNameException iqe) { + throw new XPathException(letVarName.getLine(), letVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + letVarName.getText()); + } + clause.inputSequence= inputSequence; + clauses.add(clause); + } + ) + | + // XQ4: sequence destructuring + #( + seqDestAST:SEQ_DESTRUCTURE + { + ForLetClause seqClause = new ForLetClause(); + seqClause.ast = seqDestAST; + seqClause.type = FLWORClause.ClauseType.LET_SEQ_DESTRUCTURE; + seqClause.destructureVarNames = new ArrayList(); + seqClause.destructureVarTypes = new ArrayList(); + String[] seqVarNames = seqDestAST.getText().split(",", -1); + int seqTypedIdx = 0; + boolean[] seqHasType = new boolean[seqVarNames.length]; + for (int dv = 0; dv < seqVarNames.length; dv++) { + String svn = seqVarNames[dv]; + seqHasType[dv] = svn.endsWith("+"); + if (seqHasType[dv]) svn = svn.substring(0, svn.length() - 1); + try { + seqClause.destructureVarNames.add( + QName.parse(staticContext, svn, null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(seqDestAST.getLine(), seqDestAST.getColumn(), + ErrorCodes.XPST0081, "No namespace defined for prefix " + svn); + } + seqClause.destructureVarTypes.add(null); + } + PathExpr seqInput = new PathExpr(context); + seqInput.setASTNode(exprFlowControl_AST_in); + } + ( + #( + DESTRUCTURE_VAR_TYPE + #( + "as" + { + SequenceType seqVarType = new SequenceType(); + while (seqTypedIdx < seqHasType.length && !seqHasType[seqTypedIdx]) seqTypedIdx++; + } + sequenceType [seqVarType] + { + if (seqTypedIdx < seqClause.destructureVarTypes.size()) { + seqClause.destructureVarTypes.set(seqTypedIdx, seqVarType); + } + seqTypedIdx++; + } + ) + ) + )* + ( + #( + "as" + { seqClause.sequenceType = new SequenceType(); } + sequenceType [seqClause.sequenceType] + ) + )? + step=expr [seqInput] + { + seqClause.inputSequence = seqInput; + clauses.add(seqClause); + } + ) + | + // XQ4: array destructuring + #( + arrDestAST:ARRAY_DESTRUCTURE + { + ForLetClause arrClause = new ForLetClause(); + arrClause.ast = arrDestAST; + arrClause.type = FLWORClause.ClauseType.LET_ARRAY_DESTRUCTURE; + arrClause.destructureVarNames = new ArrayList(); + arrClause.destructureVarTypes = new ArrayList(); + String[] arrVarNames = arrDestAST.getText().split(",", -1); + int arrTypedIdx = 0; + boolean[] arrHasType = new boolean[arrVarNames.length]; + for (int dv = 0; dv < arrVarNames.length; dv++) { + String avn = arrVarNames[dv]; + arrHasType[dv] = avn.endsWith("+"); + if (arrHasType[dv]) avn = avn.substring(0, avn.length() - 1); + try { + arrClause.destructureVarNames.add( + QName.parse(staticContext, avn, null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(arrDestAST.getLine(), arrDestAST.getColumn(), + ErrorCodes.XPST0081, "No namespace defined for prefix " + avn); + } + arrClause.destructureVarTypes.add(null); + } + PathExpr arrInput = new PathExpr(context); + arrInput.setASTNode(exprFlowControl_AST_in); + } + ( + #( + DESTRUCTURE_VAR_TYPE + #( + "as" + { + SequenceType arrVarType = new SequenceType(); + while (arrTypedIdx < arrHasType.length && !arrHasType[arrTypedIdx]) arrTypedIdx++; + } + sequenceType [arrVarType] + { + if (arrTypedIdx < arrClause.destructureVarTypes.size()) { + arrClause.destructureVarTypes.set(arrTypedIdx, arrVarType); + } + arrTypedIdx++; + } + ) + ) + )* ( #( "as" - { clause.sequenceType= new SequenceType(); } - sequenceType [clause.sequenceType] + { arrClause.sequenceType = new SequenceType(); } + sequenceType [arrClause.sequenceType] ) )? - step=expr [inputSequence] + step=expr [arrInput] { - try { - clause.varName = QName.parse(staticContext, letVarName.getText(), null); - } catch (final IllegalQNameException iqe) { - throw new XPathException(letVarName.getLine(), letVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + letVarName.getText()); + arrClause.inputSequence = arrInput; + clauses.add(arrClause); + } + ) + | + // XQ4: map destructuring + #( + mapDestAST:MAP_DESTRUCTURE + { + ForLetClause mapClause = new ForLetClause(); + mapClause.ast = mapDestAST; + mapClause.type = FLWORClause.ClauseType.LET_MAP_DESTRUCTURE; + mapClause.destructureVarNames = new ArrayList(); + mapClause.destructureVarTypes = new ArrayList(); + String[] mapVarNames = mapDestAST.getText().split(",", -1); + int mapTypedIdx = 0; + boolean[] mapHasType = new boolean[mapVarNames.length]; + for (int dv = 0; dv < mapVarNames.length; dv++) { + String mvn = mapVarNames[dv]; + mapHasType[dv] = mvn.endsWith("+"); + if (mapHasType[dv]) mvn = mvn.substring(0, mvn.length() - 1); + try { + mapClause.destructureVarNames.add( + QName.parse(staticContext, mvn, null)); + } catch (final IllegalQNameException iqe) { + throw new XPathException(mapDestAST.getLine(), mapDestAST.getColumn(), + ErrorCodes.XPST0081, "No namespace defined for prefix " + mvn); + } + mapClause.destructureVarTypes.add(null); } - clause.inputSequence= inputSequence; - clauses.add(clause); + PathExpr mapInput = new PathExpr(context); + mapInput.setASTNode(exprFlowControl_AST_in); + } + ( + #( + DESTRUCTURE_VAR_TYPE + #( + "as" + { + SequenceType mapVarType = new SequenceType(); + while (mapTypedIdx < mapHasType.length && !mapHasType[mapTypedIdx]) mapTypedIdx++; + } + sequenceType [mapVarType] + { + if (mapTypedIdx < mapClause.destructureVarTypes.size()) { + mapClause.destructureVarTypes.set(mapTypedIdx, mapVarType); + } + mapTypedIdx++; + } + ) + ) + )* + ( + #( + "as" + { mapClause.sequenceType = new SequenceType(); } + sequenceType [mapClause.sequenceType] + ) + )? + step=expr [mapInput] + { + mapClause.inputSequence = mapInput; + clauses.add(mapClause); } ) )+ @@ -1884,7 +2761,7 @@ throws PermissionDeniedException, EXistException, XPathException ( { groupSpecExpr = new PathExpr(context); - groupSpecExpr.setASTNode(expr_AST_in); + groupSpecExpr.setASTNode(exprFlowControl_AST_in); } step=expr [groupSpecExpr] ) @@ -1915,7 +2792,7 @@ throws PermissionDeniedException, EXistException, XPathException ( { PathExpr orderSpecExpr= new PathExpr(context); - orderSpecExpr.setASTNode(expr_AST_in); + orderSpecExpr.setASTNode(exprFlowControl_AST_in); } step=expr [orderSpecExpr] { @@ -1981,7 +2858,7 @@ throws PermissionDeniedException, EXistException, XPathException w:"where" { whereExpr= new PathExpr(context); - whereExpr.setASTNode(expr_AST_in); + whereExpr.setASTNode(exprFlowControl_AST_in); } step=expr [whereExpr] { @@ -1994,422 +2871,176 @@ throws PermissionDeniedException, EXistException, XPathException ) | #( - co:"count" - countVarName:VARIABLE_BINDING + wh:"while" + { + PathExpr whileExpr = new PathExpr(context); + whileExpr.setASTNode(exprFlowControl_AST_in); + } + step=expr [whileExpr] { ForLetClause clause = new ForLetClause(); - clause.ast = co; - try { - clause.varName = QName.parse(staticContext, countVarName.getText(), null); - } catch (final IllegalQNameException iqe) { - throw new XPathException(countVarName.getLine(), countVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + countVarName.getText()); - } - clause.type = FLWORClause.ClauseType.COUNT; - clause.inputSequence = null; + clause.ast = wh; + clause.type = FLWORClause.ClauseType.WHILE; + clause.inputSequence = whileExpr; clauses.add(clause); } ) - )+ - step=expr [(PathExpr) action] - { - for (int i= clauses.size() - 1; i >= 0; i--) { - ForLetClause clause= (ForLetClause) clauses.get(i); - FLWORClause expr; - switch (clause.type) { - case LET: - expr = new LetExpr(context); - expr.setASTNode(expr_AST_in); - break; - case GROUPBY: - expr = new GroupByClause(context); - break; - case ORDERBY: - expr = new OrderByClause(context, clause.orderSpecs); - break; - case WHERE: - expr = new WhereClause(context, new DebuggableExpression(clause.inputSequence)); - break; - case COUNT: - expr = new CountClause(context, clause.varName); - break; - case WINDOW: - expr = new WindowExpr(context, clause.windowType, clause.windowConditions.get(0), clause.windowConditions.size() > 1 ? clause.windowConditions.get(1) : null); - break; - default: - expr = new ForExpr(context, clause.allowEmpty); - break; - } - expr.setASTNode(clause.ast); - if (clause.type == FLWORClause.ClauseType.FOR || clause.type == FLWORClause.ClauseType.LET - || clause.type == FLWORClause.ClauseType.WINDOW) { - final BindingExpression bind = (BindingExpression)expr; - bind.setVariable(clause.varName); - bind.setSequenceType(clause.sequenceType); - bind.setInputSequence(clause.inputSequence); - if (clause.type == FLWORClause.ClauseType.FOR) { - ((ForExpr) bind).setPositionalVariable(clause.posVar); - } - } else if (clause.type == FLWORClause.ClauseType.GROUPBY) { - if (clause.groupSpecs != null) { - GroupSpec specs[] = new GroupSpec[clause.groupSpecs.size()]; - int k = 0; - for (GroupSpec groupSpec : clause.groupSpecs) { - specs[k++]= groupSpec; - } - ((GroupByClause)expr).setGroupSpecs(specs); - } - } - if (!(action instanceof FLWORClause)) - expr.setReturnExpression(new DebuggableExpression(action)); - else { - expr.setReturnExpression(action); - ((FLWORClause)action).setPreviousClause(expr); - } - - action= expr; - } - - path.add(action); - step = action; - } - ) - | - // instance of: - #( - "instance" - { - PathExpr expr = new PathExpr(context); - expr.setASTNode(expr_AST_in); - SequenceType type= new SequenceType(); - } - step=expr [expr] - sequenceType [type] - { - step = new InstanceOfExpression(context, expr, type); - step.setASTNode(expr_AST_in); - path.add(step); - } - ) - | - // treat as: - #( - "treat" - { - PathExpr expr = new PathExpr(context); - expr.setASTNode(expr_AST_in); - SequenceType type= new SequenceType(); - } - step=expr [expr] - sequenceType [type] - { - step = new TreatAsExpression(context, expr, type); - step.setASTNode(expr_AST_in); - path.add(step); - } - ) - | - // switch - #( - switchAST:"switch" - { - PathExpr operand = new PathExpr(context); - operand.setASTNode(expr_AST_in); - } - step=expr [operand] - { - SwitchExpression switchExpr = new SwitchExpression(context, operand); - switchExpr.setASTNode(switchAST); - path.add(switchExpr); - } - ( - { - List caseOperands = new ArrayList(2); - PathExpr returnExpr = new PathExpr(context); - returnExpr.setASTNode(expr_AST_in); - } - (( - { - PathExpr caseOperand = new PathExpr(context); - caseOperand.setASTNode(expr_AST_in); - } - "case" - expr [caseOperand] - { caseOperands.add(caseOperand); } - )+ - #( - "return" - step= expr [returnExpr] - { switchExpr.addCase(caseOperands, returnExpr); } - )) - )+ - ( - "default" - { - PathExpr returnExpr = new PathExpr(context); - returnExpr.setASTNode(expr_AST_in); - } - step=expr [returnExpr] - { - switchExpr.setDefault(returnExpr); - } - ) - { step = switchExpr; } - ) - | - // typeswitch - #( - "typeswitch" - { - PathExpr operand = new PathExpr(context); - operand.setASTNode(expr_AST_in); - } - step=expr [operand] - { - TypeswitchExpression tswitch = new TypeswitchExpression(context, operand); - tswitch.setASTNode(expr_AST_in); - path.add(tswitch); - } - ( - { - PathExpr returnExpr = new PathExpr(context); - returnExpr.setASTNode(expr_AST_in); - QName qn = null; - List types = new ArrayList(2); - SequenceType type = new SequenceType(); - } - #( - "case" - ( - var:VARIABLE_BINDING - { - try { - qn = QName.parse(staticContext, var.getText()); - } catch (final IllegalQNameException iqe) { - throw new XPathException(var.getLine(), var.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + var.getText()); - } - } - )? - ( - sequenceType[type] - { - types.add(type); - type = new SequenceType(); - } - )+ - // Need return as root in following to disambiguate - // e.g. ( case a xs:integer ( * 3 3 ) ) - // which gives xs:integer* and no operator left for 3 3 ... - // Now ( case a xs:integer ( return ( + 3 3 ) ) ) /ljo - #( - "return" - step= expr [returnExpr] - { - SequenceType[] atype = new SequenceType[types.size()]; - atype = types.toArray(atype); - tswitch.addCase(atype, qn, returnExpr); - } - ) - ) - - )+ - ( - "default" - { - PathExpr returnExpr = new PathExpr(context); - returnExpr.setASTNode(expr_AST_in); - QName qn = null; - } - ( - dvar:VARIABLE_BINDING + | + #( + co:"count" + countVarName:VARIABLE_BINDING { + ForLetClause clause = new ForLetClause(); + clause.ast = co; try { - qn = QName.parse(staticContext, dvar.getText()); + clause.varName = QName.parse(staticContext, countVarName.getText(), null); } catch (final IllegalQNameException iqe) { - throw new XPathException(dvar.getLine(), dvar.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + dvar.getText()); + throw new XPathException(countVarName.getLine(), countVarName.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + countVarName.getText()); } + clause.type = FLWORClause.ClauseType.COUNT; + clause.inputSequence = null; + clauses.add(clause); } - )? - step=expr [returnExpr] - { - tswitch.setDefault(qn, returnExpr); + ) + )+ + step=expr [(PathExpr) action] + { + for (int i= clauses.size() - 1; i >= 0; i--) { + ForLetClause clause= (ForLetClause) clauses.get(i); + FLWORClause expr; + switch (clause.type) { + case LET: + expr = new LetExpr(context); + expr.setASTNode(exprFlowControl_AST_in); + break; + case GROUPBY: + expr = new GroupByClause(context); + break; + case ORDERBY: + expr = new OrderByClause(context, clause.orderSpecs); + break; + case WHERE: + expr = new WhereClause(context, new DebuggableExpression(clause.inputSequence)); + break; + case WHILE: + expr = new WhileClause(context, new DebuggableExpression(clause.inputSequence)); + break; + case COUNT: + expr = new CountClause(context, clause.varName); + break; + case WINDOW: + expr = new WindowExpr(context, clause.windowType, clause.windowConditions.get(0), clause.windowConditions.size() > 1 ? clause.windowConditions.get(1) : null); + break; + case FOR_MEMBER: + expr = new ForMemberExpr(context); + break; + case FOR_KEY: + expr = new ForKeyValueExpr(context, FLWORClause.ClauseType.FOR_KEY); + break; + case FOR_VALUE: + expr = new ForKeyValueExpr(context, FLWORClause.ClauseType.FOR_VALUE); + break; + case FOR_KEY_VALUE: + expr = new ForKeyValueExpr(context, FLWORClause.ClauseType.FOR_KEY_VALUE); + break; + case LET_SEQ_DESTRUCTURE: + case LET_ARRAY_DESTRUCTURE: + case LET_MAP_DESTRUCTURE: + { + LetDestructureExpr.DestructureMode dmode; + if (clause.type == FLWORClause.ClauseType.LET_SEQ_DESTRUCTURE) { + dmode = LetDestructureExpr.DestructureMode.SEQUENCE; + } else if (clause.type == FLWORClause.ClauseType.LET_ARRAY_DESTRUCTURE) { + dmode = LetDestructureExpr.DestructureMode.ARRAY; + } else { + dmode = LetDestructureExpr.DestructureMode.MAP; + } + LetDestructureExpr dexpr = new LetDestructureExpr(context, dmode); + dexpr.setASTNode(clause.ast); + for (int j = 0; j < clause.destructureVarNames.size(); j++) { + dexpr.addVariable( + (QName) clause.destructureVarNames.get(j), + clause.destructureVarTypes.size() > j ? + (SequenceType) clause.destructureVarTypes.get(j) : null); + } + dexpr.setInputSequence(clause.inputSequence); + if (clause.sequenceType != null) { + dexpr.setOverallType(clause.sequenceType); + } + expr = dexpr; + break; + } + default: + expr = new ForExpr(context, clause.allowEmpty); + break; + } + expr.setASTNode(clause.ast); + if (clause.type == FLWORClause.ClauseType.FOR || clause.type == FLWORClause.ClauseType.LET + || clause.type == FLWORClause.ClauseType.WINDOW + || clause.type == FLWORClause.ClauseType.FOR_MEMBER + || clause.type == FLWORClause.ClauseType.FOR_KEY + || clause.type == FLWORClause.ClauseType.FOR_VALUE + || clause.type == FLWORClause.ClauseType.FOR_KEY_VALUE) { + final BindingExpression bind = (BindingExpression)expr; + bind.setVariable(clause.varName); + bind.setSequenceType(clause.sequenceType); + bind.setInputSequence(clause.inputSequence); + if (clause.type == FLWORClause.ClauseType.FOR) { + ((ForExpr) bind).setPositionalVariable(clause.posVar); + } else if (clause.type == FLWORClause.ClauseType.FOR_MEMBER) { + ((ForMemberExpr) bind).setPositionalVariable(clause.posVar); + } else if (clause.type == FLWORClause.ClauseType.FOR_KEY + || clause.type == FLWORClause.ClauseType.FOR_VALUE + || clause.type == FLWORClause.ClauseType.FOR_KEY_VALUE) { + ((ForKeyValueExpr) bind).setPositionalVariable(clause.posVar); + if (clause.valueVarName != null) { + ((ForKeyValueExpr) bind).setValueVariable(clause.valueVarName); + if (clause.valueSequenceType != null) { + ((ForKeyValueExpr) bind).setValueSequenceType(clause.valueSequenceType); + } + } + } + } else if (clause.type == FLWORClause.ClauseType.GROUPBY) { + if (clause.groupSpecs != null) { + GroupSpec specs[] = new GroupSpec[clause.groupSpecs.size()]; + int k = 0; + for (GroupSpec groupSpec : clause.groupSpecs) { + specs[k++]= groupSpec; + } + ((GroupByClause)expr).setGroupSpecs(specs); } - ) - { step = tswitch; } - ) - | - // logical operator: or - #( - "or" - { - PathExpr left= new PathExpr(context); - left.setASTNode(expr_AST_in); - } - step=expr [left] - { - PathExpr right= new PathExpr(context); - right.setASTNode(expr_AST_in); - } - step=expr [right] - ) - { - OpOr or= new OpOr(context); - or.addPath(left); - or.addPath(right); - path.addPath(or); - step = or; - } - | - // logical operator: and - #( - "and" - { - PathExpr left= new PathExpr(context); - left.setASTNode(expr_AST_in); - - PathExpr right= new PathExpr(context); - right.setASTNode(expr_AST_in); - } - step=expr [left] - step=expr [right] - ) - { - OpAnd and= new OpAnd(context); - and.addPath(left); - and.addPath(right); - path.addPath(and); - step = and; - } - | - // union expressions: | and union - #( - UNION - { - PathExpr left= new PathExpr(context); - left.setASTNode(expr_AST_in); - - PathExpr right= new PathExpr(context); - right.setASTNode(expr_AST_in); + } + if (!(action instanceof FLWORClause)) + expr.setReturnExpression(new DebuggableExpression(action)); + else { + expr.setReturnExpression(action); + ((FLWORClause)action).setPreviousClause(expr); } - step=expr [left] - step=expr [right] - ) - { - Union union= new Union(context, left, right); - path.add(union); - step = union; - } - | - // intersections: - #( "intersect" - { - PathExpr left = new PathExpr(context); - left.setASTNode(expr_AST_in); - PathExpr right = new PathExpr(context); - right.setASTNode(expr_AST_in); - } - step=expr [left] - step=expr [right] - ) - { - Intersect intersect = new Intersect(context, left, right); - path.add(intersect); - step = intersect; - } - | - #( "except" - { - PathExpr left = new PathExpr(context); - left.setASTNode(expr_AST_in); + action= expr; + } - PathExpr right = new PathExpr(context); - right.setASTNode(expr_AST_in); - } - step=expr [left] - step=expr [right] - ) - { - Except intersect = new Except(context, left, right); - path.add(intersect); - step = intersect; - } - | - // absolute path expression starting with a / - #( - ABSOLUTE_SLASH - { - RootNode root= new RootNode(context); - path.add(root); - } - ( step=expr [path] )? - ) - | - // absolute path expression starting with // - #( - ABSOLUTE_DSLASH - { - RootNode root= new RootNode(context); - path.add(root); + path.add(action); + step = action; } - ( - step=expr [path] - { - if (step instanceof LocationStep) { - LocationStep s= (LocationStep) step; - if (s.getAxis() == Constants.ATTRIBUTE_AXIS || - (s.getTest().getType() == Type.ATTRIBUTE && s.getAxis() == Constants.CHILD_AXIS)) - // combines descendant-or-self::node()/attribute:* - s.setAxis(Constants.DESCENDANT_ATTRIBUTE_AXIS); - else { - s.setAxis(Constants.DESCENDANT_SELF_AXIS); - s.setAbbreviated(true); - } - } else - step.setPrimaryAxis(Constants.DESCENDANT_SELF_AXIS); - } - )? ) | - // range expression: to + // instance of: #( - "to" - { - PathExpr start= new PathExpr(context); - start.setASTNode(expr_AST_in); - - PathExpr end= new PathExpr(context); - end.setASTNode(expr_AST_in); - - List args= new ArrayList(2); - args.add(start); - args.add(end); - } - step=expr [start] - step=expr [end] + "instance" { - RangeExpression range= new RangeExpression(context); - range.setASTNode(expr_AST_in); - range.setArguments(args); - path.addPath(range); - step = range; + PathExpr expr = new PathExpr(context); + expr.setASTNode(exprFlowControl_AST_in); + SequenceType type= new SequenceType(); } - ) - | - step=generalComp [path] - | - step=valueComp [path] - | - step=nodeComp [path] - | - step=primaryExpr [path] - | - step=pathExpr [path] - | - step=extensionExpr [path] - | - step=numericExpr [path] - | - step=updateExpr [path] + step=expr [expr] + sequenceType [type] + { + step = new InstanceOfExpression(context, expr, type); + step.setASTNode(exprFlowControl_AST_in); + path.add(step); + } + ) ; /** @@ -2495,14 +3126,63 @@ throws PermissionDeniedException, EXistException, XPathException step=postfixExpr [step] { path.add(step); } | + ql:QNAME_LITERAL + { + final String qlText = ql.getText(); + final QName qlQName; + try { + qlQName = QName.parse(staticContext, qlText); + } catch (final IllegalQNameException iqe) { + throw new XPathException(ql.getLine(), ql.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + qlText); + } + step = new LiteralValue(context, new QNameValue(context, qlQName)); + step.setASTNode(ql); + } + step=postfixExpr [step] + { path.add(step); } + | step=inlineFunctionDecl [path] step=postfixExpr [step] { path.add(step); } | + step=focusFunctionDecl [path] + step=postfixExpr [step] + { path.add(step); } + | step = lookup [null] step=postfixExpr [step] { path.add(step); } | + #( + stAST:STRING_TEMPLATE + { + StringConstructor st = new StringConstructor(context); + st.setASTNode(stAST); + } + ( + stContent:STRING_TEMPLATE_CONTENT + { + // Unescape {{ -> {, }} -> }, `` -> ` + String raw = stContent.getText(); + raw = raw.replace("{{", "{").replace("}}", "}").replace("``", "`"); + st.addContent(raw); + } + | + { + PathExpr stInterpolation = new PathExpr(context); + stInterpolation.setASTNode(primaryExpr_AST_in); + } + expr[stInterpolation] + { + st.addInterpolation(stInterpolation.simplify()); + } + )* + { + path.add(st); + step = st; + } + ) + | #( scAST:STRING_CONSTRUCTOR_START { @@ -3024,21 +3704,30 @@ throws XPathException | i:INTEGER_LITERAL { - step= new LiteralValue(context, new IntegerValue(i.getText())); + String itext = i.getText().replace("_", ""); + java.math.BigInteger intVal; + if (itext.startsWith("0x") || itext.startsWith("0X")) { + intVal = new java.math.BigInteger(itext.substring(2), 16); + } else if (itext.startsWith("0b") || itext.startsWith("0B")) { + intVal = new java.math.BigInteger(itext.substring(2), 2); + } else { + intVal = new java.math.BigInteger(itext); + } + step= new LiteralValue(context, new IntegerValue(intVal)); step.setASTNode(i); } | ( dec:DECIMAL_LITERAL { - step= new LiteralValue(context, new DecimalValue(dec.getText())); + step= new LiteralValue(context, new DecimalValue(dec.getText().replace("_", ""))); step.setASTNode(dec); } | dbl:DOUBLE_LITERAL { step= new LiteralValue(context, - new DoubleValue(Double.parseDouble(dbl.getText()))); + new DoubleValue(Double.parseDouble(dbl.getText().replace("_", "")))); step.setASTNode(dbl); } ) @@ -3137,6 +3826,19 @@ throws PermissionDeniedException, EXistException, XPathException ( step = lookup [step] | + #( + fam:FILTER_AM + { + PathExpr filterPred = new PathExpr(context); + filterPred.setASTNode(postfixExpr_AST_in); + } + expr [filterPred] + { + step = new FilterExprAM(context, step, filterPred.simplify()); + step.setASTNode(fam); + } + ) + | #( PREDICATE { @@ -3212,6 +3914,55 @@ throws PermissionDeniedException, EXistException, XPathException ( pos:INTEGER_VALUE { position = Integer.parseInt(pos.getText()); } | + // XQ4: string literal as key selector (?"first value") + strKey:STRING_LITERAL + { + lookupExpr.add(new LiteralValue(context, new StringValue(strKey.getText()))); + } + | + // XQ4: decimal literal as key selector (?1.2) + decKey:DECIMAL_LITERAL + { + lookupExpr.add(new LiteralValue(context, new DecimalValue(decKey.getText().replace("_", "")))); + } + | + // XQ4: double literal as key selector (?1.2e0) + dblKey:DOUBLE_LITERAL + { + lookupExpr.add(new LiteralValue(context, new DoubleValue(Double.parseDouble(dblKey.getText().replace("_", ""))))); + } + | + // XQ4: variable reference as key selector (?$var) + varKey:VARIABLE_REF + { + final QName varQn; + try { + varQn = QName.parse(staticContext, varKey.getText(), null); + } catch (final IllegalQNameException iqe) { + throw new XPathException(varKey.getLine(), varKey.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + varKey.getText()); + } + lookupExpr.add(new VariableReference(context, varQn)); + } + | + // XQ4: context item as key selector (?.) + ctxKey:SELF + { + lookupExpr.add(new ContextItemExpression(context)); + } + | + // XQ4: QName literal as key selector (?#name) + qnKey:QNAME_LITERAL + { + final String qnText = qnKey.getText(); + final QName qnQName; + try { + qnQName = QName.parse(staticContext, qnText); + } catch (final IllegalQNameException iqe) { + throw new XPathException(qnKey.getLine(), qnKey.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + qnText); + } + lookupExpr.add(new LiteralValue(context, new QNameValue(context, qnQName))); + } + | ( expr [lookupExpr] )+ )? { @@ -3254,6 +4005,27 @@ throws PermissionDeniedException, EXistException, XPathException isPartial = true; } | + #( + kw:KEYWORD_ARG + ( + QUESTION { + // Keyword argument with placeholder value: name := ? + params.add(new KeywordArgumentExpression(context, kw.getText(), + new Function.Placeholder(context))); + isPartial = true; + } + | + { + PathExpr kwExpr = new PathExpr(context); + kwExpr.setASTNode(functionCall_AST_in); + } + expr [kwExpr] + { + params.add(new KeywordArgumentExpression(context, kw.getText(), kwExpr)); + } + ) + ) + | expr [pathExpr] { params.add(pathExpr); } ) )* @@ -3288,7 +4060,7 @@ throws PermissionDeniedException, EXistException, XPathException } catch (final IllegalQNameException iqe) { throw new XPathException(name.getLine(), name.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + name.getText()); } - NamedFunctionReference ref = new NamedFunctionReference(context, qname, Integer.parseInt(arity.getText())); + NamedFunctionReference ref = new NamedFunctionReference(context, qname, Integer.parseInt(arity.getText().replace("_", ""))); step = ref; } ) @@ -3321,6 +4093,14 @@ throws PermissionDeniedException, EXistException "ancestor" { axis= Constants.ANCESTOR_AXIS; } | "ancestor-or-self" { axis= Constants.ANCESTOR_SELF_AXIS; } + | + "following-or-self" { axis= Constants.FOLLOWING_OR_SELF_AXIS; } + | + "preceding-or-self" { axis= Constants.PRECEDING_OR_SELF_AXIS; } + | + "following-sibling-or-self" { axis= Constants.FOLLOWING_SIBLING_OR_SELF_AXIS; } + | + "preceding-sibling-or-self" { axis= Constants.PRECEDING_SIBLING_OR_SELF_AXIS; } ; valueComp [PathExpr path] @@ -3818,6 +4598,140 @@ throws PermissionDeniedException, EXistException, XPathException ) ; +mappingArrowOp [PathExpr path] +returns [Expression step] +throws PermissionDeniedException, EXistException, XPathException +{ + step= null; +}: + #( + mapArrowAST:MAPPING_ARROW_OP + { + PathExpr leftExpr = new PathExpr(context); + leftExpr.setASTNode(mappingArrowOp_AST_in); + } + expr [leftExpr] + { + MappingArrowOperator op = new MappingArrowOperator(context, leftExpr.simplify()); + op.setASTNode(mapArrowAST); + path.add(op); + step = op; + + PathExpr nameExpr = new PathExpr(context); + nameExpr.setASTNode(mappingArrowOp_AST_in); + String name = null; + } + ( + eq:EQNAME + { name = eq.toString(); } + | + expr [nameExpr] + ) + { List params = new ArrayList(5); } + ( + { + PathExpr pathExpr = new PathExpr(context); + pathExpr.setASTNode(mappingArrowOp_AST_in); + } + expr [pathExpr] { params.add(pathExpr.simplify()); } + )* + { + if (name == null) { + op.setArrowFunction(nameExpr, params); + } else { + op.setArrowFunction(name, params); + } + } + ) + ; + +pipelineOp [PathExpr path] +returns [Expression step] +throws PermissionDeniedException, EXistException, XPathException +{ + step = null; +}: + #( + pipeAST:PIPELINE_OP + { + PathExpr leftExpr = new PathExpr(context); + leftExpr.setASTNode(pipelineOp_AST_in); + } + expr [leftExpr] + { + PathExpr rightExpr = new PathExpr(context); + rightExpr.setASTNode(pipelineOp_AST_in); + } + expr [rightExpr] + { + step = new PipelineExpression(context, leftExpr.simplify(), rightExpr.simplify()); + step.setASTNode(pipeAST); + path.add(step); + } + ) + ; + +methodCallOp [PathExpr path] +returns [Expression step] +throws PermissionDeniedException, EXistException, XPathException +{ + step = null; +}: + #( + mcAST:METHOD_CALL_OP + { + PathExpr leftExpr = new PathExpr(context); + leftExpr.setASTNode(methodCallOp_AST_in); + } + expr [leftExpr] + mn:NCNAME + { + MethodCallOperator op = new MethodCallOperator(context, leftExpr.simplify()); + op.setASTNode(mcAST); + path.add(op); + step = op; + + List params = new ArrayList(5); + } + ( + { + PathExpr pathExpr = new PathExpr(context); + pathExpr.setASTNode(methodCallOp_AST_in); + } + expr [pathExpr] { params.add(pathExpr.simplify()); } + )* + { + op.setMethod(mn.getText(), params); + } + ) + ; + +otherwiseExpr [PathExpr path] +returns [Expression step] +throws PermissionDeniedException, EXistException, XPathException +{ + step = null; +}: + #( + owAST:LITERAL_otherwise + { + PathExpr leftExpr = new PathExpr(context); + leftExpr.setASTNode(otherwiseExpr_AST_in); + } + expr [leftExpr] + { + PathExpr rightExpr = new PathExpr(context); + rightExpr.setASTNode(otherwiseExpr_AST_in); + } + expr [rightExpr] + { + step = new OtherwiseExpression(context, leftExpr.simplify(), rightExpr.simplify()); + step.setASTNode(owAST); + path.add(step); + } + ) + ; + typeCastExpr [PathExpr path] returns [Expression step] throws PermissionDeniedException, EXistException, XPathException @@ -3832,25 +4746,72 @@ throws PermissionDeniedException, EXistException, XPathException Cardinality cardinality= Cardinality.EXACTLY_ONE; } step=expr [expr] - t:ATOMIC_TYPE ( - QUESTION - { cardinality= Cardinality.ZERO_OR_ONE; } - )? - { - try { - QName qn= QName.parse(staticContext, t.getText()); - int code= Type.getType(qn); - CastExpression castExpr= new CastExpression(context, expr, code, cardinality); + #( + CHOICE_TYPE + { + List choiceTypes = new ArrayList(); + } + ( + ct:ATOMIC_TYPE + { + try { + QName qn = QName.parse(staticContext, ct.getText()); + choiceTypes.add(Type.getType(qn)); + } catch (final XPathException e) { + throw new XPathException(ct.getLine(), ct.getColumn(), ErrorCodes.XPST0051, "Unknown simple type " + ct.getText()); + } catch (final IllegalQNameException e) { + throw new XPathException(ct.getLine(), ct.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + ct.getText()); + } + } + )+ + ) + ( + QUESTION + { cardinality= Cardinality.ZERO_OR_ONE; } + )? + { + int[] types = new int[choiceTypes.size()]; + for (int ci = 0; ci < choiceTypes.size(); ci++) { types[ci] = choiceTypes.get(ci); } + ChoiceCastExpression castExpr = new ChoiceCastExpression(context, expr, types, cardinality); castExpr.setASTNode(castAST); path.add(castExpr); step = castExpr; - } catch (final XPathException e) { - throw new XPathException(t.getLine(), t.getColumn(), ErrorCodes.XPST0051, "Unknown simple type " + t.getText()); - } catch (final IllegalQNameException e) { - throw new XPathException(t.getLine(), t.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + t.getText()); } - } + | + t:ATOMIC_TYPE + ( + QUESTION + { cardinality= Cardinality.ZERO_OR_ONE; } + )? + { + try { + QName qn= QName.parse(staticContext, t.getText()); + int code= Type.getType(qn); + CastExpression castExpr= new CastExpression(context, expr, code, cardinality); + castExpr.setASTNode(castAST); + path.add(castExpr); + step = castExpr; + } catch (final XPathException e) { + throw new XPathException(t.getLine(), t.getColumn(), ErrorCodes.XPST0051, "Unknown simple type " + t.getText()); + } catch (final IllegalQNameException e) { + throw new XPathException(t.getLine(), t.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + t.getText()); + } + } + | + enumCast:ENUM_TYPE + ( + QUESTION + { cardinality= Cardinality.ZERO_OR_ONE; } + )? + { + String[] enumVals = enumCast.getText().split(",", -1); + EnumCastExpression enumCastExpr = new EnumCastExpression(context, expr, enumVals, cardinality, false); + enumCastExpr.setASTNode(castAST); + path.add(enumCastExpr); + step = enumCastExpr; + } + ) ) | #( @@ -3861,25 +4822,72 @@ throws PermissionDeniedException, EXistException, XPathException Cardinality cardinality= Cardinality.EXACTLY_ONE; } step=expr [expr] - t2:ATOMIC_TYPE ( - QUESTION - { cardinality= Cardinality.ZERO_OR_ONE; } - )? - { - try { - QName qn= QName.parse(staticContext, t2.getText()); - int code= Type.getType(qn); - CastableExpression castExpr= new CastableExpression(context, expr, code, cardinality); - castExpr.setASTNode(castAST); + #( + CHOICE_TYPE + { + List choiceTypes2 = new ArrayList(); + } + ( + ct2:ATOMIC_TYPE + { + try { + QName qn = QName.parse(staticContext, ct2.getText()); + choiceTypes2.add(Type.getType(qn)); + } catch (final XPathException e) { + throw new XPathException(ct2.getLine(), ct2.getColumn(), ErrorCodes.XPST0051, "Unknown simple type " + ct2.getText()); + } catch (final IllegalQNameException e) { + throw new XPathException(ct2.getLine(), ct2.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + ct2.getText()); + } + } + )+ + ) + ( + QUESTION + { cardinality= Cardinality.ZERO_OR_ONE; } + )? + { + int[] types2 = new int[choiceTypes2.size()]; + for (int ci = 0; ci < choiceTypes2.size(); ci++) { types2[ci] = choiceTypes2.get(ci); } + ChoiceCastableExpression castExpr = new ChoiceCastableExpression(context, expr, types2, cardinality); + castExpr.setASTNode(castableAST); path.add(castExpr); step = castExpr; - } catch (final XPathException e) { - throw new XPathException(t2.getLine(), t2.getColumn(), ErrorCodes.XPST0051, "Unknown simple type " + t2.getText()); - } catch (final IllegalQNameException e) { - throw new XPathException(t2.getLine(), t2.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + t2.getText()); } - } + | + t2:ATOMIC_TYPE + ( + QUESTION + { cardinality= Cardinality.ZERO_OR_ONE; } + )? + { + try { + QName qn= QName.parse(staticContext, t2.getText()); + int code= Type.getType(qn); + CastableExpression castExpr= new CastableExpression(context, expr, code, cardinality); + castExpr.setASTNode(castableAST); + path.add(castExpr); + step = castExpr; + } catch (final XPathException e) { + throw new XPathException(t2.getLine(), t2.getColumn(), ErrorCodes.XPST0051, "Unknown simple type " + t2.getText()); + } catch (final IllegalQNameException e) { + throw new XPathException(t2.getLine(), t2.getColumn(), ErrorCodes.XPST0081, "No namespace defined for prefix " + t2.getText()); + } + } + | + enumCastable:ENUM_TYPE + ( + QUESTION + { cardinality= Cardinality.ZERO_OR_ONE; } + )? + { + String[] enumVals2 = enumCastable.getText().split(",", -1); + EnumCastExpression enumCastExpr2 = new EnumCastExpression(context, expr, enumVals2, cardinality, true); + enumCastExpr2.setASTNode(castableAST); + path.add(enumCastExpr2); + step = enumCastExpr2; + } + ) ) ; diff --git a/exist-core/src/test/java/org/exist/xquery/ReservedKeywordsAsNCNamesTest.java b/exist-core/src/test/java/org/exist/xquery/ReservedKeywordsAsNCNamesTest.java new file mode 100644 index 00000000000..1d693030c1c --- /dev/null +++ b/exist-core/src/test/java/org/exist/xquery/ReservedKeywordsAsNCNamesTest.java @@ -0,0 +1,115 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.test.ExistXmldbEmbeddedServer; +import org.junit.ClassRule; +import org.junit.Test; +import org.xmldb.api.base.ResourceSet; +import org.xmldb.api.base.XMLDBException; +import org.xmldb.api.modules.XQueryService; + +import static org.junit.Assert.assertEquals; + +/** + * Verifies that XQuery keywords can be used as variable names (NCNames). + * + * The reservedKeywords grammar rule lists keywords that are also valid NCNames. + * Keywords used in grammar rules but missing from this list cause XPST0003 + * parse errors when used as variable names. + */ +public class ReservedKeywordsAsNCNamesTest { + + @ClassRule + public static final ExistXmldbEmbeddedServer existEmbeddedServer = + new ExistXmldbEmbeddedServer(false, true, true); + + private ResourceSet execute(final String xquery) throws XMLDBException { + final XQueryService xqs = existEmbeddedServer.getRoot().getService(XQueryService.class); + return xqs.query(xquery); + } + + @Test + public void ascendingDescendingAsVariableNames() throws XMLDBException { + final ResourceSet result = execute( + "let $ascending := 1, $descending := 2 return $ascending + $descending"); + assertEquals("3", result.getResource(0).getContent().toString()); + } + + @Test + public void greatestLeastAsVariableNames() throws XMLDBException { + final ResourceSet result = execute( + "let $greatest := 10, $least := 1 return $greatest - $least"); + assertEquals("9", result.getResource(0).getContent().toString()); + } + + @Test + public void satisfiesAsVariableName() throws XMLDBException { + final ResourceSet result = execute( + "let $satisfies := 'ok' return $satisfies"); + assertEquals("ok", result.getResource(0).getContent().toString()); + } + + @Test + public void castableAsVariableName() throws XMLDBException { + final ResourceSet result = execute( + "let $castable := true() return $castable"); + assertEquals("true", result.getResource(0).getContent().toString()); + } + + @Test + public void idivAsVariableName() throws XMLDBException { + final ResourceSet result = execute( + "let $idiv := 42 return $idiv"); + assertEquals("42", result.getResource(0).getContent().toString()); + } + + @Test + public void processingInstructionAsVariableName() throws XMLDBException { + final ResourceSet result = execute( + "let $processing-instruction := 'test' return $processing-instruction"); + assertEquals("test", result.getResource(0).getContent().toString()); + } + + @Test + public void schemaAttributeAsVariableName() throws XMLDBException { + final ResourceSet result = execute( + "let $schema-attribute := 'sa' return $schema-attribute"); + assertEquals("sa", result.getResource(0).getContent().toString()); + } + + @Test + public void allowingAsVariableName() throws XMLDBException { + final ResourceSet result = execute( + "let $allowing := 'yes' return $allowing"); + assertEquals("yes", result.getResource(0).getContent().toString()); + } + + @Test + public void allKeywordsTogether() throws XMLDBException { + final ResourceSet result = execute( + "let $ascending := 1, $descending := 2, $greatest := 3, $least := 4,\n" + + " $satisfies := 5, $castable := 6, $idiv := 7\n" + + "return $ascending + $descending + $greatest + $least + $satisfies + $castable + $idiv"); + assertEquals("28", result.getResource(0).getContent().toString()); + } +} From 2916f59de33b0de1e460eef6fddd3a7df3757b29 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Sat, 4 Apr 2026 09:52:12 -0400 Subject: [PATCH 006/326] [feature] Add XQuery 4.0 expression classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New expression classes that implement the evaluation logic for XQ4 syntax features parsed by the grammar: - PipelineExpression: evaluate left expr, bind to context for right - FocusFunction: anonymous function with implicit context item (.) - KeywordArgumentExpression: wraps named args for function dispatch - MappingArrowOperator: => operator (function application) - MethodCallOperator: =?> operator (method-style dispatch) - OtherwiseExpression: return left if non-empty, else right - FilterExprAM: ?[predicate] — array/map member filter - ForMemberExpr: for member $x in $array — iterate array members - ForKeyValueExpr: for key/value pair iteration - WhileClause: while (cond) in FLWOR — conditional iteration - LetDestructureExpr: let destructuring bindings - StringConstructor: XQ4 string template interpolation - ChoiceCastExpression/ChoiceCastableExpression: union/choice type casts - EnumCastExpression: enumeration type casts Each class extends Expression/AbstractExpression and implements eval() with proper context handling and dependency tracking. Spec: QT4 XQuery 4.0 §3 (Expressions) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../exist/xquery/ChoiceCastExpression.java | 137 +++++++ .../xquery/ChoiceCastableExpression.java | 128 +++++++ .../org/exist/xquery/EnumCastExpression.java | 141 ++++++++ .../java/org/exist/xquery/FilterExprAM.java | 242 +++++++++++++ .../java/org/exist/xquery/FocusFunction.java | 140 ++++++++ .../org/exist/xquery/ForKeyValueExpr.java | 308 ++++++++++++++++ .../java/org/exist/xquery/ForMemberExpr.java | 239 +++++++++++++ .../xquery/KeywordArgumentExpression.java | 85 +++++ .../org/exist/xquery/LetDestructureExpr.java | 335 ++++++++++++++++++ .../exist/xquery/MappingArrowOperator.java | 207 +++++++++++ .../org/exist/xquery/MethodCallOperator.java | 211 +++++++++++ .../org/exist/xquery/OtherwiseExpression.java | 90 +++++ .../org/exist/xquery/PipelineExpression.java | 106 ++++++ .../java/org/exist/xquery/WhileClause.java | 136 +++++++ 14 files changed, 2505 insertions(+) create mode 100644 exist-core/src/main/java/org/exist/xquery/ChoiceCastExpression.java create mode 100644 exist-core/src/main/java/org/exist/xquery/ChoiceCastableExpression.java create mode 100644 exist-core/src/main/java/org/exist/xquery/EnumCastExpression.java create mode 100644 exist-core/src/main/java/org/exist/xquery/FilterExprAM.java create mode 100644 exist-core/src/main/java/org/exist/xquery/FocusFunction.java create mode 100644 exist-core/src/main/java/org/exist/xquery/ForKeyValueExpr.java create mode 100644 exist-core/src/main/java/org/exist/xquery/ForMemberExpr.java create mode 100644 exist-core/src/main/java/org/exist/xquery/KeywordArgumentExpression.java create mode 100644 exist-core/src/main/java/org/exist/xquery/LetDestructureExpr.java create mode 100644 exist-core/src/main/java/org/exist/xquery/MappingArrowOperator.java create mode 100644 exist-core/src/main/java/org/exist/xquery/MethodCallOperator.java create mode 100644 exist-core/src/main/java/org/exist/xquery/OtherwiseExpression.java create mode 100644 exist-core/src/main/java/org/exist/xquery/PipelineExpression.java create mode 100644 exist-core/src/main/java/org/exist/xquery/WhileClause.java diff --git a/exist-core/src/main/java/org/exist/xquery/ChoiceCastExpression.java b/exist-core/src/main/java/org/exist/xquery/ChoiceCastExpression.java new file mode 100644 index 00000000000..1f58834103f --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/ChoiceCastExpression.java @@ -0,0 +1,137 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.dom.persistent.DocumentSet; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.*; + +/** + * Implements cast as (T1 | T2 | ...) from XQuery 4.0. + * Tries each target type in order and returns the first successful cast. + */ +public class ChoiceCastExpression extends AbstractExpression { + + private final int[] targetTypes; + private final Cardinality cardinality; + private Expression expression; + + public ChoiceCastExpression(final XQueryContext context, final Expression expr, + final int[] targetTypes, final Cardinality cardinality) { + super(context); + this.targetTypes = targetTypes; + this.cardinality = cardinality; + this.expression = expr; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + contextInfo.setParent(this); + expression.analyze(contextInfo); + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) throws XPathException { + final Sequence seq = Atomize.atomize(expression.eval(contextSequence, contextItem)); + if (seq.isEmpty()) { + if (cardinality.atLeastOne()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Type error: empty sequence is not allowed here"); + } + return Sequence.EMPTY_SEQUENCE; + } + if (seq.hasMany()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "cardinality error: sequence with more than one item is not allowed here"); + } + + final Item item = seq.itemAt(0); + XPathException lastError = null; + + for (final int targetType : targetTypes) { + try { + return item.convertTo(targetType); + } catch (final XPathException e) { + lastError = e; + } + } + + throw new XPathException(this, ErrorCodes.FORG0001, + "Cannot cast " + Type.getTypeName(item.getType()) + + " to any of the choice types", lastError); + } + + @Override + public int returnsType() { + return Type.ANY_ATOMIC_TYPE; + } + + @Override + public Cardinality getCardinality() { + return Cardinality.ZERO_OR_ONE; + } + + @Override + public void dump(final ExpressionDumper dumper) { + expression.dump(dumper); + dumper.display(" cast as ("); + for (int i = 0; i < targetTypes.length; i++) { + if (i > 0) { + dumper.display(" | "); + } + dumper.display(Type.getTypeName(targetTypes[i])); + } + dumper.display(")"); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append(expression.toString()).append(" cast as ("); + for (int i = 0; i < targetTypes.length; i++) { + if (i > 0) { + sb.append(" | "); + } + sb.append(Type.getTypeName(targetTypes[i])); + } + sb.append(")"); + return sb.toString(); + } + + @Override + public int getDependencies() { + return expression.getDependencies() | Dependency.CONTEXT_ITEM; + } + + @Override + public void setContextDocSet(final DocumentSet contextSet) { + super.setContextDocSet(contextSet); + expression.setContextDocSet(contextSet); + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + expression.resetState(postOptimization); + } + +} diff --git a/exist-core/src/main/java/org/exist/xquery/ChoiceCastableExpression.java b/exist-core/src/main/java/org/exist/xquery/ChoiceCastableExpression.java new file mode 100644 index 00000000000..4d867b21e44 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/ChoiceCastableExpression.java @@ -0,0 +1,128 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.dom.persistent.DocumentSet; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.*; + +/** + * Implements castable as (T1 | T2 | ...) from XQuery 4.0. + * Returns true if the value can be cast to any of the target types. + */ +public class ChoiceCastableExpression extends AbstractExpression { + + private final int[] targetTypes; + private final Cardinality requiredCardinality; + private final Expression expression; + + public ChoiceCastableExpression(final XQueryContext context, final Expression expr, + final int[] targetTypes, final Cardinality requiredCardinality) { + super(context); + this.expression = expr; + this.targetTypes = targetTypes; + this.requiredCardinality = requiredCardinality; + } + + @Override + public int returnsType() { + return Type.BOOLEAN; + } + + @Override + public Cardinality getCardinality() { + return Cardinality.EXACTLY_ONE; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + contextInfo.setParent(this); + expression.analyze(contextInfo); + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) throws XPathException { + final Sequence seq = Atomize.atomize(expression.eval(contextSequence, contextItem)); + if (seq.isEmpty()) { + return BooleanValue.valueOf( + requiredCardinality.isSuperCardinalityOrEqualOf(Cardinality.EMPTY_SEQUENCE)); + } + if (!requiredCardinality.isSuperCardinalityOrEqualOf(seq.getCardinality())) { + return BooleanValue.FALSE; + } + + final Item item = seq.itemAt(0); + for (final int targetType : targetTypes) { + try { + item.convertTo(targetType); + return BooleanValue.TRUE; + } catch (final XPathException e) { + // try next type + } + } + return BooleanValue.FALSE; + } + + @Override + public void dump(final ExpressionDumper dumper) { + expression.dump(dumper); + dumper.display(" castable as ("); + for (int i = 0; i < targetTypes.length; i++) { + if (i > 0) { + dumper.display(" | "); + } + dumper.display(Type.getTypeName(targetTypes[i])); + } + dumper.display(")"); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append(expression.toString()).append(" castable as ("); + for (int i = 0; i < targetTypes.length; i++) { + if (i > 0) { + sb.append(" | "); + } + sb.append(Type.getTypeName(targetTypes[i])); + } + sb.append(")"); + return sb.toString(); + } + + @Override + public int getDependencies() { + return Dependency.CONTEXT_SET + Dependency.CONTEXT_ITEM; + } + + @Override + public void setContextDocSet(final DocumentSet contextSet) { + super.setContextDocSet(contextSet); + expression.setContextDocSet(contextSet); + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + expression.resetState(postOptimization); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/EnumCastExpression.java b/exist-core/src/main/java/org/exist/xquery/EnumCastExpression.java new file mode 100644 index 00000000000..bf0fc6ce7b2 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/EnumCastExpression.java @@ -0,0 +1,141 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.dom.persistent.DocumentSet; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.*; + +/** + * Implements cast as enum("a","b","c") and castable as enum("a","b","c") from XQuery 4.0. + */ +public class EnumCastExpression extends AbstractExpression { + + private final String[] enumValues; + private final Cardinality cardinality; + private final Expression expression; + private final boolean isCastable; + + public EnumCastExpression(final XQueryContext context, final Expression expr, + final String[] enumValues, final Cardinality cardinality, + final boolean isCastable) { + super(context); + this.expression = expr; + this.enumValues = enumValues; + this.cardinality = cardinality; + this.isCastable = isCastable; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + contextInfo.setParent(this); + expression.analyze(contextInfo); + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) throws XPathException { + final Sequence seq = Atomize.atomize(expression.eval(contextSequence, contextItem)); + + if (seq.isEmpty()) { + if (isCastable) { + return BooleanValue.valueOf( + cardinality.isSuperCardinalityOrEqualOf(Cardinality.EMPTY_SEQUENCE)); + } + if (cardinality.atLeastOne()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Type error: empty sequence is not allowed here"); + } + return Sequence.EMPTY_SEQUENCE; + } + + final String value = seq.itemAt(0).getStringValue(); + + for (final String enumVal : enumValues) { + if (enumVal.equals(value)) { + if (isCastable) { + return BooleanValue.TRUE; + } + return new StringValue(this, value); + } + } + + if (isCastable) { + return BooleanValue.FALSE; + } + throw new XPathException(this, ErrorCodes.FORG0001, + "Cannot cast '" + value + "' to enum type"); + } + + @Override + public int returnsType() { + return isCastable ? Type.BOOLEAN : Type.STRING; + } + + @Override + public Cardinality getCardinality() { + return isCastable ? Cardinality.EXACTLY_ONE : Cardinality.ZERO_OR_ONE; + } + + @Override + public void dump(final ExpressionDumper dumper) { + expression.dump(dumper); + dumper.display(isCastable ? " castable as enum(" : " cast as enum("); + for (int i = 0; i < enumValues.length; i++) { + if (i > 0) { + dumper.display(", "); + } + dumper.display("\"" + enumValues[i] + "\""); + } + dumper.display(")"); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append(expression.toString()).append(isCastable ? " castable as enum(" : " cast as enum("); + for (int i = 0; i < enumValues.length; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append("\"").append(enumValues[i]).append("\""); + } + sb.append(")"); + return sb.toString(); + } + + @Override + public int getDependencies() { + return expression.getDependencies() | Dependency.CONTEXT_ITEM; + } + + @Override + public void setContextDocSet(final DocumentSet contextSet) { + super.setContextDocSet(contextSet); + expression.setContextDocSet(contextSet); + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + expression.resetState(postOptimization); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/FilterExprAM.java b/exist-core/src/main/java/org/exist/xquery/FilterExprAM.java new file mode 100644 index 00000000000..f07af305e12 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/FilterExprAM.java @@ -0,0 +1,242 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.xquery.functions.array.ArrayType; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.AtomicValue; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.NumericValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceIterator; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Implements the XQuery 4.0 array/map filter expression ({@code ?[predicate]}). + * + *

For arrays, iterates over members and keeps those where the predicate + * evaluates to true with the context item set to each member. + * Numeric predicates select by position (1-based).

+ * + *

For maps, iterates over entries and keeps those where the predicate + * evaluates to true with the context item set to + * {@code map { "key": key, "value": value }} for each entry. + * Numeric predicates select by position in insertion order.

+ */ +public class FilterExprAM extends AbstractExpression { + + private Expression contextExpr; + private Expression predicate; + + public FilterExprAM(final XQueryContext context, final Expression contextExpr, final Expression predicate) { + super(context); + this.contextExpr = contextExpr; + this.predicate = predicate; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + contextExpr.analyze(contextInfo); + final AnalyzeContextInfo predicateInfo = new AnalyzeContextInfo(contextInfo); + predicate.analyze(predicateInfo); + } + + @Override + public Sequence eval(Sequence contextSequence, final Item contextItem) throws XPathException { + if (contextItem != null) { + contextSequence = contextItem.toSequence(); + } + final Sequence input = contextExpr.eval(contextSequence, null); + + if (input.isEmpty()) { + return input; + } + + final Item item = input.itemAt(0); + if (Type.subTypeOf(item.getType(), Type.ARRAY_ITEM)) { + return filterArray((ArrayType) item); + } else if (Type.subTypeOf(item.getType(), Type.MAP_ITEM)) { + return filterMap((AbstractMapType) item); + } else { + throw new XPathException(this, ErrorCodes.XPTY0004, + "?[] filter requires an array or map, got " + Type.getTypeName(item.getType())); + } + } + + private ArrayType filterArray(final ArrayType array) throws XPathException { + final int size = array.getSize(); + + // Build a context sequence of all member items for position()/last() + final ValueSequence contextSeq = new ValueSequence(size); + final List members = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + final Sequence member = array.get(i); + members.add(member); + // For context sequence, we need each member as an item. + // If a member is a sequence, wrap it — but for position/last to work + // we need exactly `size` items in the context sequence. + if (member.isEmpty()) { + // Empty sequence member: use empty sequence as placeholder + contextSeq.add(AtomicValue.EMPTY_VALUE); + } else if (member.getItemCount() == 1) { + contextSeq.add(member.itemAt(0)); + } else { + // Multi-item member: use first item as representative for context + contextSeq.add(member.itemAt(0)); + } + } + + final int savedPos = context.getContextPosition(); + final Sequence savedSeq = context.getContextSequence(); + try { + final ArrayType result = new ArrayType(context, new ArrayList<>()); + for (int i = 0; i < size; i++) { + final Sequence member = members.get(i); + context.setContextSequencePosition(i, contextSeq); + + final Sequence predResult = predicate.eval(member, null); + if (isSelected(predResult, i + 1)) { + result.add(member); + } + } + return result; + } finally { + context.setContextSequencePosition(savedPos, savedSeq); + } + } + + private AbstractMapType filterMap(final AbstractMapType map) throws XPathException { + final Sequence keys = map.keys(); + final int size = keys.getItemCount(); + + // Build entry maps and context sequence for position/last + final ValueSequence contextSeq = new ValueSequence(size); + final List keyList = new ArrayList<>(size); + final List entryMaps = new ArrayList<>(size); + + for (final SequenceIterator i = keys.iterate(); i.hasNext(); ) { + final AtomicValue key = (AtomicValue) i.nextItem(); + keyList.add(key); + final Sequence value = map.get(key); + + final MapType entryMap = new MapType(context, null); + entryMap.add(new StringValue(this, "key"), key.toSequence()); + entryMap.add(new StringValue(this, "value"), value); + entryMaps.add(entryMap); + contextSeq.add(entryMap); + } + + final int savedPos = context.getContextPosition(); + final Sequence savedSeq = context.getContextSequence(); + try { + final MapType result = new MapType(context, null); + for (int i = 0; i < size; i++) { + context.setContextSequencePosition(i, contextSeq); + final AbstractMapType entryMap = entryMaps.get(i); + + final Sequence predResult = predicate.eval(entryMap.toSequence(), null); + if (isSelected(predResult, i + 1)) { + result.add(keyList.get(i), map.get(keyList.get(i))); + } + } + return result; + } finally { + context.setContextSequencePosition(savedPos, savedSeq); + } + } + + /** + * Determines whether a member/entry at the given 1-based position is selected + * by the predicate result, following XQ4 array/map filter semantics: + * - If the result is a single numeric value, select if it equals the position. + * - If the result is a multi-item all-numeric sequence, select if any value + * equals the position (XQ4 extension for ?[] filters). + * - If the result is a multi-item sequence mixing numeric and non-numeric, + * raise FORG0006. + * - Otherwise, evaluate effective boolean value. + */ + private boolean isSelected(final Sequence predResult, final int position) throws XPathException { + if (predResult.isEmpty()) { + return false; + } + + // Single numeric value: positional predicate + if (predResult.hasOne() && Type.subTypeOfUnion(predResult.itemAt(0).getType(), Type.NUMERIC)) { + final double pos = ((NumericValue) predResult.itemAt(0)).getDouble(); + return pos == position; + } + + // Multi-item sequence starting with numeric: check all items are numeric + if (predResult.getItemCount() > 1 && + Type.subTypeOfUnion(predResult.itemAt(0).getType(), Type.NUMERIC)) { + for (final SequenceIterator i = predResult.iterate(); i.hasNext(); ) { + final Item item = i.nextItem(); + if (!Type.subTypeOfUnion(item.getType(), Type.NUMERIC)) { + throw new XPathException((Expression) null, ErrorCodes.FORG0006, + "Mixed numeric and non-numeric values in filter predicate"); + } + final double pos = ((NumericValue) item).getDouble(); + if (pos == position) { + return true; + } + } + return false; + } + + // Boolean predicate + return predResult.effectiveBooleanValue(); + } + + @Override + public int returnsType() { + return Type.ITEM; + } + + @Override + public Cardinality getCardinality() { + return Cardinality.EXACTLY_ONE; + } + + @Override + public void dump(final ExpressionDumper dumper) { + contextExpr.dump(dumper); + dumper.display("?["); + predicate.dump(dumper); + dumper.display("]"); + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + contextExpr.resetState(postOptimization); + predicate.resetState(postOptimization); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/FocusFunction.java b/exist-core/src/main/java/org/exist/xquery/FocusFunction.java new file mode 100644 index 00000000000..28d930a3102 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/FocusFunction.java @@ -0,0 +1,140 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.dom.persistent.DocumentSet; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.*; + +import java.util.ArrayDeque; +import java.util.List; + +/** + * Implements XQuery 4.0 focus functions: {@code fn { expr }} and {@code function { expr }}. + * + *

A focus function is an inline function with an implicit single parameter + * of type {@code item()*}. When called, the argument is bound as the context + * item for the body expression.

+ * + *

Formally: {@code fn { EXPR }} is equivalent to + * {@code function($dot as item()*) as item()* { EXPR }} where EXPR is + * evaluated with the context value set to {@code $dot}.

+ */ +public class FocusFunction extends AbstractExpression { + + public static final String FOCUS_PARAM_NAME = ".focus"; + + private final UserDefinedFunction function; + private final ArrayDeque calls = new ArrayDeque<>(); + private AnalyzeContextInfo cachedContextInfo; + + public FocusFunction(final XQueryContext context, final UserDefinedFunction function) { + super(context); + this.function = function; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + cachedContextInfo = new AnalyzeContextInfo(contextInfo); + cachedContextInfo.addFlag(SINGLE_STEP_EXECUTION); + cachedContextInfo.setParent(this); + } + + @Override + public void dump(final ExpressionDumper dumper) { + dumper.display("fn "); + function.dump(dumper); + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) + throws XPathException { + final List closureVars = context.getLocalStack(); + + final FunctionCall call = new FocusFunctionCall(context, function); + call.getFunction().setClosureVariables(closureVars); + call.setLocation(function.getLine(), function.getColumn()); + call.analyze(new AnalyzeContextInfo(cachedContextInfo)); + + calls.push(call); + + return new FunctionReference(this, call); + } + + @Override + public int returnsType() { + return Type.FUNCTION; + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + calls.clear(); + function.resetState(postOptimization); + } + + /** + * A specialized FunctionCall that sets the argument as context item + * before evaluating the function body. + */ + public static class FocusFunctionCall extends FunctionCall { + + public FocusFunctionCall(final XQueryContext context, final UserDefinedFunction function) { + super(context, function); + } + + @Override + public Sequence evalFunction(final Sequence contextSequence, final Item contextItem, + final Sequence[] seq, final DocumentSet[] contextDocs) throws XPathException { + // The focus function's single argument becomes the context item + // for the body evaluation. + final Sequence focusArg = (seq != null && seq.length > 0) ? seq[0] : Sequence.EMPTY_SEQUENCE; + + context.stackEnter(this); + final LocalVariable mark = context.markLocalVariables(true); + if (getFunction().getClosureVariables() != null) { + context.restoreStack(getFunction().getClosureVariables()); + } + try { + // Bind the implicit parameter + final UserDefinedFunction func = getFunction(); + if (!func.getParameters().isEmpty()) { + final LocalVariable var = new LocalVariable( + func.getParameters().get(0)); + var.setValue(focusArg); + context.declareVariableBinding(var); + } + + // Evaluate the body with the argument as context + final Expression body = func.getFunctionBody(); + if (focusArg.getItemCount() == 1) { + return body.eval(focusArg, focusArg.itemAt(0)); + } else { + return body.eval(focusArg, null); + } + } finally { + context.popLocalVariables(mark); + context.stackLeave(this); + } + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/ForKeyValueExpr.java b/exist-core/src/main/java/org/exist/xquery/ForKeyValueExpr.java new file mode 100644 index 00000000000..e2956b36d3b --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/ForKeyValueExpr.java @@ -0,0 +1,308 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.dom.QName; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.value.*; + +import java.util.HashSet; +import java.util.Set; + +/** + * Implements the XQuery 4.0 "for key", "for value", and "for key/value" clauses. + * + *

{@code for key $k in map-expr} iterates over the keys of a map.

+ *

{@code for value $v in map-expr} iterates over the values of a map.

+ *

{@code for key $k value $v in map-expr} iterates over key-value pairs.

+ */ +public class ForKeyValueExpr extends BindingExpression { + + private final ClauseType clauseType; + private QName positionalVariable = null; + private QName valueVariable = null; + private SequenceType valueSequenceType = null; + + public ForKeyValueExpr(final XQueryContext context, final ClauseType clauseType) { + super(context); + this.clauseType = clauseType; + } + + public void setPositionalVariable(final QName variable) { + positionalVariable = variable; + } + + public void setValueVariable(final QName variable) { + valueVariable = variable; + } + + public void setValueSequenceType(final SequenceType type) { + valueSequenceType = type; + } + + @Override + public ClauseType getType() { + return clauseType; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + super.analyze(contextInfo); + final LocalVariable mark = context.markLocalVariables(false); + try { + contextInfo.setParent(this); + final AnalyzeContextInfo varContextInfo = new AnalyzeContextInfo(contextInfo); + inputSequence.analyze(varContextInfo); + final LocalVariable inVar = new LocalVariable(varName); + inVar.setSequenceType(sequenceType); + inVar.setStaticType(Type.ITEM); + context.declareVariableBinding(inVar); + if (valueVariable != null) { + final LocalVariable valVar = new LocalVariable(valueVariable); + valVar.setSequenceType(valueSequenceType); + valVar.setStaticType(Type.ITEM); + context.declareVariableBinding(valVar); + } + if (positionalVariable != null) { + final LocalVariable posVar = new LocalVariable(positionalVariable); + posVar.setSequenceType(POSITIONAL_VAR_TYPE); + posVar.setStaticType(Type.INTEGER); + context.declareVariableBinding(posVar); + } + + final AnalyzeContextInfo newContextInfo = new AnalyzeContextInfo(contextInfo); + newContextInfo.addFlag(SINGLE_STEP_EXECUTION); + returnExpr.analyze(newContextInfo); + } finally { + context.popLocalVariables(mark); + } + } + + @Override + public Sequence eval(Sequence contextSequence, final Item contextItem) + throws XPathException { + if (context.getProfiler().isEnabled()) { + context.getProfiler().start(this); + context.getProfiler().message(this, Profiler.DEPENDENCIES, + "DEPENDENCIES", Dependency.getDependenciesName(this.getDependencies())); + if (contextSequence != null) { + context.getProfiler().message(this, Profiler.START_SEQUENCES, + "CONTEXT SEQUENCE", contextSequence); + } + } + context.expressionStart(this); + + final LocalVariable mark = context.markLocalVariables(false); + final Sequence resultSequence = new ValueSequence(unordered); + try { + final Sequence in = inputSequence.eval(contextSequence, null); + + if (in.isEmpty()) { + // Empty map produces no iterations + } else if (in.getItemCount() != 1 || !(in.itemAt(0) instanceof AbstractMapType)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "for " + clauseLabel() + + " expression requires a single map, got " + + Type.getTypeName(in.getItemType())); + } else { + final AbstractMapType map = (AbstractMapType) in.itemAt(0); + final LocalVariable var = createVariable(varName); + var.setSequenceType(sequenceType); + context.declareVariableBinding(var); + + LocalVariable valVar = null; + if (valueVariable != null) { + valVar = new LocalVariable(valueVariable); + valVar.setSequenceType(valueSequenceType); + context.declareVariableBinding(valVar); + } + + LocalVariable at = null; + if (positionalVariable != null) { + at = new LocalVariable(positionalVariable); + at.setSequenceType(POSITIONAL_VAR_TYPE); + context.declareVariableBinding(at); + } + + final Sequence keys = map.keys(); + int pos = 0; + try { + for (final SequenceIterator i = keys.iterate(); i.hasNext() && !WhileClause.isTerminated(); ) { + context.proceed(this); + final AtomicValue key = (AtomicValue) i.nextItem(); + pos++; + + final Sequence bindValue; + if (clauseType == ClauseType.FOR_VALUE) { + bindValue = map.get(key); + } else { + // FOR_KEY or FOR_KEY_VALUE: primary var is key + bindValue = key; + } + var.setValue(bindValue); + + if (valVar != null) { + valVar.setValue(map.get(key)); + } + + if (positionalVariable != null) { + at.setValue(new IntegerValue(this, pos)); + } + if (sequenceType != null) { + var.checkType(); + } + if (valVar != null && valueSequenceType != null) { + valVar.checkType(); + } + + final Sequence returnResult; + if (returnExpr instanceof OrderByClause) { + returnResult = returnExpr.eval(bindValue, null); + } else { + returnResult = returnExpr.eval(null, null); + } + resultSequence.addAll(returnResult); + var.destroy(context, resultSequence); + } + } catch (final WhileClause.WhileTerminationException e) { + // while clause signaled end of iteration + } + if (getPreviousClause() == null && WhileClause.isTerminated()) { + WhileClause.clearTerminated(); + } + } + } finally { + context.popLocalVariables(mark, resultSequence); + } + + if (callPostEval()) { + final Sequence postResult = postEval(resultSequence); + context.expressionEnd(this); + if (context.getProfiler().isEnabled()) { + context.getProfiler().end(this, "", postResult); + } + return postResult; + } + + context.expressionEnd(this); + if (context.getProfiler().isEnabled()) { + context.getProfiler().end(this, "", resultSequence); + } + return resultSequence; + } + + private String clauseLabel() { + switch (clauseType) { + case FOR_KEY: return "key"; + case FOR_VALUE: return "value"; + case FOR_KEY_VALUE: return "key/value"; + default: return "key"; + } + } + + private boolean callPostEval() { + FLWORClause prev = getPreviousClause(); + while (prev != null) { + switch (prev.getType()) { + case LET: + case FOR: + case FOR_MEMBER: + case FOR_KEY: + case FOR_VALUE: + case FOR_KEY_VALUE: + return false; + case ORDERBY: + case GROUPBY: + return true; + default: + break; + } + prev = prev.getPreviousClause(); + } + return true; + } + + @Override + public void dump(final ExpressionDumper dumper) { + dumper.display("for " + clauseLabel() + " ", line); + dumper.startIndent(); + dumper.display("$").display(varName); + if (valueVariable != null) { + dumper.display(" value $").display(valueVariable); + } + if (sequenceType != null) { + dumper.display(" as ").display(sequenceType); + } + dumper.display(" in "); + inputSequence.dump(dumper); + dumper.endIndent().nl(); + if (returnExpr instanceof LetExpr) { + dumper.display(" ", returnExpr.getLine()); + } else { + dumper.display("return", returnExpr.getLine()); + } + dumper.startIndent(); + returnExpr.dump(dumper); + dumper.endIndent().nl(); + } + + @Override + public String toString() { + final StringBuilder result = new StringBuilder(); + result.append("for ").append(clauseLabel()).append(" "); + result.append("$").append(varName); + if (valueVariable != null) { + result.append(" value $").append(valueVariable); + } + if (sequenceType != null) { + result.append(" as ").append(sequenceType); + } + result.append(" in "); + result.append(inputSequence.toString()); + result.append(" "); + if (returnExpr instanceof LetExpr) { + result.append(" "); + } else { + result.append("return "); + } + result.append(returnExpr.toString()); + return result.toString(); + } + + @Override + public Set getTupleStreamVariables() { + final Set variables = new HashSet<>(); + final QName variable = getVariable(); + if (variable != null) { + variables.add(variable); + } + if (valueVariable != null) { + variables.add(valueVariable); + } + final LocalVariable startVar = getStartVariable(); + if (startVar != null) { + variables.add(startVar.getQName()); + } + return variables; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/ForMemberExpr.java b/exist-core/src/main/java/org/exist/xquery/ForMemberExpr.java new file mode 100644 index 00000000000..522cb213331 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/ForMemberExpr.java @@ -0,0 +1,239 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.dom.QName; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.functions.array.ArrayType; +import org.exist.xquery.value.*; + +import java.util.HashSet; +import java.util.Set; + +/** + * Implements the XQuery 4.0 "for member" clause in FLWOR expressions. + * + *

{@code for member $m in $array-expr} iterates over the members of an array, + * binding each member (which is a sequence) to the variable.

+ */ +public class ForMemberExpr extends BindingExpression { + + private QName positionalVariable = null; + + public ForMemberExpr(final XQueryContext context) { + super(context); + } + + public void setPositionalVariable(final QName variable) { + positionalVariable = variable; + } + + @Override + public ClauseType getType() { + return ClauseType.FOR_MEMBER; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + super.analyze(contextInfo); + final LocalVariable mark = context.markLocalVariables(false); + try { + contextInfo.setParent(this); + final AnalyzeContextInfo varContextInfo = new AnalyzeContextInfo(contextInfo); + inputSequence.analyze(varContextInfo); + final LocalVariable inVar = new LocalVariable(varName); + inVar.setSequenceType(sequenceType); + inVar.setStaticType(Type.ITEM); + context.declareVariableBinding(inVar); + if (positionalVariable != null) { + final LocalVariable posVar = new LocalVariable(positionalVariable); + posVar.setSequenceType(POSITIONAL_VAR_TYPE); + posVar.setStaticType(Type.INTEGER); + context.declareVariableBinding(posVar); + } + + final AnalyzeContextInfo newContextInfo = new AnalyzeContextInfo(contextInfo); + newContextInfo.addFlag(SINGLE_STEP_EXECUTION); + returnExpr.analyze(newContextInfo); + } finally { + context.popLocalVariables(mark); + } + } + + @Override + public Sequence eval(Sequence contextSequence, final Item contextItem) + throws XPathException { + if (context.getProfiler().isEnabled()) { + context.getProfiler().start(this); + context.getProfiler().message(this, Profiler.DEPENDENCIES, + "DEPENDENCIES", Dependency.getDependenciesName(this.getDependencies())); + if (contextSequence != null) { + context.getProfiler().message(this, Profiler.START_SEQUENCES, + "CONTEXT SEQUENCE", contextSequence); + } + } + context.expressionStart(this); + + final LocalVariable mark = context.markLocalVariables(false); + final Sequence resultSequence = new ValueSequence(unordered); + try { + final Sequence in = inputSequence.eval(contextSequence, null); + + if (!(in instanceof ArrayType)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "for member expression requires an array, got " + + Type.getTypeName(in.getItemType())); + } + + final ArrayType array = (ArrayType) in; + final LocalVariable var = createVariable(varName); + var.setSequenceType(sequenceType); + context.declareVariableBinding(var); + + LocalVariable at = null; + if (positionalVariable != null) { + at = new LocalVariable(positionalVariable); + at.setSequenceType(POSITIONAL_VAR_TYPE); + context.declareVariableBinding(at); + } + + try { + for (int i = 0; i < array.getSize() && !WhileClause.isTerminated(); i++) { + context.proceed(this); + final Sequence member = array.get(i); + var.setValue(member); + if (positionalVariable != null) { + at.setValue(new IntegerValue(this, i + 1)); + } + if (sequenceType == null) { + var.checkType(); + } + + final Sequence returnResult; + if (returnExpr instanceof OrderByClause) { + returnResult = returnExpr.eval(member, null); + } else { + returnResult = returnExpr.eval(null, null); + } + resultSequence.addAll(returnResult); + var.destroy(context, resultSequence); + } + } catch (final WhileClause.WhileTerminationException e) { + // while clause signaled end of iteration + } + if (getPreviousClause() == null && WhileClause.isTerminated()) { + WhileClause.clearTerminated(); + } + } finally { + context.popLocalVariables(mark, resultSequence); + } + + if (callPostEval()) { + final Sequence postResult = postEval(resultSequence); + context.expressionEnd(this); + if (context.getProfiler().isEnabled()) { + context.getProfiler().end(this, "", postResult); + } + return postResult; + } + + context.expressionEnd(this); + if (context.getProfiler().isEnabled()) { + context.getProfiler().end(this, "", resultSequence); + } + return resultSequence; + } + + private boolean callPostEval() { + FLWORClause prev = getPreviousClause(); + while (prev != null) { + switch (prev.getType()) { + case LET: + case FOR: + case FOR_MEMBER: + return false; + case ORDERBY: + case GROUPBY: + return true; + default: + break; + } + prev = prev.getPreviousClause(); + } + return true; + } + + @Override + public void dump(final ExpressionDumper dumper) { + dumper.display("for member ", line); + dumper.startIndent(); + dumper.display("$").display(varName); + if (sequenceType != null) { + dumper.display(" as ").display(sequenceType); + } + dumper.display(" in "); + inputSequence.dump(dumper); + dumper.endIndent().nl(); + if (returnExpr instanceof LetExpr) { + dumper.display(" ", returnExpr.getLine()); + } else { + dumper.display("return", returnExpr.getLine()); + } + dumper.startIndent(); + returnExpr.dump(dumper); + dumper.endIndent().nl(); + } + + @Override + public String toString() { + final StringBuilder result = new StringBuilder(); + result.append("for member "); + result.append("$").append(varName); + if (sequenceType != null) { + result.append(" as ").append(sequenceType); + } + result.append(" in "); + result.append(inputSequence.toString()); + result.append(" "); + if (returnExpr instanceof LetExpr) { + result.append(" "); + } else { + result.append("return "); + } + result.append(returnExpr.toString()); + return result.toString(); + } + + @Override + public Set getTupleStreamVariables() { + final Set variables = new HashSet<>(); + final QName variable = getVariable(); + if (variable != null) { + variables.add(variable); + } + final LocalVariable startVar = getStartVariable(); + if (startVar != null) { + variables.add(startVar.getQName()); + } + return variables; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/KeywordArgumentExpression.java b/exist-core/src/main/java/org/exist/xquery/KeywordArgumentExpression.java new file mode 100644 index 00000000000..6bd237072a9 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/KeywordArgumentExpression.java @@ -0,0 +1,85 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.Type; + +/** + * Wraps a function argument expression with a keyword name for XQuery 4.0 + * keyword argument syntax: {@code fn:slice($input, start := 3)}. + * + *

This is a transient wrapper used during function call construction. + * The keyword name is used to match the argument to the correct parameter + * position in the function signature.

+ */ +public class KeywordArgumentExpression extends AbstractExpression { + + private final String keywordName; + private final Expression argument; + + public KeywordArgumentExpression(final XQueryContext context, final String keywordName, + final Expression argument) { + super(context); + this.keywordName = keywordName; + this.argument = argument; + } + + public String getKeywordName() { + return keywordName; + } + + public Expression getArgument() { + return argument; + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) + throws XPathException { + return argument.eval(contextSequence, contextItem); + } + + @Override + public int returnsType() { + return argument.returnsType(); + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + argument.analyze(contextInfo); + } + + @Override + public void dump(final ExpressionDumper dumper) { + dumper.display(keywordName); + dumper.display(" := "); + argument.dump(dumper); + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + argument.resetState(postOptimization); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/LetDestructureExpr.java b/exist-core/src/main/java/org/exist/xquery/LetDestructureExpr.java new file mode 100644 index 00000000000..9aedcb6f144 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/LetDestructureExpr.java @@ -0,0 +1,335 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.dom.QName; +import org.exist.xquery.functions.array.ArrayType; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.*; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Implements XQuery 4.0 let destructuring: + *
    + *
  • {@code let $($x, $y) := (1, 2)} — sequence destructuring
    • + *
  • {@code let $[$x, $y] := [1, 2]} — array destructuring
    • + *
  • {@code let ${$x, $y} := map{'x':1,'y':2}} — map destructuring
    • + *
+ */ +public class LetDestructureExpr extends AbstractFLWORClause { + + public enum DestructureMode { + SEQUENCE, ARRAY, MAP + } + + private final DestructureMode mode; + private final List varNames; + private final List varTypes; + private Expression inputSequence; + + public LetDestructureExpr(final XQueryContext context, final DestructureMode mode) { + super(context); + this.mode = mode; + this.varNames = new ArrayList<>(); + this.varTypes = new ArrayList<>(); + } + + public void addVariable(final QName name, final SequenceType type) { + varNames.add(name); + varTypes.add(type); + } + + public void setInputSequence(final Expression seq) { + this.inputSequence = seq.simplify(); + } + + public void setOverallType(final SequenceType type) { + // Reserved for future type checking of overall destructure type + } + + @Override + public ClauseType getType() { + switch (mode) { + case SEQUENCE: return ClauseType.LET_SEQ_DESTRUCTURE; + case ARRAY: return ClauseType.LET_ARRAY_DESTRUCTURE; + case MAP: return ClauseType.LET_MAP_DESTRUCTURE; + default: return ClauseType.LET; + } + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + final LocalVariable mark = context.markLocalVariables(false); + try { + contextInfo.setParent(this); + final AnalyzeContextInfo varContextInfo = new AnalyzeContextInfo(contextInfo); + inputSequence.analyze(varContextInfo); + + for (int i = 0; i < varNames.size(); i++) { + final LocalVariable var = new LocalVariable(varNames.get(i)); + if (varTypes.get(i) != null) { + var.setSequenceType(varTypes.get(i)); + } + context.declareVariableBinding(var); + } + + context.setContextSequencePosition(0, null); + returnExpr.analyze(contextInfo); + } finally { + context.popLocalVariables(mark); + } + } + + @Override + public Sequence eval(Sequence contextSequence, final Item contextItem) throws XPathException { + context.expressionStart(this); + context.pushDocumentContext(); + try { + final LocalVariable mark = context.markLocalVariables(false); + Sequence resultSequence = null; + try { + final Sequence input = inputSequence.eval(contextSequence, null); + + switch (mode) { + case SEQUENCE: + bindSequenceVars(input); + break; + case ARRAY: + bindArrayVars(input); + break; + case MAP: + bindMapVars(input); + break; + default: + throw new XPathException(this, ErrorCodes.ERROR, "Unknown destructure mode: " + mode); + } + + resultSequence = returnExpr.eval(contextSequence, null); + } finally { + context.popLocalVariables(mark, resultSequence); + } + if (resultSequence == null) { + return Sequence.EMPTY_SEQUENCE; + } + if (getPreviousClause() == null) { + resultSequence = postEval(resultSequence); + } + return resultSequence; + } finally { + context.popDocumentContext(); + context.expressionEnd(this); + } + } + + private void bindSequenceVars(final Sequence input) throws XPathException { + for (int i = 0; i < varNames.size(); i++) { + final LocalVariable var = createVariable(varNames.get(i)); + final SequenceType type = varTypes.get(i); + if (type != null) { + var.setSequenceType(type); + } + context.declareVariableBinding(var); + + if (i < input.getItemCount()) { + var.setValue(input.itemAt(i).toSequence()); + } else { + var.setValue(Sequence.EMPTY_SEQUENCE); + } + if (type != null) { + checkVarType(var, type); + } + } + } + + private void bindArrayVars(final Sequence input) throws XPathException { + if (input.isEmpty()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Array destructuring requires an array, got empty sequence"); + } + final Item item = input.itemAt(0); + if (!Type.subTypeOf(item.getType(), Type.ARRAY_ITEM)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Array destructuring requires an array, got " + + Type.getTypeName(item.getType())); + } + final ArrayType array = (ArrayType) item; + for (int i = 0; i < varNames.size(); i++) { + final LocalVariable var = createVariable(varNames.get(i)); + final SequenceType type = varTypes.get(i); + if (type != null) { + var.setSequenceType(type); + } + context.declareVariableBinding(var); + + if (i < array.getSize()) { + var.setValue(array.get(i)); + } else { + var.setValue(Sequence.EMPTY_SEQUENCE); + } + if (type != null) { + checkVarType(var, type); + } + } + } + + private void bindMapVars(final Sequence input) throws XPathException { + if (input.isEmpty()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Map destructuring requires a map, got empty sequence"); + } + final Item item = input.itemAt(0); + if (!Type.subTypeOf(item.getType(), Type.MAP_ITEM)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Map destructuring requires a map, got " + + Type.getTypeName(item.getType())); + } + final AbstractMapType map = (AbstractMapType) item; + for (int i = 0; i < varNames.size(); i++) { + final QName qn = varNames.get(i); + final LocalVariable var = createVariable(qn); + final SequenceType type = varTypes.get(i); + if (type != null) { + var.setSequenceType(type); + } + context.declareVariableBinding(var); + + final Sequence value = map.get(new StringValue(this, qn.getLocalPart())); + if (value != null && !value.isEmpty()) { + var.setValue(value); + } else { + var.setValue(Sequence.EMPTY_SEQUENCE); + } + if (type != null) { + checkVarType(var, type); + } + } + } + + private void checkVarType(final LocalVariable var, final SequenceType type) throws XPathException { + final Sequence val = var.getValue(); + if (val == null) { + return; + } + final Cardinality actualCard; + if (val.isEmpty()) { + actualCard = Cardinality.EMPTY_SEQUENCE; + } else if (val.hasMany()) { + actualCard = Cardinality._MANY; + } else { + actualCard = Cardinality.EXACTLY_ONE; + } + if (!type.getCardinality().isSuperCardinalityOrEqualOf(actualCard)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Invalid cardinality for variable $" + var.getQName() + + ". Expected " + type.getCardinality().getHumanDescription() + + ", got " + actualCard.getHumanDescription(), val); + } + if (!Type.subTypeOf(type.getPrimaryType(), Type.NODE) && + !val.isEmpty() && + !Type.subTypeOf(val.getItemType(), type.getPrimaryType())) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Invalid type for variable $" + var.getQName() + + ". Expected " + Type.getTypeName(type.getPrimaryType()) + + ", got " + Type.getTypeName(val.getItemType()), val); + } + } + + @Override + public void dump(final ExpressionDumper dumper) { + dumper.display("let "); + switch (mode) { + case SEQUENCE: dumper.display("$("); break; + case ARRAY: dumper.display("$["); break; + case MAP: dumper.display("${"); break; + default: break; + } + for (int i = 0; i < varNames.size(); i++) { + if (i > 0) dumper.display(", "); + dumper.display("$").display(varNames.get(i).getLocalPart()); + } + switch (mode) { + case SEQUENCE: dumper.display(")"); break; + case ARRAY: dumper.display("]"); break; + case MAP: dumper.display("}"); break; + default: break; + } + dumper.display(" := "); + inputSequence.dump(dumper); + dumper.nl().display("return "); + returnExpr.dump(dumper); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("let "); + switch (mode) { + case SEQUENCE: sb.append("$("); break; + case ARRAY: sb.append("$["); break; + case MAP: sb.append("${"); break; + default: break; + } + for (int i = 0; i < varNames.size(); i++) { + if (i > 0) sb.append(", "); + sb.append("$").append(varNames.get(i).getLocalPart()); + } + switch (mode) { + case SEQUENCE: sb.append(")"); break; + case ARRAY: sb.append("]"); break; + case MAP: sb.append("}"); break; + default: break; + } + sb.append(" := ").append(inputSequence.toString()); + sb.append(" return ").append(returnExpr.toString()); + return sb.toString(); + } + + @Override + public void accept(final ExpressionVisitor visitor) { + // No specific visitor method for destructure - use default + } + + @Override + public boolean allowMixedNodesInReturn() { + return true; + } + + @Override + public Set getTupleStreamVariables() { + return new HashSet<>(varNames); + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + inputSequence.resetState(postOptimization); + } + + @Override + public int getDependencies() { + return Dependency.CONTEXT_SET; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/MappingArrowOperator.java b/exist-core/src/main/java/org/exist/xquery/MappingArrowOperator.java new file mode 100644 index 00000000000..1e738abc9c4 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/MappingArrowOperator.java @@ -0,0 +1,207 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.dom.QName; +import org.exist.dom.QName.IllegalQNameException; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.FunctionReference; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +import java.util.ArrayList; +import java.util.List; + +/** + * Implements the XQuery 4.0 mapping arrow operator (=!>). + * + * Unlike the fat arrow (=>), which passes the entire left-hand sequence + * as the first argument, the mapping arrow iterates over each item in + * the sequence and passes each one individually, concatenating the results. + * + * {@code (1, 2, 3) =!> string()} is equivalent to {@code (1, 2, 3) ! string(.)}. + */ +public class MappingArrowOperator extends AbstractExpression { + + private QName qname = null; + private Expression leftExpr; + private FunctionCall fcall = null; + private Expression funcSpec = null; + private List parameters; + private AnalyzeContextInfo cachedContextInfo; + + public MappingArrowOperator(final XQueryContext context, final Expression leftExpr) throws XPathException { + super(context); + this.leftExpr = leftExpr; + } + + public void setArrowFunction(final String fname, final List params) throws XPathException { + try { + this.qname = QName.parse(context, fname, context.getDefaultFunctionNamespace()); + this.parameters = params; + } catch (final IllegalQNameException e) { + throw new XPathException(this, ErrorCodes.XPST0081, "No namespace defined for prefix " + fname); + } + } + + public void setArrowFunction(final PathExpr funcSpec, final List params) { + this.funcSpec = funcSpec.simplify(); + this.parameters = params; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + if (qname != null) { + fcall = NamedFunctionReference.lookupFunction(this, context, qname, parameters.size() + 1); + } + this.cachedContextInfo = contextInfo; + leftExpr.analyze(contextInfo); + if (fcall != null) { + fcall.analyze(contextInfo); + } + if (funcSpec != null) { + funcSpec.analyze(contextInfo); + } + } + + @Override + public Sequence eval(Sequence contextSequence, final Item contextItem) throws XPathException { + if (contextItem != null) { + contextSequence = contextItem.toSequence(); + } + final Sequence inputSeq = leftExpr.eval(contextSequence, null); + + if (inputSeq.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + final ValueSequence result = new ValueSequence(); + for (int i = 0; i < inputSeq.getItemCount(); i++) { + final Item item = inputSeq.itemAt(i); + final Sequence itemSeq = item.toSequence(); + + final FunctionReference fref; + if (fcall != null) { + fref = new FunctionReference(this, fcall); + } else { + final Sequence funcSeq = funcSpec.eval(itemSeq, null); + if (funcSeq.getCardinality() != Cardinality.EXACTLY_ONE) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Expected exactly one item for the function to be called, got " + funcSeq.getItemCount() + + ". Expression: " + ExpressionDumper.dump(funcSpec)); + } + final Item item0 = funcSeq.itemAt(0); + if (!Type.subTypeOf(item0.getType(), Type.FUNCTION)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Type error: expected function, got " + Type.getTypeName(item0.getType())); + } + fref = (FunctionReference) item0; + } + try { + final List fparams = new ArrayList<>(parameters.size() + 1); + fparams.add(new ContextParam(context, itemSeq)); + fparams.addAll(parameters); + + fref.setArguments(fparams); + fref.analyze(new AnalyzeContextInfo(cachedContextInfo)); + result.addAll(fref.eval(null)); + } finally { + fref.close(); + } + } + return result; + } + + @Override + public int returnsType() { + return fcall == null ? Type.ITEM : fcall.returnsType(); + } + + @Override + public Cardinality getCardinality() { + return Cardinality.ZERO_OR_MORE; + } + + @Override + public void dump(final ExpressionDumper dumper) { + leftExpr.dump(dumper); + dumper.display(" =!> "); + if (fcall != null) { + dumper.display(fcall.getFunction().getName()).display('('); + } else { + funcSpec.dump(dumper); + } + for (int i = 0; i < parameters.size(); i++) { + if (i > 0) { + dumper.display(", "); + parameters.get(i).dump(dumper); + } + } + dumper.display(')'); + } + + @Override + public void resetState(boolean postOptimization) { + super.resetState(postOptimization); + leftExpr.resetState(postOptimization); + if (fcall != null) { + fcall.resetState(postOptimization); + } + if (funcSpec != null) { + funcSpec.resetState(postOptimization); + } + for (Expression param : parameters) { + param.resetState(postOptimization); + } + } + + private class ContextParam extends Function.Placeholder { + private final Sequence sequence; + + ContextParam(XQueryContext context, Sequence sequence) { + super(context); + this.sequence = sequence; + } + + @Override + public void analyze(AnalyzeContextInfo contextInfo) throws XPathException { + // no-op: context param is pre-evaluated + } + + @Override + public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathException { + return sequence; + } + + @Override + public int returnsType() { + return sequence.getItemType(); + } + + @Override + public void dump(ExpressionDumper dumper) { + // no-op: context param has no source representation + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/MethodCallOperator.java b/exist-core/src/main/java/org/exist/xquery/MethodCallOperator.java new file mode 100644 index 00000000000..b302c8bdac3 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/MethodCallOperator.java @@ -0,0 +1,211 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.*; + +import java.util.ArrayList; +import java.util.List; + +/** + * Implements the XQuery 4.0 method call operator (=?>). + * + * {@code $map =?> method(args)} looks up the key "method" in the map, + * retrieves the function stored there, and calls it with the map as + * the first argument followed by any additional arguments. + * + * For each item in the left-hand sequence: + *
    + *
  1. The item must be a map (XPTY0004 otherwise)
    2. + *
  2. The method name is looked up as a key in the map
    3. + *
  3. The value must be exactly one function (XPTY0004 otherwise)
    4. + *
  4. The function is called with the map as first argument + additional args
    5. + *
+ * + * Like the mapping arrow (=!>), it processes each item individually + * and concatenates results. + */ +public class MethodCallOperator extends AbstractExpression { + + private Expression leftExpr; + private String methodName; + private List parameters; + private AnalyzeContextInfo cachedContextInfo; + + public MethodCallOperator(final XQueryContext context, final Expression leftExpr) throws XPathException { + super(context); + this.leftExpr = leftExpr; + } + + public void setMethod(final String methodName, final List params) { + this.methodName = methodName; + this.parameters = params; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + this.cachedContextInfo = contextInfo; + leftExpr.analyze(contextInfo); + if (parameters != null) { + for (final Expression param : parameters) { + param.analyze(contextInfo); + } + } + } + + @Override + public Sequence eval(Sequence contextSequence, final Item contextItem) throws XPathException { + if (contextItem != null) { + contextSequence = contextItem.toSequence(); + } + final Sequence inputSeq = leftExpr.eval(contextSequence, null); + + if (inputSeq.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + final ValueSequence result = new ValueSequence(); + for (int i = 0; i < inputSeq.getItemCount(); i++) { + final Item item = inputSeq.itemAt(i); + + // The item must be a map + if (!Type.subTypeOf(item.getType(), Type.MAP_ITEM)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Method call operator (=?>) requires a map, got " + + Type.getTypeName(item.getType())); + } + + final AbstractMapType map = (AbstractMapType) item; + + // Look up the method name as a key in the map + final Sequence methodValue = map.get(new StringValue(this, methodName)); + if (methodValue == null || methodValue.isEmpty()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Method '" + methodName + "' not found in map"); + } + + if (methodValue.getItemCount() != 1) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Method '" + methodName + "' must be a single function, got " + + methodValue.getItemCount() + " items"); + } + + final Item methodItem = methodValue.itemAt(0); + if (!Type.subTypeOf(methodItem.getType(), Type.FUNCTION)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Method '" + methodName + "' is not a function, got " + + Type.getTypeName(methodItem.getType())); + } + + final FunctionReference fref = (FunctionReference) methodItem; + + // Check arity: function must accept at least 1 argument (the map itself) + final int expectedArity = (parameters != null ? parameters.size() : 0) + 1; + if (fref.getSignature().getArgumentCount() == 0) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Method '" + methodName + "' has arity 0 and cannot accept the map as first argument"); + } + + try { + final List fparams = new ArrayList<>(expectedArity); + fparams.add(new ContextParam(context, item.toSequence())); + if (parameters != null) { + fparams.addAll(parameters); + } + + fref.setArguments(fparams); + fref.analyze(new AnalyzeContextInfo(cachedContextInfo)); + result.addAll(fref.eval(null)); + } finally { + fref.close(); + } + } + return result; + } + + @Override + public int returnsType() { + return Type.ITEM; + } + + @Override + public Cardinality getCardinality() { + return Cardinality.ZERO_OR_MORE; + } + + @Override + public void dump(final ExpressionDumper dumper) { + leftExpr.dump(dumper); + dumper.display(" =?> ").display(methodName).display('('); + if (parameters != null) { + for (int i = 0; i < parameters.size(); i++) { + if (i > 0) { + dumper.display(", "); + } + parameters.get(i).dump(dumper); + } + } + dumper.display(')'); + } + + @Override + public void resetState(boolean postOptimization) { + super.resetState(postOptimization); + leftExpr.resetState(postOptimization); + if (parameters != null) { + for (Expression param : parameters) { + param.resetState(postOptimization); + } + } + } + + private class ContextParam extends Function.Placeholder { + private final Sequence sequence; + + ContextParam(XQueryContext context, Sequence sequence) { + super(context); + this.sequence = sequence; + } + + @Override + public void analyze(AnalyzeContextInfo contextInfo) throws XPathException { + // no-op: context param is pre-evaluated + } + + @Override + public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathException { + return sequence; + } + + @Override + public int returnsType() { + return sequence.getItemType(); + } + + @Override + public void dump(ExpressionDumper dumper) { + // no-op: context param has no source representation + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/OtherwiseExpression.java b/exist-core/src/main/java/org/exist/xquery/OtherwiseExpression.java new file mode 100644 index 00000000000..760ab147c54 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/OtherwiseExpression.java @@ -0,0 +1,90 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.Item; + +/** + * Implements the XQuery 4.0 "otherwise" operator. + * + * {@code E1 otherwise E2} returns E1 if it is non-empty, otherwise E2. + */ +public class OtherwiseExpression extends AbstractExpression { + + private Expression left; + private Expression right; + + public OtherwiseExpression(final XQueryContext context, final Expression left, final Expression right) { + super(context); + this.left = left; + this.right = right; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + left.analyze(new AnalyzeContextInfo(contextInfo)); + right.analyze(new AnalyzeContextInfo(contextInfo)); + } + + @Override + public Sequence eval(Sequence contextSequence, final Item contextItem) throws XPathException { + if (contextItem != null) { + contextSequence = contextItem.toSequence(); + } + final Sequence leftResult = left.eval(contextSequence, null); + if (leftResult != null && !leftResult.isEmpty()) { + return leftResult; + } + return right.eval(contextSequence, null); + } + + @Override + public int returnsType() { + return left.returnsType(); + } + + @Override + public Cardinality getCardinality() { + return Cardinality.ZERO_OR_MORE; + } + + @Override + public void dump(final ExpressionDumper dumper) { + left.dump(dumper); + dumper.display(" otherwise "); + right.dump(dumper); + } + + @Override + public String toString() { + return left.toString() + " otherwise " + right.toString(); + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + left.resetState(postOptimization); + right.resetState(postOptimization); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/PipelineExpression.java b/exist-core/src/main/java/org/exist/xquery/PipelineExpression.java new file mode 100644 index 00000000000..5c746c1127f --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/PipelineExpression.java @@ -0,0 +1,106 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.ValueSequence; + +/** + * Implements the XQuery 4.0 pipeline operator "->". + * + * The expression {@code E1 -> E2} evaluates E1, then evaluates E2 with the + * result of E1 as the context value, position 1, and last 1. + */ +public class PipelineExpression extends AbstractExpression { + + private Expression left; + private Expression right; + + public PipelineExpression(final XQueryContext context, final Expression left, final Expression right) { + super(context); + this.left = left; + this.right = right; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + left.analyze(new AnalyzeContextInfo(contextInfo)); + right.analyze(new AnalyzeContextInfo(contextInfo)); + } + + @Override + public Sequence eval(Sequence contextSequence, final Item contextItem) throws XPathException { + if (contextItem != null) { + contextSequence = contextItem.toSequence(); + } + final Sequence leftResult = left.eval(contextSequence, null); + + // Pipeline: set context position=0 (position()=1) and a single-item + // context sequence so last()=1, per XQ4 spec. + final Sequence singletonContext; + if (leftResult.isEmpty()) { + singletonContext = Sequence.EMPTY_SEQUENCE; + } else { + singletonContext = new ValueSequence(1); + singletonContext.add(leftResult.itemAt(0)); + } + final int savedPos = context.getContextPosition(); + final Sequence savedSeq = context.getContextSequence(); + context.setContextSequencePosition(0, singletonContext); + try { + return right.eval(leftResult, null); + } finally { + context.setContextSequencePosition(savedPos, savedSeq); + } + } + + @Override + public int returnsType() { + return right.returnsType(); + } + + @Override + public Cardinality getCardinality() { + return Cardinality.ZERO_OR_MORE; + } + + @Override + public void dump(final ExpressionDumper dumper) { + left.dump(dumper); + dumper.display(" -> "); + right.dump(dumper); + } + + @Override + public String toString() { + return left.toString() + " -> " + right.toString(); + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + left.resetState(postOptimization); + right.resetState(postOptimization); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/WhileClause.java b/exist-core/src/main/java/org/exist/xquery/WhileClause.java new file mode 100644 index 00000000000..e2c9d7a41df --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/WhileClause.java @@ -0,0 +1,136 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery; + +import org.exist.dom.QName; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; + +import java.util.HashSet; +import java.util.Set; + +/** + * Implements the XQuery 4.0 while clause in FLWOR expressions. + * + *

The while clause evaluates a condition for each tuple in the stream. + * If the condition is true, the tuple is retained; if false, the tuple + * and all subsequent tuples are discarded (iteration stops).

+ */ +public class WhileClause extends AbstractFLWORClause { + + /** + * Thread-local flag that signals all enclosing binding expressions + * in the same FLWOR to stop iteration after the current item. + */ + private static final ThreadLocal terminated = ThreadLocal.withInitial(() -> false); + + private final Expression whileExpr; + + /** + * Lightweight control-flow exception used to signal the immediately + * enclosing for/let binding expression to stop iteration. + */ + public static class WhileTerminationException extends XPathException { + public WhileTerminationException() { + super((Expression) null, "while clause terminated"); + } + } + + public static boolean isTerminated() { + return terminated.get(); + } + + public static void clearTerminated() { + terminated.set(false); + } + + public WhileClause(final XQueryContext context, final Expression whileExpr) { + super(context); + this.whileExpr = whileExpr; + } + + @Override + public ClauseType getType() { + return ClauseType.WHILE; + } + + public Expression getWhileExpr() { + return whileExpr; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + contextInfo.setParent(this); + final AnalyzeContextInfo newContextInfo = new AnalyzeContextInfo(contextInfo); + newContextInfo.setFlags(contextInfo.getFlags() | IN_PREDICATE | IN_WHERE_CLAUSE); + newContextInfo.setContextId(getExpressionId()); + whileExpr.analyze(newContextInfo); + + final AnalyzeContextInfo returnContextInfo = new AnalyzeContextInfo(contextInfo); + returnContextInfo.addFlag(SINGLE_STEP_EXECUTION); + returnExpr.analyze(returnContextInfo); + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) throws XPathException { + final Sequence condResult = whileExpr.eval(null, null); + if (condResult.effectiveBooleanValue()) { + return returnExpr.eval(null, null); + } + terminated.set(true); + throw new WhileTerminationException(); + } + + @Override + public Sequence postEval(final Sequence seq) throws XPathException { + if (returnExpr instanceof FLWORClause flworClause) { + return flworClause.postEval(seq); + } + return super.postEval(seq); + } + + @Override + public void dump(final ExpressionDumper dumper) { + dumper.display("while", whileExpr.getLine()); + dumper.startIndent(); + whileExpr.dump(dumper); + dumper.endIndent().nl(); + } + + @Override + public void resetState(final boolean postOptimization) { + super.resetState(postOptimization); + whileExpr.resetState(postOptimization); + returnExpr.resetState(postOptimization); + } + + @Override + public Set getTupleStreamVariables() { + final Set vars = new HashSet<>(); + final LocalVariable startVar = getStartVariable(); + if (startVar != null) { + vars.add(startVar.getQName()); + } + return vars; + } +} From 17e77ab7263ce352ab2c51188e0cb47c3e657132 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Sat, 4 Apr 2026 13:12:35 -0400 Subject: [PATCH 007/326] [feature] Improve core F&O functions for XQuery 4.0 compliance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Broadened signatures and improved compliance for existing functions: - fn:compare: broadened to anyAtomicType, numeric total order, duration/datetime ordering per QT4 spec - fn:deep-equal: text node merging across comments/PIs, options map support, BigInteger overflow fix for fn:round - fn:head/fn:tail: expanded to fn:foot/fn:trunk (XQ4 aliases) - fn:max/fn:min: duration comparison support, decimal precision - fn:doc/fn:doc-available: security-gated file:// URI resolution, fn:doc#2 overload - fn:unparsed-text: BOM stripping, fn:unparsed-text-lines fix - fn:matches/fn:replace/fn:tokenize: regex enhancements - fn:path#2: output format parameter - fn:analyze-string: reflection proxy for Saxon compatibility - fn:parse-json: option validation, empty sequence args, xs:integer for JSON integers - fn:load-xquery-module: content option (XQ4) - fn:format-number: negative exponent zero-padding, map overload, char:rendition pattern - fn:format-date/fn:format-time: comprehensive improvements - Collations: supplementary codepoint comparison - RangeSequence: primitive long storage optimization - Error codes: W3C alignment across casting and value types - JSON serialization: XDM mode bypass fix, duplicate key detection Spec: QT4 XQuery 4.0 §14 (Functions and Operators) XQTS: +111 tests across method-json, fn-compare, fn-deep-equal, fn-round, fn-max/fn-min test sets Co-Authored-By: Claude Opus 4.6 (1M context) --- .../org/exist/xquery/parser/XQueryTree.g | 6 +- .../main/java/org/exist/util/Collations.java | 159 +++++++++- .../util/serializer/XQuerySerializer.java | 21 +- .../util/serializer/json/JSONSerializer.java | 4 +- .../exist/xquery/AttributeConstructor.java | 2 +- .../java/org/exist/xquery/CastExpression.java | 7 +- .../org/exist/xquery/CastableExpression.java | 5 +- .../java/org/exist/xquery/DecimalFormat.java | 32 +- .../org/exist/xquery/DocumentConstructor.java | 2 +- .../xquery/DynamicAttributeConstructor.java | 2 +- .../java/org/exist/xquery/ErrorCodes.java | 8 + .../org/exist/xquery/FunctionFactory.java | 9 +- .../java/org/exist/xquery/RangeSequence.java | 115 ++++--- .../java/org/exist/xquery/XQueryContext.java | 7 + .../xquery/functions/fn/FunAnalyzeString.java | 57 ++-- .../exist/xquery/functions/fn/FunCompare.java | 178 +++++++++-- .../xquery/functions/fn/FunDeepEqual.java | 118 ++++++-- .../org/exist/xquery/functions/fn/FunDoc.java | 20 +- .../xquery/functions/fn/FunDocAvailable.java | 19 +- .../xquery/functions/fn/FunHeadTail.java | 83 +++-- .../functions/fn/FunInScopePrefixes.java | 4 + .../exist/xquery/functions/fn/FunMatches.java | 37 ++- .../org/exist/xquery/functions/fn/FunMax.java | 286 ++++++++---------- .../org/exist/xquery/functions/fn/FunMin.java | 286 ++++++++---------- .../exist/xquery/functions/fn/FunPath.java | 8 +- .../exist/xquery/functions/fn/FunReplace.java | 251 +++++++++++++-- .../exist/xquery/functions/fn/FunRound.java | 6 +- .../xquery/functions/fn/FunRoundBase.java | 44 ++- .../xquery/functions/fn/FunTokenize.java | 122 ++++++-- .../xquery/functions/fn/FunUnparsedText.java | 74 ++++- .../org/exist/xquery/functions/fn/JSON.java | 157 +++++++++- .../xquery/functions/fn/LoadXQueryModule.java | 63 +++- .../xquery/functions/fn/ParsingFunctions.java | 19 ++ .../org/exist/xquery/regex/RegexUtil.java | 29 ++ .../java/org/exist/xquery/util/DocUtils.java | 59 +++- .../exist/xquery/util/NumberFormatter_en.java | 3 +- .../xquery/value/AbstractDateTimeValue.java | 29 +- .../org/exist/xquery/value/AtomicValue.java | 4 +- .../xquery/value/AtomicValueComparator.java | 4 +- .../xquery/value/Base64BinaryValueType.java | 3 +- .../xquery/value/DayTimeDurationValue.java | 2 +- .../org/exist/xquery/value/DecimalValue.java | 2 +- .../org/exist/xquery/value/DurationValue.java | 4 +- .../xquery/value/SequenceComparator.java | 2 +- .../org/exist/xquery/value/StringValue.java | 52 ++-- .../src/test/xquery/xquery3/replace.xqm | 2 +- 46 files changed, 1739 insertions(+), 667 deletions(-) diff --git a/exist-core/src/main/antlr/org/exist/xquery/parser/XQueryTree.g b/exist-core/src/main/antlr/org/exist/xquery/parser/XQueryTree.g index 20308296806..399fa264dc6 100644 --- a/exist-core/src/main/antlr/org/exist/xquery/parser/XQueryTree.g +++ b/exist-core/src/main/antlr/org/exist/xquery/parser/XQueryTree.g @@ -267,14 +267,16 @@ throws PermissionDeniedException, EXistException, XPathException v:VERSION_DECL { final String version = v.getText(); - if (version.equals("3.1")) { + if (version.equals("4.0")) { + context.setXQueryVersion(40); + } else if (version.equals("3.1")) { context.setXQueryVersion(31); } else if (version.equals("3.0")) { context.setXQueryVersion(30); } else if (version.equals("1.0")) { context.setXQueryVersion(10); } else { - throw new XPathException(v, ErrorCodes.XQST0031, "Wrong XQuery version: require 1.0, 3.0 or 3.1"); + throw new XPathException(v, ErrorCodes.XQST0031, "Wrong XQuery version: require 1.0, 3.0, 3.1, or 4.0"); } } ( enc:STRING_LITERAL )? diff --git a/exist-core/src/main/java/org/exist/util/Collations.java b/exist-core/src/main/java/org/exist/util/Collations.java index 2d03138a291..ecb09f43cbe 100644 --- a/exist-core/src/main/java/org/exist/util/Collations.java +++ b/exist-core/src/main/java/org/exist/util/Collations.java @@ -346,7 +346,24 @@ public static boolean equals(@Nullable final Collator collator, final String s1, */ public static int compare(@Nullable final Collator collator, final String s1,final String s2) { if (collator == null) { - return s1 == null ? (s2 == null ? 0 : -1) : s1.compareTo(s2); + if (s1 == null) { + return s2 == null ? 0 : -1; + } + // Compare by Unicode codepoints, not UTF-16 code units. + // String.compareTo() compares char (UTF-16) values, which gives wrong + // ordering for supplementary characters (U+10000+) encoded as surrogate pairs. + int i1 = 0, i2 = 0; + while (i1 < s1.length() && i2 < s2.length()) { + final int cp1 = s1.codePointAt(i1); + final int cp2 = s2.codePointAt(i2); + if (cp1 != cp2) { + return cp1 - cp2; + } + i1 += Character.charCount(cp1); + i2 += Character.charCount(cp2); + } + // Shorter string is less; equal length means equal + return (s1.length() - i1) - (s2.length() - i2); } else { return collator.compare(s1, s2); } @@ -371,10 +388,16 @@ public static boolean startsWith(@Nullable final Collator collator, final String return true; } else if (s1.isEmpty()) { return false; - } else { + } else if (collator instanceof RuleBasedCollator rbc) { final SearchIterator searchIterator = - new StringSearch(s2, new StringCharacterIterator(s1), (RuleBasedCollator) collator); + new StringSearch(s2, new StringCharacterIterator(s1), rbc); return searchIterator.first() == 0; + } else { + // Fallback for non-RuleBasedCollator (e.g., HtmlAsciiCaseInsensitiveCollator) + if (s1.length() >= s2.length()) { + return collator.compare(s1.substring(0, s2.length()), s2) == 0; + } + return false; } } } @@ -398,9 +421,9 @@ public static boolean endsWith(@Nullable final Collator collator, final String s return true; } else if (s1.isEmpty()) { return false; - } else { + } else if (collator instanceof RuleBasedCollator rbc) { final SearchIterator searchIterator = - new StringSearch(s2, new StringCharacterIterator(s1), (RuleBasedCollator) collator); + new StringSearch(s2, new StringCharacterIterator(s1), rbc); int lastPos = SearchIterator.DONE; int lastLen = 0; for (int pos = searchIterator.first(); pos != SearchIterator.DONE; @@ -410,6 +433,12 @@ public static boolean endsWith(@Nullable final Collator collator, final String s } return lastPos > SearchIterator.DONE && lastPos + lastLen == s1.length(); + } else { + // Fallback for non-RuleBasedCollator + if (s1.length() >= s2.length()) { + return collator.compare(s1.substring(s1.length() - s2.length()), s2) == 0; + } + return false; } } } @@ -433,10 +462,18 @@ public static boolean contains(@Nullable final Collator collator, final String s return true; } else if (s1.isEmpty()) { return false; - } else { + } else if (collator instanceof RuleBasedCollator rbc) { final SearchIterator searchIterator = - new StringSearch(s2, new StringCharacterIterator(s1), (RuleBasedCollator) collator); + new StringSearch(s2, new StringCharacterIterator(s1), rbc); return searchIterator.first() >= 0; + } else { + // Fallback for non-RuleBasedCollator + for (int i = 0; i <= s1.length() - s2.length(); i++) { + if (collator.compare(s1.substring(i, i + s2.length()), s2) == 0) { + return true; + } + } + return false; } } } @@ -459,10 +496,18 @@ public static int indexOf(@Nullable final Collator collator, final String s1, fi return 0; } else if (s1.isEmpty()) { return -1; - } else { + } else if (collator instanceof RuleBasedCollator rbc) { final SearchIterator searchIterator = - new StringSearch(s2, new StringCharacterIterator(s1), (RuleBasedCollator) collator); + new StringSearch(s2, new StringCharacterIterator(s1), rbc); return searchIterator.first(); + } else { + // Fallback for non-RuleBasedCollator + for (int i = 0; i <= s1.length() - s2.length(); i++) { + if (collator.compare(s1.substring(i, i + s2.length()), s2) == 0) { + return i; + } + } + return -1; } } } @@ -809,21 +854,105 @@ private static Collator getSamiskCollator() throws Exception { return collator; } - private static Collator getHtmlAsciiCaseInsensitiveCollator() throws Exception { + private static Collator getHtmlAsciiCaseInsensitiveCollator() { Collator collator = htmlAsciiCaseInsensitiveCollator.get(); if (collator == null) { - collator = new RuleBasedCollator("&a=A &b=B &c=C &d=D &e=E &f=F &g=G &h=H " - + "&i=I &j=J &k=K &l=L &m=M &n=N &o=O &p=P &q=Q &r=R &s=S &t=T " - + "&u=U &v=V &w=W &x=X &y=Y &z=Z"); - collator.setStrength(Collator.PRIMARY); + // XQ4 html-ascii-case-insensitive: ASCII letters A-Z fold to a-z, + // all other characters compare by Unicode codepoint order. + // Cannot use RuleBasedCollator with PRIMARY strength because that + // makes ALL case/accent differences irrelevant, not just ASCII. htmlAsciiCaseInsensitiveCollator.compareAndSet(null, - collator.freeze()); + new HtmlAsciiCaseInsensitiveCollator()); collator = htmlAsciiCaseInsensitiveCollator.get(); } return collator; } + /** + * Custom Collator for HTML ASCII case-insensitive comparison. + * Folds only ASCII letters A-Z to a-z, then compares by Unicode codepoint. + * Non-ASCII characters are compared by their codepoint value without folding. + */ + private static final class HtmlAsciiCaseInsensitiveCollator extends Collator { + + @Override + public int compare(final String source, final String target) { + int i1 = 0, i2 = 0; + while (i1 < source.length() && i2 < target.length()) { + int cp1 = source.codePointAt(i1); + int cp2 = target.codePointAt(i2); + // Fold ASCII uppercase to lowercase only + if (cp1 >= 'A' && cp1 <= 'Z') { + cp1 += 32; + } + if (cp2 >= 'A' && cp2 <= 'Z') { + cp2 += 32; + } + if (cp1 != cp2) { + return cp1 - cp2; + } + i1 += Character.charCount(cp1); + i2 += Character.charCount(cp2); + } + return (source.length() - i1) - (target.length() - i2); + } + + @Override + public CollationKey getCollationKey(final String source) { + throw new UnsupportedOperationException("CollationKey not supported for HTML ASCII case-insensitive collation"); + } + + @Override + public RawCollationKey getRawCollationKey(final String source, final RawCollationKey key) { + throw new UnsupportedOperationException("RawCollationKey not supported for HTML ASCII case-insensitive collation"); + } + + @Override + public int setVariableTop(final String varTop) { + return 0; + } + + @Override + public int getVariableTop() { + return 0; + } + + @Override + public void setVariableTop(final int varTop) { + } + + @Override + public VersionInfo getVersion() { + return VersionInfo.getInstance(1); + } + + @Override + public VersionInfo getUCAVersion() { + return VersionInfo.getInstance(1); + } + + @Override + public int hashCode() { + return HtmlAsciiCaseInsensitiveCollator.class.hashCode(); + } + + @Override + public Collator freeze() { + return this; + } + + @Override + public boolean isFrozen() { + return true; + } + + @Override + public Collator cloneAsThawed() { + return new HtmlAsciiCaseInsensitiveCollator(); + } + } + private static Collator getXqtsAsciiCaseBlindCollator() throws Exception { Collator collator = xqtsAsciiCaseBlindCollator.get(); if (collator == null) { diff --git a/exist-core/src/main/java/org/exist/util/serializer/XQuerySerializer.java b/exist-core/src/main/java/org/exist/util/serializer/XQuerySerializer.java index 366e3866cbc..acfb8d16de6 100644 --- a/exist-core/src/main/java/org/exist/util/serializer/XQuerySerializer.java +++ b/exist-core/src/main/java/org/exist/util/serializer/XQuerySerializer.java @@ -103,12 +103,25 @@ private void serializeXML(final Sequence sequence, final int start, final int ho } private void serializeJSON(final Sequence sequence, final long compilationTime, final long executionTime) throws SAXException, XPathException { - // backwards compatibility: if the sequence contains a single element, we assume - // it should be transformed to JSON following the rules of the old JSON writer - if (sequence.hasOne() && (Type.subTypeOf(sequence.getItemType(), Type.DOCUMENT) || Type.subTypeOf(sequence.getItemType(), Type.ELEMENT))) { + // XDM serialization: use JSONSerializer for maps and arrays (W3C JSON output method). + // For element/document nodes, use the legacy XML-to-JSON conversion path for + // backward compatibility with eXist's traditional JSON serialization. + // TODO (eXist 8.0): Remove legacy XML-to-JSON conversion. + // The legacy path is deprecated in 7.0 — use fn:serialize($map, map{"method":"json"}) instead. + final boolean isXdmMapOrArray = sequence.hasOne() + && (sequence.getItemType() == Type.MAP_ITEM || sequence.getItemType() == Type.ARRAY_ITEM); + + if (isXdmMapOrArray || (!sequence.hasOne()) + || Type.subTypeOfUnion(sequence.getItemType(), Type.ANY_ATOMIC_TYPE)) { + // Maps, arrays, sequences, and atomic values: use W3C JSONSerializer + final JSONSerializer serializer = new JSONSerializer(broker, outputProperties); + serializer.serialize(sequence, writer); + } else if (sequence.hasOne() + && (Type.subTypeOf(sequence.getItemType(), Type.DOCUMENT) || Type.subTypeOf(sequence.getItemType(), Type.ELEMENT))) { + // Legacy path: single element/document → XML-to-JSON conversion serializeXML(sequence, 1, 1, false, false, compilationTime, executionTime); } else { - JSONSerializer serializer = new JSONSerializer(broker, outputProperties); + final JSONSerializer serializer = new JSONSerializer(broker, outputProperties); serializer.serialize(sequence, writer); } } diff --git a/exist-core/src/main/java/org/exist/util/serializer/json/JSONSerializer.java b/exist-core/src/main/java/org/exist/util/serializer/json/JSONSerializer.java index bd1f01a9454..7728633368a 100644 --- a/exist-core/src/main/java/org/exist/util/serializer/json/JSONSerializer.java +++ b/exist-core/src/main/java/org/exist/util/serializer/json/JSONSerializer.java @@ -64,7 +64,9 @@ public void serialize(Sequence sequence, Writer writer) throws SAXException { if ("yes".equals(outputProperties.getProperty(OutputKeys.INDENT, "no"))) { generator.useDefaultPrettyPrinter(); } - if ("yes".equals(outputProperties.getProperty(EXistOutputKeys.ALLOW_DUPLICATE_NAMES, "yes"))) { + // allow-duplicate-names=no (default per W3C) → enable strict detection + // allow-duplicate-names=yes → disable strict detection (allow duplicates) + if ("no".equals(outputProperties.getProperty(EXistOutputKeys.ALLOW_DUPLICATE_NAMES, "no"))) { generator.enable(JsonGenerator.Feature.STRICT_DUPLICATE_DETECTION); } else { generator.disable(JsonGenerator.Feature.STRICT_DUPLICATE_DETECTION); diff --git a/exist-core/src/main/java/org/exist/xquery/AttributeConstructor.java b/exist-core/src/main/java/org/exist/xquery/AttributeConstructor.java index bb1720e67d9..2dcbfe4652c 100644 --- a/exist-core/src/main/java/org/exist/xquery/AttributeConstructor.java +++ b/exist-core/src/main/java/org/exist/xquery/AttributeConstructor.java @@ -56,7 +56,7 @@ public void addValue(String value) { public void addEnclosedExpr(Expression expr) throws XPathException { if(isNamespaceDecl) - {throw new XPathException(this, "enclosed expressions are not allowed in namespace " + + {throw new XPathException(this, ErrorCodes.XQST0022, "enclosed expressions are not allowed in namespace " + "declaration attributes");} contents.add(expr); } diff --git a/exist-core/src/main/java/org/exist/xquery/CastExpression.java b/exist-core/src/main/java/org/exist/xquery/CastExpression.java index 8911c5c6144..a2453eaaddc 100644 --- a/exist-core/src/main/java/org/exist/xquery/CastExpression.java +++ b/exist-core/src/main/java/org/exist/xquery/CastExpression.java @@ -84,12 +84,13 @@ public Sequence eval(final Sequence contextSequence, final Item contextItem) thr } } - // Should be handled by the parser - if (requiredType == Type.ANY_ATOMIC_TYPE || (requiredType == Type.NOTATION && expression.returnsType() != Type.NOTATION)) { + // XPST0080: cannot cast to abstract or special types + if (requiredType == Type.ANY_ATOMIC_TYPE || requiredType == Type.ANY_SIMPLE_TYPE + || (requiredType == Type.NOTATION && expression.returnsType() != Type.NOTATION)) { throw new XPathException(this, ErrorCodes.XPST0080, "cannot cast to " + Type.getTypeName(requiredType)); } - if (requiredType == Type.ANY_SIMPLE_TYPE || expression.returnsType() == Type.ANY_SIMPLE_TYPE || requiredType == Type.UNTYPED || expression.returnsType() == Type.UNTYPED) { + if (expression.returnsType() == Type.ANY_SIMPLE_TYPE || requiredType == Type.UNTYPED || expression.returnsType() == Type.UNTYPED) { throw new XPathException(this, ErrorCodes.XPST0051, "cannot cast to " + Type.getTypeName(requiredType)); } diff --git a/exist-core/src/main/java/org/exist/xquery/CastableExpression.java b/exist-core/src/main/java/org/exist/xquery/CastableExpression.java index 9a0769f9653..e923da9fe08 100644 --- a/exist-core/src/main/java/org/exist/xquery/CastableExpression.java +++ b/exist-core/src/main/java/org/exist/xquery/CastableExpression.java @@ -93,10 +93,11 @@ public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathExc {context.getProfiler().message(this, Profiler.START_SEQUENCES, "CONTEXT ITEM", contextItem.toSequence());} } - if (requiredType == Type.ANY_ATOMIC_TYPE || (requiredType == Type.NOTATION && expression.returnsType() != Type.NOTATION)) + if (requiredType == Type.ANY_ATOMIC_TYPE || requiredType == Type.ANY_SIMPLE_TYPE + || (requiredType == Type.NOTATION && expression.returnsType() != Type.NOTATION)) {throw new XPathException(this, ErrorCodes.XPST0080, "cannot convert to " + Type.getTypeName(requiredType));} - if (requiredType == Type.ANY_SIMPLE_TYPE || expression.returnsType() == Type.ANY_SIMPLE_TYPE || requiredType == Type.UNTYPED || expression.returnsType() == Type.UNTYPED) + if (expression.returnsType() == Type.ANY_SIMPLE_TYPE || requiredType == Type.UNTYPED || expression.returnsType() == Type.UNTYPED) {throw new XPathException(this, ErrorCodes.XPST0051, "cannot convert to " + Type.getTypeName(requiredType));} Sequence result; diff --git a/exist-core/src/main/java/org/exist/xquery/DecimalFormat.java b/exist-core/src/main/java/org/exist/xquery/DecimalFormat.java index 46a54962ad5..93c4d4b1fff 100644 --- a/exist-core/src/main/java/org/exist/xquery/DecimalFormat.java +++ b/exist-core/src/main/java/org/exist/xquery/DecimalFormat.java @@ -47,7 +47,7 @@ public class DecimalFormat { ); - // used both in the picture string, and in the formatted number + // Markers: used in the picture string to identify active elements public final int decimalSeparator; public final int exponentSeparator; public final int groupingSeparator; @@ -55,18 +55,38 @@ public class DecimalFormat { public final int perMille; public final int zeroDigit; - // used in the picture string + // used in the picture string only public final int digit; public final int patternSeparator; - //used in the result of formatting the number, but not in the picture string + // used in the result of formatting the number, but not in the picture string public final String infinity; public final String NaN; public final int minusSign; + // XQ4 renditions: output strings for properties that support char:rendition. + // When marker != rendition, the marker is used for picture parsing and the + // rendition string appears in the formatted output. + public final String decimalSeparatorRendition; + public final String exponentSeparatorRendition; + public final String groupingSeparatorRendition; + public final String percentRendition; + public final String perMilleRendition; + public DecimalFormat(final int decimalSeparator, final int exponentSeparator, final int groupingSeparator, final int percent, final int perMille, final int zeroDigit, final int digit, final int patternSeparator, final String infinity, final String NaN, final int minusSign) { + this(decimalSeparator, exponentSeparator, groupingSeparator, percent, perMille, + zeroDigit, digit, patternSeparator, infinity, NaN, minusSign, + null, null, null, null, null); + } + + public DecimalFormat(final int decimalSeparator, final int exponentSeparator, final int groupingSeparator, + final int percent, final int perMille, final int zeroDigit, final int digit, + final int patternSeparator, final String infinity, final String NaN, final int minusSign, + final String decimalSeparatorRendition, final String exponentSeparatorRendition, + final String groupingSeparatorRendition, final String percentRendition, + final String perMilleRendition) { this.decimalSeparator = decimalSeparator; this.exponentSeparator = exponentSeparator; this.groupingSeparator = groupingSeparator; @@ -78,5 +98,11 @@ public DecimalFormat(final int decimalSeparator, final int exponentSeparator, fi this.infinity = infinity; this.NaN = NaN; this.minusSign = minusSign; + // Renditions default to the marker character as a string + this.decimalSeparatorRendition = decimalSeparatorRendition != null ? decimalSeparatorRendition : new String(Character.toChars(decimalSeparator)); + this.exponentSeparatorRendition = exponentSeparatorRendition != null ? exponentSeparatorRendition : new String(Character.toChars(exponentSeparator)); + this.groupingSeparatorRendition = groupingSeparatorRendition != null ? groupingSeparatorRendition : new String(Character.toChars(groupingSeparator)); + this.percentRendition = percentRendition != null ? percentRendition : new String(Character.toChars(percent)); + this.perMilleRendition = perMilleRendition != null ? perMilleRendition : new String(Character.toChars(perMille)); } } diff --git a/exist-core/src/main/java/org/exist/xquery/DocumentConstructor.java b/exist-core/src/main/java/org/exist/xquery/DocumentConstructor.java index 3495fed460f..a67eaee3544 100644 --- a/exist-core/src/main/java/org/exist/xquery/DocumentConstructor.java +++ b/exist-core/src/main/java/org/exist/xquery/DocumentConstructor.java @@ -90,7 +90,7 @@ public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathExc while(next != null) { context.proceed(this, builder); if (next.getType() == Type.ATTRIBUTE || next.getType() == Type.NAMESPACE) - {throw new XPathException(this, "Found a node of type " + + {throw new XPathException(this, ErrorCodes.XPTY0004, "Found a node of type " + Type.getTypeName(next.getType()) + " inside a document constructor");} // if item is an atomic value, collect the string values of all // following atomic values and seperate them by a space. diff --git a/exist-core/src/main/java/org/exist/xquery/DynamicAttributeConstructor.java b/exist-core/src/main/java/org/exist/xquery/DynamicAttributeConstructor.java index 168c2da95a6..583a4afc9ef 100644 --- a/exist-core/src/main/java/org/exist/xquery/DynamicAttributeConstructor.java +++ b/exist-core/src/main/java/org/exist/xquery/DynamicAttributeConstructor.java @@ -99,7 +99,7 @@ public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathExc final Sequence nameSeq = qnameExpr.eval(contextSequence, contextItem); if(!nameSeq.hasOne()) - {throw new XPathException(this, "The name expression should evaluate to a single value");} + {throw new XPathException(this, ErrorCodes.XPTY0004, "The name expression should evaluate to a single value");} final Item qnItem = nameSeq.itemAt(0); QName qn; diff --git a/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java b/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java index 23226a155f2..4205d01484f 100644 --- a/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java +++ b/exist-core/src/main/java/org/exist/xquery/ErrorCodes.java @@ -176,6 +176,9 @@ public class ErrorCodes { public static final ErrorCode FORX0002 = new W3CErrorCode("FORX0002", "Invalid regular expression."); public static final ErrorCode FORX0003 = new W3CErrorCode("FORX0003", "Regular expression matches zero-length string."); public static final ErrorCode FORX0004 = new W3CErrorCode("FORX0004", "Invalid replacement string."); + public static final ErrorCode FOCV0001 = new W3CErrorCode("FOCV0001", "CSV quote error."); + public static final ErrorCode FOCV0002 = new W3CErrorCode("FOCV0002", "Invalid CSV delimiter."); + public static final ErrorCode FOCV0003 = new W3CErrorCode("FOCV0003", "Conflicting CSV delimiters."); public static final ErrorCode FOTY0012 = new W3CErrorCode("FOTY0012", "Argument node does not have a typed value."); public static final ErrorCode FOTY0013 = new W3CErrorCode("FOTY0013", "The argument to fn:data() contains a function item."); @@ -211,6 +214,7 @@ public class ErrorCodes { public static final ErrorCode FTDY0020 = new W3CErrorCode("FTDY0020", ""); public static final ErrorCode FODC0006 = new W3CErrorCode("FODC0006", "String passed to fn:parse-xml is not a well-formed XML document."); + public static final ErrorCode FODC0011 = new W3CErrorCode("FODC0011", "HTML parsing error."); public static final ErrorCode FOAP0001 = new W3CErrorCode("FOAP0001", "Wrong number of arguments"); @@ -241,6 +245,10 @@ public class ErrorCodes { public static final ErrorCode FOXT0004 = new W3CErrorCode("FOXT0004", "XSLT transformation has been disabled"); public static final ErrorCode FOXT0006 = new W3CErrorCode("FOXT0006", "XSLT output contains non-accepted characters"); + // Invisible XML errors + public static final ErrorCode FOIX0001 = new W3CErrorCode("FOIX0001", "Invalid ixml grammar"); + public static final ErrorCode FOIX0002 = new W3CErrorCode("FOIX0002", "ixml parse error"); + public static final ErrorCode XTSE0165 = new W3CErrorCode("XTSE0165","It is a static error if the processor is not able to retrieve the resource identified by the URI reference [ in the href attribute of xsl:include or xsl:import] , or if the resource that is retrieved does not contain a stylesheet module conforming to this specification."); /* eXist specific XQuery and XPath errors diff --git a/exist-core/src/main/java/org/exist/xquery/FunctionFactory.java b/exist-core/src/main/java/org/exist/xquery/FunctionFactory.java index adcf7d3d5cb..a1602539964 100644 --- a/exist-core/src/main/java/org/exist/xquery/FunctionFactory.java +++ b/exist-core/src/main/java/org/exist/xquery/FunctionFactory.java @@ -245,7 +245,14 @@ private static CastExpression castExpression(XQueryContext context, ErrorCodes.XPST0017, "Wrong number of arguments for constructor function"); } final Expression arg = params.getFirst(); - final int code = Type.getType(qname); + final int code; + try { + code = Type.getType(qname); + } catch (final XPathException e) { + // Unknown type name in xs: namespace → XPST0017 (no such function) + throw new XPathException(ast.getLine(), ast.getColumn(), + ErrorCodes.XPST0017, "Unknown constructor function: " + qname.getStringValue()); + } final CastExpression castExpr = new CastExpression(context, arg, code, Cardinality.ZERO_OR_ONE); castExpr.setLocation(ast.getLine(), ast.getColumn()); return castExpr; diff --git a/exist-core/src/main/java/org/exist/xquery/RangeSequence.java b/exist-core/src/main/java/org/exist/xquery/RangeSequence.java index c23c663067e..eb3ecfa6507 100644 --- a/exist-core/src/main/java/org/exist/xquery/RangeSequence.java +++ b/exist-core/src/main/java/org/exist/xquery/RangeSequence.java @@ -21,8 +21,6 @@ */ package org.exist.xquery; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.exist.dom.persistent.NodeSet; import org.exist.xquery.value.AbstractSequence; import org.exist.xquery.value.IntegerValue; @@ -32,18 +30,40 @@ import org.exist.xquery.value.SequenceIterator; import org.exist.xquery.value.Type; -import java.math.BigInteger; - +/** + * An immutable, lazy sequence representing an integer range (start to end). + * Stores only the start and end values as primitive longs — no intermediate + * IntegerValue objects are created until accessed. Operations like count(), + * isEmpty(), itemAt(), and subsequence() are O(1). + */ public class RangeSequence extends AbstractSequence { - private final static Logger LOG = LogManager.getLogger(AbstractSequence.class); - - private final IntegerValue start; - private final IntegerValue end; + private final long start; + private final long end; + private final long size; public RangeSequence(final IntegerValue start, final IntegerValue end) { + this(start.getLong(), end.getLong()); + } + + public RangeSequence(final long start, final long end) { this.start = start; this.end = end; + if (start <= end) { + final long diff = end - start; + // Overflow protection: if diff < 0, the range is too large + this.size = (diff >= 0) ? diff + 1 : Long.MAX_VALUE; + } else { + this.size = 0; + } + } + + public long getStart() { + return start; + } + + public long getEnd() { + return end; } @Override @@ -62,16 +82,16 @@ public int getItemType() { @Override public SequenceIterator iterate() { - return new RangeSequenceIterator(start.getLong(), end.getLong()); + return new RangeSequenceIterator(start, end); } @Override public SequenceIterator unorderedIterator() { - return new RangeSequenceIterator(start.getLong(), end.getLong()); + return new RangeSequenceIterator(start, end); } public SequenceIterator iterateInReverse() { - return new ReverseRangeSequenceIterator(start.getLong(), end.getLong()); + return new ReverseRangeSequenceIterator(start, end); } private static class RangeSequenceIterator implements SequenceIterator { @@ -148,39 +168,30 @@ public long skip(final long n) { @Override public long getItemCountLong() { - if (start.compareTo(end) > 0) { - return 0; - } - try { - return ((IntegerValue) end.minus(start)).getLong() + 1; - } catch (final XPathException e) { - LOG.warn("Unexpected exception when processing result of range expression: {}", e.getMessage(), e); - return 0; - } + return size; } @Override public boolean isEmpty() { - return getItemCountLong() == 0; + return size == 0; } @Override public boolean hasOne() { - return getItemCountLong() == 1; + return size == 1; } @Override public boolean hasMany() { - return getItemCountLong() > 1; + return size > 1; } @Override public Cardinality getCardinality() { - final long itemCount = getItemCountLong(); - if (itemCount <= 0) { + if (size == 0) { return Cardinality.EMPTY_SEQUENCE; } - if (itemCount == 1) { + if (size == 1) { return Cardinality.EXACTLY_ONE; } return Cardinality._MANY; @@ -188,12 +199,26 @@ public Cardinality getCardinality() { @Override public Item itemAt(final int pos) { - if (pos < getItemCountLong()) { - return new IntegerValue(start.getLong() + pos); + if (pos >= 0 && pos < size) { + return new IntegerValue(start + pos); } return null; } + @Override + public boolean contains(final Item item) { + if (item instanceof IntegerValue) { + final long val = ((IntegerValue) item).getLong(); + return val >= start && val <= end; + } + return false; + } + + @Override + public boolean containsReference(final Item item) { + return false; // primitives don't have reference identity + } + @Override public NodeSet toNodeSet() throws XPathException { throw new XPathException(this, "Type error: the sequence cannot be converted into" + @@ -211,37 +236,7 @@ public void removeDuplicates() { } @Override - public boolean containsReference(final Item item) { - return start == item || end == item; - } - - @Override - public boolean contains(final Item item) { - if (item instanceof IntegerValue) { - try { - final BigInteger other = item.toJavaObject(BigInteger.class); - return other.compareTo(start.toJavaObject(BigInteger.class)) >= 0 - && other.compareTo(end.toJavaObject(BigInteger.class)) <= 0; - } catch (final XPathException e) { - LOG.warn(e.getMessage(), e); - return false; - } - } - return false; + public String toString() { + return "Range(" + start + " to " + end + ")"; } - - /** - * Generates a string representation of the Range Sequence. - * - * Range sequences can potentially be - * very large, so we generate a summary here - * rather than evaluating to generate a (possibly) - * huge sequence of objects. - * - * @return a string representation of the range sequence. - */ - @Override - public String toString() { - return "Range(" + start + " to " + end + ")"; - } } diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java index b3721c34179..4153e3cf5da 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java @@ -2730,6 +2730,13 @@ private ExternalModule compileOrBorrowModule(final String namespaceURI, final St * @return The compiled module, or null if the source is not a module * @throws XPathException if the module could not be loaded (XQST0059) or compiled (XPST0003) */ + /** + * Compile a module from a Source. Public wrapper for fn:load-xquery-module content option. + */ + public @Nullable ExternalModule compileModuleFromSource(final String namespaceURI, final Source source) throws XPathException { + return compileModule(namespaceURI, null, "content", source); + } + private @Nullable ExternalModule compileModule(String namespaceURI, final String prefix, final String location, final Source source) throws XPathException { if (LOG.isDebugEnabled()) { diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunAnalyzeString.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunAnalyzeString.java index 8fe035492a7..e9d8eb9764f 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunAnalyzeString.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunAnalyzeString.java @@ -158,25 +158,46 @@ private void analyzeString(final MemTreeBuilder builder, final String input, Str private void match(final MemTreeBuilder builder, final RegexIterator regexIterator) throws net.sf.saxon.trans.XPathException { builder.startElement(QN_MATCH, null); - regexIterator.processMatchingSubstring(new RegexIterator.MatchHandler() { - @Override - public void characters(final CharSequence s) { - builder.characters(s); - } - - @Override - public void onGroupStart(final int groupNumber) throws net.sf.saxon.trans.XPathException { - final AttributesImpl attributes = new AttributesImpl(); - attributes.addAttribute("", QN_NR.getLocalPart(), QN_NR.getLocalPart(), "int", Integer.toString(groupNumber)); - - builder.startElement(QN_GROUP, attributes); - } - - @Override - public void onGroupEnd(final int groupNumber) throws net.sf.saxon.trans.XPathException { - builder.endElement(); + // Use reflection to avoid compile-time dependency on RegexIterator$MatchHandler, + // which is stripped from the XQTS runner assembly JAR by sbt's merge strategy. + // When running in the normal eXist server (or on the next branch with full Saxon), + // the proxy delegates to Saxon's own group traversal logic. + try { + final Class handlerClass = Class.forName("net.sf.saxon.regex.RegexIterator$MatchHandler"); + final Object handler = java.lang.reflect.Proxy.newProxyInstance( + handlerClass.getClassLoader(), + new Class[]{ handlerClass }, + (proxy, method, args) -> { + switch (method.getName()) { + case "characters": + builder.characters((CharSequence) args[0]); + break; + case "onGroupStart": + final AttributesImpl attrs = new AttributesImpl(); + attrs.addAttribute("", QN_NR.getLocalPart(), QN_NR.getLocalPart(), + "int", Integer.toString((Integer) args[0])); + builder.startElement(QN_GROUP, attrs); + break; + case "onGroupEnd": + builder.endElement(); + break; + } + return null; + }); + final java.lang.reflect.Method processMethod = regexIterator.getClass().getMethod( + "processMatchingSubstring", handlerClass); + processMethod.invoke(regexIterator, handler); + } catch (final ClassNotFoundException e) { + // MatchHandler unavailable — output match text without group decomposition + builder.characters(regexIterator.getRegexGroup(0)); + } catch (final java.lang.reflect.InvocationTargetException e) { + if (e.getCause() instanceof net.sf.saxon.trans.XPathException) { + throw (net.sf.saxon.trans.XPathException) e.getCause(); } - }); + builder.characters(regexIterator.getRegexGroup(0)); + } catch (final Exception e) { + builder.characters(regexIterator.getRegexGroup(0)); + } builder.endElement(); } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunCompare.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunCompare.java index d2cd6e102c7..2b547f8a8a3 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunCompare.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunCompare.java @@ -32,14 +32,23 @@ import org.exist.xquery.Profiler; import org.exist.xquery.XPathException; import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.AbstractDateTimeValue; +import org.exist.xquery.value.AtomicValue; +import org.exist.xquery.value.DoubleValue; +import org.exist.xquery.value.DurationValue; +import org.exist.xquery.value.FloatValue; import org.exist.xquery.value.FunctionReturnSequenceType; import org.exist.xquery.value.FunctionParameterSequenceType; import org.exist.xquery.value.IntegerValue; import org.exist.xquery.value.Item; +import org.exist.xquery.value.NumericValue; import org.exist.xquery.value.Sequence; import org.exist.xquery.value.SequenceType; import org.exist.xquery.value.Type; +import java.math.BigDecimal; +import java.math.BigInteger; + import javax.annotation.Nullable; /** @@ -52,44 +61,33 @@ public class FunCompare extends CollatingFunction { public final static FunctionSignature[] signatures = { new FunctionSignature ( new QName("compare", Function.BUILTIN_FUNCTION_NS), - "Returns the collatable comparison between $string-1 and $string-2, using $collation-uri. " + - "-1 if $string-1 is inferior to $string-2, 0 if $string-1 is equal " + - "to $string-2, 1 if $string-1 is superior to $string-2. " + - "If either comparand is the empty sequence, the empty sequence is " + - "returned. " + - "Please remember to specify the collation in the context or use, " + - "the three argument version if you don't want the system default.", + "Returns -1, 0, or 1, depending on whether $value-1 is less than, equal to, " + + "or greater than $value-2. " + + "If either comparand is the empty sequence, the empty sequence is returned.", new SequenceType[] { - new FunctionParameterSequenceType("string-1", Type.STRING, - Cardinality.ZERO_OR_ONE, "The first string"), - new FunctionParameterSequenceType("string-2", Type.STRING, - Cardinality.ZERO_OR_ONE, "The second string") + new FunctionParameterSequenceType("value-1", Type.ANY_ATOMIC_TYPE, + Cardinality.ZERO_OR_ONE, "The first value"), + new FunctionParameterSequenceType("value-2", Type.ANY_ATOMIC_TYPE, + Cardinality.ZERO_OR_ONE, "The second value") }, new FunctionReturnSequenceType(Type.INTEGER, Cardinality.ZERO_OR_ONE, - "-1 if $string-1 is inferior to $string-2, " + - "0 if $string-1 is equal to $string-2, " + - "1 if $string-1 is superior to $string-2. " + - "If either comparand is the empty sequence, the empty sequence is returned.")), + "-1, 0, or 1 depending on comparison result")), new FunctionSignature ( new QName("compare", Function.BUILTIN_FUNCTION_NS), - "Returns the collatable comparison between $string-1 and $string-2, using $collation-uri. " + - "-1 if $string-1 is inferior to $string-2, 0 if $string-1 is equal " + - "to $string-2, 1 if $string-1 is superior to $string-2. " + + "Returns -1, 0, or 1, depending on whether $value-1 is less than, equal to, " + + "or greater than $value-2, using the specified collation. " + "If either comparand is the empty sequence, the empty sequence is returned. " + THIRD_REL_COLLATION_ARG_EXAMPLE, new SequenceType[] { - new FunctionParameterSequenceType("string-1", Type.STRING, - Cardinality.ZERO_OR_ONE, "The first string"), - new FunctionParameterSequenceType("string-2", Type.STRING, - Cardinality.ZERO_OR_ONE, "The second string"), + new FunctionParameterSequenceType("value-1", Type.ANY_ATOMIC_TYPE, + Cardinality.ZERO_OR_ONE, "The first value"), + new FunctionParameterSequenceType("value-2", Type.ANY_ATOMIC_TYPE, + Cardinality.ZERO_OR_ONE, "The second value"), new FunctionParameterSequenceType("collation-uri", Type.STRING, Cardinality.EXACTLY_ONE, "The relative collation URI") }, new FunctionReturnSequenceType(Type.INTEGER, Cardinality.ZERO_OR_ONE, - "-1 if $string-1 is inferior to $string-2, " + - "0 if $string-1 is equal to $string-2, " + - "1 if $string-1 is superior to $string-2. " + - "If either comparand is the empty sequence, the empty sequence is returned.")) + "-1, 0, or 1 depending on comparison result")) }; public FunCompare(XQueryContext context, FunctionSignature signature) { @@ -123,14 +121,132 @@ public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathExc return result; } - static int compare(final Item item1, final Item item2, @Nullable final Collator collator) throws XPathException { - final int comparison = Collations.compare(collator, item1.getStringValue(), item2.getStringValue()); - if (comparison == Constants.EQUAL) { + public static int compare(final Item item1, final Item item2, @Nullable final Collator collator) throws XPathException { + final AtomicValue v1 = item1.atomize(); + final AtomicValue v2 = item2.atomize(); + + // For string-like types, use collation-aware comparison + if (isStringLike(v1.getType()) && isStringLike(v2.getType())) { + return normalizeComparison(Collations.compare(collator, v1.getStringValue(), v2.getStringValue())); + } + + // XQ4 numeric total order: compare by exact mathematical magnitude + if (v1 instanceof NumericValue && v2 instanceof NumericValue) { + return numericTotalOrder((NumericValue) v1, (NumericValue) v2); + } + + // XQ4 duration total order: months first, then seconds + if (v1 instanceof DurationValue && v2 instanceof DurationValue) { + return durationTotalOrder((DurationValue) v1, (DurationValue) v2); + } + + // XQ4 date/time total order: normalize to millis for types where + // XMLGregorianCalendar.compare() may return INDETERMINATE + if (v1 instanceof AbstractDateTimeValue && v2 instanceof AbstractDateTimeValue + && v1.getType() == v2.getType()) { + return dateTimeTotalOrder((AbstractDateTimeValue) v1, (AbstractDateTimeValue) v2); + } + + // For other atomic types, use natural ordering via compareTo + return normalizeComparison(v1.compareTo(collator, v2)); + } + + /** + * XQ4 numeric total order for fn:compare. + * Float is promoted to double. NaN == NaN (and NaN < everything). + * -0.0 == +0.0. Doubles and decimals compared by exact mathematical magnitude. + */ + static int numericTotalOrder(final NumericValue v1, final NumericValue v2) throws XPathException { + // Promote float to double + final double d1 = v1.getDouble(); + final double d2 = v2.getDouble(); + + final boolean nan1 = Double.isNaN(d1); + final boolean nan2 = Double.isNaN(d2); + + // NaN equals NaN, NaN < everything else + if (nan1 && nan2) { return Constants.EQUAL; - } else if (comparison < 0) { + } + if (nan1) { return Constants.INFERIOR; - } else { + } + if (nan2) { return Constants.SUPERIOR; } + + // Handle infinities + if (Double.isInfinite(d1) || Double.isInfinite(d2)) { + if (d1 == d2) { + return Constants.EQUAL; + } + return d1 < d2 ? Constants.INFERIOR : Constants.SUPERIOR; + } + + // -0.0 == +0.0 + if (d1 == 0.0 && d2 == 0.0) { + return Constants.EQUAL; + } + + // Compare by exact mathematical magnitude using BigDecimal + final BigDecimal bd1 = toBigDecimal(v1); + final BigDecimal bd2 = toBigDecimal(v2); + return normalizeComparison(bd1.compareTo(bd2)); + } + + private static BigDecimal toBigDecimal(final NumericValue v) throws XPathException { + if (v instanceof org.exist.xquery.value.DecimalValue) { + return ((org.exist.xquery.value.DecimalValue) v).getValue(); + } + if (v instanceof IntegerValue) { + // Use string representation — getValue() truncates to long for big integers + return new BigDecimal(v.getStringValue()); + } + // Double or Float — use exact decimal representation (no rounding) + return new BigDecimal(v.getDouble()); + } + + /** + * XQ4 duration total order for fn:compare. + * Compares months component first, then seconds component. + * This provides a total order even for xs:duration values where + * months and seconds are both present (which XMLGregorianCalendar + * considers INDETERMINATE). + */ + static int durationTotalOrder(final DurationValue v1, final DurationValue v2) { + final BigInteger months1 = v1.monthsValueSigned(); + final BigInteger months2 = v2.monthsValueSigned(); + final int monthsCmp = months1.compareTo(months2); + if (monthsCmp != 0) { + return normalizeComparison(monthsCmp); + } + final BigDecimal seconds1 = v1.secondsValueSigned(); + final BigDecimal seconds2 = v2.secondsValueSigned(); + return normalizeComparison(seconds1.compareTo(seconds2)); + } + + /** + * XQ4 date/time total order for fn:compare. + * Uses getTimeInMillis() to normalize both values to a common + * representation, avoiding INDETERMINATE results from + * XMLGregorianCalendar.compare() on partial date/time types. + */ + static int dateTimeTotalOrder(final AbstractDateTimeValue v1, final AbstractDateTimeValue v2) { + final long ms1 = v1.getTimeInMillis(); + final long ms2 = v2.getTimeInMillis(); + return normalizeComparison(Long.compare(ms1, ms2)); + } + + private static int normalizeComparison(final int cmp) { + if (cmp == 0) { + return Constants.EQUAL; + } + return cmp < 0 ? Constants.INFERIOR : Constants.SUPERIOR; + } + + private static boolean isStringLike(final int type) { + return Type.subTypeOf(type, Type.STRING) + || type == Type.UNTYPED_ATOMIC + || Type.subTypeOf(type, Type.ANY_URI); } } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDeepEqual.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDeepEqual.java index 6e6e0285dc2..8138c198a2c 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDeepEqual.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDeepEqual.java @@ -35,6 +35,7 @@ import org.exist.xquery.Dependency; import org.exist.xquery.Function; import org.exist.xquery.FunctionSignature; +import org.exist.xquery.InlineFunction; import org.exist.xquery.Profiler; import org.exist.xquery.ValueComparison; import org.exist.xquery.XPathException; @@ -43,6 +44,7 @@ import org.exist.xquery.functions.map.AbstractMapType; import org.exist.xquery.value.AtomicValue; import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionReference; import org.exist.xquery.value.FunctionReturnSequenceType; import org.exist.xquery.value.FunctionParameterSequenceType; import org.exist.xquery.value.Item; @@ -55,6 +57,8 @@ import org.w3c.dom.Node; import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; /** * Implements the fn:deep-equal library function. @@ -225,6 +229,29 @@ public static int deepCompare(final Item item1, final Item item2, @Nullable fina } } + // XQ4: Function items compared by function-identity semantics + if (Type.subTypeOf(item1.getType(), Type.FUNCTION) || Type.subTypeOf(item2.getType(), Type.FUNCTION)) { + if (!Type.subTypeOf(item1.getType(), Type.FUNCTION) || !Type.subTypeOf(item2.getType(), Type.FUNCTION)) { + return Constants.INFERIOR; + } + if (item1 == item2) { + return Constants.EQUAL; + } + // Named functions with same name and arity are equal + if (item1 instanceof FunctionReference ref1 && item2 instanceof FunctionReference ref2) { + final org.exist.dom.QName name1 = ref1.getSignature().getName(); + final org.exist.dom.QName name2 = ref2.getSignature().getName(); + if (name1 != null && name2 != null + && name1 != InlineFunction.INLINE_FUNCTION_QNAME + && name2 != InlineFunction.INLINE_FUNCTION_QNAME + && name1.equals(name2) + && ref1.getSignature().getArgumentCount() == ref2.getSignature().getArgumentCount()) { + return Constants.EQUAL; + } + } + return Constants.INFERIOR; + } + final boolean item1IsAtomic = Type.subTypeOf(item1.getType(), Type.ANY_ATOMIC_TYPE); final boolean item2IsAtomic = Type.subTypeOf(item2.getType(), Type.ANY_ATOMIC_TYPE); if (item1IsAtomic || item2IsAtomic) { @@ -370,44 +397,75 @@ private static int compareElements(final Node a, final Node b, @Nullable final C } private static int compareContents(Node a, Node b, @Nullable final Collator collator) { - a = findNextTextOrElementNode(a.getFirstChild()); - b = findNextTextOrElementNode(b.getFirstChild()); - while (!(a == null || b == null)) { - final int nodeTypeA = getEffectiveNodeType(a); - final int nodeTypeB = getEffectiveNodeType(b); - if (nodeTypeA != nodeTypeB) { - return Constants.INFERIOR; - } - switch (nodeTypeA) { - case Node.TEXT_NODE: - final String nodeValueA = getNodeValue(a); - final String nodeValueB = getNodeValue(b); - final int textComparison = safeCompare(nodeValueA, nodeValueB, collator); + // XQ4: merge adjacent text nodes (split by ignored comments/PIs) + final List childrenA = mergeTextNodes(a); + final List childrenB = mergeTextNodes(b); + + if (childrenA.size() != childrenB.size()) { + return childrenA.size() < childrenB.size() ? Constants.INFERIOR : Constants.SUPERIOR; + } + + for (int i = 0; i < childrenA.size(); i++) { + final Object ca = childrenA.get(i); + final Object cb = childrenB.get(i); + + if (ca instanceof String sa && cb instanceof String sb) { + final int textComparison = safeCompare(sa, sb, collator); if (textComparison != Constants.EQUAL) { return textComparison; } - break; - case Node.ELEMENT_NODE: - final int elementComparison = compareElements(a, b, collator); - if (elementComparison != Constants.EQUAL) { - return elementComparison; + } else if (ca instanceof Node na && cb instanceof Node nb) { + if (getEffectiveNodeType(na) != getEffectiveNodeType(nb)) { + return Constants.INFERIOR; + } + if (getEffectiveNodeType(na) != getEffectiveNodeType(nb)) { + return Constants.INFERIOR; + } + if (getEffectiveNodeType(na) == Node.ELEMENT_NODE) { + final int cmp = compareElements(na, nb, collator); + if (cmp != Constants.EQUAL) { + return cmp; + } } - break; - default: - throw new RuntimeException("unexpected node type " + nodeTypeA); + } else { + return Constants.INFERIOR; } - a = findNextTextOrElementNode(a.getNextSibling()); - b = findNextTextOrElementNode(b.getNextSibling()); } + return Constants.EQUAL; + } - // NOTE(AR): intentional reference equality check - if (a == b) { - return Constants.EQUAL; // both null - } else if (a == null) { - return Constants.INFERIOR; - } else { - return Constants.SUPERIOR; + /** + * Collect significant children for deep-equal comparison. + * Per XQ3.1 spec §15.3.1: "children are compared after removing all + * comment and processing-instruction nodes" — but text nodes are + * NOT merged (elements with split text nodes differ from single text). + */ + static List mergeTextNodes(final Node parent) { + final List result = new ArrayList<>(); + StringBuilder textAccum = null; + Node child = parent.getFirstChild(); + while (child != null) { + final int nodeType = getEffectiveNodeType(child); + if (nodeType == Node.TEXT_NODE) { + // XQ4: merge adjacent text nodes (split by ignored comments/PIs) + if (textAccum == null) { + textAccum = new StringBuilder(); + } + textAccum.append(getNodeValue(child)); + } else if (nodeType == Node.ELEMENT_NODE) { + if (textAccum != null) { + result.add(textAccum.toString()); + textAccum = null; + } + result.add(child); + } + // Skip comments and PIs per spec — text continues to merge + child = child.getNextSibling(); } + if (textAccum != null) { + result.add(textAccum.toString()); + } + return result; } private static String getNodeValue(final Node n) { diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDoc.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDoc.java index 65b663f4848..ad8829c0d42 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDoc.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDoc.java @@ -64,11 +64,27 @@ public class FunDoc extends Function { "the document node of $document-uri") ); + // XQuery 4.0: fn:doc with options map + public final static FunctionSignature signatureWithOptions = + new FunctionSignature( + new QName("doc", Function.BUILTIN_FUNCTION_NS), + "Returns the document node of $document-uri with options. " + + XMLDBModule.ANY_URI, + new SequenceType[] { + new FunctionParameterSequenceType("document-uri", Type.STRING, + Cardinality.ZERO_OR_ONE, "The document URI"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, + Cardinality.EXACTLY_ONE, "Options map") + }, + new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.ZERO_OR_ONE, + "the document node of $document-uri") + ); + // fixit! - security warning private UpdateListener listener = null; - public FunDoc(XQueryContext context) { - super(context, signature); + public FunDoc(XQueryContext context, FunctionSignature sig) { + super(context, sig); } public Sequence eval(Sequence contextSequence, Item contextItem) diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDocAvailable.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDocAvailable.java index be44565be0a..e1f5591b4fa 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDocAvailable.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunDocAvailable.java @@ -62,8 +62,23 @@ public class FunDocAvailable extends Function { new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true() if the document is available, false() otherwise")); - public FunDocAvailable(final XQueryContext context) { - super(context, signature); + // XQuery 4.0: fn:doc-available with options map + public static final FunctionSignature signatureWithOptions = + new FunctionSignature( + new QName("doc-available", Function.BUILTIN_FUNCTION_NS), + "Returns whether or not the document is available, with options. " + + XMLDBModule.ANY_URI, + new SequenceType[]{ + new FunctionParameterSequenceType("document-uri", Type.STRING, + Cardinality.ZERO_OR_ONE, "The document URI"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, + Cardinality.EXACTLY_ONE, "Options map") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "true() if the document is available, false() otherwise")); + + public FunDocAvailable(final XQueryContext context, final FunctionSignature sig) { + super(context, sig); } @Override diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunHeadTail.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunHeadTail.java index c01a4110863..bd3949aca2a 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunHeadTail.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunHeadTail.java @@ -35,27 +35,46 @@ import org.exist.xquery.value.Sequence; import org.exist.xquery.value.SequenceType; import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; public class FunHeadTail extends BasicFunction { - public final static FunctionSignature[] signatures = { - new FunctionSignature( - new QName("head", Function.BUILTIN_FUNCTION_NS), - "The function returns the value of the expression $arg[1], i.e. the first item in the " + - "passed in sequence.", - new SequenceType[] { - new FunctionParameterSequenceType("arg", Type.ITEM, Cardinality.ZERO_OR_MORE, "") - }, - new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_ONE, "the first item or the empty sequence")), - new FunctionSignature( - new QName("tail", Function.BUILTIN_FUNCTION_NS), - "The function returns the value of the expression subsequence($sequence, 2), i.e. a new sequence containing " + - "all items of the input sequence except the first.", - new SequenceType[] { - new FunctionParameterSequenceType("sequence", Type.ITEM, Cardinality.ZERO_OR_MORE, "The source sequence") - }, - new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "the resulting sequence")) }; - + public final static FunctionSignature FN_HEAD = new FunctionSignature( + new QName("head", Function.BUILTIN_FUNCTION_NS), + "The function returns the value of the expression $arg[1], i.e. the first item in the " + + "passed in sequence.", + new SequenceType[] { + new FunctionParameterSequenceType("arg", Type.ITEM, Cardinality.ZERO_OR_MORE, "") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_ONE, "the first item or the empty sequence")); + + public final static FunctionSignature FN_TAIL = new FunctionSignature( + new QName("tail", Function.BUILTIN_FUNCTION_NS), + "The function returns the value of the expression subsequence($sequence, 2), i.e. a new sequence containing " + + "all items of the input sequence except the first.", + new SequenceType[] { + new FunctionParameterSequenceType("sequence", Type.ITEM, Cardinality.ZERO_OR_MORE, "The source sequence") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "the resulting sequence")); + + public final static FunctionSignature FN_FOOT = new FunctionSignature( + new QName("foot", Function.BUILTIN_FUNCTION_NS), + "Returns the last item in a sequence.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_ONE, "the last item or the empty sequence")); + + public final static FunctionSignature FN_TRUNK = new FunctionSignature( + new QName("trunk", Function.BUILTIN_FUNCTION_NS), + "Returns all but the last item in a sequence.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "all items except the last")); + + public final static FunctionSignature[] signatures = { FN_HEAD, FN_TAIL, FN_FOOT, FN_TRUNK }; + public FunHeadTail(XQueryContext context, FunctionSignature signature) { super(context, signature); } @@ -64,24 +83,36 @@ public FunHeadTail(XQueryContext context, FunctionSignature signature) { public void analyze(AnalyzeContextInfo contextInfo) throws XPathException { super.analyze(contextInfo); if (getContext().getXQueryVersion()<30) { - throw new XPathException(this, ErrorCodes.EXXQDY0003, "Function " + + throw new XPathException(this, ErrorCodes.EXXQDY0003, "Function " + getSignature().getName() + " is only supported for xquery version \"3.0\" and later."); } } - + @Override public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathException { final Sequence seq = args[0]; - Sequence tmp; if (seq.isEmpty()) { - tmp = Sequence.EMPTY_SEQUENCE; - } else if (isCalledAs("head")) { - tmp = seq.itemAt(0).toSequence(); + return Sequence.EMPTY_SEQUENCE; + } + if (isCalledAs("head")) { + return seq.itemAt(0).toSequence(); + } else if (isCalledAs("tail")) { + return seq.tail(); + } else if (isCalledAs("foot")) { + return seq.itemAt(seq.getItemCount() - 1).toSequence(); } else { - tmp = seq.tail(); + // trunk: all items except the last + final int count = seq.getItemCount(); + if (count <= 1) { + return Sequence.EMPTY_SEQUENCE; + } + final ValueSequence result = new ValueSequence(count - 1); + for (int i = 0; i < count - 1; i++) { + result.add(seq.itemAt(i)); + } + return result; } - return tmp; } } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunInScopePrefixes.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunInScopePrefixes.java index a0a248dface..3ba1ec23f08 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunInScopePrefixes.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunInScopePrefixes.java @@ -79,6 +79,10 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce final ValueSequence result = new ValueSequence(); for (final String prefix : prefixes.keySet()) { + // Per XQuery spec §14.2: "xmlns" must not be included in the result + if ("xmlns".equals(prefix)) { + continue; + } //The predefined namespaces (e.g. "exist" for temporary nodes) could have been removed from the static context if (!(context.getURIForPrefix(prefix) == null && ("exist".equals(prefix) || "xs".equals(prefix) || "xsi".equals(prefix) || diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMatches.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMatches.java index 6f06bd772ce..c84fc8879af 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMatches.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMatches.java @@ -63,7 +63,8 @@ public final class FunMatches extends Function implements Optimizable, IndexUseR private static final FunctionParameterSequenceType FS_PARAM_INPUT = optParam("input", Type.STRING, "The input string"); private static final FunctionParameterSequenceType FS_PARAM_PATTERN = param("pattern", Type.STRING, "The pattern"); - private static final FunctionParameterSequenceType FS_PARAM_FLAGS = param("flags", Type.STRING, "The flags"); + private static final FunctionParameterSequenceType FS_PARAM_FLAGS = + new FunctionParameterSequenceType("flags", Type.STRING, Cardinality.ZERO_OR_ONE, "The flags"); private static final String FS_MATCHES_NAME = "matches"; private static final String FS_DESCRIPTION = @@ -138,7 +139,7 @@ public void setArguments(final List arguments) throws XPathException if (arguments.size() >= 3) { Expression arg = arguments.get(2); - arg = new DynamicCardinalityCheck(context, Cardinality.EXACTLY_ONE, arg, + arg = new DynamicCardinalityCheck(context, Cardinality.ZERO_OR_ONE, arg, new Error(Error.FUNC_PARAM_CARDINALITY, "3", getSignature())); if (!Type.subTypeOf(arg.returnsType(), Type.ANY_ATOMIC_TYPE)) { arg = new Atomize(context, arg); @@ -212,7 +213,8 @@ public NodeSet preSelect(final Sequence contextSequence, final boolean useContex final int flags; if (getSignature().getArgumentCount() == 3) { - final String flagsArg = getArgument(2).eval(contextSequence, null).getStringValue(); + final Sequence flagsSeq = getArgument(2).eval(contextSequence, null); + final String flagsArg = flagsSeq.isEmpty() ? "" : flagsSeq.getStringValue(); flags = parseFlags(this, flagsArg); } else { flags = 0; @@ -382,7 +384,8 @@ private Sequence evalWithIndex(final Sequence contextSequence, final Item contex final int flags; if (getSignature().getArgumentCount() == 3) { - final String flagsArg = getArgument(2).eval(contextSequence, contextItem).getStringValue(); + final Sequence flagsSeq = getArgument(2).eval(contextSequence, contextItem); + final String flagsArg = flagsSeq.isEmpty() ? "" : flagsSeq.getStringValue(); flags = parseFlags(this, flagsArg); } else { flags = 0; @@ -497,7 +500,8 @@ private Sequence evalGeneric(final Sequence contextSequence, final Item contextI final String xmlRegexFlags; if (getSignature().getArgumentCount() == 3) { - xmlRegexFlags = getArgument(2).eval(contextSequence, contextItem).getStringValue(); + final Sequence flagsSeq = getArgument(2).eval(contextSequence, contextItem); + xmlRegexFlags = flagsSeq.isEmpty() ? "" : flagsSeq.getStringValue(); } else { xmlRegexFlags = ""; } @@ -512,7 +516,16 @@ private Sequence evalGeneric(final Sequence contextSequence, final Item contextI } - private boolean matchXmlRegex(final String string, final String pattern, final String flags) throws XPathException { + private boolean matchXmlRegex(String string, String pattern, String flags) throws XPathException { + // XQ4: 'c' flag — strip regex comments before compilation + final boolean hasCommentFlag = flags.indexOf('c') >= 0 && flags.indexOf('q') < 0; + if (flags.indexOf('c') >= 0) { + flags = flags.replace("c", ""); + } + if (hasCommentFlag) { + pattern = FunReplace.stripRegexComments(pattern); + } + try { List warnings = new ArrayList<>(1); RegularExpression regex = context.getBroker().getBrokerPool() @@ -526,6 +539,18 @@ private boolean matchXmlRegex(final String string, final String pattern, final S return regex.containsMatch(string); } catch (final net.sf.saxon.trans.XPathException e) { + // Fallback: if the pattern uses \p{Is} Unicode block names that + // Saxon doesn't recognize, convert to Java's \p{In} and use Java regex + if (pattern.contains("\\p{Is") || pattern.contains("\\P{Is")) { + try { + final String javaPattern = org.exist.xquery.regex.RegexUtil.translateRegexp( + this, pattern, flags.contains("x"), flags.contains("i")); + int javaFlags = org.exist.xquery.regex.RegexUtil.parseFlags(this, flags); + return Pattern.compile(javaPattern, javaFlags).matcher(string).find(); + } catch (final XPathException | PatternSyntaxException ignored) { + // fallback failed, throw original Saxon error + } + } switch (e.getErrorCodeLocalPart()) { case "FORX0001" -> throw new XPathException(this, ErrorCodes.FORX0001, "Invalid regular expression: " + e.getMessage()); case "FORX0002" -> throw new XPathException(this, ErrorCodes.FORX0002, "Invalid regular expression: " + e.getMessage()); diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMax.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMax.java index 31fe3ee95b3..41e356645b1 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMax.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMax.java @@ -23,8 +23,8 @@ import com.ibm.icu.text.Collator; import org.exist.dom.QName; -import org.exist.util.Collations; import org.exist.xquery.Cardinality; +import org.exist.xquery.Constants; import org.exist.xquery.Dependency; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Function; @@ -33,191 +33,145 @@ import org.exist.xquery.XPathException; import org.exist.xquery.XQueryContext; import org.exist.xquery.value.AtomicValue; -import org.exist.xquery.value.ComputableValue; -import org.exist.xquery.value.DoubleValue; import org.exist.xquery.value.DurationValue; -import org.exist.xquery.value.FloatValue; import org.exist.xquery.value.FunctionParameterSequenceType; import org.exist.xquery.value.FunctionReturnSequenceType; import org.exist.xquery.value.Item; import org.exist.xquery.value.NumericValue; -import org.exist.xquery.value.QNameValue; import org.exist.xquery.value.Sequence; import org.exist.xquery.value.SequenceIterator; import org.exist.xquery.value.SequenceType; import org.exist.xquery.value.Type; /** + * Implementation of fn:max with XQuery 4.0 semantics. + * Uses fn:compare-based mutual comparability (XQ4 numeric total order, + * duration total order, date/time total order). + * * @author Wolfgang Meier */ public class FunMax extends CollatingFunction { - protected static final String FUNCTION_DESCRIPTION_COMMON_1 = - "Selects an item from the input sequence $arg whose value is " + - "greater than or equal to the value of every other item in the " + - "input sequence. If there are two or more such items, then the " + - "specific item whose value is returned is implementation dependent.\n\n" + - "The following rules are applied to the input sequence:\n\n" + - "- Values of type xs:untypedAtomic in $arg are cast to xs:double.\n" + - "- Numeric and xs:anyURI values are converted to the least common " + - "type that supports the 'ge' operator by a combination of type " + - "promotion and subtype substitution. See Section B.1 Type " + - "PromotionXP and Section B.2 Operator MappingXP.\n\n" + - "The items in the resulting sequence may be reordered in an arbitrary " + - "order. The resulting sequence is referred to below as the converted " + - "sequence. This function returns an item from the converted sequence " + - "rather than the input sequence.\n\n" + - "If the converted sequence is empty, the empty sequence is returned.\n\n" + - "All items in $arg must be numeric or derived from a single base type " + - "for which the 'ge' operator is defined. In addition, the values in the " + - "sequence must have a total order. If date/time values do not have a " + - "timezone, they are considered to have the implicit timezone provided " + - "by the dynamic context for purposes of comparison. Duration values " + - "must either all be xs:yearMonthDuration values or must all be " + - "xs:dayTimeDuration values.\n\n" + - "If any of these conditions is not met, then a type error is raised [err:FORG0006].\n\n" + - "If the converted sequence contains the value NaN, the value NaN is returned.\n\n" + - "If the items in the value of $arg are of type xs:string or types " + - "derived by restriction from xs:string, then the determination of " + - "the item with the largest value is made according to the collation " + - "that is used."; - protected static final String FUNCTION_DESCRIPTION_2_PARAM = - "If the type of the items in $arg is not xs:string " + - "and $collation-uri is specified, the collation is ignored.\n\n"; - protected static final String FUNCTION_DESCRIPTION_COMMON_2 = - "The collation used by the invocation of this function is " + - "determined according to the rules in 7.3.1 Collations."; - - - public final static FunctionSignature[] signatures = { - new FunctionSignature( - new QName("max", Function.BUILTIN_FUNCTION_NS), - FUNCTION_DESCRIPTION_COMMON_1 + - FUNCTION_DESCRIPTION_COMMON_2, - new SequenceType[] { - new FunctionParameterSequenceType("arg", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The input sequence") - }, - new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_ONE, "the max value") - ), - new FunctionSignature( - new QName("max", Function.BUILTIN_FUNCTION_NS), - FUNCTION_DESCRIPTION_COMMON_1 + FUNCTION_DESCRIPTION_2_PARAM + - FUNCTION_DESCRIPTION_COMMON_2, - new SequenceType[] { - new FunctionParameterSequenceType("arg", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The input sequence"), - new FunctionParameterSequenceType("collation-uri", Type.STRING, Cardinality.EXACTLY_ONE, "The collation URI") - }, - new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_ONE, "the max value") - ) - }; - - public FunMax(XQueryContext context, FunctionSignature signature) { - super(context, signature); - } - - /* (non-Javadoc) - * @see org.exist.xquery.Expression#eval(org.exist.dom.persistent.DocumentSet, org.exist.xquery.value.Sequence, org.exist.xquery.value.Item) - */ - public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathException { + public final static FunctionSignature[] signatures = { + new FunctionSignature( + new QName("max", Function.BUILTIN_FUNCTION_NS), + "Returns the maximum value from the input sequence, using XQ4 comparison semantics.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, + Cardinality.ZERO_OR_MORE, "The input sequence") + }, + new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_ONE, + "the maximum value") + ), + new FunctionSignature( + new QName("max", Function.BUILTIN_FUNCTION_NS), + "Returns the maximum value from the input sequence, using the specified collation.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, + Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("collation", Type.STRING, + Cardinality.ZERO_OR_ONE, "The collation URI") + }, + new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_ONE, + "the maximum value") + ) + }; + + public FunMax(XQueryContext context, FunctionSignature signature) { + super(context, signature); + } + + public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathException { if (context.getProfiler().isEnabled()) { - context.getProfiler().start(this); - context.getProfiler().message(this, Profiler.DEPENDENCIES, "DEPENDENCIES", Dependency.getDependenciesName(this.getDependencies())); + context.getProfiler().start(this); + context.getProfiler().message(this, Profiler.DEPENDENCIES, "DEPENDENCIES", + Dependency.getDependenciesName(this.getDependencies())); if (contextSequence != null) - {context.getProfiler().message(this, Profiler.START_SEQUENCES, "CONTEXT SEQUENCE", contextSequence);} + {context.getProfiler().message(this, Profiler.START_SEQUENCES, + "CONTEXT SEQUENCE", contextSequence);} if (contextItem != null) - {context.getProfiler().message(this, Profiler.START_SEQUENCES, "CONTEXT ITEM", contextItem.toSequence());} - } - + {context.getProfiler().message(this, Profiler.START_SEQUENCES, + "CONTEXT ITEM", contextItem.toSequence());} + } + Sequence result; - final Sequence arg = getArgument(0).eval(contextSequence, contextItem); - if(arg.isEmpty()) - {result = Sequence.EMPTY_SEQUENCE;} - else { - boolean computableProcessing = false; - //TODO : test if a range index is defined *iff* it is compatible with the collator - final Collator collator = getCollator(contextSequence, contextItem, 2); - final SequenceIterator iter = arg.unorderedIterator(); - AtomicValue max = null; - while (iter.hasNext()) { - final Item item = iter.nextItem(); - - if (item instanceof QNameValue) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(item.getType()), arg);} - - AtomicValue value = item.atomize(); - - //Duration values must either all be xs:yearMonthDuration values or must all be xs:dayTimeDuration values. - if (Type.subTypeOf(value.getType(), Type.DURATION)) { - value = ((DurationValue)value).wrap(); - if (value.getType() == Type.YEAR_MONTH_DURATION) { - if (max != null && max.getType() != Type.YEAR_MONTH_DURATION) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(max.getType()) + - " and " + Type.getTypeName(value.getType()), value);} - - } else if (value.getType() == Type.DAY_TIME_DURATION) { - if (max != null && max.getType() != Type.DAY_TIME_DURATION) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(max.getType()) + - " and " + Type.getTypeName(value.getType()), value);} - - } else - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(value.getType()), value);} - - //Any value of type xdt:untypedAtomic is cast to xs:double - } else if (value.getType() == Type.UNTYPED_ATOMIC) - {value = value.convertTo(Type.DOUBLE);} - - if (max == null) - {max = value;} - - else { - if (Type.getCommonSuperType(max.getType(), value.getType()) == Type.ANY_ATOMIC_TYPE) { - throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(max.getType()) + - " and " + Type.getTypeName(value.getType()), max); - } - //Any value of type xdt:untypedAtomic is cast to xs:double - if (value.getType() == Type.UNTYPED_ATOMIC) - {value = value.convertTo(Type.DOUBLE);} - - //Numeric tests - if (Type.subTypeOfUnion(value.getType(), Type.NUMERIC)) { - //Don't mix comparisons - if (!Type.subTypeOfUnion(max.getType(), Type.NUMERIC)) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(max.getType()) + - " and " + Type.getTypeName(value.getType()), max);} - if (((NumericValue) value).isNaN()) { - //Type NaN correctly - value = value.promote(max); - if (value.getType() == Type.FLOAT) - {max = FloatValue.NaN;} - else - {max = DoubleValue.NaN;} - //although result will be NaN, we need to continue on order to type correctly - continue; - } else - {max = max.promote(value);} - } - //Ugly test - if (max instanceof ComputableValue && value instanceof ComputableValue) { - //Type value correctly - value = value.promote(max); - max = (ComputableValue) max.max(collator, value); - computableProcessing = true; - } else { - if (computableProcessing) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(max.getType()) + - " and " + Type.getTypeName(value.getType()), max);} - if (Collations.compare(collator, value.getStringValue(), max.getStringValue()) > 0) - {max = value;} - } + final Sequence arg = getArgument(0).eval(contextSequence, contextItem); + if (arg.isEmpty()) { + result = Sequence.EMPTY_SEQUENCE; + } else { + final Collator collator = getOptionalCollator(contextSequence, contextItem); + result = findMax(arg, collator); + } + + if (context.getProfiler().isEnabled()) + {context.getProfiler().end(this, "", result);} + + return result; + } + + /** + * Get collator, handling empty sequence for XQ4 optional collation parameter. + */ + private Collator getOptionalCollator(Sequence contextSequence, Item contextItem) + throws XPathException { + if (getArgumentCount() == 2) { + final Sequence collationSeq = getArgument(1).eval(contextSequence, contextItem); + if (!collationSeq.isEmpty()) { + final String collationURI = collationSeq.getStringValue(); + return context.getCollator(collationURI, ErrorCodes.FOCH0002); + } + } + return context.getDefaultCollator(); + } + + private Sequence findMax(Sequence arg, Collator collator) throws XPathException { + final SequenceIterator iter = arg.unorderedIterator(); + AtomicValue max = null; + boolean hasNaN = false; + AtomicValue nanValue = null; + + while (iter.hasNext()) { + final Item item = iter.nextItem(); + AtomicValue value = item.atomize(); + + // Cast untypedAtomic to double + if (value.getType() == Type.UNTYPED_ATOMIC) { + value = value.convertTo(Type.DOUBLE); + } + + // Wrap duration subtypes + if (Type.subTypeOf(value.getType(), Type.DURATION)) { + value = ((DurationValue) value).wrap(); + } + + // Track NaN: if any value is NaN, result is NaN + if (value instanceof NumericValue && ((NumericValue) value).isNaN()) { + if (!hasNaN) { + hasNaN = true; + nanValue = value; + } + continue; + } + + if (max == null) { + max = value; + } else { + try { + final int cmp = FunCompare.compare(value, max, collator); + if (cmp > 0) { + max = value; + } + } catch (final XPathException e) { + throw new XPathException(this, ErrorCodes.FORG0006, + "Cannot compare " + Type.getTypeName(max.getType()) + + " and " + Type.getTypeName(value.getType()), value); } - } - result = max; + } } - if (context.getProfiler().isEnabled()) - {context.getProfiler().end(this, "", result);} - - return result; - - } + if (hasNaN) { + return nanValue; + } + return max; + } } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMin.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMin.java index c98ce39133a..10d58c59c4d 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMin.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunMin.java @@ -23,8 +23,8 @@ import com.ibm.icu.text.Collator; import org.exist.dom.QName; -import org.exist.util.Collations; import org.exist.xquery.Cardinality; +import org.exist.xquery.Constants; import org.exist.xquery.Dependency; import org.exist.xquery.ErrorCodes; import org.exist.xquery.Function; @@ -33,193 +33,145 @@ import org.exist.xquery.XPathException; import org.exist.xquery.XQueryContext; import org.exist.xquery.value.AtomicValue; -import org.exist.xquery.value.ComputableValue; -import org.exist.xquery.value.DoubleValue; import org.exist.xquery.value.DurationValue; -import org.exist.xquery.value.FloatValue; import org.exist.xquery.value.FunctionParameterSequenceType; import org.exist.xquery.value.FunctionReturnSequenceType; import org.exist.xquery.value.Item; import org.exist.xquery.value.NumericValue; -import org.exist.xquery.value.QNameValue; import org.exist.xquery.value.Sequence; import org.exist.xquery.value.SequenceIterator; import org.exist.xquery.value.SequenceType; import org.exist.xquery.value.Type; /** + * Implementation of fn:min with XQuery 4.0 semantics. + * Uses fn:compare-based mutual comparability (XQ4 numeric total order, + * duration total order, date/time total order). + * * @author Wolfgang Meier */ public class FunMin extends CollatingFunction { - protected static final String FUNCTION_DESCRIPTION_COMMON_1 = - - "Selects an item from the input sequence $arg whose value is " + - "less than or equal to the value of every other item in the " + - "input sequence. If there are two or more such items, then " + - "the specific item whose value is returned is implementation dependent.\n\n" + - "The following rules are applied to the input sequence:\n\n" + - "- Values of type xs:untypedAtomic in $arg are cast to xs:double.\n" + - "- Numeric and xs:anyURI values are converted to the least common " + - "type that supports the 'le' operator by a combination of type promotion " + - "and subtype substitution. See Section B.1 Type PromotionXP and " + - "Section B.2 Operator MappingXP.\n\n" + - - "The items in the resulting sequence may be reordered in an arbitrary " + - "order. The resulting sequence is referred to below as the converted " + - "sequence. This function returns an item from the converted sequence " + - "rather than the input sequence.\n\n" + - - "If the converted sequence is empty, the empty sequence is returned.\n\n" + - - "All items in $arg must be numeric or derived from a single base type " + - "for which the 'le' operator is defined. In addition, the values in the " + - "sequence must have a total order. If date/time values do not have a " + - "timezone, they are considered to have the implicit timezone provided " + - "by the dynamic context for the purpose of comparison. Duration values " + - "must either all be xs:yearMonthDuration values or must all be " + - "xs:dayTimeDuration values.\n\n" + - - "If any of these conditions is not met, a type error is raised [err:FORG0006].\n\n" + - - "If the converted sequence contains the value NaN, the value NaN is returned.\n\n" + - - "If the items in the value of $arg are of type xs:string or types derived " + - "by restriction from xs:string, then the determination of the item with " + - "the smallest value is made according to the collation that is used. "; - protected static final String FUNCTION_DESCRIPTION_2_PARAM = - "If the type of the items in $arg is not xs:string and $collation is " + - "specified, the collation is ignored.\n\n"; - protected static final String FUNCTION_DESCRIPTION_COMMON_2 = - "The collation used by the invocation of this function is determined " + - "according to the rules in 7.3.1 Collations."; - - public final static FunctionSignature[] signatures = { - new FunctionSignature( - new QName("min", Function.BUILTIN_FUNCTION_NS), - FUNCTION_DESCRIPTION_COMMON_1 + - FUNCTION_DESCRIPTION_COMMON_2, - new SequenceType[] { new FunctionParameterSequenceType("arg", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The input sequence")}, - new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_ONE, "the minimum value") - ), - new FunctionSignature( - new QName("min", Function.BUILTIN_FUNCTION_NS), - FUNCTION_DESCRIPTION_COMMON_1 + FUNCTION_DESCRIPTION_2_PARAM + - FUNCTION_DESCRIPTION_COMMON_2, - new SequenceType[] { - new FunctionParameterSequenceType("arg", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The input sequence"), - new FunctionParameterSequenceType("collation-uri", Type.STRING, Cardinality.EXACTLY_ONE, "The collation URI") - }, - new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_ONE, "the minimum value") - ) - }; - - public FunMin(XQueryContext context, FunctionSignature signature) { - super(context, signature); - } - - /* (non-Javadoc) - * @see org.exist.xquery.Expression#eval(org.exist.dom.persistent.DocumentSet, org.exist.xquery.value.Sequence, org.exist.xquery.value.Item) - */ - public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathException { + public final static FunctionSignature[] signatures = { + new FunctionSignature( + new QName("min", Function.BUILTIN_FUNCTION_NS), + "Returns the minimum value from the input sequence, using XQ4 comparison semantics.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, + Cardinality.ZERO_OR_MORE, "The input sequence") + }, + new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_ONE, + "the minimum value") + ), + new FunctionSignature( + new QName("min", Function.BUILTIN_FUNCTION_NS), + "Returns the minimum value from the input sequence, using the specified collation.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, + Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("collation", Type.STRING, + Cardinality.ZERO_OR_ONE, "The collation URI") + }, + new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_ONE, + "the minimum value") + ) + }; + + public FunMin(XQueryContext context, FunctionSignature signature) { + super(context, signature); + } + + public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathException { if (context.getProfiler().isEnabled()) { - context.getProfiler().start(this); - context.getProfiler().message(this, Profiler.DEPENDENCIES, "DEPENDENCIES", Dependency.getDependenciesName(this.getDependencies())); + context.getProfiler().start(this); + context.getProfiler().message(this, Profiler.DEPENDENCIES, "DEPENDENCIES", + Dependency.getDependenciesName(this.getDependencies())); if (contextSequence != null) - {context.getProfiler().message(this, Profiler.START_SEQUENCES, "CONTEXT SEQUENCE", contextSequence);} + {context.getProfiler().message(this, Profiler.START_SEQUENCES, + "CONTEXT SEQUENCE", contextSequence);} if (contextItem != null) - {context.getProfiler().message(this, Profiler.START_SEQUENCES, "CONTEXT ITEM", contextItem.toSequence());} + {context.getProfiler().message(this, Profiler.START_SEQUENCES, + "CONTEXT ITEM", contextItem.toSequence());} } - - boolean computableProcessing = false; + Sequence result; - final Sequence arg = getArgument(0).eval(contextSequence, contextItem); - if (arg.isEmpty()) - {result = Sequence.EMPTY_SEQUENCE;} - else { - //TODO : test if a range index is defined *iff* it is compatible with the collator - final Collator collator = getCollator(contextSequence, contextItem, 2); - final SequenceIterator iter = arg.unorderedIterator(); - AtomicValue min = null; - while (iter.hasNext()) { - final Item item = iter.nextItem(); - if (item instanceof QNameValue) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(item.getType()), arg);} - AtomicValue value = item.atomize(); - - //Duration values must either all be xs:yearMonthDuration values or must all be xs:dayTimeDuration values. - if (Type.subTypeOf(value.getType(), Type.DURATION)) { - value = ((DurationValue)value).wrap(); - if (value.getType() == Type.YEAR_MONTH_DURATION) { - if (min != null && min.getType() != Type.YEAR_MONTH_DURATION) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(min.getType()) + - " and " + Type.getTypeName(value.getType()), value);} - - } else if (value.getType() == Type.DAY_TIME_DURATION) { - if (min != null && min.getType() != Type.DAY_TIME_DURATION) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(min.getType()) + - " and " + Type.getTypeName(value.getType()), value);} - - } else - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(value.getType()), value);} - - //Any value of type xdt:untypedAtomic is cast to xs:double - } else if (value.getType() == Type.UNTYPED_ATOMIC) - {value = value.convertTo(Type.DOUBLE);} - - if (min == null) - {min = value;} - else { - if (Type.getCommonSuperType(min.getType(), value.getType()) == Type.ANY_ATOMIC_TYPE) { - throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(min.getType()) + - " and " + Type.getTypeName(value.getType()), value); - } - //Any value of type xdt:untypedAtomic is cast to xs:double - if (value.getType() == Type.ANY_ATOMIC_TYPE) - {value = value.convertTo(Type.DOUBLE);} - //Numeric tests - if (Type.subTypeOfUnion(value.getType(), Type.NUMERIC)) { - //Don't mix comparisons - if (!Type.subTypeOfUnion(min.getType(), Type.NUMERIC)) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(min.getType()) + - " and " + Type.getTypeName(value.getType()), min);} - if (((NumericValue) value).isNaN()) { - //Type NaN correctly - value = value.promote(min); - if (value.getType() == Type.FLOAT) - {min = FloatValue.NaN;} - else - {min = DoubleValue.NaN;} - //although result will be NaN, we need to continue on order to type correctly - continue; - } - min = min.promote(value); - } - //Ugly test - if (value instanceof ComputableValue) { - if (!(min instanceof ComputableValue)) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(min.getType()) + - " and " + Type.getTypeName(value.getType()), min);} - //Type value correctly - value = value.promote(min); - min = min.min(collator, value); - computableProcessing = true; - } else { - if (computableProcessing) - {throw new XPathException(this, ErrorCodes.FORG0006, "Cannot compare " + Type.getTypeName(min.getType()) + - " and " + Type.getTypeName(value.getType()), value);} - if (Collations.compare(collator, value.getStringValue(), min.getStringValue()) < 0) - {min = value;} - } - } - } - result = min; + final Sequence arg = getArgument(0).eval(contextSequence, contextItem); + if (arg.isEmpty()) { + result = Sequence.EMPTY_SEQUENCE; + } else { + final Collator collator = getOptionalCollator(contextSequence, contextItem); + result = findMin(arg, collator); + } + + if (context.getProfiler().isEnabled()) + {context.getProfiler().end(this, "", result);} + + return result; + } + + /** + * Get collator, handling empty sequence for XQ4 optional collation parameter. + */ + private Collator getOptionalCollator(Sequence contextSequence, Item contextItem) + throws XPathException { + if (getArgumentCount() == 2) { + final Sequence collationSeq = getArgument(1).eval(contextSequence, contextItem); + if (!collationSeq.isEmpty()) { + final String collationURI = collationSeq.getStringValue(); + return context.getCollator(collationURI, ErrorCodes.FOCH0002); + } } - - if (context.getProfiler().isEnabled()) - {context.getProfiler().end(this, "", result);} - - return result; + return context.getDefaultCollator(); } + private Sequence findMin(Sequence arg, Collator collator) throws XPathException { + final SequenceIterator iter = arg.unorderedIterator(); + AtomicValue min = null; + boolean hasNaN = false; + AtomicValue nanValue = null; + + while (iter.hasNext()) { + final Item item = iter.nextItem(); + AtomicValue value = item.atomize(); + + // Cast untypedAtomic to double + if (value.getType() == Type.UNTYPED_ATOMIC) { + value = value.convertTo(Type.DOUBLE); + } + + // Wrap duration subtypes + if (Type.subTypeOf(value.getType(), Type.DURATION)) { + value = ((DurationValue) value).wrap(); + } + + // Track NaN: if any value is NaN, result is NaN + if (value instanceof NumericValue && ((NumericValue) value).isNaN()) { + if (!hasNaN) { + hasNaN = true; + nanValue = value; + } + continue; + } + + if (min == null) { + min = value; + } else { + try { + final int cmp = FunCompare.compare(value, min, collator); + if (cmp < 0) { + min = value; + } + } catch (final XPathException e) { + throw new XPathException(this, ErrorCodes.FORG0006, + "Cannot compare " + Type.getTypeName(min.getType()) + + " and " + Type.getTypeName(value.getType()), value); + } + } + } + + if (hasNaN) { + return nanValue; + } + return min; + } } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunPath.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunPath.java index 5d05e48535e..8544c8230ff 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunPath.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunPath.java @@ -48,7 +48,11 @@ public class FunPath extends BasicFunction { public static final FunctionSignature[] FS_PATH_SIGNATURES = { functionSignature(FunPath.FN_PATH_NAME, FunPath.FN_PATH_DESCRIPTION, FunPath.FN_PATH_RETURN), functionSignature(FunPath.FN_PATH_NAME, FunPath.FN_PATH_DESCRIPTION, FunPath.FN_PATH_RETURN, - new FunctionParameterSequenceType("node", Type.NODE, Cardinality.ZERO_OR_ONE, "The node for which to calculate a path expression")) + new FunctionParameterSequenceType("node", Type.NODE, Cardinality.ZERO_OR_ONE, "The node for which to calculate a path expression")), + // XQuery 4.0: fn:path with options map + functionSignature(FunPath.FN_PATH_NAME, FunPath.FN_PATH_DESCRIPTION, FunPath.FN_PATH_RETURN, + new FunctionParameterSequenceType("node", Type.NODE, Cardinality.ZERO_OR_ONE, "The node for which to calculate a path expression"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, Cardinality.EXACTLY_ONE, "Options map (e.g., format)")) }; public FunPath(final XQueryContext context, final FunctionSignature signature) { @@ -66,6 +70,8 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro sequence = Objects.requireNonNullElse(contextSequence, Sequence.EMPTY_SEQUENCE); } else { sequence = args[0]; + // XQuery 4.0: 2-arg fn:path($node, $options) — options map accepted + // but currently only the default EQName format is supported } if (sequence.isEmpty()) { diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunReplace.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunReplace.java index 6dea523469a..1a6ca8e18d3 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunReplace.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunReplace.java @@ -23,6 +23,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import net.sf.saxon.Configuration; import net.sf.saxon.functions.Replace; @@ -30,9 +32,12 @@ import org.exist.dom.QName; import org.exist.xquery.*; import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReference; +import org.exist.xquery.value.Item; import org.exist.xquery.value.Sequence; import org.exist.xquery.value.StringValue; import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; import static org.exist.xquery.FunctionDSL.*; import static org.exist.xquery.regex.RegexUtil.*; @@ -72,7 +77,9 @@ public class FunReplace extends BasicFunction { private static final FunctionParameterSequenceType FS_TOKENIZE_PARAM_INPUT = optParam("input", Type.STRING, "The input string"); private static final FunctionParameterSequenceType FS_TOKENIZE_PARAM_PATTERN = param("pattern", Type.STRING, "The pattern to match"); - private static final FunctionParameterSequenceType FS_TOKENIZE_PARAM_REPLACEMENT = param("replacement", Type.STRING, "The string to replace the pattern with"); + private static final FunctionParameterSequenceType FS_TOKENIZE_PARAM_REPLACEMENT = + new FunctionParameterSequenceType("replacement", Type.ITEM, Cardinality.ZERO_OR_ONE, + "The replacement string, function, or empty sequence"); static final FunctionSignature [] FS_REPLACE = functionSignatures( FS_REPLACE_NAME, @@ -88,7 +95,7 @@ public class FunReplace extends BasicFunction { FS_TOKENIZE_PARAM_INPUT, FS_TOKENIZE_PARAM_PATTERN, FS_TOKENIZE_PARAM_REPLACEMENT, - param("flags", Type.STRING, Cardinality.EXACTLY_ONE, "The flags") + param("flags", Type.STRING, Cardinality.ZERO_OR_ONE, "The flags") ) ) ); @@ -104,36 +111,70 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro if (stringArg.isEmpty()) { result = StringValue.EMPTY_STRING; } else { - final String flags; - if (args.length == 4) { + String flags; + if (args.length == 4 && !args[3].isEmpty()) { flags = args[3].itemAt(0).getStringValue(); } else { flags = ""; } + + // XQ4: 'c' flag — strip regex comments (#...#) before compilation + // When 'q' (literal) flag is present, 'c' is ignored + final boolean hasCommentFlag = flags.indexOf('c') >= 0 && flags.indexOf('q') < 0; + if (flags.indexOf('c') >= 0) { + flags = flags.replace("c", ""); + } + final String string = stringArg.getStringValue(); - final String pattern = args[1].itemAt(0).getStringValue(); - final String replace = args[2].itemAt(0).getStringValue(); + String pattern = args[1].itemAt(0).getStringValue(); - final Configuration config = context.getBroker().getBrokerPool().getSaxonConfiguration(); + if (hasCommentFlag) { + pattern = stripRegexComments(pattern); + } + // XQ4: 3rd arg can be empty sequence (treated as empty string) or a function + final Sequence replacementArg = args[2]; + final boolean isFunctionReplacement = !replacementArg.isEmpty() + && Type.subTypeOf(replacementArg.itemAt(0).getType(), Type.FUNCTION); + final String replace; + if (isFunctionReplacement) { + replace = null; // handled below + } else if (replacementArg.isEmpty()) { + replace = ""; + } else { + replace = replacementArg.itemAt(0).getStringValue(); + } + + final Configuration config = context.getBroker().getBrokerPool().getSaxonConfiguration(); final List warnings = new ArrayList<>(1); try { final RegularExpression regularExpression = config.compileRegularExpression(pattern, flags, "XP30", warnings); - if (regularExpression.matches("")) { + final boolean canMatchEmpty = regularExpression.matches(""); + + // XQ 3.1: FORX0003 if regex can match empty string + // XQ 4.0: empty-matching regex is allowed + if (canMatchEmpty && context.getXQueryVersion() < 40 && !isFunctionReplacement) { throw new XPathException(this, ErrorCodes.FORX0003, "regular expression could match empty string"); } - //TODO(AR) cache the regular expression... might be possible through Saxon config - - if (!hasLiteral(flags)) { - final String msg = Replace.checkReplacement(replace); - if (msg != null) { - throw new XPathException(this, ErrorCodes.FORX0004, msg); + if (isFunctionReplacement) { + result = evalFunctionReplacement(string, pattern, flags, + (FunctionReference) replacementArg.itemAt(0)); + } else if (canMatchEmpty) { + // XQ4: empty-matching regex allowed — use Java regex fallback + // since Saxon's replace() doesn't handle empty matches well + result = evalEmptyMatchReplace(string, pattern, replace, flags); + } else { + if (!hasLiteral(flags)) { + final String msg = Replace.checkReplacement(replace); + if (msg != null) { + throw new XPathException(this, ErrorCodes.FORX0004, msg); + } } + final CharSequence res = regularExpression.replace(string, replace); + result = new StringValue(this, res.toString()); } - final CharSequence res = regularExpression.replace(string, replace); - result = new StringValue(this, res.toString()); } catch (final net.sf.saxon.trans.XPathException e) { switch (e.getErrorCodeLocalPart()) { @@ -145,7 +186,183 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro } } } - + return result; } + + /** + * XQ4: Handle replacement when the regex can match the empty string. + * Uses Java regex with XPath-to-Java translation for proper empty-match handling. + */ + private Sequence evalEmptyMatchReplace(final String input, final String pattern, + final String replace, final String flags) throws XPathException { + final String javaPattern = translateRegexp( + this, pattern, hasIgnoreWhitespace(flags), hasCaseInsensitive(flags)); + final int javaFlags = parseFlags(this, flags); + final Pattern compiled = Pattern.compile(javaPattern, javaFlags); + final Matcher matcher = compiled.matcher(input); + + final StringBuilder sb = new StringBuilder(); + int lastEnd = 0; + while (matcher.find()) { + sb.append(input, lastEnd, matcher.start()); + + // Apply XPath-style replacement ($0, $1, etc.) + sb.append(applyXPathReplacement(replace, matcher)); + + lastEnd = matcher.end(); + + // Advance past empty match to prevent infinite loop + if (matcher.start() == matcher.end()) { + if (lastEnd < input.length()) { + sb.append(input.charAt(lastEnd)); + lastEnd++; + matcher.region(lastEnd, input.length()); + } else { + break; + } + } + } + sb.append(input, lastEnd, input.length()); + return new StringValue(this, sb.toString()); + } + + /** + * Apply XPath-style replacement string ($0, $1, etc.) using a Java Matcher. + */ + private static String applyXPathReplacement(final String replacement, final Matcher matcher) { + final StringBuilder result = new StringBuilder(); + for (int i = 0; i < replacement.length(); i++) { + final char ch = replacement.charAt(i); + if (ch == '$' && i + 1 < replacement.length()) { + i++; + int groupNum = 0; + boolean hasDigit = false; + while (i < replacement.length() && Character.isDigit(replacement.charAt(i))) { + groupNum = groupNum * 10 + (replacement.charAt(i) - '0'); + hasDigit = true; + i++; + } + i--; // back up one + if (hasDigit && groupNum <= matcher.groupCount()) { + final String g = matcher.group(groupNum); + if (g != null) { + result.append(g); + } + } else if (hasDigit) { + // Group doesn't exist, output empty + } + } else if (ch == '\\' && i + 1 < replacement.length()) { + i++; + result.append(replacement.charAt(i)); + } else { + result.append(ch); + } + } + return result.toString(); + } + + /** + * XQ4: Evaluate fn:replace with a function replacement parameter. + * The function receives (match, groups*) and returns the replacement string. + */ + private Sequence evalFunctionReplacement(final String input, final String pattern, + final String flags, final FunctionReference func) throws XPathException { + // Use Java regex for function replacement since Saxon's replace() only accepts strings + final String javaPattern = translateRegexp( + this, pattern, hasIgnoreWhitespace(flags), hasCaseInsensitive(flags)); + int javaFlags = parseFlags(this, flags); + final Pattern compiled = Pattern.compile(javaPattern, javaFlags); + final Matcher matcher = compiled.matcher(input); + + final StringBuilder sb = new StringBuilder(); + int lastEnd = 0; + while (matcher.find()) { + sb.append(input, lastEnd, matcher.start()); + + // Build arguments: (match, group1, group2, ...) + final int groupCount = matcher.groupCount(); + final Sequence[] funcArgs = new Sequence[2]; + funcArgs[0] = new StringValue(this, matcher.group()); + final ValueSequence groups = new ValueSequence(groupCount); + for (int i = 1; i <= groupCount; i++) { + final String g = matcher.group(i); + groups.add(g != null ? new StringValue(this, g) : StringValue.EMPTY_STRING); + } + funcArgs[1] = groups; + + final Sequence replacement = func.evalFunction(null, null, funcArgs); + if (!replacement.isEmpty()) { + sb.append(replacement.getStringValue()); + } + + lastEnd = matcher.end(); + + // Prevent infinite loop on empty match + if (matcher.start() == matcher.end()) { + if (lastEnd < input.length()) { + sb.append(input.charAt(lastEnd)); + lastEnd++; + // Reset matcher position + matcher.region(lastEnd, input.length()); + } else { + break; + } + } + } + sb.append(input, lastEnd, input.length()); + return new StringValue(this, sb.toString()); + } + + /** + * XQ4: Strip regex comments (c flag). + * Removes text between # markers: #comment# becomes empty. + * A # at end of pattern (no closing #) is treated as end-of-line comment. + * Escaped \# is preserved. + */ + static String stripRegexComments(final String pattern) { + final StringBuilder result = new StringBuilder(pattern.length()); + boolean inComment = false; + boolean inCharClass = false; + for (int i = 0; i < pattern.length(); i++) { + final char ch = pattern.charAt(i); + if (ch == '\\' && i + 1 < pattern.length()) { + final char next = pattern.charAt(i + 1); + if (!inComment) { + if (next == '#') { + // \# in c-flag mode is a literal # — output just # + result.append('#'); + } else { + result.append(ch); + result.append(next); + } + } + i++; // skip escaped character + } else if (inCharClass) { + // Inside [...] character class, # is literal + if (ch == ']') { + inCharClass = false; + } + if (!inComment) { + result.append(ch); + } + } else if (ch == '[' && !inComment) { + inCharClass = true; + result.append(ch); + } else if (ch == '#' && !inCharClass) { + inComment = !inComment; + } else if (!inComment) { + result.append(ch); + } + } + return result.toString(); + } + + private static boolean hasCaseInsensitive(final String flags) { + return flags.contains("i"); + } + + private static boolean hasIgnoreWhitespace(final String flags) { + return flags.contains("x"); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunRound.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunRound.java index 4ad0bb8cf7b..2b4286f5c15 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunRound.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunRound.java @@ -68,7 +68,11 @@ public class FunRound extends FunRoundBase { optParam("arg", Type.NUMERIC, "The input number")), functionSignature(FN_NAME, FunRound.description, FunRound.returnType, optParam("arg", Type.NUMERIC, "The input number"), - optParam("precision", Type.INTEGER, "The input number")) + optParam("precision", Type.INTEGER, "The precision")), + functionSignature(FN_NAME, FunRound.description, FunRound.returnType, + optParam("arg", Type.NUMERIC, "The input number"), + optParam("precision", Type.INTEGER, "The precision"), + optParam("mode", Type.STRING, "The rounding mode")) }; public FunRound(final XQueryContext context, final FunctionSignature signature) { diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunRoundBase.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunRoundBase.java index ba7b1050a3b..85080cd0e17 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunRoundBase.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunRoundBase.java @@ -25,6 +25,7 @@ import org.exist.xquery.value.*; import java.math.RoundingMode; +import java.util.Map; import java.util.Objects; /** @@ -45,6 +46,18 @@ public int returnsType() { abstract protected RoundingMode getFunctionRoundingMode(NumericValue value); + private static final Map ROUNDING_MODE_MAP = Map.of( + "floor", "FLOOR", + "ceiling", "CEILING", + "toward-zero", "DOWN", + "away-from-zero", "UP", + "half-to-floor", "HALF_FLOOR", + "half-to-ceiling", "HALF_CEILING", + "half-toward-zero", "HALF_DOWN", + "half-away-from-zero", "HALF_UP", + "half-to-even", "HALF_EVEN" + ); + @Override public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { @@ -60,9 +73,15 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro value = (NumericValue) item.convertTo(Type.NUMERIC); } - final RoundingMode roundingMode = getFunctionRoundingMode(value); + // Determine rounding mode: 3-arg form overrides the function default + final RoundingMode roundingMode; + if (args.length > 2 && !args[2].isEmpty()) { + roundingMode = parseRoundingMode(args[2].getStringValue(), value); + } else { + roundingMode = getFunctionRoundingMode(value); + } - if (args.length > 1) { + if (args.length > 1 && !args[1].isEmpty()) { final Item precisionItem = args[1].itemAt(0); if (precisionItem instanceof IntegerValue precision) { return convertValue(precision, value, roundingMode, this); @@ -72,6 +91,27 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro return convertValue(IntegerValue.ZERO, value, roundingMode, this); } + private RoundingMode parseRoundingMode(final String mode, final NumericValue value) throws XPathException { + // XQ4 rounding modes that map directly to Java RoundingMode + switch (mode) { + case "floor": return RoundingMode.FLOOR; + case "ceiling": return RoundingMode.CEILING; + case "toward-zero": return RoundingMode.DOWN; + case "away-from-zero": return RoundingMode.UP; + case "half-to-even": return RoundingMode.HALF_EVEN; + case "half-away-from-zero": return RoundingMode.HALF_UP; + case "half-toward-zero": return RoundingMode.HALF_DOWN; + // half-to-floor and half-to-ceiling need special handling based on sign + case "half-to-floor": + return value.isNegative() ? RoundingMode.HALF_UP : RoundingMode.HALF_DOWN; + case "half-to-ceiling": + return value.isNegative() ? RoundingMode.HALF_DOWN : RoundingMode.HALF_UP; + default: + throw new XPathException(this, ErrorCodes.XPTY0004, + "Unknown rounding mode: '" + mode + "'"); + } + } + /** * Apply necessary conversions to/from decimal to perform rounding in decimal * diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunTokenize.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunTokenize.java index f31b8b645f0..aec9f7d867a 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunTokenize.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunTokenize.java @@ -45,7 +45,8 @@ public class FunTokenize extends BasicFunction { private static final QName FS_TOKENIZE_NAME = new QName("tokenize", Function.BUILTIN_FUNCTION_NS); private final static FunctionParameterSequenceType FS_TOKENIZE_PARAM_INPUT = optParam("input", Type.STRING, "The input string"); - private final static FunctionParameterSequenceType FS_TOKENIZE_PARAM_PATTERN = param("pattern", Type.STRING, "The tokenization pattern"); + private final static FunctionParameterSequenceType FS_TOKENIZE_PARAM_PATTERN = + new FunctionParameterSequenceType("pattern", Type.STRING, Cardinality.ZERO_OR_ONE, "The tokenization pattern"); public final static FunctionSignature[] FS_TOKENIZE = functionSignatures( FS_TOKENIZE_NAME, @@ -62,7 +63,7 @@ public class FunTokenize extends BasicFunction { arity( FS_TOKENIZE_PARAM_INPUT, FS_TOKENIZE_PARAM_PATTERN, - param("flags", Type.STRING,"The flags") + new FunctionParameterSequenceType("flags", Type.STRING, Cardinality.ZERO_OR_ONE, "The flags") ) ) ); @@ -82,47 +83,112 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro if (string.isEmpty()) { result = Sequence.EMPTY_SEQUENCE; } else { - final int flags; - if (args.length == 3) { - flags = parseFlags(this, args[2].itemAt(0).getStringValue()); - } else { - flags = 0; - } + // XQ4: pattern can be empty sequence — treat as 1-arg whitespace form + final boolean useWhitespaceTokenization = args.length == 1 + || (args.length >= 2 && args[1].isEmpty()); - final String pattern; - if (args.length == 1) { - pattern = " "; + if (useWhitespaceTokenization) { string = FunNormalizeSpace.normalize(string); + if (string.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + final String[] tokens = string.split(" "); + result = new ValueSequence(); + for (final String token : tokens) { + result.add(new StringValue(this, token)); + } } else { - if(hasLiteral(flags)) { - // no need to change anything - pattern = args[1].itemAt(0).getStringValue(); + // XQ4: flags can be empty sequence + String flagsStr = ""; + if (args.length == 3 && !args[2].isEmpty()) { + flagsStr = args[2].itemAt(0).getStringValue(); + } + + // XQ4: 'c' flag — strip regex comments + final boolean hasCommentFlag = flagsStr.indexOf('c') >= 0 && flagsStr.indexOf('q') < 0; + if (flagsStr.indexOf('c') >= 0) { + flagsStr = flagsStr.replace("c", ""); + } + final int flags = parseFlags(this, flagsStr); + + String rawPattern = args[1].itemAt(0).getStringValue(); + if (hasCommentFlag) { + rawPattern = FunReplace.stripRegexComments(rawPattern); + } + final String pattern; + if (hasLiteral(flags)) { + pattern = rawPattern; } else { final boolean ignoreWhitespace = hasIgnoreWhitespace(flags); final boolean caseBlind = hasCaseInsensitive(flags); - pattern = translateRegexp(this, args[1].itemAt(0).getStringValue(), ignoreWhitespace, caseBlind); + pattern = translateRegexp(this, rawPattern, ignoreWhitespace, caseBlind); } - } - try { - final Pattern pat = PatternFactory.getInstance().getPattern(pattern, flags); - if (pat.matcher("").matches()) { - throw new XPathException(this, ErrorCodes.FORX0003, "regular expression could match empty string"); + try { + final Pattern pat = PatternFactory.getInstance().getPattern(pattern, flags); + + if (pat.matcher("").matches()) { + // XQ 3.1: FORX0003 if regex can match empty string + // XQ 4.0: empty-matching regex is allowed + if (context.getXQueryVersion() < 40) { + throw new XPathException(this, ErrorCodes.FORX0003, "regular expression could match empty string"); + } + // XQ4: empty-matching regex allowed — tokenize between each character + result = tokenizeEmptyMatch(string, pat); + } else { + final String[] tokens = pat.split(string, -1); + result = new ValueSequence(); + for (final String token : tokens) { + result.add(new StringValue(this, token)); + } + } + + } catch (final PatternSyntaxException e) { + throw new XPathException(this, ErrorCodes.FORX0001, "Invalid regular expression: " + e.getMessage(), new StringValue(this, pattern), e); } + } + } + } - final String[] tokens = pat.split(string, -1); - result = new ValueSequence(); + return result; + } - for (final String token : tokens) { - result.add(new StringValue(this, token)); - } + /** + * XQ4: Handle tokenization when the regex matches the empty string. + * Per spec: zero-length matches at start/end of string do not produce + * leading/trailing empty tokens. Empty matches advance past one character. + */ + private Sequence tokenizeEmptyMatch(final String input, final Pattern pat) throws XPathException { + final ValueSequence result = new ValueSequence(); + final java.util.regex.Matcher matcher = pat.matcher(input); + int lastEnd = 0; + while (matcher.find()) { + final boolean isEmpty = matcher.start() == matcher.end(); + + // Skip zero-length match at end of string + if (isEmpty && matcher.start() >= input.length()) { + break; + } + + // Add token: text from end of last match to start of this match + result.add(new StringValue(this, input.substring(lastEnd, matcher.start()))); + lastEnd = matcher.end(); - } catch (final PatternSyntaxException e) { - throw new XPathException(this, ErrorCodes.FORX0001, "Invalid regular expression: " + e.getMessage(), new StringValue(this, pattern), e); + // For empty match, advance matcher past one character to prevent infinite loop. + // The skipped character becomes part of the next token (not consumed). + if (isEmpty) { + final int nextPos = lastEnd + 1; + if (nextPos <= input.length()) { + matcher.region(nextPos, input.length()); + } else { + break; } } } - + // Add trailing token + if (lastEnd <= input.length()) { + result.add(new StringValue(this, input.substring(lastEnd))); + } return result; } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunUnparsedText.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunUnparsedText.java index e4e134e9919..e1062936021 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FunUnparsedText.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FunUnparsedText.java @@ -37,6 +37,7 @@ import java.net.URISyntaxException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.nio.charset.UnsupportedCharsetException; import static java.nio.charset.StandardCharsets.UTF_8; import static org.exist.xquery.FunctionDSL.*; @@ -87,7 +88,7 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce } else if (isCalledAs("unparsed-text-available")) { return BooleanValue.valueOf(contentAvailable(href, encoding)); } else { - return new StringValue(this, readContent(href, encoding)); + return new StringValue(this, stripBOM(readContent(href, encoding))); } } return Sequence.EMPTY_SEQUENCE; @@ -96,7 +97,7 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce private boolean contentAvailable(final String uri, final String encoding) { final Charset charset; try { - charset = encoding != null ? Charset.forName(encoding) : UTF_8; + charset = encoding != null ? resolveCharset(encoding) : UTF_8; } catch (final IllegalArgumentException e) { return false; } @@ -120,7 +121,7 @@ private boolean contentAvailable(final String uri, final String encoding) { private String readContent(final String uri, final String encoding) throws XPathException { final Charset charset; try { - charset = encoding != null ? Charset.forName(encoding) : UTF_8; + charset = encoding != null ? resolveCharset(encoding) : UTF_8; } catch (final IllegalArgumentException e) { throw new XPathException(this, ErrorCodes.FOUT1190, e.getMessage()); } @@ -175,7 +176,12 @@ private Sequence readLines(final String uriParam, final String encoding) throws try (final BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, charset))) { String line; + boolean firstLine = true; while ((line = reader.readLine()) != null) { + if (firstLine) { + line = stripBOM(line); + firstLine = false; + } result.add(new StringValue(this, line)); } } @@ -199,7 +205,7 @@ private Charset getCharset(final String encoding, final Source source) throws XP } } else { try { - charset = Charset.forName(encoding); + charset = resolveCharset(encoding); } catch (final IllegalArgumentException e) { throw new XPathException(this, ErrorCodes.FOUT1190, e.getMessage()); } @@ -207,14 +213,70 @@ private Charset getCharset(final String encoding, final Source source) throws XP return charset; } + /** + * Resolve a charset name, mapping common aliases that Java doesn't recognize. + */ + /** + * Strip the Unicode BOM (U+FEFF) from the beginning of a string. + * Per XQuery spec: "If the text resource has a BOM, the BOM is excluded from the result." + */ + private static String stripBOM(final String s) { + if (s != null && !s.isEmpty() && s.charAt(0) == '\uFEFF') { + return s.substring(1); + } + return s; + } + + private static Charset resolveCharset(final String encoding) { + try { + return Charset.forName(encoding); + } catch (final UnsupportedCharsetException e) { + if ("iso-8859".equalsIgnoreCase(encoding)) { + return Charset.forName("iso-8859-1"); + } + throw e; + } + } + private Source getSource(final String uriParam) throws XPathException { try { - final URI uri = new URI(uriParam); + URI uri = new URI(uriParam); if (uri.getFragment() != null) { throw new XPathException(this, ErrorCodes.FOUT1170, "href argument may not contain fragment identifier"); } - final Source source = SourceFactory.getSource(context.getBroker(), "", uri.toASCIIString(), false); + // Resolve relative URIs against file: base URI directory + boolean resolvedFromBaseUri = false; + if (!uri.isAbsolute()) { + final AnyURIValue baseXdmUri = context.getBaseURI(); + if (baseXdmUri != null && !baseXdmUri.equals(AnyURIValue.EMPTY_URI)) { + String baseStr = baseXdmUri.toURI().toString(); + if (baseStr.startsWith("file:")) { + final int lastSlash = baseStr.lastIndexOf('/'); + if (lastSlash >= 0) { + baseStr = baseStr.substring(0, lastSlash + 1); + } + uri = new URI(baseStr).resolve(uri); + resolvedFromBaseUri = true; + } + } + } + + final String resolvedUri = uri.toASCIIString(); + + // Only use direct file: access for URIs resolved from a relative path + // against a file: base URI. Absolute file: URIs (e.g., file:///etc/passwd) + // must go through SourceFactory which enforces security checks. + if (resolvedFromBaseUri && resolvedUri.startsWith("file:")) { + final String filePath = resolvedUri.replaceFirst("^file:(?://[^/]*)?", ""); + final java.nio.file.Path path = java.nio.file.Paths.get(filePath); + if (java.nio.file.Files.isReadable(path)) { + return new FileSource(path, false); + } + throw new XPathException(this, ErrorCodes.FOUT1170, "Could not find source for: " + uriParam); + } + + final Source source = SourceFactory.getSource(context.getBroker(), "", resolvedUri, false); if (source == null) { throw new XPathException(this, ErrorCodes.FOUT1170, "Could not find source for: " + uriParam); } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java index 42912e0569d..8f8a75d716e 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java @@ -64,7 +64,7 @@ public class JSON extends BasicFunction { ), arity( FS_PARAM_JSON_TEXT, - param("options", Type.MAP_ITEM, "Parsing options") + optParam("options", Type.MAP_ITEM, "Parsing options") ) ) ); @@ -121,18 +121,71 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce context.getXQueryVersion()); } // process options if present - // TODO: jackson does not allow access to raw string, so option "unescape" is not supported boolean liberal = false; String handleDuplicates = OPTION_DUPLICATES_USE_LAST; if (getArgumentCount() == 2) { final MapType options = (MapType)args[1].itemAt(0); + + // Validate deprecated options → XPTY0004 + final Sequence validateOpt = options.get(new StringValue("validate")); + if (validateOpt != null && validateOpt.hasOne()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "The 'validate' option is not supported"); + } + // XQuery 4.0: 'spec' option controls JSON spec version (RFC7159, ECMA-404, etc.) + // Accepted but not yet enforced — we always parse per RFC 7159 + final Sequence specOpt = options.get(new StringValue("spec")); + // (no validation needed — all spec values are accepted) + + // Validate liberal option — must be boolean final Sequence liberalOpt = options.get(new StringValue(OPTION_LIBERAL)); - if (liberalOpt.hasOne()) { - liberal = liberalOpt.itemAt(0).convertTo(Type.BOOLEAN).effectiveBooleanValue(); + if (liberalOpt != null && liberalOpt.hasOne()) { + final Item liberalItem = liberalOpt.itemAt(0); + if (liberalItem.getType() != Type.BOOLEAN) { + // Try to convert; if the value is a non-boolean string, reject + if (Type.subTypeOf(liberalItem.getType(), Type.STRING)) { + final String val = liberalItem.getStringValue(); + if (!"true".equals(val) && !"false".equals(val) && !"1".equals(val) && !"0".equals(val)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option 'liberal' must be a boolean, got: " + val); + } + } + liberal = liberalItem.convertTo(Type.BOOLEAN).effectiveBooleanValue(); + } else { + liberal = liberalItem.convertTo(Type.BOOLEAN).effectiveBooleanValue(); + } } + + // Validate duplicates option final Sequence duplicateOpt = options.get(new StringValue(OPTION_DUPLICATES)); - if (duplicateOpt.hasOne()) { + if (duplicateOpt != null && duplicateOpt.hasOne()) { handleDuplicates = duplicateOpt.itemAt(0).getStringValue(); + if (!OPTION_DUPLICATES_USE_FIRST.equals(handleDuplicates) + && !OPTION_DUPLICATES_USE_LAST.equals(handleDuplicates) + && !OPTION_DUPLICATES_REJECT.equals(handleDuplicates)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Invalid value for 'duplicates' option: " + handleDuplicates); + } + } + + // Validate fallback option — must be a function with arity 1 + final Sequence fallbackOpt = options.get(new StringValue("fallback")); + if (fallbackOpt != null && fallbackOpt.hasOne()) { + final Item fallbackItem = fallbackOpt.itemAt(0); + if (!(fallbackItem instanceof FunctionReference)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option 'fallback' must be a function, got: " + Type.getTypeName(fallbackItem.getType())); + } + } + + // Validate number-parser option — must be a function + final Sequence numberParserOpt = options.get(new StringValue("number-parser")); + if (numberParserOpt != null && numberParserOpt.hasOne()) { + final Item npItem = numberParserOpt.itemAt(0); + if (!(npItem instanceof FunctionReference)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option 'number-parser' must be a function, got: " + Type.getTypeName(npItem.getType())); + } } } @@ -212,24 +265,94 @@ private Sequence parseResource(Sequence href, String handleDuplicates, JsonFacto } try { String url = href.getStringValue(); + + // Check dynamically available text resources first (XQTS runner registers these) + try (final java.io.Reader dynReader = context.getDynamicallyAvailableTextResource( + url, java.nio.charset.StandardCharsets.UTF_8)) { + if (dynReader != null) { + final StringBuilder sb = new StringBuilder(); + final char[] buf = new char[4096]; + int read; + while ((read = dynReader.read(buf)) > 0) { + sb.append(buf, 0, read); + } + try (final JsonParser parser = factory.createParser(sb.toString())) { + final Item result = readValue(context, parser, handleDuplicates); + return result == null ? Sequence.EMPTY_SEQUENCE : result.toSequence(); + } catch (final java.io.IOException jsonErr) { + throw new XPathException(this, ErrorCodes.FOJS0001, jsonErr.getMessage()); + } + } + } catch (final java.io.IOException e) { + // Not a dynamic resource, fall through to URL resolution + } + boolean resolvedFromBaseUri = false; if (url.indexOf(':') == Constants.STRING_NOT_FOUND) { - url = XmldbURI.EMBEDDED_SERVER_URI_PREFIX + url; + // Relative URI: resolve against static base URI + final String resolved = resolveAgainstBaseUri(url); + if (resolved != null && resolved.startsWith("file:")) { + url = resolved; + resolvedFromBaseUri = true; + } else { + url = XmldbURI.EMBEDDED_SERVER_URI_PREFIX + url; + } + } + // Only use direct file: access for URIs resolved from a relative path. + // Absolute file: URIs go through SourceFactory for security. + if (resolvedFromBaseUri && url.startsWith("file:")) { + // Extract path from file: URI: file:/path, file://host/path, file:///path + final String filePath = url.replaceFirst("^file:(?://[^/]*)?", ""); + final java.nio.file.Path path = java.nio.file.Paths.get(filePath); + if (java.nio.file.Files.isReadable(path)) { + try (final InputStream is = java.nio.file.Files.newInputStream(path)) { + try (final JsonParser parser = factory.createParser(is)) { + final Item result = readValue(context, parser, handleDuplicates); + return result == null ? Sequence.EMPTY_SEQUENCE : result.toSequence(); + } catch (final IOException jsonErr) { + // JSON parsing error, not file I/O + throw new XPathException(this, ErrorCodes.FOJS0001, jsonErr.getMessage()); + } + } + } + throw new XPathException(this, ErrorCodes.FOUT1170, "failed to load json doc from file: " + filePath); } + final Source source = SourceFactory.getSource(context.getBroker(), "", url, false); if (source == null) { throw new XPathException(this, ErrorCodes.FOUT1170, "failed to load json doc from URI " + url); } - try (final InputStream is = source.getInputStream(); - final JsonParser parser = factory.createParser(is)) { - - final Item result = readValue(context, parser, handleDuplicates); - return result == null ? Sequence.EMPTY_SEQUENCE : result.toSequence(); + try (final InputStream is = source.getInputStream()) { + try (final JsonParser parser = factory.createParser(is)) { + final Item result = readValue(context, parser, handleDuplicates); + return result == null ? Sequence.EMPTY_SEQUENCE : result.toSequence(); + } catch (final IOException jsonErr) { + throw new XPathException(this, ErrorCodes.FOJS0001, jsonErr.getMessage()); + } } } catch (IOException | PermissionDeniedException e) { throw new XPathException(this, ErrorCodes.FOUT1170, e.getMessage()); } } + private String resolveAgainstBaseUri(final String relativePath) { + try { + final AnyURIValue baseXdmUri = context.getBaseURI(); + if (baseXdmUri != null && !baseXdmUri.equals(AnyURIValue.EMPTY_URI)) { + String baseStr = baseXdmUri.toURI().toString(); + // Strip filename to get directory URI + final int lastSlash = baseStr.lastIndexOf('/'); + if (lastSlash >= 0) { + baseStr = baseStr.substring(0, lastSlash + 1); + } + final java.net.URI baseUri = new java.net.URI(baseStr); + return baseUri.resolve(relativePath).toString(); + } + } catch (final java.net.URISyntaxException | XPathException e) { + // fall through + } + return null; + } + /** * Generate an XDM from the tokens delivered by the JSON parser. * @@ -267,10 +390,18 @@ private static Item readValue(XQueryContext context, JsonParser parser, Item par next = BooleanValue.TRUE; break; case VALUE_NUMBER_FLOAT: - case VALUE_NUMBER_INT: - // according to spec, all numbers are converted to double + // JSON fractional numbers → xs:double next = new StringValue(parser.getText()).convertTo(Type.DOUBLE); break; + case VALUE_NUMBER_INT: + // XQuery 4.0: JSON integers → xs:integer (was xs:double in 3.1) + try { + next = new IntegerValue(parser.getLongValue()); + } catch (final Exception e) { + // Fallback to double for very large integers + next = new StringValue(parser.getText()).convertTo(Type.DOUBLE); + } + break; case VALUE_NULL: next = null; break; diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/LoadXQueryModule.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/LoadXQueryModule.java index f2d409ebeb9..70a3d2bb678 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/LoadXQueryModule.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/LoadXQueryModule.java @@ -27,13 +27,18 @@ import io.lacuna.bifurcan.Map; import io.lacuna.bifurcan.Maps; import org.exist.dom.QName; +import org.exist.source.StringSource; import org.exist.xquery.*; import org.exist.xquery.Module; import org.exist.xquery.functions.map.AbstractMapType; import org.exist.xquery.functions.map.MapType; import org.exist.xquery.parser.XQueryAST; +import org.exist.xquery.parser.XQueryLexer; +import org.exist.xquery.parser.XQueryParser; +import org.exist.xquery.parser.XQueryTreeParser; import org.exist.xquery.value.*; +import java.io.Reader; import java.util.*; import static org.exist.xquery.functions.map.MapType.newLinearMap; @@ -98,6 +103,7 @@ public class LoadXQueryModule extends BasicFunction { public final static StringValue OPTIONS_VARIABLES = new StringValue("variables"); public final static StringValue OPTIONS_CONTEXT_ITEM = new StringValue("context-item"); public final static StringValue OPTIONS_VENDOR = new StringValue("vendor-options"); + public final static StringValue OPTIONS_CONTENT = new StringValue("content"); public final static StringValue RESULT_FUNCTIONS = new StringValue("functions"); public final static StringValue RESULT_VARIABLES = new StringValue("variables"); @@ -116,6 +122,7 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce String xqVersion = getXQueryVersion(context.getXQueryVersion()); AbstractMapType externalVars = new MapType(this, context); Sequence contextItem = Sequence.EMPTY_SEQUENCE; + String contentSource = null; // evaluate options if (getArgumentCount() == 2) { @@ -144,6 +151,12 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce throw new XPathException(this, ErrorCodes.XPTY0004, "Option 'context-item' must contain zero or one " + "items"); } + + // XQ4: content option — compile module from provided source string + final Sequence contentOption = map.get(OPTIONS_CONTENT); + if (!contentOption.isEmpty()) { + contentSource = contentOption.getStringValue(); + } } // create temporary context so main context is not polluted @@ -154,15 +167,21 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce tempContext.prepareForExecution(); Module[] loadedModules = null; - try { - loadedModules = tempContext.importModule(targetNamespace, null, locationHints); - } catch (final XPathException e) { - if (e.getErrorCode() == ErrorCodes.XQST0059) { - // importModule may throw exception if no location is given and module cannot be resolved - throw new XPathException(this, ErrorCodes.FOQM0002, "Module with URI " + targetNamespace + " not found"); + if (contentSource != null) { + // XQ4: compile module from content string + final ExternalModule contentModule = compileModuleFromContent( + targetNamespace, contentSource, tempContext); + loadedModules = new Module[] { contentModule }; + } else { + try { + loadedModules = tempContext.importModule(targetNamespace, null, locationHints); + } catch (final XPathException e) { + if (e.getErrorCode() == ErrorCodes.XQST0059) { + throw new XPathException(this, ErrorCodes.FOQM0002, "Module with URI " + targetNamespace + " not found"); + } + throw new XPathException(this, ErrorCodes.FOQM0003, "Error found when importing module: " + e.getMessage()); } - throw new XPathException(this, ErrorCodes.FOQM0003, "Error found when importing module: " + e.getMessage()); } // not found, raise error @@ -284,6 +303,36 @@ public static void addFunctionRefsFromModule(final Expression parent, final XQue } } + /** + * XQ4: Compile a library module from a content string. + * Uses XQueryContext.compileModuleFromSource() which handles all the + * parsing, AST walking, and module registration. + */ + private ExternalModule compileModuleFromContent(final String targetNamespace, + final String content, final XQueryContext tempContext) throws XPathException { + final StringSource source = new StringSource(content); + try { + final ExternalModule module = tempContext.compileModuleFromSource(targetNamespace, source); + if (module == null) { + throw new XPathException(this, ErrorCodes.FOQM0005, + "Content string is not a library module"); + } + // Verify the module's namespace matches the target + if (!module.getNamespaceURI().equals(targetNamespace)) { + throw new XPathException(this, ErrorCodes.FOQM0001, + "Module namespace '" + module.getNamespaceURI() + + "' does not match target namespace '" + targetNamespace + "'"); + } + return module; + } catch (final XPathException e) { + if (e.getErrorCode() == ErrorCodes.FOQM0001 || e.getErrorCode() == ErrorCodes.FOQM0005) { + throw e; + } + throw new XPathException(this, ErrorCodes.FOQM0003, + "Error compiling module content: " + e.getMessage(), e); + } + } + private static String getXQueryVersion(final int version) { return String.valueOf(version / 10) + '.' + version % 10; } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/ParsingFunctions.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/ParsingFunctions.java index 48a8353d83c..0ebd4188059 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/ParsingFunctions.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/ParsingFunctions.java @@ -49,6 +49,9 @@ public class ParsingFunctions extends BasicFunction { protected static final Logger logger = LogManager.getLogger(ParsingFunctions.class); + protected static final FunctionParameterSequenceType OPTIONS_PARAMETER = new FunctionParameterSequenceType( + "options", Type.MAP_ITEM, Cardinality.EXACTLY_ONE, "Options map"); + public final static FunctionSignature[] signatures = { new FunctionSignature( new QName("parse-xml", Function.BUILTIN_FUNCTION_NS), @@ -64,6 +67,22 @@ public class ParsingFunctions extends BasicFunction { + "Returns the document node with the parsed document fragment.", new SequenceType[] { TO_BE_PARSED_PARAMETER }, PARSE_RESULT_TYPE + ), + // XQuery 4.0: fn:parse-xml with options map + new FunctionSignature( + new QName("parse-xml", Function.BUILTIN_FUNCTION_NS), + "Parse an XML document with options. " + + "Returns the document node with the parsed document.", + new SequenceType[] { TO_BE_PARSED_PARAMETER, OPTIONS_PARAMETER }, + PARSE_RESULT_TYPE + ), + // XQuery 4.0: fn:parse-xml-fragment with options map + new FunctionSignature( + new QName("parse-xml-fragment", Function.BUILTIN_FUNCTION_NS), + "Parse an XML fragment with options. " + + "Returns the document node with the parsed document fragment.", + new SequenceType[] { TO_BE_PARSED_PARAMETER, OPTIONS_PARAMETER }, + PARSE_RESULT_TYPE ) }; diff --git a/exist-core/src/main/java/org/exist/xquery/regex/RegexUtil.java b/exist-core/src/main/java/org/exist/xquery/regex/RegexUtil.java index d54ca496c01..31c372db37f 100644 --- a/exist-core/src/main/java/org/exist/xquery/regex/RegexUtil.java +++ b/exist-core/src/main/java/org/exist/xquery/regex/RegexUtil.java @@ -155,7 +155,36 @@ public static String translateRegexp(final Expression context, final String patt final List warnings = new ArrayList<>(); return JDK15RegexTranslator.translate(pattern, options, flagbits, warnings); } catch (final RegexSyntaxException e) { + // Fallback: if the pattern uses \p{Is} Unicode block names that + // the bundled Saxon regex translator doesn't recognize, convert them to + // Java's \p{In} syntax and try compiling directly. + if (pattern.contains("\\p{Is") || pattern.contains("\\P{Is")) { + final String javaPattern = convertUnicodeBlockNames(pattern); + try { + int flags = Pattern.UNICODE_CHARACTER_CLASS; + if (ignoreWhitespace) { + flags |= Pattern.COMMENTS; + } + if (caseBlind) { + flags |= Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE; + } + Pattern.compile(javaPattern, flags); + return javaPattern; + } catch (final java.util.regex.PatternSyntaxException ignored) { + // fallback failed, throw original error + } + } throw new XPathException(context, ErrorCodes.FORX0002, "Conversion from XPath F&O 3.0 regular expression syntax to Java regular expression syntax failed: " + e.getMessage(), new StringValue(pattern), e); } } + + /** + * Convert XML Schema/XPath \p{Is} and \P{Is} Unicode block + * property escapes to Java's \p{In} and \P{In} syntax. + */ + private static String convertUnicodeBlockNames(final String pattern) { + return pattern + .replaceAll("\\\\p\\{Is([^}]+)}", "\\\\p{In$1}") + .replaceAll("\\\\P\\{Is([^}]+)}", "\\\\P{In$1}"); + } } diff --git a/exist-core/src/main/java/org/exist/xquery/util/DocUtils.java b/exist-core/src/main/java/org/exist/xquery/util/DocUtils.java index 22e3e15721c..be77f2c4a94 100644 --- a/exist-core/src/main/java/org/exist/xquery/util/DocUtils.java +++ b/exist-core/src/main/java/org/exist/xquery/util/DocUtils.java @@ -99,10 +99,19 @@ private static Sequence getDocumentByPath(final XQueryContext context, final Str Sequence doc = getFromDynamicallyAvailableDocuments(context, path, expression); if (doc == null) { if (PTN_PROTOCOL_PREFIX.matcher(path).matches() && !path.startsWith("xmldb:")) { - /* URL */ - doc = getDocumentByPathFromURL(context, path, expression); + /* URL — use SourceFactory (has security checks) */ + doc = getDocumentByPathFromURL(context, path, expression, false); + } else if (!PTN_PROTOCOL_PREFIX.matcher(path).matches()) { + // Relative URI: resolve against static base URI per XQuery spec §2.1.2 + final String resolved = resolveAgainstBaseUri(context, path); + if (resolved != null && resolved.startsWith("file:")) { + doc = getDocumentByPathFromURL(context, resolved, expression, true); + } else { + /* Database documents */ + doc = getDocumentByPathFromDB(context, path, expression); + } } else { - /* Database documents */ + /* Database documents (xmldb: prefix) */ doc = getDocumentByPathFromDB(context, path, expression); } } @@ -110,6 +119,29 @@ private static Sequence getDocumentByPath(final XQueryContext context, final Str return doc; } + /** + * Resolve a relative URI against the static base URI. + * + * @return the resolved URI string, or null if resolution is not possible + */ + private static @Nullable String resolveAgainstBaseUri(final XQueryContext context, final String relativePath) { + try { + final AnyURIValue baseXdmUri = context.getBaseURI(); + if (baseXdmUri != null && !baseXdmUri.equals(AnyURIValue.EMPTY_URI)) { + String baseStr = baseXdmUri.toURI().toString(); + // Strip filename to get directory URI + final int lastSlash = baseStr.lastIndexOf('/'); + if (lastSlash >= 0) { + baseStr = baseStr.substring(0, lastSlash + 1); + } + return new URI(baseStr).resolve(relativePath).toString(); + } + } catch (final URISyntaxException | XPathException e) { + // fall through + } + return null; + } + private static @Nullable Sequence getFromDynamicallyAvailableDocuments(final XQueryContext context, final String path) throws XPathException { return getFromDynamicallyAvailableDocuments(context, path, null); } @@ -134,11 +166,28 @@ private static Sequence getDocumentByPath(final XQueryContext context, final Str } private static Sequence getDocumentByPathFromURL(final XQueryContext context, final String path) throws XPathException, PermissionDeniedException { - return getDocumentByPathFromURL(context, path, null); + return getDocumentByPathFromURL(context, path, null, false); } - private static Sequence getDocumentByPathFromURL(final XQueryContext context, final String path, final Expression expression) throws XPathException, PermissionDeniedException { + private static Sequence getDocumentByPathFromURL(final XQueryContext context, final String path, final Expression expression, final boolean resolvedFromBaseUri) throws XPathException, PermissionDeniedException { try { + // Only use direct file: access for URIs resolved from a relative path + // against a file: base URI. Absolute file: URIs go through SourceFactory + // which enforces security checks (e.g., blocking file:///etc/passwd). + if (resolvedFromBaseUri && path.startsWith("file:")) { + final String filePath = path.replaceFirst("^file:(?://[^/]*)?", ""); + final java.nio.file.Path nioPath = java.nio.file.Paths.get(filePath); + if (java.nio.file.Files.isReadable(nioPath)) { + try (final java.io.InputStream fis = java.nio.file.Files.newInputStream(nioPath)) { + final org.exist.dom.memtree.DocumentImpl memtreeDoc = parse( + context.getBroker().getBrokerPool(), context, fis, expression); + memtreeDoc.setDocumentURI(path); + return memtreeDoc; + } + } + return Sequence.EMPTY_SEQUENCE; + } + final Source source = SourceFactory.getSource(context.getBroker(), "", path, false); if (source == null) { return Sequence.EMPTY_SEQUENCE; diff --git a/exist-core/src/main/java/org/exist/xquery/util/NumberFormatter_en.java b/exist-core/src/main/java/org/exist/xquery/util/NumberFormatter_en.java index bf51c0b26f1..47a8c337ef5 100644 --- a/exist-core/src/main/java/org/exist/xquery/util/NumberFormatter_en.java +++ b/exist-core/src/main/java/org/exist/xquery/util/NumberFormatter_en.java @@ -36,7 +36,8 @@ public NumberFormatter_en(final Locale locale) { @Override public String getOrdinalSuffix(long number) { - if (number > 10 && number < 20) + final long lastTwo = number % 100; + if (lastTwo > 10 && lastTwo < 20) {return "th";} final long mod = number % 10; if (mod == 1) diff --git a/exist-core/src/main/java/org/exist/xquery/value/AbstractDateTimeValue.java b/exist-core/src/main/java/org/exist/xquery/value/AbstractDateTimeValue.java index 4b4f36150e8..5573b3966cd 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/AbstractDateTimeValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/AbstractDateTimeValue.java @@ -186,6 +186,31 @@ protected XMLGregorianCalendar getImplicitCalendar() { implicitCalendar.setMonth(12); implicitCalendar.setDay(31); break; + case Type.G_DAY: + // Per XPath spec §10.4, use reference date 1972-12 for gDay comparison + implicitCalendar.setYear(1972); + implicitCalendar.setMonth(12); + implicitCalendar.setTime(0, 0, 0); + break; + case Type.G_MONTH: + // Per XPath spec §10.4, use reference date 1972-xx-01 for gMonth + implicitCalendar.setYear(1972); + implicitCalendar.setDay(1); + implicitCalendar.setTime(0, 0, 0); + break; + case Type.G_YEAR: + implicitCalendar.setMonth(1); + implicitCalendar.setDay(1); + implicitCalendar.setTime(0, 0, 0); + break; + case Type.G_MONTH_DAY: + implicitCalendar.setYear(1972); + implicitCalendar.setTime(0, 0, 0); + break; + case Type.G_YEAR_MONTH: + implicitCalendar.setDay(1); + implicitCalendar.setTime(0, 0, 0); + break; default: } implicitCalendar = implicitCalendar.normalize(); // the comparison routines will normalize it anyway, just do it once here @@ -401,11 +426,11 @@ public ComputableValue plus(ComputableValue other) throws XPathException { } public ComputableValue mult(ComputableValue other) throws XPathException { - throw new XPathException(getExpression(), "multiplication is not supported for type " + Type.getTypeName(getType())); + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "multiplication is not supported for type " + Type.getTypeName(getType())); } public ComputableValue div(ComputableValue other) throws XPathException { - throw new XPathException(getExpression(), "division is not supported for type " + Type.getTypeName(getType())); + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "division is not supported for type " + Type.getTypeName(getType())); } public int conversionPreference(Class javaClass) { diff --git a/exist-core/src/main/java/org/exist/xquery/value/AtomicValue.java b/exist-core/src/main/java/org/exist/xquery/value/AtomicValue.java index cc6331f4017..568b57080b0 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/AtomicValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/AtomicValue.java @@ -230,14 +230,14 @@ public NodeSet toNodeSet() throws XPathException { if (!effectiveBooleanValue()) return NodeSet.EMPTY_SET; */ - throw new XPathException(getExpression(), + throw new XPathException(getExpression(), ErrorCodes.XPTY0019, "cannot convert " + Type.getTypeName(getType()) + "('" + getStringValue() + "')" + " to a node set"); } @Override public MemoryNodeSet toMemNodeSet() throws XPathException { - throw new XPathException(getExpression(), + throw new XPathException(getExpression(), ErrorCodes.XPTY0019, "cannot convert " + Type.getTypeName(getType()) + "('" + getStringValue() + "')" + " to a node set"); } diff --git a/exist-core/src/main/java/org/exist/xquery/value/AtomicValueComparator.java b/exist-core/src/main/java/org/exist/xquery/value/AtomicValueComparator.java index 1eda756ca15..48059df743e 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/AtomicValueComparator.java +++ b/exist-core/src/main/java/org/exist/xquery/value/AtomicValueComparator.java @@ -74,7 +74,9 @@ public int compare(final AtomicValue o1, final AtomicValue o2) { return o1.compareTo(collator, o2); } catch (final XPathException e) { LOG.error(e.getMessage(), e); - throw new ClassCastException(e.getMessage()); + final ClassCastException cce = new ClassCastException(e.getMessage()); + cce.initCause(e); + throw cce; } } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/Base64BinaryValueType.java b/exist-core/src/main/java/org/exist/xquery/value/Base64BinaryValueType.java index 3e96607b5a8..9f6c1027a77 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/Base64BinaryValueType.java +++ b/exist-core/src/main/java/org/exist/xquery/value/Base64BinaryValueType.java @@ -22,6 +22,7 @@ package org.exist.xquery.value; import org.exist.util.io.Base64OutputStream; +import org.exist.xquery.ErrorCodes; import org.exist.xquery.Expression; import org.exist.xquery.XPathException; @@ -50,7 +51,7 @@ private Matcher getMatcher(final String toMatch) { @Override public void verifyString(String str) throws XPathException { if (!getMatcher(str).matches()) { - throw new XPathException((Expression) null, "FORG0001: Invalid base64 data"); + throw new XPathException((Expression) null, ErrorCodes.FORG0001, "Invalid base64 data"); } } diff --git a/exist-core/src/main/java/org/exist/xquery/value/DayTimeDurationValue.java b/exist-core/src/main/java/org/exist/xquery/value/DayTimeDurationValue.java index cf960f108f2..59a7e09c953 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DayTimeDurationValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DayTimeDurationValue.java @@ -216,7 +216,7 @@ public ComputableValue plus(ComputableValue other) throws XPathException { try { return super.plus(other); } catch (IllegalArgumentException e) { - throw new XPathException(getExpression(), "Operand to plus should be of type xdt:dayTimeDuration, xs:time, " + + throw new XPathException(getExpression(), ErrorCodes.XPTY0004, "Operand to plus should be of type xdt:dayTimeDuration, xs:time, " + "xs:date or xs:dateTime; got: " + Type.getTypeName(other.getType())); } diff --git a/exist-core/src/main/java/org/exist/xquery/value/DecimalValue.java b/exist-core/src/main/java/org/exist/xquery/value/DecimalValue.java index d69144666b9..0beeaf53d6b 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DecimalValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DecimalValue.java @@ -256,7 +256,7 @@ public AtomicValue convertTo(final int requiredType) throws XPathException { case Type.UNSIGNED_SHORT: case Type.UNSIGNED_BYTE: case Type.POSITIVE_INTEGER: - return new IntegerValue(getExpression(), value.longValue(), requiredType); + return new IntegerValue(getExpression(), value.toBigInteger(), requiredType); case Type.BOOLEAN: return value.signum() == 0 ? BooleanValue.FALSE : BooleanValue.TRUE; default: diff --git a/exist-core/src/main/java/org/exist/xquery/value/DurationValue.java b/exist-core/src/main/java/org/exist/xquery/value/DurationValue.java index 192d8bf8537..536850d8486 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/DurationValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/DurationValue.java @@ -214,7 +214,7 @@ protected BigDecimal secondsValue() { ).add(zeroIfNull((BigDecimal) duration.getField(DatatypeConstants.SECONDS))); } - protected BigDecimal secondsValueSigned() { + public BigDecimal secondsValueSigned() { BigDecimal x = secondsValue(); if (duration.getSign() < 0) { x = x.negate(); @@ -229,7 +229,7 @@ protected BigInteger monthsValue() { .add(zeroIfNull((BigInteger) duration.getField(DatatypeConstants.MONTHS))); } - protected BigInteger monthsValueSigned() { + public BigInteger monthsValueSigned() { BigInteger x = monthsValue(); if (duration.getSign() < 0) { x = x.negate(); diff --git a/exist-core/src/main/java/org/exist/xquery/value/SequenceComparator.java b/exist-core/src/main/java/org/exist/xquery/value/SequenceComparator.java index a64b1e65692..39eeb0e4ddb 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/SequenceComparator.java +++ b/exist-core/src/main/java/org/exist/xquery/value/SequenceComparator.java @@ -71,7 +71,7 @@ public int compare(final Sequence o1, final Sequence o2) { } final int o1Count = o1.getItemCount(); - final int o2Count = o1.getItemCount(); + final int o2Count = o2.getItemCount(); if (o1Count < o2Count) { return -1; diff --git a/exist-core/src/main/java/org/exist/xquery/value/StringValue.java b/exist-core/src/main/java/org/exist/xquery/value/StringValue.java index 9b2fccf0c83..10e28830e79 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/StringValue.java +++ b/exist-core/src/main/java/org/exist/xquery/value/StringValue.java @@ -134,40 +134,27 @@ public static String collapseWhitespace(final CharSequence in) { if (in.isEmpty()) { return in.toString(); } - int i = 0; - // this method is performance critical, so first test if we need to collapse at all - for (; i < in.length(); i++) { - final char c = in.charAt(i); - if (XMLChar.isSpace(c)) { - if (i + 1 < in.length() && XMLChar.isSpace(in.charAt(i + 1))) { - break; - } - } - } - if (i == in.length()) { - // no whitespace to collapse, just return - return in.toString(); - } - - // start to collapse whitespace + // XML Schema "collapse" facet: + // 1. Replace #x9, #xA, #xD with #x20 + // 2. Collapse consecutive #x20 to single #x20 + // 3. Strip leading and trailing #x20 final StringBuilder sb = new StringBuilder(in.length()); - sb.append(in.subSequence(0, i + 1)); - boolean inWhitespace = true; - for (; i < in.length(); i++) { + boolean lastWasSpace = true; // treat start as space to strip leading + for (int i = 0; i < in.length(); i++) { final char c = in.charAt(i); if (XMLChar.isSpace(c)) { - if (inWhitespace) { - // remove the whitespace - } else { + if (!lastWasSpace) { sb.append(' '); - inWhitespace = true; + lastWasSpace = true; } + // else: skip consecutive whitespace } else { sb.append(c); - inWhitespace = false; + lastWasSpace = false; } } - if (sb.charAt(sb.length() - 1) == ' ') { + // Strip trailing space + if (sb.length() > 0 && sb.charAt(sb.length() - 1) == ' ') { sb.deleteCharAt(sb.length() - 1); } return sb.toString(); @@ -379,15 +366,14 @@ private void checkType() throws XPathException { case Type.LANGUAGE: final Matcher matcher = langPattern.matcher(value); if (!matcher.matches()) { - throw new XPathException(getExpression(), - "Type error: string " - + value - + " is not valid for type xs:language"); + throw new XPathException(getExpression(), ErrorCodes.FORG0001, + "String '" + value + "' is not valid for type xs:language"); } return; case Type.NAME: if (QName.isQName(value) != VALID.val) { - throw new XPathException(getExpression(), "Type error: string " + value + " is not a valid xs:Name"); + throw new XPathException(getExpression(), ErrorCodes.FORG0001, + "String '" + value + "' is not a valid xs:Name"); } return; case Type.NCNAME: @@ -395,12 +381,14 @@ private void checkType() throws XPathException { case Type.IDREF: case Type.ENTITY: if (!XMLNames.isNCName(value)) { - throw new XPathException(getExpression(), "Type error: string " + value + " is not a valid " + Type.getTypeName(type)); + throw new XPathException(getExpression(), ErrorCodes.FORG0001, + "String '" + value + "' is not a valid " + Type.getTypeName(type)); } return; case Type.NMTOKEN: if (!XMLNames.isNmToken(value)) { - throw new XPathException(getExpression(), "Type error: string " + value + " is not a valid xs:NMTOKEN"); + throw new XPathException(getExpression(), ErrorCodes.FORG0001, + "String '" + value + "' is not a valid xs:NMTOKEN"); } } } diff --git a/exist-core/src/test/xquery/xquery3/replace.xqm b/exist-core/src/test/xquery/xquery3/replace.xqm index 05943a5437d..b4672ced57d 100644 --- a/exist-core/src/test/xquery/xquery3/replace.xqm +++ b/exist-core/src/test/xquery/xquery3/replace.xqm @@ -19,7 +19,7 @@ : License along with this library; if not, write to the Free Software : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA :) -xquery version "3.1"; +xquery version "4.0"; module namespace rt="http://exist-db.org/xquery/test/replace"; From 4202cef2c4da047fc0d9a320c9cb4d5920778cf8 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Sat, 4 Apr 2026 09:52:27 -0400 Subject: [PATCH 008/326] [feature] Add XQuery 4.0 type infrastructure and expression modifications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the existing type system and expression classes for XQ4: Type system: - Type.CHOICE: union/choice type constant - Type.ENUM: enumeration type constant - SequenceType: XQ4 record type support, named function parameters - FunctionParameterSequenceType: default value support for params - Constants: XQ4 axis constants (ancestor-or-self::*, etc.) Expression modifications: - Function: support default parameter values (XQ4 §4.15) - FunctionFactory: keyword argument dispatch - UserDefinedFunction: default parameter evaluation - ForExpr/LetExpr: for-member and while clause integration - FLWORClause: while clause chaining - TryCatchExpression: finally clause support - SwitchExpression: XQ4 fall-through semantics - StringConstructor: XQ4 string template evaluation - XQueryContext: XQ4 version detection, xquery version "4.0" - LocationStep: combined axis support Spec: QT4 XQuery 4.0 §2.5 (Types), §4 (Modules) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../main/java/org/exist/xquery/Constants.java | 12 +- .../java/org/exist/xquery/FLWORClause.java | 3 +- .../main/java/org/exist/xquery/ForExpr.java | 20 +- .../main/java/org/exist/xquery/Function.java | 27 +- .../org/exist/xquery/FunctionFactory.java | 210 ++++++++++- .../main/java/org/exist/xquery/LetExpr.java | 9 +- .../java/org/exist/xquery/LocationStep.java | 97 +++++ .../exist/xquery/StaticXQueryException.java | 25 +- .../org/exist/xquery/StringConstructor.java | 6 +- .../org/exist/xquery/SwitchExpression.java | 79 +++-- .../org/exist/xquery/TryCatchExpression.java | 214 ++++++----- .../org/exist/xquery/UserDefinedFunction.java | 48 ++- .../java/org/exist/xquery/XQueryContext.java | 35 +- .../value/FunctionParameterSequenceType.java | 14 + .../org/exist/xquery/value/SequenceType.java | 335 +++++++++++++++--- .../java/org/exist/xquery/value/Type.java | 13 +- 16 files changed, 945 insertions(+), 202 deletions(-) diff --git a/exist-core/src/main/java/org/exist/xquery/Constants.java b/exist-core/src/main/java/org/exist/xquery/Constants.java index 7a5069d7416..62f16a2d304 100644 --- a/exist-core/src/main/java/org/exist/xquery/Constants.java +++ b/exist-core/src/main/java/org/exist/xquery/Constants.java @@ -46,7 +46,11 @@ public interface Constants { "following-sibling", "namespace", "self", - "attribute-descendant" + "attribute-descendant", + "following-or-self", + "preceding-or-self", + "following-sibling-or-self", + "preceding-sibling-or-self" }; /** @@ -73,6 +77,12 @@ public interface Constants { //combines /descendant-or-self::node()/attribute:* int DESCENDANT_ATTRIBUTE_AXIS = 13; + /** XQuery 4.0 axes */ + int FOLLOWING_OR_SELF_AXIS = 14; + int PRECEDING_OR_SELF_AXIS = 15; + int FOLLOWING_SIBLING_OR_SELF_AXIS = 16; + int PRECEDING_SIBLING_OR_SELF_AXIS = 17; + /** * Node types */ diff --git a/exist-core/src/main/java/org/exist/xquery/FLWORClause.java b/exist-core/src/main/java/org/exist/xquery/FLWORClause.java index d56ed4777d2..ea632d51e17 100644 --- a/exist-core/src/main/java/org/exist/xquery/FLWORClause.java +++ b/exist-core/src/main/java/org/exist/xquery/FLWORClause.java @@ -34,7 +34,8 @@ public interface FLWORClause extends Expression { enum ClauseType { - FOR, LET, GROUPBY, ORDERBY, WHERE, SOME, EVERY, COUNT, WINDOW + FOR, LET, GROUPBY, ORDERBY, WHERE, WHILE, SOME, EVERY, COUNT, WINDOW, FOR_MEMBER, FOR_KEY, FOR_VALUE, FOR_KEY_VALUE, + LET_SEQ_DESTRUCTURE, LET_ARRAY_DESTRUCTURE, LET_MAP_DESTRUCTURE } /** diff --git a/exist-core/src/main/java/org/exist/xquery/ForExpr.java b/exist-core/src/main/java/org/exist/xquery/ForExpr.java index 1a5eab2f4dd..577784de185 100644 --- a/exist-core/src/main/java/org/exist/xquery/ForExpr.java +++ b/exist-core/src/main/java/org/exist/xquery/ForExpr.java @@ -176,15 +176,23 @@ public Sequence eval(Sequence contextSequence, Item contextItem) // Loop through each variable binding int p = 0; - if (in.isEmpty() && allowEmpty) { - processItem(var, AtomicValue.EMPTY_VALUE, Sequence.EMPTY_SEQUENCE, resultSequence, at, p); - } else { - for (final SequenceIterator i = in.iterate(); i.hasNext(); p++) { - processItem(var, i.nextItem(), in, resultSequence, at, p); + try { + if (in.isEmpty() && allowEmpty) { + processItem(var, AtomicValue.EMPTY_VALUE, Sequence.EMPTY_SEQUENCE, resultSequence, at, p); + } else { + for (final SequenceIterator i = in.iterate(); i.hasNext() && !WhileClause.isTerminated(); p++) { + processItem(var, i.nextItem(), in, resultSequence, at, p); + } } + } catch (final WhileClause.WhileTerminationException e) { + // while clause signaled end of iteration for this for loop + } + // clear terminated flag if this is the outermost for + if (isOuterFor && WhileClause.isTerminated()) { + WhileClause.clearTerminated(); } } finally { - // restore the local variable stack + // restore the local variable stack context.popLocalVariables(mark, resultSequence); } diff --git a/exist-core/src/main/java/org/exist/xquery/Function.java b/exist-core/src/main/java/org/exist/xquery/Function.java index 161cba2957b..a22837100ab 100644 --- a/exist-core/src/main/java/org/exist/xquery/Function.java +++ b/exist-core/src/main/java/org/exist/xquery/Function.java @@ -212,10 +212,29 @@ public void setParent(final Expression parent) { * @throws XPathException if an error occurs setting the arguments */ public void setArguments(final List arguments) throws XPathException { - if ((!mySignature.isVariadic()) && arguments.size() != mySignature.getArgumentCount()) { - throw new XPathException(this, ErrorCodes.XPST0017, - "Number of arguments of function " + getName() + " doesn't match function signature (expected " - + mySignature.getArgumentCount() + ", got " + arguments.size() + ')'); + final int argCount = mySignature.getArgumentCount(); + if ((!mySignature.isVariadic()) && arguments.size() != argCount) { + // XQ4: Allow fewer arguments if trailing params have default values + if (arguments.size() < argCount) { + boolean hasDefaults = true; + final SequenceType[] argTypes = mySignature.getArgumentTypes(); + for (int i = arguments.size(); i < argCount; i++) { + if (!(argTypes[i] instanceof FunctionParameterSequenceType) || + !((FunctionParameterSequenceType) argTypes[i]).hasDefaultValue()) { + hasDefaults = false; + break; + } + } + if (!hasDefaults) { + throw new XPathException(this, ErrorCodes.XPST0017, + "Number of arguments of function " + getName() + " doesn't match function signature (expected " + + argCount + ", got " + arguments.size() + ')'); + } + } else { + throw new XPathException(this, ErrorCodes.XPST0017, + "Number of arguments of function " + getName() + " doesn't match function signature (expected " + + argCount + ", got " + arguments.size() + ')'); + } } steps.clear(); diff --git a/exist-core/src/main/java/org/exist/xquery/FunctionFactory.java b/exist-core/src/main/java/org/exist/xquery/FunctionFactory.java index adcf7d3d5cb..07d6a924516 100644 --- a/exist-core/src/main/java/org/exist/xquery/FunctionFactory.java +++ b/exist-core/src/main/java/org/exist/xquery/FunctionFactory.java @@ -54,6 +54,17 @@ public static Expression createFunction(XQueryContext context, XQueryAST ast, Pa } catch(final QName.IllegalQNameException xpe) { throw new XPathException(ast, ErrorCodes.XPST0081, "Invalid qname " + ast.getText() + ". " + xpe.getMessage()); } + // XQ4 (PR2200): for unprefixed function calls, check if there's a + // no-namespace user-defined function that should override fn: + if (context.getXQueryVersion() >= 40 + && !ast.getText().contains(":") + && Namespaces.XPATH_FUNCTIONS_NS.equals(qname.getNamespaceURI())) { + final QName noNsName = new QName(ast.getText(), ""); + final UserDefinedFunction noNsFunc = context.resolveFunction(noNsName, params.size()); + if (noNsFunc != null) { + qname = noNsName; + } + } return createFunction(context, qname, ast, parent, params); } @@ -240,12 +251,25 @@ private static GeneralComparison equals(XQueryContext context, XQueryAST ast, private static CastExpression castExpression(XQueryContext context, XQueryAST ast, List params, QName qname) throws XPathException { - if (params.size() != 1) { + final Expression arg; + if (params.size() == 1) { + arg = params.getFirst(); + } else if (params.isEmpty() && context.getXQueryVersion() >= 31) { + // XQ4 focus constructor: xs:type() uses context item as argument + arg = new ContextItemExpression(context); + ((ContextItemExpression) arg).setLocation(ast.getLine(), ast.getColumn()); + } else { throw new XPathException(ast.getLine(), ast.getColumn(), ErrorCodes.XPST0017, "Wrong number of arguments for constructor function"); } - final Expression arg = params.getFirst(); - final int code = Type.getType(qname); + final int code; + try { + code = Type.getType(qname); + } catch (final XPathException e) { + // Unknown type name in xs: namespace → XPST0017 (no such function) + throw new XPathException(ast.getLine(), ast.getColumn(), + ErrorCodes.XPST0017, "Unknown constructor function: " + qname.getStringValue()); + } final CastExpression castExpr = new CastExpression(context, arg, code, Cardinality.ZERO_OR_ONE); castExpr.setLocation(ast.getLine(), ast.getColumn()); return castExpr; @@ -305,10 +329,34 @@ private static Function functionCall(final XQueryContext context, * @param throwOnNotFound true to throw an XPST0017 if the functions is not found, false to just return null */ private static @Nullable Function getInternalModuleFunction(final XQueryContext context, - final XQueryAST ast, final List params, QName qname, Module module, + final XQueryAST ast, List params, QName qname, Module module, final boolean throwOnNotFound) throws XPathException { //For internal modules: create a new function instance from the class - FunctionDef def = ((InternalModule) module).getFunctionDef(qname, params.size()); + final boolean hasKeywordArgs = hasKeywordArguments(params); + FunctionDef def = null; + + // When keyword args are present, skip the initial arity-based lookup because + // params.size() may not match the correct overload. Instead, resolve keyword + // args against all signatures (largest arity first) to find the right one. + if (hasKeywordArgs) { + final List funcs = ((InternalModule) module).getFunctionsByName(qname); + // Sort by arity descending — keyword args typically target the largest overload + funcs.sort((a, b) -> b.getArgumentCount() - a.getArgumentCount()); + for (final FunctionSignature sig : funcs) { + final List resolved = resolveKeywordArguments(context, params, sig, ast); + if (resolved != null) { + def = ((InternalModule) module).getFunctionDef(qname, sig.getArgumentCount()); + if (def != null) { + params = resolved; + break; + } + } + } + } + + if (def == null && !hasKeywordArgs) { + def = ((InternalModule) module).getFunctionDef(qname, params.size()); + } //TODO: rethink: xsl namespace function should search xpath one too if (def == null && Namespaces.XSL_NS.equals(qname.getNamespaceURI())) { //Search xpath namespace @@ -360,7 +408,12 @@ private static Function functionCall(final XQueryContext context, "Access to deprecated functions is not allowed. Call to '" + qname.getStringValue() + "()' denied. " + def.getSignature().getDeprecated()); } final Function fn = Function.createFunction(context, ast, module, def); - fn.setArguments(params); + if (hasKeywordArgs) { + final List resolved = resolveKeywordArguments(context, params, def.getSignature(), ast); + fn.setArguments(resolved != null ? resolved : params); + } else { + fn.setArguments(params); + } fn.setASTNode(ast); return new InternalFunctionCall(fn); } @@ -370,11 +423,36 @@ private static Function functionCall(final XQueryContext context, */ private static FunctionCall getUserDefinedFunction(XQueryContext context, XQueryAST ast, List params, QName qname) throws XPathException { final FunctionCall fc; - final UserDefinedFunction func = context.resolveFunction(qname, params.size()); + final boolean hasKeywordArgs = hasKeywordArguments(params); + + // Count positional arguments to determine resolution arity + int positionalCount = params.size(); + if (hasKeywordArgs) { + positionalCount = 0; + for (final Expression param : params) { + if (param instanceof KeywordArgumentExpression) { + break; + } + positionalCount++; + } + } + + UserDefinedFunction func = context.resolveFunction(qname, params.size()); + + // If keyword args and no exact match, try resolving with positional count + if (func == null && hasKeywordArgs && positionalCount != params.size()) { + func = context.resolveFunction(qname, positionalCount); + } + if (func != null) { fc = new FunctionCall(context, func); fc.setLocation(ast.getLine(), ast.getColumn()); - fc.setArguments(params); + if (hasKeywordArgs) { + final List resolved = resolveKeywordArguments(context, params, func.getSignature(), ast); + fc.setArguments(resolved != null ? resolved : params); + } else { + fc.setArguments(params); + } } else { //Create a forward reference which will be resolved later fc = new FunctionCall(context, qname, params); @@ -482,4 +560,120 @@ public static FunctionCall wrap(XQueryContext context, Function call) throws XPa wrappedCall.setArguments(wrapperArgs); return wrappedCall; } + + /** + * Check if any parameter is a keyword argument. + */ + private static boolean hasKeywordArguments(final List params) { + for (final Expression param : params) { + if (param instanceof KeywordArgumentExpression) { + return true; + } + } + return false; + } + + /** + * Resolve keyword arguments to positional arguments using the function signature. + * + * Keyword arguments (name := value) are matched to the corresponding parameter + * position in the function signature. Positional arguments must come before + * keyword arguments. Gaps between positional and keyword arguments are filled + * with empty sequence expressions for optional parameters. Returns null if + * resolution fails. + */ + private static @Nullable List resolveKeywordArguments( + final XQueryContext context, + final List params, final FunctionSignature signature, + final XQueryAST ast) throws XPathException { + final SequenceType[] argTypes = signature.getArgumentTypes(); + if (argTypes == null) { + return null; + } + + // Find where keyword arguments start + int firstKeyword = -1; + for (int i = 0; i < params.size(); i++) { + if (params.get(i) instanceof KeywordArgumentExpression) { + firstKeyword = i; + break; + } + } + if (firstKeyword < 0) { + return params; // no keyword args + } + + // Build the resolved argument list + final List resolved = new ArrayList<>(argTypes.length); + + // Copy positional arguments + for (int i = 0; i < firstKeyword; i++) { + resolved.add(params.get(i)); + } + + // Fill remaining positions with nulls (to be filled by keyword args) + for (int i = firstKeyword; i < argTypes.length; i++) { + resolved.add(null); + } + + // Match keyword arguments to parameter positions + for (int i = firstKeyword; i < params.size(); i++) { + final Expression param = params.get(i); + if (!(param instanceof KeywordArgumentExpression)) { + throw new XPathException(ast.getLine(), ast.getColumn(), + ErrorCodes.XPST0003, + "Positional arguments must not follow keyword arguments"); + } + final KeywordArgumentExpression kwArg = (KeywordArgumentExpression) param; + final String kwName = kwArg.getKeywordName(); + + // Find matching parameter by name + int matchPos = -1; + for (int j = firstKeyword; j < argTypes.length; j++) { + if (argTypes[j] instanceof org.exist.xquery.value.FunctionParameterSequenceType) { + final String paramName = ((org.exist.xquery.value.FunctionParameterSequenceType) argTypes[j]) + .getAttributeName(); + if (kwName.equals(paramName)) { + matchPos = j; + break; + } + } + } + + if (matchPos < 0) { + return null; // no matching parameter found — signature mismatch + } + if (resolved.get(matchPos) != null) { + throw new XPathException(ast.getLine(), ast.getColumn(), + ErrorCodes.XPST0003, + "Duplicate keyword argument: " + kwName); + } + resolved.set(matchPos, kwArg.getArgument()); + } + + // Fill gaps: for parameters that allow empty sequences or have defaults, + // supply an empty sequence expression. This enables keyword arguments to + // skip optional positional parameters in overloaded built-in functions. + for (int i = 0; i < resolved.size(); i++) { + if (resolved.get(i) == null) { + if (argTypes[i] instanceof org.exist.xquery.value.FunctionParameterSequenceType) { + final org.exist.xquery.value.FunctionParameterSequenceType pst = + (org.exist.xquery.value.FunctionParameterSequenceType) argTypes[i]; + if (pst.hasDefaultValue()) { + resolved.set(i, pst.getDefaultValue()); + } else if (pst.getCardinality().isSuperCardinalityOrEqualOf( + org.exist.xquery.Cardinality.EMPTY_SEQUENCE)) { + // Parameter allows empty — fill with empty sequence + resolved.set(i, new PathExpr(context)); + } else { + return null; // required parameter missing + } + } else { + return null; // can't determine if parameter is optional + } + } + } + + return resolved; + } } diff --git a/exist-core/src/main/java/org/exist/xquery/LetExpr.java b/exist-core/src/main/java/org/exist/xquery/LetExpr.java index 278e7d18295..b18f6d5f257 100644 --- a/exist-core/src/main/java/org/exist/xquery/LetExpr.java +++ b/exist-core/src/main/java/org/exist/xquery/LetExpr.java @@ -108,7 +108,14 @@ public Sequence eval(Sequence contextSequence, Item contextItem) var.setContextDocs(inputSequence.getContextDocSet()); registerUpdateListener(in); - resultSequence = returnExpr.eval(contextSequence, null); + try { + resultSequence = returnExpr.eval(contextSequence, null); + } catch (final WhileClause.WhileTerminationException e) { + resultSequence = Sequence.EMPTY_SEQUENCE; + } + if (getPreviousClause() == null && WhileClause.isTerminated()) { + WhileClause.clearTerminated(); + } if (sequenceType != null) { Cardinality actualCardinality; diff --git a/exist-core/src/main/java/org/exist/xquery/LocationStep.java b/exist-core/src/main/java/org/exist/xquery/LocationStep.java index 624795add20..db87581b741 100644 --- a/exist-core/src/main/java/org/exist/xquery/LocationStep.java +++ b/exist-core/src/main/java/org/exist/xquery/LocationStep.java @@ -443,6 +443,16 @@ public Sequence eval(Sequence contextSequence, final Item contextItem) result = getSiblings(context, contextSequence); break; + case Constants.FOLLOWING_OR_SELF_AXIS: + case Constants.PRECEDING_OR_SELF_AXIS: + result = getOrSelfAxis(context, contextSequence); + break; + + case Constants.FOLLOWING_SIBLING_OR_SELF_AXIS: + case Constants.PRECEDING_SIBLING_OR_SELF_AXIS: + result = getSiblingOrSelfAxis(context, contextSequence); + break; + default: throw new IllegalArgumentException("Unsupported axis specified"); } @@ -1003,6 +1013,93 @@ private Sequence getPrecedingOrFollowing(final XQueryContext context, final Sequ } } + /** + * XQ4: Evaluate following-or-self or preceding-or-self axis. + * Combines self:: with following:: or preceding:: and returns + * results in document order. + */ + private Sequence getOrSelfAxis(final XQueryContext context, final Sequence contextSequence) + throws XPathException { + // Evaluate self:: axis + final int savedAxis = axis; + axis = Constants.SELF_AXIS; + final Sequence selfResult = getSelf(context, contextSequence); + + // Evaluate the base axis (following or preceding) + axis = (savedAxis == Constants.FOLLOWING_OR_SELF_AXIS) + ? Constants.FOLLOWING_AXIS : Constants.PRECEDING_AXIS; + final Sequence baseResult = getPrecedingOrFollowing(context, contextSequence); + + axis = savedAxis; + + // Merge results + if (selfResult.isEmpty()) { + return baseResult; + } + if (baseResult.isEmpty()) { + return selfResult; + } + final ValueSequence combined = new ValueSequence(); + if (savedAxis == Constants.PRECEDING_OR_SELF_AXIS) { + // preceding comes first in document order, then self + combined.addAll(baseResult); + combined.addAll(selfResult); + } else { + // self comes first, then following + combined.addAll(selfResult); + combined.addAll(baseResult); + } + combined.sortInDocumentOrder(); + combined.removeDuplicates(); + return combined; + } + + /** + * XQ4: Evaluate following-sibling-or-self or preceding-sibling-or-self axis. + * Combines self:: with following-sibling:: or preceding-sibling:: and returns + * results in document order. + */ + private Sequence getSiblingOrSelfAxis(final XQueryContext context, final Sequence contextSequence) + throws XPathException { + // Evaluate self:: axis + final int savedAxis = axis; + axis = Constants.SELF_AXIS; + final Sequence selfResult = getSelf(context, contextSequence); + + // Evaluate the base sibling axis — guard against document nodes + // which don't have siblings and cause ArrayIndexOutOfBounds + axis = (savedAxis == Constants.FOLLOWING_SIBLING_OR_SELF_AXIS) + ? Constants.FOLLOWING_SIBLING_AXIS : Constants.PRECEDING_SIBLING_AXIS; + Sequence baseResult; + try { + baseResult = getSiblings(context, contextSequence); + } catch (final ArrayIndexOutOfBoundsException e) { + // Document nodes don't have siblings + baseResult = Sequence.EMPTY_SEQUENCE; + } + + axis = savedAxis; + + // Merge results + if (selfResult.isEmpty()) { + return baseResult; + } + if (baseResult.isEmpty()) { + return selfResult; + } + final ValueSequence combined = new ValueSequence(); + if (savedAxis == Constants.PRECEDING_SIBLING_OR_SELF_AXIS) { + combined.addAll(baseResult); + combined.addAll(selfResult); + } else { + combined.addAll(selfResult); + combined.addAll(baseResult); + } + combined.sortInDocumentOrder(); + combined.removeDuplicates(); + return combined; + } + /** * If the optimizer has determined that the first filter after this step is a simple positional * predicate and can be optimized, try to precompute the position and return it to limit the diff --git a/exist-core/src/main/java/org/exist/xquery/StaticXQueryException.java b/exist-core/src/main/java/org/exist/xquery/StaticXQueryException.java index 682be4dfff1..36494f688cc 100644 --- a/exist-core/src/main/java/org/exist/xquery/StaticXQueryException.java +++ b/exist-core/src/main/java/org/exist/xquery/StaticXQueryException.java @@ -30,19 +30,19 @@ public StaticXQueryException(String message) { } public StaticXQueryException(final Expression expression, String message) { - super(expression, message); + super(expression, ErrorCodes.XPST0003, message); } public StaticXQueryException(int line, int column, String message) { - super(line, column, message); + super(line, column, ErrorCodes.XPST0003, message); } - + public StaticXQueryException(Throwable cause) { this((Expression) null, cause); } - + public StaticXQueryException(final Expression expression, Throwable cause) { - super(expression, cause); + super(expression, ErrorCodes.XPST0003, cause.getMessage(), cause); } public StaticXQueryException(String message, Throwable cause) { @@ -50,11 +50,20 @@ public StaticXQueryException(String message, Throwable cause) { } public StaticXQueryException(final Expression expression, String message, Throwable cause) { - super(expression, message, cause); + super(expression, ErrorCodes.XPST0003, message, cause); } - //TODO add in ErrorCode and ErrorVal public StaticXQueryException(int line, int column, String message, Throwable cause) { - super(line, column, message, cause); + super(line, column, ErrorCodes.XPST0003, message); + initCause(cause); + } + + public StaticXQueryException(int line, int column, ErrorCodes.ErrorCode errorCode, String message) { + super(line, column, errorCode, message); + } + + public StaticXQueryException(int line, int column, ErrorCodes.ErrorCode errorCode, String message, Throwable cause) { + super(line, column, errorCode, message); + initCause(cause); } } \ No newline at end of file diff --git a/exist-core/src/main/java/org/exist/xquery/StringConstructor.java b/exist-core/src/main/java/org/exist/xquery/StringConstructor.java index 3d725e63c66..ba3b0fce492 100644 --- a/exist-core/src/main/java/org/exist/xquery/StringConstructor.java +++ b/exist-core/src/main/java/org/exist/xquery/StringConstructor.java @@ -159,9 +159,13 @@ public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException public String eval(final Sequence contextSequence) throws XPathException { final Sequence result = expression.eval(contextSequence, null); + // Atomize the result per spec: string constructor interpolation + // atomizes its content, joining with spaces + final Sequence atomized = Atomize.atomize(result); + final StringBuilder out = new StringBuilder(); boolean gotOne = false; - for(final SequenceIterator i = result.iterate(); i.hasNext(); ) { + for(final SequenceIterator i = atomized.iterate(); i.hasNext(); ) { final Item next = i.nextItem(); if (gotOne) { out.append(' '); diff --git a/exist-core/src/main/java/org/exist/xquery/SwitchExpression.java b/exist-core/src/main/java/org/exist/xquery/SwitchExpression.java index d75361bf784..70e263539cf 100644 --- a/exist-core/src/main/java/org/exist/xquery/SwitchExpression.java +++ b/exist-core/src/main/java/org/exist/xquery/SwitchExpression.java @@ -56,11 +56,20 @@ public Case(List caseOperands, Expression caseClause) { private Expression operand; private Case defaultClause = null; private List cases = new ArrayList<>(5); - + private boolean booleanMode = false; + public SwitchExpression(XQueryContext context, Expression operand) { super(context); this.operand = operand; } + + /** + * Set boolean mode for XQ4 omitted comparand: switch () { case boolExpr return ... } + * In boolean mode, each case operand is evaluated and its effective boolean value determines the match. + */ + public void setBooleanMode(boolean booleanMode) { + this.booleanMode = booleanMode; + } /** * Add case clause(s) with a return. @@ -88,34 +97,58 @@ public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathExc if (contextItem != null) {contextSequence = contextItem.toSequence();} + + if (booleanMode) { + // XQ4 omitted comparand: evaluate each case operand as boolean + return evalBooleanMode(contextSequence, contextItem); + } + final Sequence opSeq = operand.eval(contextSequence, null); - Sequence result = null; + if (opSeq.hasMany()) { + throw new XPathException(this, ErrorCodes.XPTY0004, "Cardinality error in switch operand ", opSeq); + } + final Collator defaultCollator = context.getDefaultCollator(); if (opSeq.isEmpty()) { - result = defaultClause.returnClause.eval(contextSequence, null); + // XQ4: empty comparand can match case () (empty case operand) + for (final Case next : cases) { + for (final Expression caseOperand : next.operands) { + final Sequence caseSeq = caseOperand.eval(contextSequence, contextItem); + if (caseSeq.isEmpty()) { + return next.returnClause.eval(contextSequence, null); + } + } + } } else { - if (opSeq.hasMany()) { - throw new XPathException(this, ErrorCodes.XPTY0004, "Cardinality error in switch operand ", opSeq); + final AtomicValue opVal = opSeq.itemAt(0).atomize(); + for (final Case next : cases) { + for (final Expression caseOperand : next.operands) { + final Sequence caseSeq = caseOperand.eval(contextSequence, contextItem); + if (context.getXQueryVersion() <= 30 && caseSeq.hasMany()) { + throw new XPathException(this, ErrorCodes.XPTY0004, "Cardinality error in switch case operand ", caseSeq); + } + // XQ4: case operand may be a sequence; match if any item equals the comparand + for (int i = 0; i < caseSeq.getItemCount(); i++) { + final AtomicValue caseVal = caseSeq.itemAt(i).atomize(); + if (FunDeepEqual.deepEquals(caseVal, opVal, defaultCollator)) { + return next.returnClause.eval(contextSequence, null); + } + } + } } - final AtomicValue opVal = opSeq.itemAt(0).atomize(); - final Collator defaultCollator = context.getDefaultCollator(); - for (final Case next : cases) { - for (final Expression caseOperand : next.operands) { - final Sequence caseSeq = caseOperand.eval(contextSequence, contextItem); - if (caseSeq.hasMany()) { - throw new XPathException(this, ErrorCodes.XPTY0004, "Cardinality error in switch case operand ", caseSeq); - } - final AtomicValue caseVal = caseSeq.isEmpty() ? AtomicValue.EMPTY_VALUE : caseSeq.itemAt(0).atomize(); - if (FunDeepEqual.deepEquals(caseVal, opVal, defaultCollator)) { - return next.returnClause.eval(contextSequence, null); - } - } - } } - if (result == null) { - result = defaultClause.returnClause.eval(contextSequence, null); + return defaultClause.returnClause.eval(contextSequence, null); + } + + private Sequence evalBooleanMode(Sequence contextSequence, Item contextItem) throws XPathException { + for (final Case next : cases) { + for (final Expression caseOperand : next.operands) { + final Sequence caseSeq = caseOperand.eval(contextSequence, contextItem); + if (caseSeq.effectiveBooleanValue()) { + return next.returnClause.eval(contextSequence, null); + } + } } - - return result; + return defaultClause.returnClause.eval(contextSequence, null); } public int returnsType() { diff --git a/exist-core/src/main/java/org/exist/xquery/TryCatchExpression.java b/exist-core/src/main/java/org/exist/xquery/TryCatchExpression.java index c11a2acf065..0712770b636 100644 --- a/exist-core/src/main/java/org/exist/xquery/TryCatchExpression.java +++ b/exist-core/src/main/java/org/exist/xquery/TryCatchExpression.java @@ -63,6 +63,7 @@ public class TryCatchExpression extends AbstractExpression { private final Expression tryTargetExpr; private final List catchClauses = new ArrayList<>(); + private Expression finallyExpr; /** * Constructor. @@ -88,6 +89,10 @@ public void addCatchClause(final List catchErrorList, final List c catchClauses.add( new CatchClause(catchErrorList, catchVars, catchExpr) ); } + public void setFinallyExpr(final Expression finallyExpr) { + this.finallyExpr = finallyExpr; + } + @Override public int getDependencies() { return Dependency.CONTEXT_SET | Dependency.CONTEXT_ITEM; @@ -126,6 +131,9 @@ public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException for (final CatchClause catchClause : catchClauses) { catchClause.getCatchExpr().analyze(contextInfo); } + if (finallyExpr != null) { + finallyExpr.analyze(contextInfo); + } } finally { // restore the local variable stack context.popLocalVariables(mark); @@ -141,107 +149,136 @@ public Sequence eval(final Sequence contextSequence, final Item contextItem) thr throw new XPathException(this, ErrorCodes.EXXQDY0003, "The try-catch expression is only available in xquery version \"3.0\" and later."); } + Sequence result = null; + Throwable pendingError = null; + try { // Evaluate 'try' expression - final Sequence tryTargetSeq = tryTargetExpr.eval(contextSequence, contextItem); - return tryTargetSeq; + result = tryTargetExpr.eval(contextSequence, contextItem); - } catch (final Throwable throwable) { + } catch (final Throwable throwable) { - final ErrorCode errorCode; + // If no catch clauses (try/finally only), re-throw after finally + if (catchClauses.isEmpty()) { + pendingError = throwable; + } else { - // fn:error throws an XPathException - if(throwable instanceof XPathException xpe){ - // Get errorcode from nicely thrown xpathexception + final ErrorCode errorCode; - if(xpe.getErrorCode() != null) { - if(xpe.getErrorCode() == ErrorCodes.ERROR) { - errorCode = extractErrorCode(xpe); + // fn:error throws an XPathException + if (throwable instanceof XPathException xpe) { + // Get errorcode from nicely thrown xpathexception + + if (xpe.getErrorCode() != null) { + if (xpe.getErrorCode() == ErrorCodes.ERROR) { + errorCode = extractErrorCode(xpe); + } else { + errorCode = xpe.getErrorCode(); + } } else { - errorCode = xpe.getErrorCode(); + // if no errorcode is found, reconstruct by parsing the error text. + errorCode = extractErrorCode(xpe); } } else { - // if no errorcode is found, reconstruct by parsing the error text. - errorCode = extractErrorCode(xpe); + // Get errorcode from all other errors and exceptions + errorCode = new JavaErrorCode(throwable); } - } else { - // Get errorcode from all other errors and exceptions - errorCode = new JavaErrorCode(throwable); - } - // We need the qname in the end - final QName errorCodeQname = errorCode.getErrorQName(); - - // Exception in thrown, catch expression will be evaluated. - // catchvars (CatchErrorCode (, CatchErrorDesc (, CatchErrorVal)?)? ) - // need to be retrieved as variables - Sequence catchResultSeq = null; - final LocalVariable mark0 = context.markLocalVariables(false); // DWES: what does this do? - - // DWES: should I use popLocalVariables - context.declareInScopeNamespace(Namespaces.W3C_XQUERY_XPATH_ERROR_PREFIX, Namespaces.W3C_XQUERY_XPATH_ERROR_NS); - context.declareInScopeNamespace(Namespaces.EXIST_XQUERY_XPATH_ERROR_PREFIX, Namespaces.EXIST_XQUERY_XPATH_ERROR_NS); - - //context.declareInScopeNamespace(null, null); - - try { - // flag used to escape loop when errorcode has matched - boolean errorMatched = false; - - // Iterate on all catch clauses - for (final CatchClause catchClause : catchClauses) { - - if (isErrorInList(errorCodeQname, catchClause.getCatchErrorList()) && !errorMatched) { - - errorMatched = true; - - // Get catch variables - final LocalVariable mark1 = context.markLocalVariables(false); // DWES: what does this do? - - try { - // Add std errors - addErrCode(errorCodeQname); - addErrDescription(throwable, errorCode); - addErrValue(throwable); - addErrModule(throwable); - addErrLineNumber(throwable); - addErrColumnNumber(throwable); - addErrAdditional(throwable); - addFunctionTrace(throwable); - addJavaTrace(throwable); - - // Evaluate catch expression - catchResultSeq = ((Expression) catchClause.getCatchExpr()).eval(contextSequence, contextItem); - - - } finally { - context.popLocalVariables(mark1, catchResultSeq); + // We need the qname in the end + final QName errorCodeQname = errorCode.getErrorQName(); + + // Exception in thrown, catch expression will be evaluated. + // catchvars (CatchErrorCode (, CatchErrorDesc (, CatchErrorVal)?)? ) + // need to be retrieved as variables + Sequence catchResultSeq = null; + final LocalVariable mark0 = context.markLocalVariables(false); + + context.declareInScopeNamespace(Namespaces.W3C_XQUERY_XPATH_ERROR_PREFIX, Namespaces.W3C_XQUERY_XPATH_ERROR_NS); + context.declareInScopeNamespace(Namespaces.EXIST_XQUERY_XPATH_ERROR_PREFIX, Namespaces.EXIST_XQUERY_XPATH_ERROR_NS); + + try { + // flag used to escape loop when errorcode has matched + boolean errorMatched = false; + + // Iterate on all catch clauses + for (final CatchClause catchClause : catchClauses) { + + if (isErrorInList(errorCodeQname, catchClause.getCatchErrorList()) && !errorMatched) { + + errorMatched = true; + + // Get catch variables + final LocalVariable mark1 = context.markLocalVariables(false); + + try { + // Add std errors + addErrCode(errorCodeQname); + addErrDescription(throwable, errorCode); + addErrValue(throwable); + addErrModule(throwable); + addErrLineNumber(throwable); + addErrColumnNumber(throwable); + addErrAdditional(throwable); + addFunctionTrace(throwable); + addJavaTrace(throwable); + + // Evaluate catch expression + catchResultSeq = ((Expression) catchClause.getCatchExpr()).eval(contextSequence, contextItem); + + + } finally { + context.popLocalVariables(mark1, catchResultSeq); + } + + } else { + // if in the end nothing is set, rethrow after loop } + } // for catch clauses + // If an error hasn't been caught, store for re-throw after finally + if (!errorMatched) { + pendingError = throwable; } else { - // if in the end nothing is set, rethrow after loop + result = catchResultSeq; } - } // for catch clauses - // If an error hasn't been caught, throw new one - if (!errorMatched) { - if (throwable instanceof XPathException) { - throw throwable; - } else { - LOG.error(throwable); - throw new XPathException(this, throwable); + } finally { + context.popLocalVariables(mark0, catchResultSeq); + } + } + } finally { + // XQ4: Evaluate finally clause (always, even if try/catch succeeded or failed) + if (finallyExpr != null) { + try { + final Sequence finallyResult = finallyExpr.eval(contextSequence, contextItem); + // If finally produces a non-empty sequence, raise XQTY0153 + if (finallyResult != null && !finallyResult.isEmpty()) { + throw new XPathException(this, ErrorCodes.XQTY0153, + "The finally clause must evaluate to an empty sequence, got " + + finallyResult.getItemCount() + " item(s)"); } + } catch (final XPathException finallyError) { + // Finally error replaces any pending error or result + context.expressionEnd(this); + throw finallyError; } - - } finally { - context.popLocalVariables(mark0, catchResultSeq); } - return catchResultSeq; + // Re-throw pending error from try body (if not caught) + if (pendingError != null) { + context.expressionEnd(this); + if (pendingError instanceof XPathException) { + throw (XPathException) pendingError; + } else { + LOG.error(pendingError); + throw new XPathException(this, pendingError); + } + } - } finally { context.expressionEnd(this); } + + return result; } @@ -384,6 +421,13 @@ public void dump(final ExpressionDumper dumper) { dumper.nl().display("}"); dumper.endIndent(); } + if (finallyExpr != null) { + dumper.nl().display("} finally {"); + dumper.startIndent(); + finallyExpr.dump(dumper); + dumper.nl().display("}"); + dumper.endIndent(); + } } /** @@ -428,6 +472,11 @@ public String toString() { result.append(catchExpr.toString()); result.append("}"); } + if (finallyExpr != null) { + result.append(" finally { "); + result.append(finallyExpr.toString()); + result.append("}"); + } return result.toString(); } @@ -436,8 +485,10 @@ public String toString() { */ @Override public int returnsType() { - // fixme! /ljo - return ((Expression) catchClauses.getFirst().getCatchExpr()).returnsType(); + if (!catchClauses.isEmpty()) { + return ((Expression) catchClauses.getFirst().getCatchExpr()).returnsType(); + } + return tryTargetExpr.returnsType(); } /* (non-Javadoc) @@ -451,6 +502,9 @@ public void resetState(final boolean postOptimization) { final Expression catchExpr = (Expression) catchClause.getCatchExpr(); catchExpr.resetState(postOptimization); } + if (finallyExpr != null) { + finallyExpr.resetState(postOptimization); + } } @Override diff --git a/exist-core/src/main/java/org/exist/xquery/UserDefinedFunction.java b/exist-core/src/main/java/org/exist/xquery/UserDefinedFunction.java index a56db1a200b..33b781868ed 100644 --- a/exist-core/src/main/java/org/exist/xquery/UserDefinedFunction.java +++ b/exist-core/src/main/java/org/exist/xquery/UserDefinedFunction.java @@ -24,8 +24,10 @@ import org.exist.dom.persistent.DocumentSet; import org.exist.dom.QName; import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.FunctionParameterSequenceType; import org.exist.xquery.value.Item; import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; import java.util.ArrayList; import java.util.List; @@ -125,31 +127,51 @@ public Sequence eval(Sequence contextSequence, Item contextItem) throws XPathExc } Sequence result = null; try { - QName varName; - LocalVariable var; - int j = 0; - for (int i = 0; i < parameters.size(); i++, j++) { - varName = parameters.get(i); - var = new LocalVariable(varName); - var.setValue(currentArguments[j]); - if (contextDocs != null) { + final SequenceType[] argTypes = getSignature().getArgumentTypes(); + + // Evaluate all argument values first, BEFORE declaring any parameters. + // Default value expressions must be evaluated in the prolog's variable scope, + // not the function body scope (XQ4 spec: default sees variables in scope at + // the function declaration, not other parameters). Context is passed so that + // default values like "." can access the context item at the call site. + final Sequence[] argValues = new Sequence[parameters.size()]; + for (int i = 0; i < parameters.size(); i++) { + if (i < currentArguments.length) { + argValues[i] = currentArguments[i]; + } else if (argTypes[i] instanceof FunctionParameterSequenceType && + ((FunctionParameterSequenceType) argTypes[i]).hasDefaultValue()) { + argValues[i] = ((FunctionParameterSequenceType) argTypes[i]) + .getDefaultValue().eval(contextSequence, contextItem); + } else { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Missing required argument $" + parameters.get(i)); + } + } + + // Now declare all parameters with their resolved values + for (int i = 0; i < parameters.size(); i++) { + final QName varName = parameters.get(i); + final LocalVariable var = new LocalVariable(varName); + + var.setValue(argValues[i]); + if (contextDocs != null && i < contextDocs.length) { var.setContextDocs(contextDocs[i]); } context.declareVariableBinding(var); Cardinality actualCardinality; - if (currentArguments[j].isEmpty()) { + if (argValues[i].isEmpty()) { actualCardinality = Cardinality.EMPTY_SEQUENCE; - } else if (currentArguments[j].hasMany()) { + } else if (argValues[i].hasMany()) { actualCardinality = Cardinality._MANY; } else { actualCardinality = Cardinality.EXACTLY_ONE; } - if (!getSignature().getArgumentTypes()[j].getCardinality().isSuperCardinalityOrEqualOf(actualCardinality)) { + if (!argTypes[i].getCardinality().isSuperCardinalityOrEqualOf(actualCardinality)) { throw new XPathException(this, ErrorCodes.XPTY0004, "Invalid cardinality for parameter $" + varName + - ". Expected " + getSignature().getArgumentTypes()[j].getCardinality().getHumanDescription() + - ", got " + currentArguments[j].getItemCount()); + ". Expected " + argTypes[i].getCardinality().getHumanDescription() + + ", got " + argValues[i].getItemCount()); } } result = body.eval(null, null); diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java index b3721c34179..600ef6b336b 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java @@ -1840,7 +1840,7 @@ public void declareFunction(final UserDefinedFunction function) throws XPathExce final QName name = function.getSignature().getName(); final String uri = name.getNamespaceURI(); - if (uri.isEmpty()) { + if (uri.isEmpty() && getXQueryVersion() < 40) { throw new XPathException(function, ErrorCodes.XQST0060, "Every declared function name must have a non-null namespace URI, " + "but function '" + name + "' does not meet this requirement."); @@ -1865,7 +1865,31 @@ public void declareFunction(final UserDefinedFunction function) throws XPathExce @Override public @Nullable UserDefinedFunction resolveFunction(final QName name, final int argCount) { final FunctionId id = new FunctionId(name, argCount); - return declaredFunctions.get(id); + final UserDefinedFunction exact = declaredFunctions.get(id); + if (exact != null) { + return exact; + } + // XQ4: Try to find a function with more params where trailing params have defaults + for (final UserDefinedFunction func : declaredFunctions.values()) { + if (func.getName().equals(name)) { + final SequenceType[] argTypes = func.getSignature().getArgumentTypes(); + if (argTypes.length > argCount) { + // Check that all params from argCount onwards have defaults + boolean allDefaulted = true; + for (int i = argCount; i < argTypes.length; i++) { + if (!(argTypes[i] instanceof FunctionParameterSequenceType) || + !((FunctionParameterSequenceType) argTypes[i]).hasDefaultValue()) { + allDefaulted = false; + break; + } + } + if (allDefaulted) { + return func; + } + } + } + } + return null; } @Override @@ -2730,6 +2754,13 @@ private ExternalModule compileOrBorrowModule(final String namespaceURI, final St * @return The compiled module, or null if the source is not a module * @throws XPathException if the module could not be loaded (XQST0059) or compiled (XPST0003) */ + /** + * Compile a module from a Source. Public wrapper for fn:load-xquery-module content option. + */ + public @Nullable ExternalModule compileModuleFromSource(final String namespaceURI, final Source source) throws XPathException { + return compileModule(namespaceURI, null, "content", source); + } + private @Nullable ExternalModule compileModule(String namespaceURI, final String prefix, final String location, final Source source) throws XPathException { if (LOG.isDebugEnabled()) { diff --git a/exist-core/src/main/java/org/exist/xquery/value/FunctionParameterSequenceType.java b/exist-core/src/main/java/org/exist/xquery/value/FunctionParameterSequenceType.java index b383b881d75..a2aa7078d52 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/FunctionParameterSequenceType.java +++ b/exist-core/src/main/java/org/exist/xquery/value/FunctionParameterSequenceType.java @@ -22,6 +22,7 @@ package org.exist.xquery.value; import org.exist.xquery.Cardinality; +import org.exist.xquery.Expression; /** * This class is used to specify the name and description of an XQuery function parameter. @@ -32,6 +33,7 @@ public class FunctionParameterSequenceType extends FunctionReturnSequenceType { private String attributeName; + private Expression defaultValue; /** * @param attributeName The name of the parameter in the FunctionSignature. @@ -79,4 +81,16 @@ public void setAttributeName(String attributeName) { this.attributeName = attributeName; } + public boolean hasDefaultValue() { + return defaultValue != null; + } + + public Expression getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(final Expression defaultValue) { + this.defaultValue = defaultValue; + } + } diff --git a/exist-core/src/main/java/org/exist/xquery/value/SequenceType.java b/exist-core/src/main/java/org/exist/xquery/value/SequenceType.java index f00c9811ea1..9016cd0c844 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/SequenceType.java +++ b/exist-core/src/main/java/org/exist/xquery/value/SequenceType.java @@ -30,6 +30,9 @@ import org.w3c.dom.Element; import org.w3c.dom.Node; +import java.util.ArrayList; +import java.util.List; + /** * Represents an XQuery SequenceType and provides methods to check * sequences and items against this type. @@ -41,6 +44,8 @@ public class SequenceType { private int primaryType = Type.ITEM; private Cardinality cardinality = Cardinality.EXACTLY_ONE; private QName nodeName = null; + private List choiceAlternatives = null; + private String[] enumValues = null; public SequenceType() { } @@ -108,6 +113,81 @@ public void setNodeName(QName qname) { this.nodeName = qname; } + public void addChoiceAlternative(final SequenceType alt) { + if (choiceAlternatives == null) { + choiceAlternatives = new ArrayList<>(); + } + choiceAlternatives.add(alt); + } + + public List getChoiceAlternatives() { + return choiceAlternatives; + } + + public boolean isChoiceType() { + return choiceAlternatives != null && !choiceAlternatives.isEmpty(); + } + + public void setEnumValues(final String[] values) { + this.enumValues = values; + this.primaryType = Type.STRING; + } + + public String[] getEnumValues() { + return enumValues; + } + + public boolean isEnumType() { + return enumValues != null; + } + + // Record type support + + /** + * Represents a field in a record type declaration. + */ + public static class RecordField { + private final String name; + private final boolean optional; + private final SequenceType fieldType; + + public RecordField(final String name, final boolean optional, final SequenceType fieldType) { + this.name = name; + this.optional = optional; + this.fieldType = fieldType; + } + + public String getName() { return name; } + public boolean isOptional() { return optional; } + public SequenceType getFieldType() { return fieldType; } + } + + private List recordFields = null; + private boolean recordExtensible = false; + + public void addRecordField(final RecordField field) { + if (recordFields == null) { + recordFields = new ArrayList<>(); + } + recordFields.add(field); + } + + public List getRecordFields() { + return recordFields; + } + + public void setRecordExtensible(final boolean extensible) { + this.recordExtensible = extensible; + } + + public boolean isRecordExtensible() { + return recordExtensible; + } + + public boolean isRecordType() { + return primaryType == Type.RECORD; + } + /** * Check the specified sequence against this SequenceType. * @@ -115,17 +195,29 @@ public void setNodeName(QName qname) { * @throws XPathException if check fails for one item in the sequence * @return true, if all items of the sequence have the same type as or a subtype of primaryType */ - public boolean checkType(final Sequence seq) throws XPathException { - if (nodeName == null) { - return Type.subTypeOf(seq.getItemType(), primaryType); + public boolean checkType(Sequence seq) throws XPathException { + if (isChoiceType()) { + Item next; + for (final SequenceIterator i = seq.iterate(); i.hasNext(); ) { + next = i.nextItem(); + if (!checkType(next)) { + return false; + } + } + return true; } - - for (final SequenceIterator i = seq.iterate(); i.hasNext(); ) { - if (!checkType(i.nextItem())) { - return false; + if (nodeName != null) { + Item next; + for (final SequenceIterator i = seq.iterate(); i.hasNext(); ) { + next = i.nextItem(); + if (!checkType(next)) { + return false; + } } + return true; + } else { + return Type.subTypeOf(seq.getItemType(), primaryType); } - return true; } /** @@ -134,55 +226,172 @@ public boolean checkType(final Sequence seq) throws XPathException { * @param item the item to check * @return true, if item is a subtype of primaryType */ - public boolean checkType(final Item item) { + public boolean checkType(Item item) { + if (isChoiceType()) { + for (final SequenceType alt : choiceAlternatives) { + if (alt.checkType(item)) { + return true; + } + } + return false; + } + if (isEnumType()) { + if (!Type.subTypeOf(item.getType(), Type.STRING)) { + return false; + } + try { + final String val = item.getStringValue(); + for (final String enumVal : enumValues) { + if (enumVal.equals(val)) { + return true; + } + } + } catch (final XPathException e) { + // cannot get string value + } + return false; + } + if (isRecordType()) { + return checkRecordType(item); + } + Node realNode = null; int type = item.getType(); if (type == Type.NODE) { - final Node realNode = ((NodeValue) item).getNode(); + realNode = ((NodeValue) item).getNode(); type = realNode.getNodeType(); } if (!Type.subTypeOf(type, primaryType)) { return false; } - if (nodeName == null) { - return true; + if (nodeName != null) { + + //TODO : how to improve performance ? + + final NodeValue nvItem = (NodeValue) item; + QName realName = null; + if (item.getType() == Type.DOCUMENT) { + // it's a document... we need to get the document element's name + final Document doc; + if (nvItem instanceof Document) { + doc = (Document) nvItem; + } else { + doc = nvItem.getOwnerDocument(); + } + if (doc != null) { + final Element elem = doc.getDocumentElement(); + if (elem != null) { + realName = new QName(elem.getLocalName(), elem.getNamespaceURI()); + } + } + } else { + // get the name of the element/attribute + realName = nvItem.getQName(); + } + + if (realName == null) { + return false; + } + + if (nodeName.getNamespaceURI() != null) { + if (!nodeName.getNamespaceURI().equals(realName.getNamespaceURI())) { + return false; + } + } + if (nodeName.getLocalPart() != null) { + return nodeName.getLocalPart().equals(realName.getLocalPart()); + } } - //TODO : how to improve performance ? - final QName realName = getRealName(item); + return true; + } - if (realName == null) { + /** + * Check if an item matches this record type declaration. + * A map matches a record type if: + * - All required fields are present + * - Each field value matches the declared type + * - If not extensible (no *), no extra keys are present + */ + private boolean checkRecordType(final Item item) { + if (!Type.subTypeOf(item.getType(), Type.MAP_ITEM)) { return false; } - if (nodeName.getNamespaceURI() != null && - !nodeName.getNamespaceURI().equals(realName.getNamespaceURI())) { - return false; + // record(*) matches any map + if (recordExtensible && (recordFields == null || recordFields.isEmpty())) { + return true; } - if (nodeName.getLocalPart() != null) { - return nodeName.getLocalPart().equals(realName.getLocalPart()); + final org.exist.xquery.functions.map.AbstractMapType map = + (org.exist.xquery.functions.map.AbstractMapType) item; + + // record() with no fields and not extensible: only empty maps match + if ((recordFields == null || recordFields.isEmpty()) && !recordExtensible) { + return map.size() == 0; } + + // Check required fields are present and types match + for (final RecordField field : recordFields) { + final AtomicValue key = new StringValue(null, field.getName()); + final boolean hasKey = map.contains(key); + + if (!hasKey && !field.isOptional()) { + return false; // required field missing + } + + if (hasKey && field.getFieldType() != null) { + try { + final Sequence value = map.get(key); + if (!field.getFieldType().matchesCardinality(value)) { + return false; + } + if (!value.isEmpty() && !field.getFieldType().checkType(value)) { + return false; + } + } catch (final XPathException e) { + return false; + } + } + } + + // If not extensible, check for extra keys + if (!recordExtensible) { + try { + final Sequence keys = map.keys(); + for (final SequenceIterator it = keys.iterate(); it.hasNext(); ) { + final String keyName = it.nextItem().getStringValue(); + boolean declared = false; + for (final RecordField field : recordFields) { + if (field.getName().equals(keyName)) { + declared = true; + break; + } + } + if (!declared) { + return false; // undeclared key in non-extensible record + } + } + } catch (final XPathException e) { + return false; + } + } + return true; } - private static QName getRealName(final Item item) { - final NodeValue nvItem = (NodeValue) item; - if (item.getType() != Type.DOCUMENT) { - // get the name of the element/attribute - return nvItem.getQName(); - } - // it's a document... we need to get the document element's name - final Document doc; - if (nvItem instanceof Document) { - doc = (Document) nvItem; - } else { - doc = nvItem.getOwnerDocument(); + /** + * Check if a sequence's cardinality matches this type's cardinality declaration. + */ + public boolean matchesCardinality(final Sequence seq) { + if (cardinality == Cardinality.ZERO_OR_MORE) { + return true; } - if (doc == null) { - return null; + final int count = seq.getItemCount(); + if (count == 0) { + return cardinality.isSuperCardinalityOrEqualOf(Cardinality.EMPTY_SEQUENCE); } - final Element elem = doc.getDocumentElement(); - if (elem == null) { - return null; + if (count == 1) { + return true; // EXACTLY_ONE, ZERO_OR_ONE, ONE_OR_MORE all accept 1 } - return new QName(elem.getLocalName(), elem.getNamespaceURI()); + // count > 1 + return cardinality == Cardinality.ONE_OR_MORE || cardinality == Cardinality.ZERO_OR_MORE; } /** @@ -197,14 +406,17 @@ public void checkType(int type) throws XPathException { return; } - // Although xs:anyURI is not a subtype of xs:string, both types are compatible + //Although xs:anyURI is not a subtype of xs:string, both types are compatible if (type == Type.ANY_URI && primaryType == Type.STRING) { return; } if (!Type.subTypeOf(type, primaryType)) { throw new XPathException((Expression) null, ErrorCodes.XPTY0004, - "Type error: expected type: " + Type.getTypeName(primaryType) + "; got: " + Type.getTypeName(type)); + "Type error: expected type: " + + Type.getTypeName(primaryType) + + "; got: " + + Type.getTypeName(type)); } } @@ -226,28 +438,49 @@ public void checkCardinality(Sequence seq) throws XPathException { } } - /** - * Used to serialize SequenceTypes, when building stack traces, for example. - * - * @return The serialized SequenceType - */ @Override public String toString() { if (cardinality == Cardinality.EMPTY_SEQUENCE) { return cardinality.toXQueryCardinalityString(); } + if (isChoiceType()) { + final StringBuilder sb = new StringBuilder("("); + for (int i = 0; i < choiceAlternatives.size(); i++) { + if (i > 0) { + sb.append(" | "); + } + sb.append(choiceAlternatives.get(i).toString()); + } + sb.append(")"); + sb.append(cardinality.toXQueryCardinalityString()); + return sb.toString(); + } + + if (isEnumType()) { + final StringBuilder sb = new StringBuilder("enum("); + for (int i = 0; i < enumValues.length; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append("\"").append(enumValues[i]).append("\""); + } + sb.append(")"); + sb.append(cardinality.toXQueryCardinalityString()); + return sb.toString(); + } + final String str; if (primaryType == Type.DOCUMENT && nodeName != null) { str = "document-node(" + nodeName.getStringValue() + ")"; } else if (primaryType == Type.ELEMENT && nodeName != null) { str = "element(" + nodeName.getStringValue() + ")"; - } else if (primaryType == Type.MAP_ITEM) { - str = "map(*)"; - } else if (primaryType == Type.ARRAY_ITEM) { - str = "array(*)"; - } else if (primaryType == Type.FUNCTION) { - str = "function(*)"; +// } else if (primaryType == Type.MAP) { +// str = "map(" + + ")"; +// } else if (primaryType == Type.ARRAY) { +// str = "array(" + + ")"; +// } else if (primaryType == Type.FUNCTION_REFERENCE) { +// str = "function(" + + ")"; } else { str = Type.getTypeName(primaryType); } diff --git a/exist-core/src/main/java/org/exist/xquery/value/Type.java b/exist-core/src/main/java/org/exist/xquery/value/Type.java index f60c60d7255..0c65c7a031a 100644 --- a/exist-core/src/main/java/org/exist/xquery/value/Type.java +++ b/exist-core/src/main/java/org/exist/xquery/value/Type.java @@ -133,9 +133,12 @@ public class Type { public final static int JAVA_OBJECT = 68; public final static int EMPTY_SEQUENCE = 69; // NOTE(AR) this types does appear in the XQ 3.1 spec - https://www.w3.org/TR/xquery-31/#id-sequencetype-syntax - private final static int[] superTypes = new int[69]; - private final static Int2ObjectOpenHashMap typeNames = new Int2ObjectOpenHashMap<>(69, Hash.FAST_LOAD_FACTOR); - private final static Object2IntOpenHashMap typeCodes = new Object2IntOpenHashMap<>(78, Hash.FAST_LOAD_FACTOR); + /* XQuery 4.0 types */ + public final static int RECORD = 70; + + private final static int[] superTypes = new int[71]; + private final static Int2ObjectOpenHashMap typeNames = new Int2ObjectOpenHashMap<>(71, Hash.FAST_LOAD_FACTOR); + private final static Object2IntOpenHashMap typeCodes = new Object2IntOpenHashMap<>(80, Hash.FAST_LOAD_FACTOR); static { typeCodes.defaultReturnValue(NO_SUCH_VALUE); } @@ -249,6 +252,9 @@ public class Type { defineSubType(FUNCTION, MAP_ITEM); defineSubType(FUNCTION, ARRAY_ITEM); + // XQ4: RECORD is a subtype of MAP + defineSubType(MAP_ITEM, RECORD); + // NODE types defineSubType(NODE, ATTRIBUTE); defineSubType(NODE, CDATA_SECTION); @@ -327,6 +333,7 @@ public class Type { defineBuiltInType(FUNCTION, "function(*)", "function"); defineBuiltInType(ARRAY_ITEM, "array(*)", "array"); defineBuiltInType(MAP_ITEM, "map(*)", "map"); // keep `map` for backward compatibility + defineBuiltInType(RECORD, "record(*)", "record"); defineBuiltInType(CDATA_SECTION, "cdata-section()"); defineBuiltInType(JAVA_OBJECT, "object"); defineBuiltInType(EMPTY_SEQUENCE, "empty-sequence()", "empty()"); // keep `empty()` for backward compatibility From 3a023d897de5525f945a0d75bac916dbbfc28d28 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Sat, 4 Apr 2026 13:12:51 -0400 Subject: [PATCH 009/326] [feature] Implement 50+ new XQuery 4.0 fn: functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New function implementations in the fn: namespace: Sequence functions: fn:characters, fn:identity, fn:void, fn:foot, fn:trunk, fn:slice, fn:items-at, fn:replicate, fn:insert-separator, fn:all-equal, fn:all-different, fn:duplicate-values, fn:index-where, fn:take-while, fn:distinct-ordered-nodes, fn:siblings Higher-order functions: fn:every, fn:some (function form), fn:highest, fn:lowest, fn:sort-by, fn:sort-with, fn:partition, fn:scan-left, fn:scan-right, fn:subsequence-where, fn:transitive-closure, fn:partial-apply, fn:op String/URI functions: fn:char, fn:graphemes, fn:decode-from-uri, fn:parse-uri, fn:build-uri, fn:expanded-QName, fn:parse-QName, fn:parse-integer, fn:divide-decimals Date/Time functions: fn:civil-timezone, fn:build-dateTime, fn:parts-of-dateTime, fn:unix-dateTime, fn:seconds Type functions: fn:schema-type, fn:atomic-type-annotation, fn:node-type-annotation, fn:element-to-map, fn:element-to-map-plan, fn:type-of, fn:is-NaN Context functions: fn:get, fn:collation, fn:collation-available, fn:message Parsing functions: fn:parse-html (Validator.nu HTML5 parser), fn:invisible-xml (Markup Blitz iXML parser), fn:parse-csv, fn:csv, fn:html-doc, fn:unparsed-binary Data functions: fn:hash, fn:function-annotations, fn:function-identity, fn:in-scope-namespaces Also: DeepEqualOptions class for fn:deep-equal options map support, FnModule registrations for all new functions. Spec: QT4 XQuery 4.0 §14 (Functions and Operators) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../xquery/functions/fn/CsvFunctions.java | 619 +++++++++++ .../exist/xquery/functions/fn/CsvParser.java | 338 ++++++ .../xquery/functions/fn/DeepEqualOptions.java | 962 ++++++++++++++++++ .../functions/fn/FnAllEqualDifferent.java | 165 +++ .../xquery/functions/fn/FnAtomicEqual.java | 214 ++++ .../exist/xquery/functions/fn/FnBuildUri.java | 335 ++++++ .../org/exist/xquery/functions/fn/FnChar.java | 218 ++++ .../xquery/functions/fn/FnCharacters.java | 77 ++ .../xquery/functions/fn/FnCivilTimezone.java | 152 +++ .../xquery/functions/fn/FnCollation.java | 94 ++ .../xquery/functions/fn/FnDateTimeParts.java | 176 ++++ .../xquery/functions/fn/FnDecodeFromUri.java | 183 ++++ .../functions/fn/FnDeepEqualOptions.java | 84 ++ .../functions/fn/FnDistinctOrderedNodes.java | 71 ++ .../xquery/functions/fn/FnDivideDecimals.java | 119 +++ .../functions/fn/FnDuplicateValues.java | 126 +++ .../xquery/functions/fn/FnElementToMap.java | 458 +++++++++ .../functions/fn/FnElementToMapPlan.java | 263 +++++ .../xquery/functions/fn/FnEverySome.java | 177 ++++ .../xquery/functions/fn/FnExpandedQName.java | 74 ++ .../xquery/functions/fn/FnFormatDates.java | 465 ++++++++- .../xquery/functions/fn/FnFormatNumbers.java | 203 +++- .../functions/fn/FnFunctionAnnotations.java | 84 ++ .../functions/fn/FnFunctionIdentity.java | 100 ++ .../org/exist/xquery/functions/fn/FnGet.java | 91 ++ .../xquery/functions/fn/FnGraphemes.java | 86 ++ .../org/exist/xquery/functions/fn/FnHash.java | 177 ++++ .../functions/fn/FnHigherOrderFun40.java | 361 +++++++ .../xquery/functions/fn/FnHighestLowest.java | 226 ++++ .../exist/xquery/functions/fn/FnHtmlDoc.java | 71 ++ .../xquery/functions/fn/FnIdentityVoid.java | 78 ++ .../functions/fn/FnInScopeNamespaces.java | 152 +++ .../functions/fn/FnInsertSeparator.java | 74 ++ .../xquery/functions/fn/FnInvisibleXml.java | 308 ++++++ .../exist/xquery/functions/fn/FnIsNaN.java | 71 ++ .../exist/xquery/functions/fn/FnItemsAt.java | 79 ++ .../exist/xquery/functions/fn/FnMessage.java | 85 ++ .../exist/xquery/functions/fn/FnModule.java | 135 ++- .../org/exist/xquery/functions/fn/FnOp.java | 404 ++++++++ .../xquery/functions/fn/FnParseHtml.java | 177 ++++ .../xquery/functions/fn/FnParseInteger.java | 142 +++ .../xquery/functions/fn/FnParseQName.java | 174 ++++ .../exist/xquery/functions/fn/FnParseUri.java | 462 +++++++++ .../xquery/functions/fn/FnPartialApply.java | 188 ++++ .../xquery/functions/fn/FnPartition.java | 140 +++ .../xquery/functions/fn/FnReplicate.java | 77 ++ .../xquery/functions/fn/FnSchemaType.java | 72 ++ .../exist/xquery/functions/fn/FnSeconds.java | 64 ++ .../exist/xquery/functions/fn/FnSiblings.java | 111 ++ .../exist/xquery/functions/fn/FnSlice.java | 149 +++ .../exist/xquery/functions/fn/FnSortBy.java | 267 +++++ .../functions/fn/FnSubsequenceMatching.java | 208 ++++ .../functions/fn/FnSubsequenceWhere.java | 181 ++++ .../functions/fn/FnTransitiveClosure.java | 143 +++ .../xquery/functions/fn/FnTypeAnnotation.java | 528 ++++++++++ .../exist/xquery/functions/fn/FnTypeOf.java | 146 +++ .../xquery/functions/fn/FnUnixDateTime.java | 86 ++ .../xquery/functions/fn/FnUnparsedBinary.java | 114 +++ 58 files changed, 11508 insertions(+), 76 deletions(-) create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/CsvFunctions.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/CsvParser.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/DeepEqualOptions.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnAllEqualDifferent.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnAtomicEqual.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnBuildUri.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnChar.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnCharacters.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnCivilTimezone.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnCollation.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnDateTimeParts.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnDecodeFromUri.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnDeepEqualOptions.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnDistinctOrderedNodes.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnDivideDecimals.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnDuplicateValues.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnElementToMap.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnElementToMapPlan.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnEverySome.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnExpandedQName.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnFunctionAnnotations.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnFunctionIdentity.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnGet.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnGraphemes.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnHash.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnHigherOrderFun40.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnHighestLowest.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnHtmlDoc.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnIdentityVoid.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnInScopeNamespaces.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnInsertSeparator.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnInvisibleXml.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnIsNaN.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnItemsAt.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnMessage.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnOp.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnParseHtml.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnParseInteger.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnParseQName.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnParseUri.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnPartialApply.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnPartition.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnReplicate.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnSchemaType.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnSeconds.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnSiblings.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnSlice.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnSortBy.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnSubsequenceMatching.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnSubsequenceWhere.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnTransitiveClosure.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnTypeAnnotation.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnTypeOf.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnUnixDateTime.java create mode 100644 exist-core/src/main/java/org/exist/xquery/functions/fn/FnUnparsedBinary.java diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/CsvFunctions.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/CsvFunctions.java new file mode 100644 index 00000000000..ea4a275ba56 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/CsvFunctions.java @@ -0,0 +1,619 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.apache.commons.io.IOUtils; +import org.exist.dom.QName; +import org.exist.dom.memtree.MemTreeBuilder; +import org.exist.security.PermissionDeniedException; +import org.exist.source.FileSource; +import org.exist.source.Source; +import org.exist.source.SourceFactory; +import org.exist.xquery.*; +import org.exist.xquery.functions.array.ArrayType; +import io.lacuna.bifurcan.IEntry; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.value.*; + +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** + * Implements XQuery 4.0 CSV functions: + * fn:csv-to-arrays, fn:parse-csv, fn:csv-to-xml, fn:csv-doc. + */ +public class CsvFunctions extends BasicFunction { + + // XQ4 namespace for CSV XML output + private static final String CSV_NS = "http://www.w3.org/2005/xpath-functions"; + + // fn:csv-to-arrays signatures + public static final FunctionSignature[] FN_CSV_TO_ARRAYS = { + new FunctionSignature( + new QName("csv-to-arrays", Function.BUILTIN_FUNCTION_NS), + "Parses a string as CSV data and returns the result as a sequence of arrays.", + new SequenceType[]{ + new FunctionParameterSequenceType("csv", Type.STRING, Cardinality.ZERO_OR_ONE, "The CSV string to parse") + }, + new FunctionReturnSequenceType(Type.ARRAY_ITEM, Cardinality.ZERO_OR_MORE, "A sequence of arrays, one per row")), + new FunctionSignature( + new QName("csv-to-arrays", Function.BUILTIN_FUNCTION_NS), + "Parses a string as CSV data and returns the result as a sequence of arrays, using the specified options.", + new SequenceType[]{ + new FunctionParameterSequenceType("csv", Type.STRING, Cardinality.ZERO_OR_ONE, "The CSV string to parse"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, Cardinality.EXACTLY_ONE, "Parsing options") + }, + new FunctionReturnSequenceType(Type.ARRAY_ITEM, Cardinality.ZERO_OR_MORE, "A sequence of arrays, one per row")) + }; + + // fn:parse-csv signatures + public static final FunctionSignature[] FN_PARSE_CSV = { + new FunctionSignature( + new QName("parse-csv", Function.BUILTIN_FUNCTION_NS), + "Parses a string as CSV data and returns the result as a map.", + new SequenceType[]{ + new FunctionParameterSequenceType("csv", Type.STRING, Cardinality.ZERO_OR_ONE, "The CSV string to parse") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.ZERO_OR_ONE, "A map with columns, column-index, rows, and get")), + new FunctionSignature( + new QName("parse-csv", Function.BUILTIN_FUNCTION_NS), + "Parses a string as CSV data and returns the result as a map, using the specified options.", + new SequenceType[]{ + new FunctionParameterSequenceType("csv", Type.STRING, Cardinality.ZERO_OR_ONE, "The CSV string to parse"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, Cardinality.EXACTLY_ONE, "Parsing options") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.ZERO_OR_ONE, "A map with columns, column-index, rows, and get")) + }; + + // fn:csv-to-xml signatures + public static final FunctionSignature[] FN_CSV_TO_XML = { + new FunctionSignature( + new QName("csv-to-xml", Function.BUILTIN_FUNCTION_NS), + "Parses a string as CSV data and returns the result as an XML document.", + new SequenceType[]{ + new FunctionParameterSequenceType("csv", Type.STRING, Cardinality.ZERO_OR_ONE, "The CSV string to parse") + }, + new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.ZERO_OR_ONE, "An XML document representing the CSV data")), + new FunctionSignature( + new QName("csv-to-xml", Function.BUILTIN_FUNCTION_NS), + "Parses a string as CSV data and returns the result as an XML document, using the specified options.", + new SequenceType[]{ + new FunctionParameterSequenceType("csv", Type.STRING, Cardinality.ZERO_OR_ONE, "The CSV string to parse"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, Cardinality.EXACTLY_ONE, "Parsing options") + }, + new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.ZERO_OR_ONE, "An XML document representing the CSV data")) + }; + + // fn:csv-doc signatures + public static final FunctionSignature[] FN_CSV_DOC = { + new FunctionSignature( + new QName("csv-doc", Function.BUILTIN_FUNCTION_NS), + "Reads CSV data from the specified URI and returns the result as a map.", + new SequenceType[]{ + new FunctionParameterSequenceType("uri", Type.STRING, Cardinality.ZERO_OR_ONE, "The URI of the CSV resource") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.ZERO_OR_ONE, "A map with columns, column-index, rows, and get")), + new FunctionSignature( + new QName("csv-doc", Function.BUILTIN_FUNCTION_NS), + "Reads CSV data from the specified URI and returns the result as a map, using the specified options.", + new SequenceType[]{ + new FunctionParameterSequenceType("uri", Type.STRING, Cardinality.ZERO_OR_ONE, "The URI of the CSV resource"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, Cardinality.EXACTLY_ONE, "Parsing options") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.ZERO_OR_ONE, "A map with columns, column-index, rows, and get")) + }; + + public CsvFunctions(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (isCalledAs("csv-doc")) { + return evalCsvDoc(args); + } + + // Empty sequence input returns empty + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + final String csv = args[0].getStringValue(); + final CsvParser.CsvOptions options = parseOptions(args); + + if (isCalledAs("csv-to-arrays")) { + return evalCsvToArrays(csv, options); + } else if (isCalledAs("parse-csv")) { + return evalParseCsv(csv, options); + } else if (isCalledAs("csv-to-xml")) { + return evalCsvToXml(csv, options); + } + throw new XPathException(this, ErrorCodes.XPST0017, "Unknown CSV function: " + getSignature().getName().getLocalPart()); + } + + // ==================== fn:csv-to-arrays ==================== + + private Sequence evalCsvToArrays(final String csv, final CsvParser.CsvOptions options) throws XPathException { + options.validate(this); + final CsvParser parser = new CsvParser(options, this); + final ValueSequence result = new ValueSequence(); + + parser.parse(csv, new CsvParser.CsvConverter() { + @Override + public void header(final List fields) { + // Header row is also returned as an array in csv-to-arrays + // (per XQ4 spec: "If header is true, the first row is treated as a header + // but still appears in the output") + // Actually per spec: if header=true, the header row is NOT included + // in the result of csv-to-arrays. + } + + @Override + public void record(final List fields) throws XPathException { + result.add(fieldsToArray(fields)); + } + + @Override + public void finish() { + } + }); + return result; + } + + // ==================== fn:parse-csv ==================== + + private Sequence evalParseCsv(final String csv, final CsvParser.CsvOptions options) throws XPathException { + options.validate(this); + final CsvParser parser = new CsvParser(options, this); + final List> allRows = new ArrayList<>(); + final List[] headerHolder = new List[]{null}; + + parser.parse(csv, new CsvParser.CsvConverter() { + @Override + public void header(final List fields) { + headerHolder[0] = fields; + } + + @Override + public void record(final List fields) { + allRows.add(fields); + } + + @Override + public void finish() { + } + }); + + // Explicit header from options overrides parsed header + final List effectiveHeader = options.explicitHeader != null + ? options.explicitHeader : headerHolder[0]; + + return buildParseCsvResult(effectiveHeader, allRows, options); + } + + private Sequence buildParseCsvResult(final List header, final List> rows, + final CsvParser.CsvOptions options) throws XPathException { + final MapType result = new MapType(this, context); + + // "columns" - sequence of column names (empty sequence if no header) + final Sequence columns; + if (header != null) { + final ValueSequence colSeq = new ValueSequence(header.size()); + for (final String h : header) { + colSeq.add(new StringValue(this, h)); + } + columns = colSeq; + } else { + columns = Sequence.EMPTY_SEQUENCE; + } + + // "column-index" - map from column name to 1-based position + // Empty names are excluded; duplicate names map to first occurrence + final MapType columnIndex = new MapType(this, context); + MapType colIdxResult = columnIndex; + if (header != null) { + final java.util.Set seen = new java.util.HashSet<>(); + for (int i = 0; i < header.size(); i++) { + final String name = header.get(i); + if (!name.isEmpty() && seen.add(name)) { + colIdxResult = (MapType) colIdxResult.put(new StringValue(this, name), + new IntegerValue(this, i + 1)); + } + } + } + + // "rows" - sequence of arrays + final ValueSequence rowSeq = new ValueSequence(rows.size()); + for (final List row : rows) { + rowSeq.add(fieldsToArray(row)); + } + + // Build the result map + MapType map = (MapType) result.put(new StringValue(this, "columns"), columns); + map = (MapType) map.put(new StringValue(this, "column-index"), colIdxResult); + map = (MapType) map.put(new StringValue(this, "rows"), rowSeq); + + // "get" - accessor function: fn($row as xs:integer, $column as item()) as xs:string + // $column can be an integer (1-based) or a string (column name) + final UserDefinedFunction getFunc = new UserDefinedFunction(context, + new FunctionSignature( + new QName("get", Function.BUILTIN_FUNCTION_NS), + null, + new SequenceType[]{ + new FunctionParameterSequenceType("row", Type.INTEGER, Cardinality.EXACTLY_ONE, "Row number (1-based)"), + new FunctionParameterSequenceType("column", Type.ITEM, Cardinality.EXACTLY_ONE, "Column number (1-based) or column name") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_ONE, "The field value"))); + getFunc.addVariable("row"); + getFunc.addVariable("column"); + getFunc.setFunctionBody(new CsvGetExpression(context, rowSeq, header)); + final FunctionCall getCall = new FunctionCall(context, getFunc); + getCall.setLocation(getLine(), getColumn()); + final FunctionReference getFuncRef = new FunctionReference(this, getCall); + map = (MapType) map.put(new StringValue(this, "get"), getFuncRef); + + return map; + } + + // ==================== fn:csv-to-xml ==================== + + private Sequence evalCsvToXml(final String csv, final CsvParser.CsvOptions options) throws XPathException { + options.validate(this); + final CsvParser parser = new CsvParser(options, this); + + final List[] headerHolder = new List[]{null}; + final List> allRecords = new ArrayList<>(); + + parser.parse(csv, new CsvParser.CsvConverter() { + @Override + public void header(final List fields) { + headerHolder[0] = fields; + } + + @Override + public void record(final List fields) { + allRecords.add(fields); + } + + @Override + public void finish() { + } + }); + + // Explicit header from options overrides parsed header + final List effectiveHeader = options.explicitHeader != null + ? options.explicitHeader : headerHolder[0]; + + context.pushDocumentContext(); + try { + final MemTreeBuilder builder = context.getDocumentBuilder(); + + builder.startElement(new QName("csv", CSV_NS), null); + + // Write columns element only if headers are present + if (effectiveHeader != null) { + builder.startElement(new QName("columns", CSV_NS), null); + for (final String col : effectiveHeader) { + builder.startElement(new QName("column", CSV_NS), null); + builder.characters(col); + builder.endElement(); + } + builder.endElement(); // + } + + // Write rows + builder.startElement(new QName("rows", CSV_NS), null); + for (final List record : allRecords) { + builder.startElement(new QName("row", CSV_NS), null); + // A row with a single empty field is an empty row (no field elements) + final boolean isEmptyRow = record.size() == 1 && record.get(0).isEmpty(); + if (!isEmptyRow) { + for (int f = 0; f < record.size(); f++) { + final String field = record.get(f); + builder.startElement(new QName("field", CSV_NS), null); + if (effectiveHeader != null && f < effectiveHeader.size() + && !effectiveHeader.get(f).isEmpty()) { + builder.addAttribute(new QName("column", null, null), effectiveHeader.get(f)); + } + if (!field.isEmpty()) { + builder.characters(field); + } + builder.endElement(); + } + } + builder.endElement(); // + } + builder.endElement(); // + + builder.endElement(); // + + return builder.getDocument(); + } finally { + context.popDocumentContext(); + } + } + + // ==================== fn:csv-doc ==================== + + private Sequence evalCsvDoc(final Sequence[] args) throws XPathException { + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + final String uri = args[0].getStringValue(); + + // Read the CSV content from the URI (same approach as fn:unparsed-text) + final String csvContent; + try { + final URI parsedUri = new URI(uri); + if (parsedUri.getFragment() != null) { + throw new XPathException(this, ErrorCodes.FODC0005, + "URI may not contain a fragment identifier: " + uri); + } + final Source source = SourceFactory.getSource(context.getBroker(), "", parsedUri.toASCIIString(), false); + if (source == null) { + throw new XPathException(this, ErrorCodes.FODC0002, + "Could not find CSV resource: " + uri); + } + if (source instanceof FileSource && !context.getBroker().getCurrentSubject().hasDbaRole()) { + throw new PermissionDeniedException("non-dba user not allowed to read from file system"); + } + final StringWriter output = new StringWriter(); + try (final InputStream is = source.getInputStream()) { + IOUtils.copy(is, output, StandardCharsets.UTF_8); + } + csvContent = output.toString(); + } catch (final IOException | PermissionDeniedException | URISyntaxException e) { + throw new XPathException(this, ErrorCodes.FODC0002, + "Error reading CSV resource: " + uri + " - " + e.getMessage()); + } + + final CsvParser.CsvOptions options = parseOptions(args); + return evalParseCsv(csvContent, options); + } + + // ==================== Shared utilities ==================== + + private CsvParser.CsvOptions parseOptions(final Sequence[] args) throws XPathException { + final CsvParser.CsvOptions options = new CsvParser.CsvOptions(); + if (args.length < 2 || args[1].isEmpty()) { + return options; + } + + final AbstractMapType map = (AbstractMapType) args[1].itemAt(0); + + // field-delimiter + final Sequence fdSeq = map.get(new StringValue(this, "field-delimiter")); + if (fdSeq != null && !fdSeq.isEmpty()) { + final String fd = fdSeq.getStringValue(); + if (fd.isEmpty()) { + throw new XPathException(this, ErrorCodes.FOCV0002, + "field-delimiter must be a single character"); + } + if (fd.codePointCount(0, fd.length()) != 1) { + throw new XPathException(this, ErrorCodes.FOCV0002, + "field-delimiter must be a single character, got: \"" + fd + "\""); + } + options.fieldDelimiter = fd.codePointAt(0); + } + + // row-delimiter + final Sequence rdSeq = map.get(new StringValue(this, "row-delimiter")); + if (rdSeq != null && !rdSeq.isEmpty()) { + if (rdSeq.getItemCount() != 1) { + throw new XPathException(this, ErrorCodes.FOCV0002, + "row-delimiter must be a single string, got " + rdSeq.getItemCount() + " items"); + } + final String rd = rdSeq.itemAt(0).getStringValue(); + if (rd.isEmpty() || rd.codePointCount(0, rd.length()) != 1) { + throw new XPathException(this, ErrorCodes.FOCV0002, + "row-delimiter must be a single character"); + } + options.rowDelimiter = rd.codePointAt(0); + } + + // quote-character + final Sequence qcSeq = map.get(new StringValue(this, "quote-character")); + if (qcSeq != null && !qcSeq.isEmpty()) { + final String qc = qcSeq.getStringValue(); + if (qc.isEmpty()) { + options.quoteChar = -1; // disable quoting + } else if (qc.codePointCount(0, qc.length()) != 1) { + throw new XPathException(this, ErrorCodes.FOCV0002, + "quote-character must be a single character or empty string"); + } else { + options.quoteChar = qc.codePointAt(0); + } + } + + // trim-whitespace + final Sequence twSeq = map.get(new StringValue(this, "trim-whitespace")); + if (twSeq != null && !twSeq.isEmpty()) { + options.trimWhitespace = twSeq.effectiveBooleanValue(); + } + + // header: boolean, "present", or sequence of explicit column names + final Sequence hdrSeq = map.get(new StringValue(this, "header")); + if (hdrSeq != null && !hdrSeq.isEmpty()) { + final Item hdrItem = hdrSeq.itemAt(0); + if (hdrItem.getType() == Type.BOOLEAN) { + options.hasHeader = hdrItem.toSequence().effectiveBooleanValue(); + } else if (hdrSeq.getItemCount() == 1) { + final String hdrStr = hdrItem.getStringValue(); + if ("true".equals(hdrStr) || "present".equals(hdrStr)) { + options.hasHeader = true; + } else if ("false".equals(hdrStr) || "absent".equals(hdrStr)) { + options.hasHeader = false; + } else { + // Single string → explicit column name + options.explicitHeader = new ArrayList<>(); + options.explicitHeader.add(hdrStr); + options.hasHeader = false; // don't consume first data row + } + } else { + // Multiple items → sequence of explicit column names + options.explicitHeader = new ArrayList<>(hdrSeq.getItemCount()); + for (int j = 0; j < hdrSeq.getItemCount(); j++) { + options.explicitHeader.add(hdrSeq.itemAt(j).getStringValue()); + } + options.hasHeader = false; // don't consume first data row + } + } + + // select-columns + final Sequence scSeq = map.get(new StringValue(this, "select-columns")); + if (scSeq != null && !scSeq.isEmpty()) { + final int count = scSeq.getItemCount(); + options.selectColumns = new int[count]; + for (int j = 0; j < count; j++) { + final int col = ((IntegerValue) scSeq.itemAt(j).convertTo(Type.INTEGER)).getInt(); + if (col < 1) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "select-columns values must be positive integers, got: " + col); + } + options.selectColumns[j] = col; + } + } + + // trim-rows + final Sequence trSeq = map.get(new StringValue(this, "trim-rows")); + if (trSeq != null && !trSeq.isEmpty()) { + options.trimRows = trSeq.effectiveBooleanValue(); + } + + // Validate no unknown option keys + final java.util.Set knownKeys = java.util.Set.of( + "field-delimiter", "row-delimiter", "quote-character", + "trim-whitespace", "header", "select-columns", "trim-rows"); + for (final IEntry entry : map) { + final String key = entry.key().getStringValue(); + if (!knownKeys.contains(key)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Unknown CSV option: '" + key + "'"); + } + } + + return options; + } + + private ArrayType fieldsToArray(final List fields) throws XPathException { + // XQ4 spec: a row with a single empty field produces an empty array + if (fields.size() == 1 && fields.get(0).isEmpty()) { + return new ArrayType(this, context, new ArrayList<>()); + } + final List items = new ArrayList<>(fields.size()); + for (final String field : fields) { + items.add(new StringValue(this, field)); + } + return new ArrayType(this, context, items); + } + + /** + * Expression body for the "get" accessor function in fn:parse-csv results. + * Implements fn($row as xs:integer, $column as xs:integer) as xs:string. + * Both row and column are 1-based indexes. + */ + private static class CsvGetExpression extends AbstractExpression { + + private final ValueSequence rows; + private final List header; + + public CsvGetExpression(final XQueryContext context, final ValueSequence rows, final List header) { + super(context); + this.rows = rows; + this.header = header; + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) throws XPathException { + final Sequence rowIdxSeq = context.resolveVariable("row").getValue(); + final Sequence colSeq = context.resolveVariable("column").getValue(); + + if (rowIdxSeq.isEmpty() || colSeq.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + final int rowIdx = ((IntegerValue) rowIdxSeq.itemAt(0).convertTo(Type.INTEGER)).getInt(); + + if (rowIdx < 1 || rowIdx > rows.getItemCount()) { + return Sequence.EMPTY_SEQUENCE; + } + + // Resolve column: integer index or string name + final Item colItem = colSeq.itemAt(0); + final int colIdx; + if (Type.subTypeOf(colItem.getType(), Type.INTEGER)) { + colIdx = ((IntegerValue) colItem.convertTo(Type.INTEGER)).getInt(); + } else { + // String column name — look up in header + final String colName = colItem.getStringValue(); + if (header == null) { + return Sequence.EMPTY_SEQUENCE; + } + int found = -1; + for (int i = 0; i < header.size(); i++) { + if (header.get(i).equals(colName)) { + found = i + 1; // 1-based + break; + } + } + if (found == -1) { + return Sequence.EMPTY_SEQUENCE; + } + colIdx = found; + } + + final ArrayType row = (ArrayType) rows.itemAt(rowIdx - 1); + if (colIdx < 1 || colIdx > row.getSize()) { + return Sequence.EMPTY_SEQUENCE; + } + + return row.get(colIdx - 1); + } + + @Override + public int returnsType() { + return Type.STRING; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + // no-op + } + + @Override + public void dump(final org.exist.xquery.util.ExpressionDumper dumper) { + dumper.display("[csv-get]"); + } + + @Override + public String toString() { + return "[csv-get]"; + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/CsvParser.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/CsvParser.java new file mode 100644 index 00000000000..3b1524108bb --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/CsvParser.java @@ -0,0 +1,338 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Expression; +import org.exist.xquery.XPathException; + +import java.util.ArrayList; +import java.util.List; + +/** + * State-machine CSV parser following the XQuery 4.0 specification. + * Parses CSV text into records (rows) of fields using SAX-like callbacks. + * + * Options supported (per XQ4 spec): + * - field-delimiter (default: comma) + * - row-delimiter (default: CRLF/LF/CR) + * - quote-character (default: double-quote; empty string disables quoting) + * - trim-whitespace (default: false) + * - header (default: false; true or "present" means first row is header) + * - select-columns (default: all) + * - trim-rows (default: false; removes trailing empty rows) + */ +public class CsvParser { + + /** + * Callback interface for CSV parsing events. + */ + public interface CsvConverter { + void header(List fields) throws XPathException; + void record(List fields) throws XPathException; + void finish() throws XPathException; + } + + private final int fieldDelimiter; + private final int rowDelimiter; + private final int quoteChar; + private final boolean trimWhitespace; + private final boolean hasHeader; + private final int[] selectColumns; + private final boolean trimRows; + private final Expression expression; + + public CsvParser(final CsvOptions options, final Expression expression) { + this.fieldDelimiter = options.fieldDelimiter; + this.rowDelimiter = options.rowDelimiter; + this.quoteChar = options.quoteChar; + this.trimWhitespace = options.trimWhitespace; + this.hasHeader = options.hasHeader; + this.selectColumns = options.selectColumns; + this.trimRows = options.trimRows; + this.expression = expression; + } + + /** + * Parse CSV text, calling the converter for each record. + */ + public void parse(final String input, final CsvConverter converter) throws XPathException { + final List> allRecords = new ArrayList<>(); + List currentRecord = new ArrayList<>(); + final StringBuilder field = new StringBuilder(); + + // State: FIELD_START, IN_UNQUOTED, IN_QUOTED, AFTER_QUOTED + int state = 0; // 0=field_start, 1=in_unquoted, 2=in_quoted, 3=after_quoted + int i = 0; + final int len = input.length(); + + while (i < len) { + final int cp = input.codePointAt(i); + final int cpLen = Character.charCount(cp); + + switch (state) { + case 0: // FIELD_START — beginning of a new field + if (cp == quoteChar && quoteChar != -1) { + state = 2; // start quoted field + i += cpLen; + } else if (cp == fieldDelimiter) { + currentRecord.add(finishField(field)); + field.setLength(0); + // remain in FIELD_START + i += cpLen; + } else if (isRowDelimiter(cp)) { + currentRecord.add(finishField(field)); + field.setLength(0); + allRecords.add(currentRecord); + currentRecord = new ArrayList<>(); + i += rowDelimiterLength(input, i, cp); + } else { + field.appendCodePoint(cp); + state = 1; // in unquoted field + i += cpLen; + } + break; + + case 1: // IN_UNQUOTED — inside an unquoted field + if (cp == quoteChar && quoteChar != -1) { + // Quote in middle of unquoted field → error + throw new XPathException(expression, ErrorCodes.FOCV0001, + "Quote character found in middle of unquoted field"); + } else if (cp == fieldDelimiter) { + currentRecord.add(finishField(field)); + field.setLength(0); + state = 0; + i += cpLen; + } else if (isRowDelimiter(cp)) { + currentRecord.add(finishField(field)); + field.setLength(0); + allRecords.add(currentRecord); + currentRecord = new ArrayList<>(); + state = 0; + i += rowDelimiterLength(input, i, cp); + } else { + field.appendCodePoint(cp); + i += cpLen; + } + break; + + case 2: // IN_QUOTED — inside a quoted field + if (cp == quoteChar) { + // Check for escaped quote (doubled) + if (i + cpLen < len && input.codePointAt(i + cpLen) == quoteChar) { + field.appendCodePoint(quoteChar); + i += cpLen * 2; + } else { + // End of quoted field + state = 3; // after closing quote + i += cpLen; + } + } else { + field.appendCodePoint(cp); + i += cpLen; + } + break; + + case 3: // AFTER_QUOTED — just saw closing quote + if (cp == fieldDelimiter) { + currentRecord.add(finishField(field)); + field.setLength(0); + state = 0; + i += cpLen; + } else if (isRowDelimiter(cp)) { + currentRecord.add(finishField(field)); + field.setLength(0); + allRecords.add(currentRecord); + currentRecord = new ArrayList<>(); + state = 0; + i += rowDelimiterLength(input, i, cp); + } else if (cp == ' ' || cp == '\t') { + // Whitespace after closing quote is allowed (ignored) + i += cpLen; + } else { + // Non-delimiter content after closing quote → error + throw new XPathException(expression, ErrorCodes.FOCV0001, + "Content after closing quote in CSV field"); + } + break; + } + } + + // Check for unterminated quotes + if (state == 2) { + throw new XPathException(expression, ErrorCodes.FOCV0001, + "Unterminated quoted field in CSV input"); + } + + // Handle last field/record (if input doesn't end with row delimiter). + // A trailing row delimiter does not create an additional empty record. + // With trim-whitespace, a trailing row delimiter followed by only whitespace + // also does not create an additional record. + if (!currentRecord.isEmpty() || state == 3) { + // We had field delimiters on this line or a quoted field — always add + currentRecord.add(finishField(field)); + allRecords.add(currentRecord); + } else if (field.length() > 0) { + final String finished = finishField(field); + if (!finished.isEmpty()) { + currentRecord.add(finished); + allRecords.add(currentRecord); + } + } + + // Trim trailing empty rows if requested + if (trimRows) { + while (!allRecords.isEmpty()) { + final List lastRow = allRecords.get(allRecords.size() - 1); + if (isEmptyRow(lastRow)) { + allRecords.remove(allRecords.size() - 1); + } else { + break; + } + } + + // Normalize column count: all rows trimmed/padded to match first row (or header) + if (!allRecords.isEmpty()) { + final int columnCount = allRecords.get(0).size(); + for (int r = 1; r < allRecords.size(); r++) { + final List row = allRecords.get(r); + if (row.size() > columnCount) { + allRecords.set(r, new ArrayList<>(row.subList(0, columnCount))); + } else { + while (row.size() < columnCount) { + row.add(""); + } + } + } + } + } + + // Process header and records + int startIdx = 0; + if (hasHeader && !allRecords.isEmpty()) { + // Headers are always trimmed (per XQ4 spec), regardless of trim-whitespace option + final List headerFields = allRecords.get(0); + final List trimmedHeader = new ArrayList<>(headerFields.size()); + for (final String h : headerFields) { + trimmedHeader.add(h.trim()); + } + converter.header(selectFields(trimmedHeader)); + startIdx = 1; + } + + for (int r = startIdx; r < allRecords.size(); r++) { + converter.record(selectFields(allRecords.get(r))); + } + + converter.finish(); + } + + private String finishField(final StringBuilder field) { + if (trimWhitespace) { + return field.toString().trim(); + } + return field.toString(); + } + + private boolean isRowDelimiter(final int cp) { + if (rowDelimiter == -1) { + // Auto-detect: CR, LF, or CRLF + return cp == '\n' || cp == '\r'; + } + return cp == rowDelimiter; + } + + private int rowDelimiterLength(final String input, final int pos, final int cp) { + if (rowDelimiter == -1) { + // Auto-detect: CRLF counts as one delimiter + if (cp == '\r' && pos + 1 < input.length() && input.charAt(pos + 1) == '\n') { + return 2; + } + return 1; + } + return Character.charCount(rowDelimiter); + } + + private List selectFields(final List fields) { + if (selectColumns == null) { + return fields; + } + final List selected = new ArrayList<>(selectColumns.length); + for (final int col : selectColumns) { + if (col >= 1 && col <= fields.size()) { + selected.add(fields.get(col - 1)); + } else { + selected.add(""); + } + } + return selected; + } + + private static boolean isEmptyRow(final List row) { + for (final String field : row) { + if (!field.isEmpty()) { + return false; + } + } + return true; + } + + /** + * Parsed CSV options from an XQuery map. + */ + public static class CsvOptions { + public int fieldDelimiter = ','; + public int rowDelimiter = -1; // -1 = auto-detect (CR/LF/CRLF) + public int quoteChar = '"'; + public boolean trimWhitespace = false; + public boolean hasHeader = false; + public List explicitHeader = null; // explicit column names from options + public int[] selectColumns = null; + public boolean trimRows = false; + + /** + * Validate options per the XQ4 spec. + */ + public void validate(final Expression expression) throws XPathException { + // Field delimiter and quote character must be different + if (quoteChar != -1 && fieldDelimiter == quoteChar) { + throw new XPathException(expression, ErrorCodes.FOCV0003, + "Field delimiter and quote character must be different"); + } + // Field delimiter and row delimiter must be different + if (rowDelimiter != -1 && fieldDelimiter == rowDelimiter) { + throw new XPathException(expression, ErrorCodes.FOCV0003, + "Field delimiter and row delimiter must be different"); + } + // When using auto-detect row delimiters, field delimiter can't be CR or LF + if (rowDelimiter == -1 && (fieldDelimiter == '\n' || fieldDelimiter == '\r')) { + throw new XPathException(expression, ErrorCodes.FOCV0003, + "Field delimiter conflicts with auto-detected row delimiter (CR/LF)"); + } + // Quote character and row delimiter must be different + if (quoteChar != -1 && rowDelimiter != -1 && quoteChar == rowDelimiter) { + throw new XPathException(expression, ErrorCodes.FOCV0003, + "Quote character and row delimiter must be different"); + } + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/DeepEqualOptions.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/DeepEqualOptions.java new file mode 100644 index 00000000000..17e1bcdcd57 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/DeepEqualOptions.java @@ -0,0 +1,962 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import com.ibm.icu.text.Collator; +import io.lacuna.bifurcan.IEntry; +import org.exist.Namespaces; +import org.exist.dom.memtree.NodeImpl; +import org.exist.dom.memtree.ReferenceNode; +import org.exist.xquery.Constants; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.InlineFunction; +import org.exist.xquery.ValueComparison; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.functions.array.ArrayType; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.value.*; + +import javax.annotation.Nullable; +import java.text.Normalizer; +import java.util.*; + +/** + * XQuery 4.0 deep-equal options and options-aware comparison engine. + * + * Holds the parsed option flags from the options map parameter and provides + * comparison methods that respect those options. + */ +public class DeepEqualOptions { + + // Valid boolean option keys (no namespace) + private static final Set VALID_BOOLEAN_OPTIONS = Set.of( + "base-uri", "comments", "debug", + "id-property", "idrefs-property", + "in-scope-namespaces", "namespace-prefixes", "nilled-property", + "processing-instructions", "timezones", "type-annotations", + "type-variety", "typed-values" + ); + + // Valid string-valued option keys + private static final Set VALID_STRING_OPTIONS = Set.of( + "collation", "whitespace" + ); + + // Valid boolean-valued option keys (not in VALID_BOOLEAN_OPTIONS) + private static final Set VALID_ORDERED_OPTIONS = Set.of( + "ordered", "map-order" + ); + + // All valid string keys (no namespace) + private static final Set ALL_VALID_KEYS; + static { + final Set keys = new HashSet<>(); + keys.addAll(VALID_BOOLEAN_OPTIONS); + keys.addAll(VALID_STRING_OPTIONS); + keys.addAll(VALID_ORDERED_OPTIONS); + ALL_VALID_KEYS = Collections.unmodifiableSet(keys); + } + + // Option flags (defaults per XQ4 spec) + public final boolean comments; // default: false + public final boolean processingInstructions; // default: false + public final boolean ordered; // default: true + public final boolean namespacePrefixes; // default: false + public final boolean inScopeNamespaces; // default: false + public final boolean baseUri; // default: false + public final boolean idProperty; // default: false + public final boolean idrefsProperty; // default: false + public final boolean nilledProperty; // default: false + public final boolean timezones; // default: true + public final boolean typeAnnotations; // default: false + public final boolean typeVariety; // default: false + public final boolean typedValues; // default: true + public final boolean debug; // default: false + public final boolean mapOrder; // default: false + public final boolean unorderedElements; // from 'ordered' key on element comparison + + public enum WhitespaceMode { PRESERVE, NORMALIZE, STRIP } + public final WhitespaceMode whitespace; // default: PRESERVE + + @Nullable + public final Collator collator; + + /** Default options (XQ3.1 compatible behavior). */ + public static final DeepEqualOptions DEFAULTS = new DeepEqualOptions( + false, false, true, false, false, false, + false, false, false, true, false, false, true, + false, false, WhitespaceMode.PRESERVE, null + ); + + private DeepEqualOptions( + boolean comments, boolean processingInstructions, boolean ordered, + boolean namespacePrefixes, boolean inScopeNamespaces, boolean baseUri, + boolean idProperty, boolean idrefsProperty, boolean nilledProperty, + boolean timezones, boolean typeAnnotations, boolean typeVariety, + boolean typedValues, boolean debug, boolean mapOrder, + WhitespaceMode whitespace, @Nullable Collator collator) { + this.comments = comments; + this.processingInstructions = processingInstructions; + this.ordered = ordered; + this.namespacePrefixes = namespacePrefixes; + this.inScopeNamespaces = inScopeNamespaces; + this.baseUri = baseUri; + this.idProperty = idProperty; + this.idrefsProperty = idrefsProperty; + this.nilledProperty = nilledProperty; + this.timezones = timezones; + this.typeAnnotations = typeAnnotations; + this.typeVariety = typeVariety; + this.typedValues = typedValues; + this.debug = debug; + this.mapOrder = mapOrder; + this.unorderedElements = !ordered; + this.whitespace = whitespace; + this.collator = collator; + } + + /** + * Parse an XQ4 options map into a DeepEqualOptions instance. + * Validates all option keys and values per the spec. + * + * @param options the options map + * @param context the XQuery context (for collation resolution) + * @return parsed options + * @throws XPathException XPTY0004 if any option key or value is invalid + */ + public static DeepEqualOptions parse(final AbstractMapType options, final XQueryContext context) throws XPathException { + boolean comments = false; + boolean processingInstructions = false; + boolean ordered = true; + boolean namespacePrefixes = false; + boolean inScopeNamespaces = false; + boolean baseUri = false; + boolean idProperty = false; + boolean idrefsProperty = false; + boolean nilledProperty = false; + boolean timezones = true; + boolean typeAnnotations = false; + boolean typeVariety = false; + boolean typedValues = true; + boolean debug = false; + boolean mapOrder = false; + WhitespaceMode whitespace = WhitespaceMode.PRESERVE; + Collator collator = context.getDefaultCollator(); + + for (final IEntry entry : options) { + final AtomicValue key = entry.key(); + + // Keys that are QNames in a namespace are ignored (vendor extensions) + if (key.getType() == Type.QNAME) { + final QNameValue qnv = (QNameValue) key; + final String ns = qnv.getQName().getNamespaceURI(); + if (ns != null && !ns.isEmpty()) { + continue; // Ignore vendor extension options + } + // QName in no namespace → error + throw new XPathException(ErrorCodes.XPTY0004, + "Option key in no namespace is not recognized: " + key.getStringValue()); + } + + final String keyStr = key.getStringValue(); + + // Validate that the key is known + if (!ALL_VALID_KEYS.contains(keyStr)) { + throw new XPathException(ErrorCodes.XPTY0004, + "Unknown deep-equal option: '" + keyStr + "'"); + } + + final Sequence value = entry.value(); + + if (VALID_BOOLEAN_OPTIONS.contains(keyStr)) { + final boolean boolVal = parseBooleanOption(keyStr, value); + switch (keyStr) { + case "comments" -> comments = boolVal; + case "processing-instructions" -> processingInstructions = boolVal; + case "namespace-prefixes" -> namespacePrefixes = boolVal; + case "in-scope-namespaces" -> inScopeNamespaces = boolVal; + case "base-uri" -> baseUri = boolVal; + case "id-property" -> idProperty = boolVal; + case "idrefs-property" -> idrefsProperty = boolVal; + case "nilled-property" -> nilledProperty = boolVal; + case "timezones" -> timezones = boolVal; + case "type-annotations" -> typeAnnotations = boolVal; + case "type-variety" -> typeVariety = boolVal; + case "typed-values" -> typedValues = boolVal; + case "debug" -> debug = boolVal; + } + } else if (VALID_ORDERED_OPTIONS.contains(keyStr)) { + final boolean boolVal = parseBooleanOption(keyStr, value); + switch (keyStr) { + case "ordered" -> ordered = boolVal; + case "map-order" -> mapOrder = boolVal; + } + } else if ("collation".equals(keyStr)) { + if (!value.isEmpty()) { + collator = context.getCollator(value.getStringValue()); + } + } else if ("whitespace".equals(keyStr)) { + if (!value.isEmpty()) { + final String wsVal = value.getStringValue(); + whitespace = switch (wsVal) { + case "preserve" -> WhitespaceMode.PRESERVE; + case "normalize" -> WhitespaceMode.NORMALIZE; + case "strip" -> WhitespaceMode.STRIP; + default -> throw new XPathException(ErrorCodes.XPTY0004, + "Invalid whitespace option value: '" + wsVal + "'"); + }; + } + } + } + + return new DeepEqualOptions( + comments, processingInstructions, ordered, + namespacePrefixes, inScopeNamespaces, baseUri, + idProperty, idrefsProperty, nilledProperty, + timezones, typeAnnotations, typeVariety, typedValues, + debug, mapOrder, whitespace, collator + ); + } + + /** + * Parse a boolean option value using XQ4 option parameter conventions. + * Accepts: xs:boolean, xs:string ("true"/"false"/"0"/"1"), + * xs:integer (0/1), or nodes (effective boolean value). + */ + private static boolean parseBooleanOption(final String key, final Sequence value) throws XPathException { + if (value.isEmpty()) { + return false; + } + + final Item item = value.itemAt(0); + + // If it's already a boolean, use it directly + if (item.getType() == Type.BOOLEAN) { + return ((BooleanValue) item).getValue(); + } + + // Try casting to xs:boolean — accepts "true"/"false"/"0"/"1" and numeric 0/1 + try { + final AtomicValue boolVal = item.atomize().convertTo(Type.BOOLEAN); + return ((BooleanValue) boolVal).getValue(); + } catch (final XPathException e) { + throw new XPathException(ErrorCodes.XPTY0004, + "Invalid value for boolean option '" + key + "': " + item.getStringValue()); + } + } + + // ======================================================================== + // Options-aware deep comparison engine + // ======================================================================== + + /** + * Deep-compare two sequences with options. + */ + public int deepCompareSeq(final Sequence sequence1, final Sequence sequence2) { + if (sequence1 == sequence2) { + return Constants.EQUAL; + } + + if (!ordered) { + return deepCompareSeqUnordered(sequence1, sequence2); + } + + final int count1 = sequence1.getItemCount(); + final int count2 = sequence2.getItemCount(); + if (count1 != count2) { + return count1 < count2 ? Constants.INFERIOR : Constants.SUPERIOR; + } + + for (int i = 0; i < count1; i++) { + final int cmp = deepCompare(sequence1.itemAt(i), sequence2.itemAt(i)); + if (cmp != Constants.EQUAL) { + return cmp; + } + } + return Constants.EQUAL; + } + + /** + * Unordered sequence comparison: every item in seq1 must match some + * item in seq2 (and vice versa, by equal counts of matches). + */ + private int deepCompareSeqUnordered(final Sequence sequence1, final Sequence sequence2) { + final int count1 = sequence1.getItemCount(); + final int count2 = sequence2.getItemCount(); + if (count1 != count2) { + return count1 < count2 ? Constants.INFERIOR : Constants.SUPERIOR; + } + + // For each item in seq1, find a matching item in seq2 + final boolean[] matched = new boolean[count2]; + for (int i = 0; i < count1; i++) { + final Item item1 = sequence1.itemAt(i); + boolean found = false; + for (int j = 0; j < count2; j++) { + if (!matched[j] && deepCompare(item1, sequence2.itemAt(j)) == Constants.EQUAL) { + matched[j] = true; + found = true; + break; + } + } + if (!found) { + return Constants.INFERIOR; + } + } + return Constants.EQUAL; + } + + /** + * Deep-compare two items with options. + */ + public int deepCompare(final Item item1, final Item item2) { + if (item1 == item2) { + return Constants.EQUAL; + } + + try { + // Array comparison + if (item1.getType() == Type.ARRAY_ITEM || item2.getType() == Type.ARRAY_ITEM) { + if (item1.getType() != item2.getType()) { + return Constants.INFERIOR; + } + final ArrayType array1 = (ArrayType) item1; + final ArrayType array2 = (ArrayType) item2; + if (array1.getSize() != array2.getSize()) { + return array1.getSize() < array2.getSize() ? Constants.INFERIOR : Constants.SUPERIOR; + } + for (int i = 0; i < array1.getSize(); i++) { + final int cmp = deepCompareSeq(array1.get(i), array2.get(i)); + if (cmp != Constants.EQUAL) { + return cmp; + } + } + return Constants.EQUAL; + } + + // Map comparison + if (item1.getType() == Type.MAP_ITEM || item2.getType() == Type.MAP_ITEM) { + if (item1.getType() != item2.getType()) { + return Constants.INFERIOR; + } + return compareMaps((AbstractMapType) item1, (AbstractMapType) item2); + } + + // Function items: identity comparison via function-identity semantics + if (Type.subTypeOf(item1.getType(), Type.FUNCTION) || Type.subTypeOf(item2.getType(), Type.FUNCTION)) { + if (!Type.subTypeOf(item1.getType(), Type.FUNCTION) || !Type.subTypeOf(item2.getType(), Type.FUNCTION)) { + return Constants.INFERIOR; + } + return compareFunctionItems(item1, item2); + } + + // Atomic values + final boolean item1IsAtomic = Type.subTypeOf(item1.getType(), Type.ANY_ATOMIC_TYPE); + final boolean item2IsAtomic = Type.subTypeOf(item2.getType(), Type.ANY_ATOMIC_TYPE); + if (item1IsAtomic || item2IsAtomic) { + if (!item1IsAtomic || !item2IsAtomic) { + return item1IsAtomic ? Constants.INFERIOR : Constants.SUPERIOR; + } + return compareAtomics((AtomicValue) item1, (AtomicValue) item2); + } + + // Node comparison + if (item1.getType() != item2.getType()) { + return Constants.INFERIOR; + } + + final NodeValue nva = (NodeValue) item1; + final NodeValue nvb = (NodeValue) item2; + if (nva == nvb) { + return Constants.EQUAL; + } + + switch (item1.getType()) { + case Type.DOCUMENT: + return compareContents( + nva instanceof org.w3c.dom.Node n1 ? n1 : ((org.exist.dom.persistent.NodeProxy) nva).getOwnerDocument(), + nvb instanceof org.w3c.dom.Node n2 ? n2 : ((org.exist.dom.persistent.NodeProxy) nvb).getOwnerDocument()); + + case Type.ELEMENT: + return compareElements(nva.getNode(), nvb.getNode()); + + case Type.ATTRIBUTE: + final int attrNameCmp = compareNames(nva.getNode(), nvb.getNode()); + if (attrNameCmp != Constants.EQUAL) { + return attrNameCmp; + } + // whitespace:normalize applies to attribute values, but strip does NOT + return safeCompare( + maybeNormalizeWSAttr(nva.getNode().getNodeValue()), + maybeNormalizeWSAttr(nvb.getNode().getNodeValue()), + collator); + + case Type.PROCESSING_INSTRUCTION: + return comparePIs(nva, nvb); + + case Type.NAMESPACE: + final int nsNameCmp = safeCompare(nva.getNode().getNodeName(), nvb.getNode().getNodeName(), null); + if (nsNameCmp != Constants.EQUAL) { + return nsNameCmp; + } + return safeCompare(nva.getStringValue(), nvb.getStringValue(), collator); + + case Type.TEXT: + return safeCompare( + maybeNormalizeWS(nva.getStringValue()), + maybeNormalizeWS(nvb.getStringValue()), + collator); + + case Type.COMMENT: + // Apply whitespace normalization to comment content if whitespace option is set + return safeCompare( + maybeNormalizeWS(nva.getStringValue()), + maybeNormalizeWS(nvb.getStringValue()), + collator); + + default: + return Constants.INFERIOR; + } + } catch (final XPathException e) { + return Constants.INFERIOR; + } + } + + /** + * Compare function items using function-identity semantics (XQ4). + * Named functions with same name and arity are equal. + * Anonymous functions use reference identity. + */ + private static int compareFunctionItems(final Item item1, final Item item2) { + if (item1 == item2) { + return Constants.EQUAL; + } + if (item1 instanceof FunctionReference ref1 && item2 instanceof FunctionReference ref2) { + final FunctionSignature sig1 = ref1.getSignature(); + final FunctionSignature sig2 = ref2.getSignature(); + final org.exist.dom.QName name1 = sig1.getName(); + final org.exist.dom.QName name2 = sig2.getName(); + // Both must be named functions (not inline/anonymous) + if (name1 != null && name2 != null + && name1 != InlineFunction.INLINE_FUNCTION_QNAME + && name2 != InlineFunction.INLINE_FUNCTION_QNAME) { + if (name1.equals(name2) && sig1.getArgumentCount() == sig2.getArgumentCount()) { + return Constants.EQUAL; + } + } + } + return Constants.INFERIOR; + } + + private int compareMaps(final AbstractMapType map1, final AbstractMapType map2) { + if (map1.size() != map2.size()) { + return map1.size() < map2.size() ? Constants.INFERIOR : Constants.SUPERIOR; + } + + for (final IEntry entry1 : map1) { + if (!map2.contains(entry1.key())) { + return Constants.SUPERIOR; + } + final int cmp = deepCompareSeq(entry1.value(), map2.get(entry1.key())); + if (cmp != Constants.EQUAL) { + return cmp; + } + } + return Constants.EQUAL; + } + + private int compareAtomics(final AtomicValue av, final AtomicValue bv) { + try { + // Whitespace normalization for string-like atomics + if (whitespace != WhitespaceMode.PRESERVE) { + if (isStringLike(av) && isStringLike(bv)) { + final String a = applyWhitespace(av.getStringValue()); + final String b = applyWhitespace(bv.getStringValue()); + if (collator != null) { + return collator.compare(a, b); + } + return a.compareTo(b); + } + } + + if (Type.subTypeOfUnion(av.getType(), Type.NUMERIC) && + Type.subTypeOfUnion(bv.getType(), Type.NUMERIC)) { + if (((NumericValue) av).isNaN() && ((NumericValue) bv).isNaN()) { + return Constants.EQUAL; + } + } + return ValueComparison.compareAtomic(collator, av, bv); + } catch (final XPathException e) { + return Constants.INFERIOR; + } + } + + private static boolean isStringLike(final AtomicValue v) { + return Type.subTypeOf(v.getType(), Type.STRING) || + v.getType() == Type.UNTYPED_ATOMIC || + v.getType() == Type.ANY_URI; + } + + private int compareElements(final org.w3c.dom.Node a, final org.w3c.dom.Node b) { + int cmp = compareNames(a, b); + if (cmp != Constants.EQUAL) { + return cmp; + } + + // Compare namespace prefixes if option is set + if (namespacePrefixes) { + cmp = safeCompare(a.getPrefix(), b.getPrefix(), null); + if (cmp != Constants.EQUAL) { + return cmp; + } + } + + cmp = compareAttributes(a, b); + if (cmp != Constants.EQUAL) { + return cmp; + } + + if (unorderedElements) { + return compareContentsUnordered(a, b); + } + + return compareContents(a, b); + } + + private int comparePIs(final NodeValue nva, final NodeValue nvb) throws XPathException { + final int nameCmp = safeCompare(nva.getNode().getNodeName(), nvb.getNode().getNodeName(), null); + if (nameCmp != Constants.EQUAL) { + return nameCmp; + } + // Apply whitespace normalization to PI data content + return safeCompare( + maybeNormalizeWS(nva.getStringValue()), + maybeNormalizeWS(nvb.getStringValue()), + collator); + } + + private int compareContents(final org.w3c.dom.Node a, final org.w3c.dom.Node b) { + final List childrenA = getSignificantChildren(a); + final List childrenB = getSignificantChildren(b); + + // Merge adjacent text nodes + final List mergedA = mergeAdjacentTextNodes(childrenA); + final List mergedB = mergeAdjacentTextNodes(childrenB); + + if (mergedA.size() != mergedB.size()) { + return mergedA.size() < mergedB.size() ? Constants.INFERIOR : Constants.SUPERIOR; + } + + for (int i = 0; i < mergedA.size(); i++) { + final Object itemA = mergedA.get(i); + final Object itemB = mergedB.get(i); + + if (itemA instanceof String sa && itemB instanceof String sb) { + // Text may already be normalized/stripped by addMergedText; apply WS normalization if PRESERVE mode + final String normA = whitespace == WhitespaceMode.PRESERVE ? sa : maybeNormalizeWS(sa); + final String normB = whitespace == WhitespaceMode.PRESERVE ? sb : maybeNormalizeWS(sb); + final int cmp = safeCompare(normA, normB, collator); + if (cmp != Constants.EQUAL) { + return cmp; + } + } else if (itemA instanceof org.w3c.dom.Node na && itemB instanceof org.w3c.dom.Node nb) { + final int typeA = getEffectiveNodeType(na); + final int typeB = getEffectiveNodeType(nb); + if (typeA != typeB) { + return Constants.INFERIOR; + } + final int cmp; + switch (typeA) { + case org.w3c.dom.Node.ELEMENT_NODE: + cmp = compareElements(na, nb); + break; + case org.w3c.dom.Node.COMMENT_NODE: + cmp = safeCompare(maybeNormalizeWS(na.getNodeValue()), + maybeNormalizeWS(nb.getNodeValue()), collator); + break; + case org.w3c.dom.Node.PROCESSING_INSTRUCTION_NODE: + final int piNameCmp = safeCompare(na.getNodeName(), nb.getNodeName(), null); + cmp = piNameCmp != Constants.EQUAL ? piNameCmp : + safeCompare(maybeNormalizeWS(na.getNodeValue()), + maybeNormalizeWS(nb.getNodeValue()), collator); + break; + default: + cmp = Constants.INFERIOR; + } + if (cmp != Constants.EQUAL) { + return cmp; + } + } else { + // Mismatched types (text vs node) + return Constants.INFERIOR; + } + } + return Constants.EQUAL; + } + + /** + * Unordered element comparison: child elements are compared as multisets. + */ + private int compareContentsUnordered(final org.w3c.dom.Node a, final org.w3c.dom.Node b) { + final List childrenA = getSignificantChildren(a); + final List childrenB = getSignificantChildren(b); + + // Separate text content and element/other nodes + final StringBuilder textA = new StringBuilder(); + final List elementsA = new ArrayList<>(); + for (final org.w3c.dom.Node n : childrenA) { + final int type = getEffectiveNodeType(n); + if (type == org.w3c.dom.Node.TEXT_NODE) { + textA.append(getNodeValue(n)); + } else { + elementsA.add(n); + } + } + + final StringBuilder textB = new StringBuilder(); + final List elementsB = new ArrayList<>(); + for (final org.w3c.dom.Node n : childrenB) { + final int type = getEffectiveNodeType(n); + if (type == org.w3c.dom.Node.TEXT_NODE) { + textB.append(getNodeValue(n)); + } else { + elementsB.add(n); + } + } + + // Compare concatenated text content + final int textCmp = safeCompare( + maybeNormalizeWS(textA.toString()), + maybeNormalizeWS(textB.toString()), + collator); + if (textCmp != Constants.EQUAL) { + return textCmp; + } + + // Compare elements as multisets + if (elementsA.size() != elementsB.size()) { + return elementsA.size() < elementsB.size() ? Constants.INFERIOR : Constants.SUPERIOR; + } + + final boolean[] matched = new boolean[elementsB.size()]; + for (final org.w3c.dom.Node na : elementsA) { + boolean found = false; + for (int j = 0; j < elementsB.size(); j++) { + if (!matched[j]) { + final int typeA = getEffectiveNodeType(na); + final int typeB = getEffectiveNodeType(elementsB.get(j)); + if (typeA == typeB) { + int cmp; + if (typeA == org.w3c.dom.Node.ELEMENT_NODE) { + cmp = compareElements(na, elementsB.get(j)); + } else if (typeA == org.w3c.dom.Node.COMMENT_NODE) { + cmp = safeCompare(na.getNodeValue(), elementsB.get(j).getNodeValue(), collator); + } else if (typeA == org.w3c.dom.Node.PROCESSING_INSTRUCTION_NODE) { + cmp = safeCompare(na.getNodeName(), elementsB.get(j).getNodeName(), null); + if (cmp == Constants.EQUAL) { + cmp = safeCompare(na.getNodeValue(), elementsB.get(j).getNodeValue(), collator); + } + } else { + cmp = Constants.INFERIOR; + } + if (cmp == Constants.EQUAL) { + matched[j] = true; + found = true; + break; + } + } + } + } + if (!found) { + return Constants.INFERIOR; + } + } + return Constants.EQUAL; + } + + /** + * Get child nodes that are significant for deep-equal comparison, + * based on the current options. + */ + private List getSignificantChildren(final org.w3c.dom.Node parent) { + final List result = new ArrayList<>(); + final boolean preserveWS = isXmlSpacePreserve(parent); + org.w3c.dom.Node child = parent.getFirstChild(); + while (child != null) { + final int type = getEffectiveNodeType(child); + switch (type) { + case org.w3c.dom.Node.ELEMENT_NODE: + result.add(child); + break; + case org.w3c.dom.Node.TEXT_NODE: + if (whitespace == WhitespaceMode.STRIP) { + // Strip whitespace-only text nodes (deep-equal strip option + // overrides xml:space="preserve" per XQ4 spec) + final String value = getNodeValue(child); + if (value != null && !value.trim().isEmpty()) { + result.add(child); + } + } else { + result.add(child); + } + break; + case org.w3c.dom.Node.COMMENT_NODE: + if (comments) { + result.add(child); + } + break; + case org.w3c.dom.Node.PROCESSING_INSTRUCTION_NODE: + if (processingInstructions) { + result.add(child); + } + break; + } + child = child.getNextSibling(); + } + return result; + } + + /** + * Merge adjacent text nodes into single String entries. + * Non-text nodes are kept as-is. This handles the case where + * comments/PIs split text differently in two trees. + */ + private List mergeAdjacentTextNodes(final List nodes) { + final List result = new ArrayList<>(); + StringBuilder currentText = null; + + for (final org.w3c.dom.Node node : nodes) { + final int type = getEffectiveNodeType(node); + if (type == org.w3c.dom.Node.TEXT_NODE) { + if (currentText == null) { + currentText = new StringBuilder(); + } + currentText.append(getNodeValue(node)); + } else { + if (currentText != null) { + addMergedText(result, currentText.toString()); + currentText = null; + } + result.add(node); + } + } + if (currentText != null) { + addMergedText(result, currentText.toString()); + } + return result; + } + + /** + * Add merged text to the result list, applying whitespace rules. + * In STRIP mode, whitespace-only text is dropped. + * In NORMALIZE mode, text that normalizes to empty is dropped. + */ + private void addMergedText(final List result, final String text) { + if (whitespace == WhitespaceMode.STRIP) { + if (!text.trim().isEmpty()) { + result.add(text); + } + } else if (whitespace == WhitespaceMode.NORMALIZE) { + final String normalized = normalizeWhitespace(text); + if (!normalized.isEmpty()) { + result.add(normalized); + } + } else { + result.add(text); + } + } + + private int compareAttributes(final org.w3c.dom.Node a, final org.w3c.dom.Node b) { + final org.w3c.dom.NamedNodeMap nnma = a.getAttributes(); + final org.w3c.dom.NamedNodeMap nnmb = b.getAttributes(); + + final int aCount = getAttrCount(nnma); + final int bCount = getAttrCount(nnmb); + + if (aCount != bCount) { + return aCount < bCount ? Constants.INFERIOR : Constants.SUPERIOR; + } + + for (int i = 0; i < nnma.getLength(); i++) { + final org.w3c.dom.Node ta = nnma.item(i); + final String nsA = ta.getNamespaceURI(); + if (nsA != null && Namespaces.XMLNS_NS.equals(nsA)) { + continue; + } + final org.w3c.dom.Node tb = ta.getLocalName() == null ? + nnmb.getNamedItem(ta.getNodeName()) : + nnmb.getNamedItemNS(ta.getNamespaceURI(), ta.getLocalName()); + if (tb == null) { + return Constants.SUPERIOR; + } + final int cmp = safeCompare( + maybeNormalizeWSAttr(ta.getNodeValue()), + maybeNormalizeWSAttr(tb.getNodeValue()), + collator); + if (cmp != Constants.EQUAL) { + return cmp; + } + } + return Constants.EQUAL; + } + + // ======================================================================== + // Utility methods + // ======================================================================== + + private String maybeNormalizeWS(@Nullable final String s) { + if (s == null || whitespace == WhitespaceMode.PRESERVE) { + return s; + } + // Both NORMALIZE and STRIP normalize text content + return normalizeWhitespace(s); + } + + /** + * Normalize whitespace for attribute values: only NORMALIZE mode applies, + * STRIP mode does NOT affect attribute values. + */ + private String maybeNormalizeWSAttr(@Nullable final String s) { + if (s == null || whitespace != WhitespaceMode.NORMALIZE) { + return s; + } + return normalizeWhitespace(s); + } + + private static String normalizeWhitespace(final String s) { + return s.strip().replaceAll("\\s+", " "); + } + + private String applyWhitespace(final String s) { + if (whitespace == WhitespaceMode.NORMALIZE) { + return normalizeWhitespace(s); + } + if (whitespace == WhitespaceMode.STRIP) { + return normalizeWhitespace(s); + } + return s; + } + + private static int getAttrCount(final org.w3c.dom.NamedNodeMap nnm) { + int count = 0; + for (int i = 0; i < nnm.getLength(); i++) { + final org.w3c.dom.Node n = nnm.item(i); + final String ns = n.getNamespaceURI(); + if (ns == null || !Namespaces.XMLNS_NS.equals(ns)) { + ++count; + } + } + return count; + } + + private static int compareNames(final org.w3c.dom.Node a, final org.w3c.dom.Node b) { + if (a.getLocalName() != null || b.getLocalName() != null) { + final int nsComparison = safeCompare(a.getNamespaceURI(), b.getNamespaceURI(), null); + if (nsComparison != Constants.EQUAL) { + return nsComparison; + } + return safeCompare(a.getLocalName(), b.getLocalName(), null); + } + return safeCompare(a.getNodeName(), b.getNodeName(), null); + } + + private static int safeCompare(@Nullable final String a, @Nullable final String b, @Nullable final Collator collator) { + if (a == b) { + return Constants.EQUAL; + } + if (a == null) { + return Constants.INFERIOR; + } + if (b == null) { + return Constants.SUPERIOR; + } + if (collator != null) { + return collator.compare(a, b); + } + return a.compareTo(b); + } + + private static String getNodeValue(final org.w3c.dom.Node n) { + if (n.getNodeType() == NodeImpl.REFERENCE_NODE) { + return ((ReferenceNode) n).getReference().getNodeValue(); + } + return n.getNodeValue(); + } + + private static int getEffectiveNodeType(final org.w3c.dom.Node n) { + int nodeType = n.getNodeType(); + if (nodeType == NodeImpl.REFERENCE_NODE) { + nodeType = ((ReferenceNode) n).getReference().getNode().getNodeType(); + } + return nodeType; + } + + /** + * Check if the given node or any ancestor has xml:space="preserve". + * Uses NamedNodeMap lookup for broader DOM compatibility. + */ + private static boolean isXmlSpacePreserve(final org.w3c.dom.Node node) { + org.w3c.dom.Node current = node; + while (current != null && current.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) { + if (current instanceof org.w3c.dom.Element elem) { + // Use Element.getAttributeNS for persistent DOM and other implementations + final String xmlSpace = elem.getAttributeNS( + "http://www.w3.org/XML/1998/namespace", "space"); + if ("preserve".equals(xmlSpace)) { + return true; + } + if ("default".equals(xmlSpace)) { + return false; + } + } + // Also check via NamedNodeMap for broader compatibility + final org.w3c.dom.NamedNodeMap attrs = current.getAttributes(); + if (attrs != null) { + org.w3c.dom.Node xmlSpace = attrs.getNamedItemNS( + "http://www.w3.org/XML/1998/namespace", "space"); + if (xmlSpace == null) { + xmlSpace = attrs.getNamedItem("xml:space"); + } + if (xmlSpace != null) { + final String val = xmlSpace.getNodeValue(); + if ("preserve".equals(val)) { + return true; + } + if ("default".equals(val)) { + return false; + } + } + } + current = current.getParentNode(); + } + return false; + } + + /** + * Deep equality using these options. + */ + public boolean deepEqualsSeq(final Sequence sequence1, final Sequence sequence2) { + return deepCompareSeq(sequence1, sequence2) == Constants.EQUAL; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnAllEqualDifferent.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnAllEqualDifferent.java new file mode 100644 index 00000000000..c8825265991 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnAllEqualDifferent.java @@ -0,0 +1,165 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import com.ibm.icu.text.Collator; +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.AtomicValue; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.NumericValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceIterator; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * Implements fn:all-equal and fn:all-different (XQuery 4.0). + */ +public class FnAllEqualDifferent extends BasicFunction { + + public static final FunctionSignature[] FN_ALL_EQUAL = { + new FunctionSignature( + new QName("all-equal", Function.BUILTIN_FUNCTION_NS), + "Returns true if all items in the supplied sequence are equal.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The values to compare") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if all values are equal")), + new FunctionSignature( + new QName("all-equal", Function.BUILTIN_FUNCTION_NS), + "Returns true if all items in the supplied sequence are equal, using the specified collation.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The values to compare"), + new FunctionParameterSequenceType("collation", Type.STRING, Cardinality.ZERO_OR_ONE, "The collation URI") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if all values are equal")) + }; + + public static final FunctionSignature[] FN_ALL_DIFFERENT = { + new FunctionSignature( + new QName("all-different", Function.BUILTIN_FUNCTION_NS), + "Returns true if no two items in the supplied sequence are equal.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The values to compare") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if all values are different")), + new FunctionSignature( + new QName("all-different", Function.BUILTIN_FUNCTION_NS), + "Returns true if no two items in the supplied sequence are equal, using the specified collation.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The values to compare"), + new FunctionParameterSequenceType("collation", Type.STRING, Cardinality.ZERO_OR_ONE, "The collation URI") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if all values are different")) + }; + + public FnAllEqualDifferent(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence values = args[0]; + if (values.getItemCount() <= 1) { + return BooleanValue.TRUE; + } + + final Collator collator = getCollator(args); + + // Collect all atomized values + final java.util.List items = new java.util.ArrayList<>(values.getItemCount()); + for (final SequenceIterator i = values.iterate(); i.hasNext(); ) { + items.add(i.nextItem().atomize()); + } + + if (isCalledAs("all-equal")) { + return allEqual(items, collator); + } else { + return allDifferent(items, collator); + } + } + + private Sequence allEqual(final java.util.List items, final Collator collator) throws XPathException { + // all-equal iff count(distinct-values) <= 1, using contextual equality + final AtomicValue first = items.get(0); + for (int i = 1; i < items.size(); i++) { + if (!contextuallyEqual(first, items.get(i), collator)) { + return BooleanValue.FALSE; + } + } + return BooleanValue.TRUE; + } + + private Sequence allDifferent(final java.util.List items, final Collator collator) throws XPathException { + // all-different iff count(distinct-values) == count + for (int i = 0; i < items.size(); i++) { + for (int j = i + 1; j < items.size(); j++) { + if (contextuallyEqual(items.get(i), items.get(j), collator)) { + return BooleanValue.FALSE; + } + } + } + return BooleanValue.TRUE; + } + + /** + * XQ4 contextual equality: two values are contextually equal if fn:compare returns 0. + * NaN is treated as equal to NaN. Errors in comparison mean values are unequal. + */ + static boolean contextuallyEqual(final AtomicValue v1, final AtomicValue v2, final Collator collator) { + try { + // NaN handling: NaN equals NaN + if (v1 instanceof NumericValue && v2 instanceof NumericValue) { + final boolean nan1 = ((NumericValue) v1).isNaN(); + final boolean nan2 = ((NumericValue) v2).isNaN(); + if (nan1 && nan2) { + return true; + } + if (nan1 || nan2) { + return false; + } + } + return FunCompare.compare(v1, v2, collator) == 0; + } catch (final Exception e) { + // Errors in comparison mean values are unequal + return false; + } + } + + private Collator getCollator(final Sequence[] args) throws XPathException { + if (args.length > 1 && !args[1].isEmpty()) { + final String collationURI = args[1].getStringValue(); + return context.getCollator(collationURI, ErrorCodes.FOCH0002); + } + return context.getDefaultCollator(); + } + +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnAtomicEqual.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnAtomicEqual.java new file mode 100644 index 00000000000..0836fb00e66 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnAtomicEqual.java @@ -0,0 +1,214 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.AbstractDateTimeValue; +import org.exist.xquery.value.AtomicValue; +import org.exist.xquery.value.BinaryValue; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.DoubleValue; +import org.exist.xquery.value.FloatValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.NumericValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * Implements fn:atomic-equal (XQuery 4.0). + * + * Compares two atomic values for equality. Unlike eq, this function: + * - Never raises a dynamic error (returns false for incomparable types) + * - NaN equals NaN + * - Does not depend on static or dynamic context + */ +public class FnAtomicEqual extends BasicFunction { + + public static final FunctionSignature FN_ATOMIC_EQUAL = new FunctionSignature( + new QName("atomic-equal", Function.BUILTIN_FUNCTION_NS), + "Compares two atomic values for equality. NaN equals NaN, and incomparable types return false.", + new SequenceType[] { + new FunctionParameterSequenceType("value1", Type.ANY_ATOMIC_TYPE, Cardinality.EXACTLY_ONE, "The first value"), + new FunctionParameterSequenceType("value2", Type.ANY_ATOMIC_TYPE, Cardinality.EXACTLY_ONE, "The second value") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if the values are equal")); + + public FnAtomicEqual(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final AtomicValue v1 = args[0].itemAt(0).atomize(); + final AtomicValue v2 = args[1].itemAt(0).atomize(); + + // Handle NaN: NaN equals NaN (across float/double) + if (isNaN(v1) && isNaN(v2)) { + return BooleanValue.TRUE; + } + if (isNaN(v1) || isNaN(v2)) { + return BooleanValue.FALSE; + } + + // Handle Infinity: float INF equals double INF (and -INF) + if (isInfinite(v1) && isInfinite(v2)) { + return BooleanValue.valueOf(toDouble(v1) == toDouble(v2)); + } + + try { + final int t1 = v1.getType(); + final int t2 = v2.getType(); + + // String-like types: string, untypedAtomic, anyURI all compare equal + if (isStringLike(t1) && isStringLike(t2)) { + return BooleanValue.valueOf(v1.getStringValue().equals(v2.getStringValue())); + } + + // Numeric: compare by mathematical value regardless of type + // Per XQ4 spec: "Two numeric values are equal if their mathematical values are equal" + if (v1 instanceof NumericValue && v2 instanceof NumericValue) { + return BooleanValue.valueOf(numericEqual((NumericValue) v1, (NumericValue) v2)); + } + + // Binary types: hexBinary and base64Binary compare equal by content + if (isBinaryType(t1) && isBinaryType(t2)) { + if (v1 instanceof BinaryValue && v2 instanceof BinaryValue) { + return BooleanValue.valueOf(v1.compareTo(null, v2) == 0); + } + return BooleanValue.FALSE; + } + + // Boolean + if (t1 == Type.BOOLEAN && t2 == Type.BOOLEAN) { + return BooleanValue.valueOf(v1.effectiveBooleanValue() == v2.effectiveBooleanValue()); + } + + // Date/time: values with timezone never equal values without timezone + if (v1 instanceof AbstractDateTimeValue && v2 instanceof AbstractDateTimeValue) { + final AbstractDateTimeValue dt1 = (AbstractDateTimeValue) v1; + final AbstractDateTimeValue dt2 = (AbstractDateTimeValue) v2; + if (dt1.hasTimezone() != dt2.hasTimezone()) { + return BooleanValue.FALSE; + } + } + + // Different base types are never equal + if (t1 != t2 && !Type.subTypeOf(t1, t2) && !Type.subTypeOf(t2, t1)) { + return BooleanValue.FALSE; + } + + // Same type — compare by value + final int cmp = v1.compareTo(null, v2); + return BooleanValue.valueOf(cmp == 0); + } catch (final XPathException | RuntimeException e) { + // Incomparable types or indeterminate ordering — return false per spec + return BooleanValue.FALSE; + } + } + + private static boolean isNaN(final AtomicValue v) { + if (v instanceof DoubleValue) { + return Double.isNaN(((DoubleValue) v).getDouble()); + } + if (v instanceof FloatValue) { + return Float.isNaN(((FloatValue) v).getValue()); + } + return false; + } + + private static boolean isInfinite(final AtomicValue v) { + if (v instanceof DoubleValue) { + return Double.isInfinite(((DoubleValue) v).getDouble()); + } + if (v instanceof FloatValue) { + return Float.isInfinite(((FloatValue) v).getValue()); + } + return false; + } + + private static double toDouble(final AtomicValue v) { + if (v instanceof DoubleValue) { + return ((DoubleValue) v).getDouble(); + } + if (v instanceof FloatValue) { + return ((FloatValue) v).getValue(); + } + return 0; + } + + static boolean numericEqual(final NumericValue v1, final NumericValue v2) throws XPathException { + // Both floating-point: use double comparison (handles 0.0 == -0.0) + if ((v1 instanceof DoubleValue || v1 instanceof FloatValue) + && (v2 instanceof DoubleValue || v2 instanceof FloatValue)) { + return v1.getDouble() == v2.getDouble(); + } + // Mixed floating-point and exact: convert to BigDecimal for exact mathematical comparison + // This handles cases like atomic-equal(16777218, xs:double("16777218")) + final java.math.BigDecimal bd1 = numericToBigDecimal(v1); + final java.math.BigDecimal bd2 = numericToBigDecimal(v2); + return bd1.compareTo(bd2) == 0; + } + + private static java.math.BigDecimal numericToBigDecimal(final NumericValue v) throws XPathException { + if (v instanceof DoubleValue) { + // Use new BigDecimal(double) for exact binary representation, + // not valueOf() which rounds via Double.toString() + return new java.math.BigDecimal(((DoubleValue) v).getDouble()); + } + if (v instanceof FloatValue) { + return new java.math.BigDecimal(((FloatValue) v).getValue()); + } + // Integer and decimal types: parse from string for exact representation + return new java.math.BigDecimal(v.getStringValue()); + } + + private static int primitiveNumericType(final int type) { + if (Type.subTypeOf(type, Type.INTEGER)) { + return Type.INTEGER; + } + if (Type.subTypeOf(type, Type.DECIMAL)) { + return Type.DECIMAL; + } + if (type == Type.FLOAT) { + return Type.FLOAT; + } + return Type.DOUBLE; + } + + private static boolean isStringLike(final int type) { + return Type.subTypeOf(type, Type.STRING) + || type == Type.UNTYPED_ATOMIC + || Type.subTypeOf(type, Type.ANY_URI); + } + + private static boolean isBinaryType(final int type) { + return type == Type.HEX_BINARY || type == Type.BASE64_BINARY; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnBuildUri.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnBuildUri.java new file mode 100644 index 00000000000..bcd257a3869 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnBuildUri.java @@ -0,0 +1,335 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceIterator; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; + +/** + * Implements fn:build-uri (XQuery 4.0). + * + * Constructs a URI from the parts provided in a map. + */ +public class FnBuildUri extends BasicFunction { + + private static final Set NON_HIERARCHICAL_SCHEMES = new HashSet<>(Arrays.asList( + "mailto", "news", "urn", "tel", "tag", "jar", "data", "javascript", "cid", "mid" + )); + + public static final FunctionSignature[] FN_BUILD_URI = { + new FunctionSignature( + new QName("build-uri", Function.BUILTIN_FUNCTION_NS), + "Constructs a URI from the parts provided.", + new SequenceType[] { + new FunctionParameterSequenceType("parts", Type.MAP_ITEM, + Cardinality.EXACTLY_ONE, "Map of URI components") + }, + new FunctionReturnSequenceType(Type.STRING, + Cardinality.EXACTLY_ONE, "The constructed URI")), + new FunctionSignature( + new QName("build-uri", Function.BUILTIN_FUNCTION_NS), + "Constructs a URI from the parts provided.", + new SequenceType[] { + new FunctionParameterSequenceType("parts", Type.MAP_ITEM, + Cardinality.EXACTLY_ONE, "Map of URI components"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, + Cardinality.ZERO_OR_ONE, "Options map") + }, + new FunctionReturnSequenceType(Type.STRING, + Cardinality.EXACTLY_ONE, "The constructed URI")) + }; + + public FnBuildUri(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final MapType parts = (MapType) args[0].itemAt(0); + + // Parse options + boolean allowDeprecated = false; + boolean omitDefaultPorts = false; + boolean uncPath = false; + if (args.length > 1 && !args[1].isEmpty()) { + final MapType options = (MapType) args[1].itemAt(0); + allowDeprecated = getBooleanOption(options, "allow-deprecated-features", false); + omitDefaultPorts = getBooleanOption(options, "omit-default-ports", false); + uncPath = getBooleanOption(options, "unc-path", false); + } + + final StringBuilder uri = new StringBuilder(); + + // Get scheme + final String scheme = getStringValue(parts, "scheme"); + + // Determine if hierarchical + boolean hierarchical = true; + final Sequence hierSeq = parts.get(new StringValue(this, "hierarchical")); + if (hierSeq != null && !hierSeq.isEmpty()) { + hierarchical = hierSeq.effectiveBooleanValue(); + } else if (scheme != null) { + hierarchical = !NON_HIERARCHICAL_SCHEMES.contains(scheme.toLowerCase()); + } + + // Add scheme + if (scheme != null) { + uri.append(scheme); + if (!hierarchical) { + uri.append(':'); + } else if ("file".equalsIgnoreCase(scheme) && uncPath) { + uri.append(":////"); + } else { + uri.append("://"); + } + } + + // Build authority from components or use authority directly + final String userinfo = getStringValue(parts, "userinfo"); + final String host = getStringValue(parts, "host"); + final Sequence portSeq = parts.get(new StringValue(this, "port")); + Integer port = null; + if (portSeq != null && !portSeq.isEmpty()) { + port = ((Number) portSeq.itemAt(0).toJavaObject(Long.class)).intValue(); + } + + // Handle deprecated password in userinfo + String effectiveUserinfo = userinfo; + if (!allowDeprecated && effectiveUserinfo != null && effectiveUserinfo.contains(":")) { + final String password = effectiveUserinfo.substring(effectiveUserinfo.indexOf(':') + 1); + if (!password.isEmpty()) { + effectiveUserinfo = null; + } + } + + // Omit default ports + if (omitDefaultPorts && port != null && scheme != null) { + if (isDefaultPort(scheme.toLowerCase(), port)) { + port = null; + } + } + + if (effectiveUserinfo != null || host != null || port != null) { + if (scheme == null) { + uri.append("//"); + } + if (effectiveUserinfo != null) { + uri.append(effectiveUserinfo).append('@'); + } + if (host != null) { + uri.append(host); + } + if (port != null) { + uri.append(':').append(port); + } + } else { + final String authority = getStringValue(parts, "authority"); + if (authority != null) { + if (scheme == null) { + uri.append("//"); + } + uri.append(authority); + } + } + + // Build path from path-segments or use path directly + final Sequence pathSegments = parts.get(new StringValue(this, "path-segments")); + if (pathSegments != null && !pathSegments.isEmpty()) { + final StringBuilder pathBuilder = new StringBuilder(); + boolean first = true; + for (final SequenceIterator i = pathSegments.iterate(); i.hasNext(); ) { + if (!first) { + pathBuilder.append('/'); + } + first = false; + final String segment = i.nextItem().getStringValue(); + if (hierarchical) { + pathBuilder.append(encodePathSegment(segment)); + } else { + pathBuilder.append(segment); + } + } + uri.append(pathBuilder); + } else { + final String path = getStringValue(parts, "path"); + if (path != null) { + uri.append(path); + } + } + + // Build query from query-parameters or use query directly + final Sequence queryParamsSeq = parts.get(new StringValue(this, "query-parameters")); + if (queryParamsSeq != null && !queryParamsSeq.isEmpty() && queryParamsSeq.itemAt(0) instanceof MapType) { + final MapType queryParams = (MapType) queryParamsSeq.itemAt(0); + final StringBuilder queryBuilder = new StringBuilder(); + boolean first = true; + for (final SequenceIterator ki = queryParams.keys().iterate(); ki.hasNext(); ) { + final StringValue key = (StringValue) ki.nextItem(); + final Sequence values = queryParams.get(key); + for (final SequenceIterator vi = values.iterate(); vi.hasNext(); ) { + if (!first) { + queryBuilder.append('&'); + } + first = false; + final String keyStr = key.getStringValue(); + final String valStr = vi.nextItem().getStringValue(); + if (keyStr.isEmpty()) { + queryBuilder.append(encodeQueryComponent(valStr)); + } else { + queryBuilder.append(encodeQueryComponent(keyStr)) + .append('=') + .append(encodeQueryComponent(valStr)); + } + } + } + if (queryBuilder.length() > 0) { + uri.append('?').append(queryBuilder); + } + } else { + final String query = getStringValue(parts, "query"); + if (query != null) { + uri.append('?').append(query); + } + } + + // Fragment + final String fragment = getStringValue(parts, "fragment"); + if (fragment != null) { + uri.append('#').append(encodeFragment(fragment)); + } + + return new StringValue(this, uri.toString()); + } + + private String getStringValue(final MapType map, final String key) throws XPathException { + final Sequence val = map.get(new StringValue(this, key)); + if (val != null && !val.isEmpty()) { + return val.getStringValue(); + } + return null; + } + + private boolean getBooleanOption(final MapType options, final String key, + final boolean defaultValue) throws XPathException { + final Sequence val = options.get(new StringValue(this, key)); + if (val != null && !val.isEmpty()) { + return val.effectiveBooleanValue(); + } + return defaultValue; + } + + private static boolean isDefaultPort(final String scheme, final int port) { + switch (scheme) { + case "http": return port == 80; + case "https": return port == 443; + case "ftp": return port == 21; + case "ssh": return port == 22; + default: return false; + } + } + + // Encode path segment: control chars + space % / ? # + [ ] + private static String encodePathSegment(final String s) { + if (s == null || s.isEmpty()) { + return s; + } + final StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + if (c < 0x20 || c == ' ' || c == '%' || c == '/' || c == '?' + || c == '#' || c == '+' || c == '[' || c == ']') { + appendPercentEncoded(sb, c); + } else { + sb.append(c); + } + } + return sb.toString(); + } + + // Encode query component: control chars + space % = & # + [ ] + private static String encodeQueryComponent(final String s) { + if (s == null || s.isEmpty()) { + return s; + } + final StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + if (c < 0x20 || c == ' ' || c == '%' || c == '=' || c == '&' + || c == '#' || c == '+' || c == '[' || c == ']') { + appendPercentEncoded(sb, c); + } else { + sb.append(c); + } + } + return sb.toString(); + } + + // Encode fragment: control chars + space % # + [ ] + private static String encodeFragment(final String s) { + if (s == null || s.isEmpty()) { + return s; + } + final StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + if (c < 0x20 || c == ' ' || c == '%' || c == '#' || c == '+' || c == '[' || c == ']') { + appendPercentEncoded(sb, c); + } else { + sb.append(c); + } + } + return sb.toString(); + } + + private static void appendPercentEncoded(final StringBuilder sb, final char c) { + if (c < 0x80) { + sb.append('%').append(String.format("%02X", (int) c)); + } else { + try { + final byte[] bytes = String.valueOf(c).getBytes("UTF-8"); + for (final byte b : bytes) { + sb.append('%').append(String.format("%02X", b & 0xFF)); + } + } catch (final UnsupportedEncodingException e) { + sb.append(c); + } + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnChar.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnChar.java new file mode 100644 index 00000000000..fb97fd1a0dc --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnChar.java @@ -0,0 +1,218 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Function; +import org.exist.xquery.value.NumericValue; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.IntegerValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; + +/** + * Implements fn:char (XQuery 4.0). + * + * Returns a string containing a single character identified by its codepoint + * or by an HTML5 character reference name. + */ +public class FnChar extends BasicFunction { + + private static final ErrorCodes.ErrorCode FOCH0005 = new ErrorCodes.ErrorCode( + "FOCH0005", "Unknown character name"); + + public static final FunctionSignature FN_CHAR = new FunctionSignature( + new QName("char", Function.BUILTIN_FUNCTION_NS), + "Returns a string containing a single character identified by codepoint or character name.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.ANY_ATOMIC_TYPE, Cardinality.EXACTLY_ONE, + "A codepoint (integer) or character name (string)") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the character")); + + private static volatile Map htmlEntities; + + public FnChar(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final var item = args[0].itemAt(0); + final int type = item.getType(); + + if (Type.subTypeOf(type, Type.INTEGER)) { + // Codepoint + final long codepoint = ((IntegerValue) item).getLong(); + if (codepoint < 1 || codepoint > 0x10FFFF) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Codepoint " + codepoint + " is not in the valid range 1 to 1114111"); + } + // Check for XML-illegal characters (surrogates, etc.) + if (!isXmlChar((int) codepoint)) { + throw new XPathException(this, FOCH0005, + "Codepoint " + codepoint + " is not a valid XML character"); + } + return new StringValue(this, new String(Character.toChars((int) codepoint))); + } else if (Type.subTypeOf(type, Type.DOUBLE) || Type.subTypeOf(type, Type.FLOAT) + || Type.subTypeOf(type, Type.DECIMAL)) { + // Numeric but not integer — try to convert + final NumericValue num = (NumericValue) item; + if (num.hasFractionalPart()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Codepoint must be an integer, got " + Type.getTypeName(type)); + } + final long codepoint = num.getLong(); + if (codepoint < 1 || codepoint > 0x10FFFF) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Codepoint " + codepoint + " is not in the valid range 1 to 1114111"); + } + if (!isXmlChar((int) codepoint)) { + throw new XPathException(this, FOCH0005, + "Codepoint " + codepoint + " is not a valid XML character"); + } + return new StringValue(this, new String(Character.toChars((int) codepoint))); + } else { + // Character name lookup + final String name = item.getStringValue(); + + // Handle backslash escapes + switch (name) { + case "\\n": return new StringValue(this, "\n"); + case "\\r": return new StringValue(this, "\r"); + case "\\t": return new StringValue(this, "\t"); + } + + // Try HTML5 named character reference first (case-sensitive per spec) + final Map entities = getHtmlEntities(); + String resolved = entities.get(name); + if (resolved != null) { + return new StringValue(this, resolved); + } + + // Try Unicode character name + try { + final int cp = Character.codePointOf(name.replace(" ", "_").replace("-", "_").toUpperCase()); + if (isXmlChar(cp)) { + return new StringValue(this, new String(Character.toChars(cp))); + } + } catch (final IllegalArgumentException e) { + // Not a Unicode name either + } + + throw new XPathException(this, FOCH0005, + "Unknown character name: " + name); + } + } + + private static boolean isXmlChar(final int cp) { + return cp == 0x9 || cp == 0xA || cp == 0xD + || (cp >= 0x20 && cp <= 0xD7FF) + || (cp >= 0xE000 && cp <= 0xFFFD) + || (cp >= 0x10000 && cp <= 0x10FFFF); + } + + private static Map getHtmlEntities() { + if (htmlEntities == null) { + synchronized (FnChar.class) { + if (htmlEntities == null) { + htmlEntities = loadHtmlEntities(); + } + } + } + return htmlEntities; + } + + private static Map loadHtmlEntities() { + final Map map = new HashMap<>(2500); + + // Load from bundled resource file + final InputStream is = FnChar.class.getResourceAsStream("html5-entities.properties"); + if (is != null) { + try (final BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + final int eq = line.indexOf('='); + if (eq > 0) { + final String entityName = line.substring(0, eq); + final String codepoints = line.substring(eq + 1); + map.put(entityName, decodeCodepoints(codepoints)); + } + } + } catch (final IOException e) { + // Fall through with partial map + } + } + + // Add a few critical aliases if the file wasn't found + if (map.isEmpty()) { + addCommonEntities(map); + } + + return map; + } + + private static String decodeCodepoints(final String spec) { + // Format: "U+XXXX" or "U+XXXX,U+YYYY" + final StringBuilder sb = new StringBuilder(); + for (final String part : spec.split(",")) { + final String trimmed = part.trim(); + if (trimmed.startsWith("U+") || trimmed.startsWith("u+")) { + final int cp = Integer.parseInt(trimmed.substring(2), 16); + sb.appendCodePoint(cp); + } + } + return sb.toString(); + } + + private static void addCommonEntities(final Map map) { + map.put("amp", "&"); + map.put("lt", "<"); + map.put("gt", ">"); + map.put("quot", "\""); + map.put("apos", "'"); + map.put("nbsp", "\u00A0"); + map.put("tab", "\t"); + map.put("newline", "\n"); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnCharacters.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnCharacters.java new file mode 100644 index 00000000000..a45f63ca623 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnCharacters.java @@ -0,0 +1,77 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +/** + * Implements fn:characters (XQuery 4.0). + * + * Splits the supplied string into a sequence of single-character strings. + */ +public class FnCharacters extends BasicFunction { + + public static final FunctionSignature FN_CHARACTERS = new FunctionSignature( + new QName("characters", Function.BUILTIN_FUNCTION_NS), + "Splits the supplied string into a sequence of single-character strings.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.STRING, Cardinality.ZERO_OR_ONE, "The string to split") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "a sequence of single-character strings")); + + public FnCharacters(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + final String str = args[0].getStringValue(); + if (str.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + final ValueSequence result = new ValueSequence(str.length()); + // Use codepoint iteration to handle surrogate pairs correctly + int i = 0; + while (i < str.length()) { + final int codepoint = str.codePointAt(i); + result.add(new StringValue(this, new String(Character.toChars(codepoint)))); + i += Character.charCount(codepoint); + } + return result; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnCivilTimezone.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnCivilTimezone.java new file mode 100644 index 00000000000..bace87f5755 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnCivilTimezone.java @@ -0,0 +1,152 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.AbstractDateTimeValue; +import org.exist.xquery.value.DayTimeDurationValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +import javax.xml.datatype.XMLGregorianCalendar; +import javax.xml.datatype.DatatypeConstants; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.zone.ZoneRulesException; + +/** + * Implements XQuery 4.0 fn:civil-timezone. + * + * fn:civil-timezone($value as xs:dateTime, $place as xs:string?) as xs:dayTimeDuration + * + * Returns the civil timezone offset for a given dateTime at a given IANA timezone location, + * accounting for daylight savings time transitions. + */ +public class FnCivilTimezone extends BasicFunction { + + private static final ErrorCodes.ErrorCode FODT0004 = new ErrorCodes.ErrorCode("FODT0004", + "No timezone data available"); + + public static final FunctionSignature[] FN_CIVIL_TIMEZONE = { + new FunctionSignature( + new QName("civil-timezone", Function.BUILTIN_FUNCTION_NS), + "Returns the civil timezone offset for a dateTime at a place.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.DATE_TIME, Cardinality.EXACTLY_ONE, "The dateTime to look up"), + new FunctionParameterSequenceType("place", Type.STRING, Cardinality.ZERO_OR_ONE, "IANA timezone name (e.g. 'America/New_York')") + }, + new FunctionReturnSequenceType(Type.DAY_TIME_DURATION, Cardinality.EXACTLY_ONE, "the civil timezone offset")), + new FunctionSignature( + new QName("civil-timezone", Function.BUILTIN_FUNCTION_NS), + "Returns the civil timezone offset for a dateTime using the default place.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.DATE_TIME, Cardinality.EXACTLY_ONE, "The dateTime to look up") + }, + new FunctionReturnSequenceType(Type.DAY_TIME_DURATION, Cardinality.EXACTLY_ONE, "the civil timezone offset")) + }; + + public FnCivilTimezone(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final AbstractDateTimeValue dtv = (AbstractDateTimeValue) args[0].itemAt(0); + final XMLGregorianCalendar cal = (XMLGregorianCalendar) dtv.calendar.clone(); + + // Determine the IANA zone + final ZoneId zone; + if (args.length > 1 && !args[1].isEmpty()) { + final String place = args[1].getStringValue(); + try { + zone = ZoneId.of(place); + } catch (final java.time.DateTimeException e) { + throw new XPathException(this, FODT0004, + "Unknown timezone: " + place); + } + } else { + // Use system default timezone as the "default place" + zone = ZoneId.systemDefault(); + } + + // Convert the dateTime to a LocalDateTime (ignoring any timezone on the value) + final int year = cal.getYear(); + final int month = cal.getMonth(); + final int day = cal.getDay(); + final int hour = cal.getHour() == DatatypeConstants.FIELD_UNDEFINED ? 0 : cal.getHour(); + final int minute = cal.getMinute() == DatatypeConstants.FIELD_UNDEFINED ? 0 : cal.getMinute(); + final int second = cal.getSecond() == DatatypeConstants.FIELD_UNDEFINED ? 0 : cal.getSecond(); + + final LocalDateTime ldt = LocalDateTime.of(year, month, day, hour, minute, second); + + // Get the offset at that local date-time in the given zone + final ZonedDateTime zdt = ldt.atZone(zone); + final ZoneOffset offset = zdt.getOffset(); + final int totalSeconds = offset.getTotalSeconds(); + + // Convert to xs:dayTimeDuration + final String dur = secondsToDayTimeDuration(totalSeconds); + return new DayTimeDurationValue(this, dur); + } + + private static String secondsToDayTimeDuration(final int totalSeconds) { + final boolean negative = totalSeconds < 0; + int abs = Math.abs(totalSeconds); + final int hours = abs / 3600; + abs %= 3600; + final int minutes = abs / 60; + final int seconds = abs % 60; + + final StringBuilder sb = new StringBuilder(); + if (negative) { + sb.append('-'); + } + sb.append("PT"); + if (hours > 0) { + sb.append(hours).append('H'); + } + if (minutes > 0) { + sb.append(minutes).append('M'); + } + if (seconds > 0) { + sb.append(seconds).append('S'); + } + // If all zero, output PT0S + if (hours == 0 && minutes == 0 && seconds == 0) { + sb.append("0S"); + } + return sb.toString(); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnCollation.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnCollation.java new file mode 100644 index 00000000000..fd8fed95284 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnCollation.java @@ -0,0 +1,94 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.value.*; + +/** + * fn:collation() — Returns the default collation URI. + * fn:collation-available($uri) — Returns true if the collation is supported. + */ +public class FnCollation extends BasicFunction { + + public static final FunctionSignature[] FN_COLLATION = { + new FunctionSignature( + new QName("collation", Function.BUILTIN_FUNCTION_NS), + "Returns the URI of the default collation.", + null, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, + "The default collation URI")), + new FunctionSignature( + new QName("collation", Function.BUILTIN_FUNCTION_NS), + "Returns the collation URI if supported, empty sequence otherwise.", + new SequenceType[] { + new FunctionParameterSequenceType("uri", Type.STRING, + Cardinality.EXACTLY_ONE, "The collation URI to check") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_ONE, + "The collation URI if supported")) + }; + + public static final FunctionSignature FN_COLLATION_AVAILABLE = new FunctionSignature( + new QName("collation-available", Function.BUILTIN_FUNCTION_NS), + "Returns true if the specified collation is supported.", + new SequenceType[] { + new FunctionParameterSequenceType("uri", Type.STRING, + Cardinality.EXACTLY_ONE, "The collation URI") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "true if the collation is supported")); + + public FnCollation(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (isCalledAs("collation")) { + if (getArgumentCount() == 1) { + // 1-arg: check if the named collation is supported + final String uri = args[0].getStringValue(); + try { + context.getCollator(uri); + return new StringValue(this, uri); + } catch (final XPathException e) { + return Sequence.EMPTY_SEQUENCE; + } + } + // 0-arg: return default collation + final String defaultCollation = context.getDefaultCollation(); + return new StringValue(this, defaultCollation != null ? defaultCollation + : org.exist.util.Collations.UNICODE_CODEPOINT_COLLATION_URI); + } else { + // collation-available + final String uri = args[0].getStringValue(); + try { + context.getCollator(uri); + return BooleanValue.TRUE; + } catch (final XPathException e) { + return BooleanValue.FALSE; + } + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDateTimeParts.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDateTimeParts.java new file mode 100644 index 00000000000..b9445ce1fd0 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDateTimeParts.java @@ -0,0 +1,176 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.value.*; + +import javax.xml.datatype.DatatypeConstants; +import java.math.BigDecimal; + +/** + * fn:build-dateTime($date, $time) — Combine xs:date + xs:time into xs:dateTime. + * fn:parts-of-dateTime($dateTime) — Decompose xs:dateTime into a map of components. + * + * The map returned by parts-of-dateTime has keys: year, month, day, hour, minute, + * seconds (as xs:decimal including fractional), timezone (as xs:dayTimeDuration). + * When the Parser branch merges, these maps will be compatible with record type checking. + */ +public class FnDateTimeParts extends BasicFunction { + + public static final FunctionSignature FN_BUILD_DATETIME = new FunctionSignature( + new QName("build-dateTime", Function.BUILTIN_FUNCTION_NS), + "Combines an xs:date and an xs:time into an xs:dateTime.", + new SequenceType[] { + new FunctionParameterSequenceType("date", Type.DATE, + Cardinality.EXACTLY_ONE, "The date component"), + new FunctionParameterSequenceType("time", Type.TIME, + Cardinality.EXACTLY_ONE, "The time component") + }, + new FunctionReturnSequenceType(Type.DATE_TIME, Cardinality.EXACTLY_ONE, + "The combined xs:dateTime")); + + public static final FunctionSignature FN_PARTS_OF_DATETIME = new FunctionSignature( + new QName("parts-of-dateTime", Function.BUILTIN_FUNCTION_NS), + "Decomposes an xs:dateTime into a map of its components.", + new SequenceType[] { + new FunctionParameterSequenceType("dateTime", Type.DATE_TIME, + Cardinality.ZERO_OR_ONE, "The dateTime to decompose") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.ZERO_OR_ONE, + "A map with keys: year, month, day, hour, minute, seconds, timezone")); + + public FnDateTimeParts(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (isCalledAs("build-dateTime")) { + return buildDateTime(args); + } else { + return partsOfDateTime(args); + } + } + + private Sequence buildDateTime(final Sequence[] args) throws XPathException { + final DateValue date = (DateValue) args[0].itemAt(0); + final TimeValue time = (TimeValue) args[1].itemAt(0); + + final int year = date.getPart(AbstractDateTimeValue.YEAR); + final int month = date.getPart(AbstractDateTimeValue.MONTH); + final int day = date.getPart(AbstractDateTimeValue.DAY); + final int hour = time.getPart(AbstractDateTimeValue.HOUR); + final int minute = time.getPart(AbstractDateTimeValue.MINUTE); + final int second = time.getPart(AbstractDateTimeValue.SECOND); + final int millis = time.getPart(AbstractDateTimeValue.MILLISECOND); + + // Timezone: both must agree or one must be absent + final Sequence dateTz = date.getTimezone(); + final Sequence timeTz = time.getTimezone(); + + String tzSuffix = ""; + if (!dateTz.isEmpty() && !timeTz.isEmpty()) { + // Both have timezones — they must be equal + final String dateTzStr = dateTz.getStringValue(); + final String timeTzStr = timeTz.getStringValue(); + if (!dateTzStr.equals(timeTzStr)) { + throw new XPathException(this, ErrorCodes.FORG0008, + "Date and time timezone offsets do not match"); + } + tzSuffix = formatTimezoneOffset(date); + } else if (!dateTz.isEmpty()) { + tzSuffix = formatTimezoneOffset(date); + } else if (!timeTz.isEmpty()) { + tzSuffix = formatTimezoneOffset(time); + } + + // Build the lexical representation + final String fracSeconds = millis > 0 ? "." + String.format("%03d", millis) : ""; + final String lexical = String.format("%04d-%02d-%02dT%02d:%02d:%02d%s%s", + year, month, day, hour, minute, second, fracSeconds, tzSuffix); + + return new DateTimeValue(this, lexical); + } + + private String formatTimezoneOffset(final AbstractDateTimeValue dt) throws XPathException { + final Sequence tz = dt.getTimezone(); + if (tz.isEmpty()) { + return ""; + } + final DayTimeDurationValue dtv = (DayTimeDurationValue) tz; + final int totalMinutes = (int) (dtv.getValueInMilliseconds() / 60000L); + if (totalMinutes == 0) { + return "Z"; + } + final int hours = totalMinutes / 60; + final int mins = Math.abs(totalMinutes % 60); + return String.format("%+03d:%02d", hours, mins); + } + + private Sequence partsOfDateTime(final Sequence[] args) throws XPathException { + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + final DateTimeValue dt = (DateTimeValue) args[0].itemAt(0); + final MapType result = new MapType(this, context); + + // year as xs:integer + result.add(new StringValue("year"), + new IntegerValue(this, dt.getPart(AbstractDateTimeValue.YEAR))); + + // month as xs:integer + result.add(new StringValue("month"), + new IntegerValue(this, dt.getPart(AbstractDateTimeValue.MONTH))); + + // day as xs:integer + result.add(new StringValue("day"), + new IntegerValue(this, dt.getPart(AbstractDateTimeValue.DAY))); + + // hour as xs:integer + result.add(new StringValue("hour"), + new IntegerValue(this, dt.getPart(AbstractDateTimeValue.HOUR))); + + // minute as xs:integer + result.add(new StringValue("minute"), + new IntegerValue(this, dt.getPart(AbstractDateTimeValue.MINUTE))); + + // seconds as xs:decimal (including fractional part) + final int sec = dt.getPart(AbstractDateTimeValue.SECOND); + final int millis = dt.getPart(AbstractDateTimeValue.MILLISECOND); + final BigDecimal seconds = BigDecimal.valueOf(sec) + .add(BigDecimal.valueOf(millis, 3)); + result.add(new StringValue("seconds"), + new DecimalValue(this, seconds)); + + // timezone as xs:dayTimeDuration (or absent) + final Sequence tz = dt.getTimezone(); + if (!tz.isEmpty()) { + result.add(new StringValue("timezone"), tz); + } + + return result; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDecodeFromUri.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDecodeFromUri.java new file mode 100644 index 00000000000..b94f7bc58c1 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDecodeFromUri.java @@ -0,0 +1,183 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** + * Implements XQuery 4.0 fn:decode-from-uri. + * + * Decodes a URI-encoded string. Replaces '+' with space. + * Invalid/incomplete percent sequences are replaced with U+FFFD. + * Resulting octets are decoded as UTF-8; invalid UTF-8 is replaced with U+FFFD. + * XML-invalid codepoints are replaced with U+FFFD. + */ +public class FnDecodeFromUri extends BasicFunction { + + private static final char REPLACEMENT = '\uFFFD'; + + public static final FunctionSignature FN_DECODE_FROM_URI = new FunctionSignature( + new QName("decode-from-uri", Function.BUILTIN_FUNCTION_NS), + "Decodes a URI-encoded string.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.STRING, Cardinality.ZERO_OR_ONE, "The URI-encoded string to decode") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the decoded string")); + + public FnDecodeFromUri(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (args[0].isEmpty()) { + return new StringValue(this, ""); + } + + final String input = args[0].getStringValue(); + + // Phase 1: decode percent-encoding and '+' to bytes, collecting raw bytes + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(input.length()); + final StringBuilder result = new StringBuilder(input.length()); + + int i = 0; + while (i < input.length()) { + final char c = input.charAt(i); + if (c == '+') { + // Flush any accumulated bytes first + flushBytes(bytes, result); + result.append(' '); + i++; + } else if (c == '%') { + // Try to read percent-encoded byte + if (i + 2 < input.length() && isAscii(input.charAt(i + 1))) { + // Two chars follow and first is ASCII — treat as percent triplet + final int hi = hexDigit(input.charAt(i + 1)); + final int lo = hexDigit(input.charAt(i + 2)); + if (hi >= 0 && lo >= 0) { + bytes.write((hi << 4) | lo); + i += 3; + } else { + // Invalid hex pair: consume all 3 chars, produce one replacement + flushBytes(bytes, result); + result.append(REPLACEMENT); + i += 3; + } + } else if (i + 1 < input.length()) { + // First char after % is non-ASCII, or only 1 char follows + // Consume % + next char, produce replacement + flushBytes(bytes, result); + result.append(REPLACEMENT); + i += 2; + } else { + // % at end of string + flushBytes(bytes, result); + result.append(REPLACEMENT); + i++; + } + } else { + flushBytes(bytes, result); + result.append(c); + i++; + } + } + flushBytes(bytes, result); + + // Phase 2: replace XML-invalid codepoints (handle surrogate pairs for supplementary chars) + final StringBuilder cleaned = new StringBuilder(result.length()); + for (int j = 0; j < result.length(); j++) { + final char ch = result.charAt(j); + if (Character.isHighSurrogate(ch) && j + 1 < result.length() + && Character.isLowSurrogate(result.charAt(j + 1))) { + // Valid surrogate pair = supplementary character (valid in XML 1.0 4th+ edition) + cleaned.append(ch); + cleaned.append(result.charAt(++j)); + } else if (isXmlValid(ch)) { + cleaned.append(ch); + } else { + cleaned.append(REPLACEMENT); + } + } + + return new StringValue(this, cleaned.toString()); + } + + /** + * Flush accumulated bytes as UTF-8, replacing invalid sequences with U+FFFD. + */ + private void flushBytes(final ByteArrayOutputStream bytes, final StringBuilder result) { + if (bytes.size() == 0) { + return; + } + final byte[] data = bytes.toByteArray(); + bytes.reset(); + + final CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE) + .replaceWith("\uFFFD"); + + final ByteBuffer bb = ByteBuffer.wrap(data); + final CharBuffer cb = CharBuffer.allocate(data.length * 2); + decoder.decode(bb, cb, true); + decoder.flush(cb); + cb.flip(); + result.append(cb); + } + + private static int hexDigit(final char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + } + + private static boolean isAscii(final char c) { + return c <= 0x7F; + } + + private static boolean isXmlValid(final char c) { + return c == 0x9 || c == 0xA || c == 0xD || + (c >= 0x20 && c <= 0xD7FF) || + (c >= 0xE000 && c <= 0xFFFD); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDeepEqualOptions.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDeepEqualOptions.java new file mode 100644 index 00000000000..05973da16c1 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDeepEqualOptions.java @@ -0,0 +1,84 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import com.ibm.icu.text.Collator; +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.value.*; + +/** + * Implements XQuery 4.0 fn:deep-equal with options parameter (string or map). + * + * Accepts either a collation URI string (XQ3.1 compatible) or an options + * map (XQ4.0) as the 3rd parameter. When an options map is provided, + * validates all option keys/values and uses the options-aware comparison + * engine in {@link DeepEqualOptions}. + */ +public class FnDeepEqualOptions extends BasicFunction { + + public static final FunctionSignature FN_DEEP_EQUAL_OPTIONS = new FunctionSignature( + new QName("deep-equal", Function.BUILTIN_FUNCTION_NS), + "Returns true() iff every item in $items-1 is deep-equal to the item " + + "at the same position in $items-2, using the specified options or collation. " + + "If both $items-1 and $items-2 are the empty sequence, returns true().", + new SequenceType[]{ + new FunctionParameterSequenceType("items-1", Type.ITEM, + Cardinality.ZERO_OR_MORE, "The first item sequence"), + new FunctionParameterSequenceType("items-2", Type.ITEM, + Cardinality.ZERO_OR_MORE, "The second item sequence"), + new FunctionParameterSequenceType("options", Type.ITEM, + Cardinality.ZERO_OR_ONE, "Collation URI string or options map") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, + "true() if the sequences are deep-equal, false() otherwise")); + + public FnDeepEqualOptions(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence items1 = args[0]; + final Sequence items2 = args[1]; + + // Parse 3rd parameter: either string (collation) or map (options) + if (args.length > 2 && !args[2].isEmpty()) { + final Item optionsItem = args[2].itemAt(0); + if (optionsItem instanceof AbstractMapType) { + // XQ4: options map — parse, validate, and use options-aware comparison + final DeepEqualOptions options = DeepEqualOptions.parse( + (AbstractMapType) optionsItem, context); + return BooleanValue.valueOf(options.deepEqualsSeq(items1, items2)); + } else { + // XQ3.1 compat: string collation URI + final Collator collator = context.getCollator(optionsItem.getStringValue()); + return BooleanValue.valueOf(FunDeepEqual.deepEqualsSeq(items1, items2, collator)); + } + } + + // No 3rd parameter — use default comparison + final Collator collator = context.getDefaultCollator(); + return BooleanValue.valueOf(FunDeepEqual.deepEqualsSeq(items1, items2, collator)); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDistinctOrderedNodes.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDistinctOrderedNodes.java new file mode 100644 index 00000000000..e8f6f151094 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDistinctOrderedNodes.java @@ -0,0 +1,71 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +/** + * Implements XQuery 4.0 fn:distinct-ordered-nodes. + * + * Returns nodes in document order with duplicates removed, equivalent to + * the "/" operator's node deduplication behavior. + */ +public class FnDistinctOrderedNodes extends BasicFunction { + + public static final FunctionSignature FN_DISTINCT_ORDERED_NODES = new FunctionSignature( + new QName("distinct-ordered-nodes", Function.BUILTIN_FUNCTION_NS), + "Returns nodes in document order with duplicates removed.", + new SequenceType[] { + new FunctionParameterSequenceType("nodes", Type.NODE, Cardinality.ZERO_OR_MORE, "The nodes to deduplicate and order") + }, + new FunctionReturnSequenceType(Type.NODE, Cardinality.ZERO_OR_MORE, "the deduplicated nodes in document order")); + + public FnDistinctOrderedNodes(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence nodes = args[0]; + if (nodes.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + // ValueSequence with noDups=true handles both document ordering and deduplication + final ValueSequence result = new ValueSequence(true); + result.addAll(nodes); + result.removeDuplicates(); + return result; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDivideDecimals.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDivideDecimals.java new file mode 100644 index 00000000000..0ebd7c732f0 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDivideDecimals.java @@ -0,0 +1,119 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.DecimalValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.IntegerValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; + +import org.exist.xquery.functions.map.MapType; + +import org.exist.xquery.value.AtomicValue; +import org.exist.xquery.value.Item; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +/** + * Implements XQuery 4.0 fn:divide-decimals. + * + * fn:divide-decimals($value, $divisor, $precision?) returns a record with + * quotient and remainder fields. + */ +public class FnDivideDecimals extends BasicFunction { + + public static final FunctionSignature[] FN_DIVIDE_DECIMALS = { + new FunctionSignature( + new QName("divide-decimals", Function.BUILTIN_FUNCTION_NS), + "Divides one decimal by another to specified precision, returning quotient and remainder.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.DECIMAL, Cardinality.EXACTLY_ONE, "The dividend"), + new FunctionParameterSequenceType("divisor", Type.DECIMAL, Cardinality.EXACTLY_ONE, "The divisor"), + new FunctionParameterSequenceType("precision", Type.INTEGER, Cardinality.ZERO_OR_ONE, "Decimal precision (default: 0)") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.EXACTLY_ONE, "record with quotient and remainder")), + new FunctionSignature( + new QName("divide-decimals", Function.BUILTIN_FUNCTION_NS), + "Divides one decimal by another returning integer quotient and remainder.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.DECIMAL, Cardinality.EXACTLY_ONE, "The dividend"), + new FunctionParameterSequenceType("divisor", Type.DECIMAL, Cardinality.EXACTLY_ONE, "The divisor") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.EXACTLY_ONE, "record with quotient and remainder")) + }; + + public FnDivideDecimals(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final BigDecimal value = toBigDecimal(args[0].itemAt(0)); + final BigDecimal divisor = toBigDecimal(args[1].itemAt(0)); + + if (divisor.compareTo(BigDecimal.ZERO) == 0) { + throw new XPathException(this, ErrorCodes.FOAR0001, "Division by zero"); + } + + int precision = 0; + if (args.length > 2 && !args[2].isEmpty()) { + precision = (int) ((IntegerValue) args[2].itemAt(0)).getLong(); + } + + // Quotient: truncate toward zero to given precision + final BigDecimal quotient = value.divide(divisor, precision, RoundingMode.DOWN); + final BigDecimal remainder = value.subtract(quotient.multiply(divisor)); + + // Build result record (map) + final MapType result = new MapType(this, context); + result.add(new StringValue(this, "quotient"), new DecimalValue(this, quotient)); + result.add(new StringValue(this, "remainder"), new DecimalValue(this, remainder)); + + return result; + } + + private BigDecimal toBigDecimal(final Item item) throws XPathException { + final AtomicValue av = item.atomize(); + if (av instanceof DecimalValue) { + return ((DecimalValue) av).getValue(); + } + // xs:integer is a subtype of xs:decimal — use string to avoid long truncation + if (av instanceof IntegerValue) { + return new BigDecimal(av.getStringValue()); + } + // Fallback: convert to decimal + return ((DecimalValue) av.convertTo(Type.DECIMAL)).getValue(); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDuplicateValues.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDuplicateValues.java new file mode 100644 index 00000000000..356b53b6826 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnDuplicateValues.java @@ -0,0 +1,126 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import com.ibm.icu.text.Collator; +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.AtomicValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceIterator; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +/** + * Implements fn:duplicate-values (XQuery 4.0). + * + * Returns the values that appear more than once in the input sequence. + */ +public class FnDuplicateValues extends BasicFunction { + + public static final FunctionSignature[] FN_DUPLICATE_VALUES = { + new FunctionSignature( + new QName("duplicate-values", Function.BUILTIN_FUNCTION_NS), + "Returns those values that appear more than once in the input sequence.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The input values") + }, + new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "the duplicate values")), + new FunctionSignature( + new QName("duplicate-values", Function.BUILTIN_FUNCTION_NS), + "Returns those values that appear more than once in the input sequence, using the specified collation.", + new SequenceType[] { + new FunctionParameterSequenceType("values", Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "The input values"), + new FunctionParameterSequenceType("collation", Type.STRING, Cardinality.ZERO_OR_ONE, "The collation URI") + }, + new FunctionReturnSequenceType(Type.ANY_ATOMIC_TYPE, Cardinality.ZERO_OR_MORE, "the duplicate values")) + }; + + public FnDuplicateValues(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence values = args[0]; + if (values.getItemCount() <= 1) { + return Sequence.EMPTY_SEQUENCE; + } + + final Collator collator = getCollator(args); + + // Use contextual equality (fn:compare = 0) per XQ4 spec + final java.util.List seen = new java.util.ArrayList<>(); + final java.util.List reported = new java.util.ArrayList<>(); + final ValueSequence result = new ValueSequence(); + + for (final SequenceIterator i = values.iterate(); i.hasNext(); ) { + final Item item = i.nextItem(); + final AtomicValue value = item.atomize(); + + boolean isDuplicate = false; + for (final AtomicValue prev : seen) { + if (FnAllEqualDifferent.contextuallyEqual(prev, value, collator)) { + isDuplicate = true; + break; + } + } + + if (isDuplicate) { + // Check if we already reported this value + boolean alreadyReported = false; + for (final AtomicValue rep : reported) { + if (FnAllEqualDifferent.contextuallyEqual(rep, value, collator)) { + alreadyReported = true; + break; + } + } + if (!alreadyReported) { + result.add(value); + reported.add(value); + } + } else { + seen.add(value); + } + } + return result; + } + + private Collator getCollator(final Sequence[] args) throws XPathException { + if (args.length > 1 && !args[1].isEmpty()) { + final String collationURI = args[1].getStringValue(); + return context.getCollator(collationURI, ErrorCodes.FOCH0002); + } + return context.getDefaultCollator(); + } + +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnElementToMap.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnElementToMap.java new file mode 100644 index 00000000000..5635586ee7a --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnElementToMap.java @@ -0,0 +1,458 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.functions.array.ArrayType; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.value.*; +import org.w3c.dom.*; + +import javax.xml.XMLConstants; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Implements XQuery 4.0 fn:element-to-map. + * + * Converts an element node to a map representation following the XQ4 spec rules + * for different content models (empty, simple, record, list, sequence, mixed). + */ +public class FnElementToMap extends BasicFunction { + + public static final FunctionSignature[] FN_ELEMENT_TO_MAP = { + new FunctionSignature( + new QName("element-to-map", Function.BUILTIN_FUNCTION_NS), + "Converts an element to a map representation.", + new SequenceType[]{ + new FunctionParameterSequenceType("element", Type.ELEMENT, Cardinality.ZERO_OR_ONE, "The element to convert") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.ZERO_OR_ONE, "The map representation")), + new FunctionSignature( + new QName("element-to-map", Function.BUILTIN_FUNCTION_NS), + "Converts an element to a map representation with options.", + new SequenceType[]{ + new FunctionParameterSequenceType("element", Type.ELEMENT, Cardinality.ZERO_OR_ONE, "The element to convert"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, Cardinality.ZERO_OR_ONE, "Options map") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.ZERO_OR_ONE, "The map representation")) + }; + + private static final String DEFAULT_ATTR_MARKER = "@"; + private static final String DEFAULT_CONTENT_KEY = "#content"; + private static final String DEFAULT_COMMENT_KEY = "#comment"; + private static final String DEFAULT_NAME_FORMAT = "eqname"; + + public FnElementToMap(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + final Node node = ((NodeValue) args[0].itemAt(0)).getNode(); + if (node.getNodeType() != Node.ELEMENT_NODE) { + throw new XPathException(this, ErrorCodes.XPTY0004, "Expected element node"); + } + + // Parse options + String nameFormat = DEFAULT_NAME_FORMAT; + String attrMarker = DEFAULT_ATTR_MARKER; + String contentKey = DEFAULT_CONTENT_KEY; + String commentKey = DEFAULT_COMMENT_KEY; + + if (args.length > 1 && !args[1].isEmpty()) { + final MapType options = (MapType) args[1].itemAt(0); + final Sequence nfSeq = options.get(new StringValue(this, "name-format")); + if (nfSeq != null && !nfSeq.isEmpty()) { + nameFormat = nfSeq.getStringValue(); + } + final Sequence amSeq = options.get(new StringValue(this, "attribute-marker")); + if (amSeq != null && !amSeq.isEmpty()) { + attrMarker = amSeq.getStringValue(); + } + final Sequence ckSeq = options.get(new StringValue(this, "content-key")); + if (ckSeq != null && !ckSeq.isEmpty()) { + contentKey = ckSeq.getStringValue(); + } + final Sequence cmSeq = options.get(new StringValue(this, "comment-key")); + if (cmSeq != null && !cmSeq.isEmpty()) { + commentKey = cmSeq.getStringValue(); + } + } + + final Options opts = new Options(nameFormat, attrMarker, contentKey, commentKey); + return convertElement((Element) node, opts); + } + + private MapType convertElement(final Element elem, final Options opts) throws XPathException { + final String elemName = formatName(elem, opts); + final Sequence value = convertContent(elem, opts); + + MapType result = new MapType(this, context); + result = (MapType) result.put(new StringValue(this, elemName), value); + return result; + } + + private Sequence convertContent(final Element elem, final Options opts) throws XPathException { + // Collect attributes (excluding xmlns and xsi:type) + final Map attrs = new LinkedHashMap<>(); + final NamedNodeMap attrNodes = elem.getAttributes(); + if (attrNodes != null) { + for (int i = 0; i < attrNodes.getLength(); i++) { + final Attr attr = (Attr) attrNodes.item(i); + final String attrName = attr.getName(); + // Skip namespace declarations and xsi:type + if (attrName.startsWith("xmlns") && (attrName.length() == 5 || attrName.charAt(5) == ':')) { + continue; + } + if ("xsi:type".equals(attrName)) { + continue; + } + if (attrName.equals("xsi:nil")) { + continue; + } + final String key = opts.attrMarker + formatAttrName(attr, opts); + attrs.put(key, attr.getValue()); + } + } + + // Check for xsi:nil + final String nilAttr = elem.getAttributeNS(XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI, "nil"); + if ("true".equals(nilAttr) || "1".equals(nilAttr)) { + if (attrs.isEmpty()) { + // Return fn:null() as QName + return new QNameValue(this, context, new QName("null", Function.BUILTIN_FUNCTION_NS, "fn")); + } else { + MapType attrMap = new MapType(this, context); + for (final Map.Entry a : attrs.entrySet()) { + attrMap = (MapType) attrMap.put(new StringValue(this, a.getKey()), new StringValue(this, a.getValue())); + } + attrMap = (MapType) attrMap.put( + new StringValue(this, opts.contentKey), + new QNameValue(this, context, new QName("null", Function.BUILTIN_FUNCTION_NS, "fn"))); + return attrMap; + } + } + + // Collect child nodes (elements, text, comments, PIs) + final List children = new ArrayList<>(); + final NodeList childNodes = elem.getChildNodes(); + for (int i = 0; i < childNodes.getLength(); i++) { + final Node child = childNodes.item(i); + switch (child.getNodeType()) { + case Node.ELEMENT_NODE: + case Node.TEXT_NODE: + case Node.CDATA_SECTION_NODE: + case Node.COMMENT_NODE: + children.add(child); + break; + default: + break; + } + } + + // Classify content model + final boolean hasElements = children.stream().anyMatch(n -> n.getNodeType() == Node.ELEMENT_NODE); + final boolean hasTextContent = children.stream().anyMatch(n -> + (n.getNodeType() == Node.TEXT_NODE || n.getNodeType() == Node.CDATA_SECTION_NODE) + && !n.getTextContent().trim().isEmpty()); + final boolean hasComments = children.stream().anyMatch(n -> n.getNodeType() == Node.COMMENT_NODE); + + // Empty element + if (children.isEmpty() || (!hasElements && !hasTextContent && !hasComments)) { + if (attrs.isEmpty()) { + return new StringValue(this, ""); + } else { + // Empty-plus: attributes only, no #content key + MapType attrMap = new MapType(this, context); + for (final Map.Entry a : attrs.entrySet()) { + attrMap = (MapType) attrMap.put(new StringValue(this, a.getKey()), new StringValue(this, a.getValue())); + } + return attrMap; + } + } + + // Simple text content (no child elements) + if (!hasElements && !hasComments) { + final String textContent = getTextContent(children); + if (attrs.isEmpty()) { + return new StringValue(this, textContent); + } else { + return buildAttrMap(attrs, new StringValue(this, textContent), opts); + } + } + + // Mixed content (has both text and element children) + if (hasTextContent && hasElements) { + return buildMixedContent(children, attrs, opts); + } + + // Element-only content — determine layout + final List childElements = new ArrayList<>(); + for (final Node child : children) { + if (child.getNodeType() == Node.ELEMENT_NODE) { + childElements.add((Element) child); + } + } + + // Check for comments interleaved with elements + if (hasComments && !hasElements) { + return buildMixedContent(children, attrs, opts); + } + + // Check if all children have the same name (list pattern) + final boolean allSameName = childElements.size() > 1 && + childElements.stream().allMatch(e -> + formatName(e, opts).equals(formatName(childElements.get(0), opts))); + + // Check if all children have unique names (record pattern) + final Map> groupedByName = new LinkedHashMap<>(); + for (final Element child : childElements) { + groupedByName.computeIfAbsent(formatName(child, opts), k -> new ArrayList<>()).add(child); + } + final boolean allUnique = groupedByName.values().stream().allMatch(l -> l.size() == 1); + + if (allSameName) { + // List layout: array of child values + return buildListContent(childElements, attrs, opts); + } else if (allUnique) { + // Record layout: map of child name → value + return buildRecordContent(childElements, attrs, children, opts); + } else { + // Sequence layout: array of child maps + return buildSequenceContent(children, attrs, opts); + } + } + + private Sequence buildAttrMap(final Map attrs, final Sequence contentValue, final Options opts) throws XPathException { + MapType attrMap = new MapType(this, context); + for (final Map.Entry a : attrs.entrySet()) { + attrMap = (MapType) attrMap.put(new StringValue(this, a.getKey()), new StringValue(this, a.getValue())); + } + attrMap = (MapType) attrMap.put(new StringValue(this, opts.contentKey), contentValue); + return attrMap; + } + + private Sequence buildListContent(final List children, final Map attrs, final Options opts) throws XPathException { + // Array of child content values + final List items = new ArrayList<>(); + for (final Element child : children) { + items.add(convertContent(child, opts)); + } + final ArrayType array = new ArrayType(this, context, items); + + if (attrs.isEmpty()) { + return array; + } else { + MapType attrMap = new MapType(this, context); + for (final Map.Entry a : attrs.entrySet()) { + attrMap = (MapType) attrMap.put(new StringValue(this, a.getKey()), new StringValue(this, a.getValue())); + } + attrMap = (MapType) attrMap.put(new StringValue(this, opts.contentKey), array); + return attrMap; + } + } + + private Sequence buildRecordContent(final List childElements, final Map attrs, + final List allChildren, final Options opts) throws XPathException { + MapType recordMap = new MapType(this, context); + + // Add attributes first + for (final Map.Entry a : attrs.entrySet()) { + recordMap = (MapType) recordMap.put(new StringValue(this, a.getKey()), new StringValue(this, a.getValue())); + } + + // Add comments if present + for (final Node child : allChildren) { + if (child.getNodeType() == Node.COMMENT_NODE) { + recordMap = (MapType) recordMap.put( + new StringValue(this, opts.commentKey), + new StringValue(this, child.getTextContent())); + } + } + + // Add child elements + for (final Element child : childElements) { + final String childName = formatName(child, opts); + final Sequence childValue = convertContent(child, opts); + recordMap = (MapType) recordMap.put(new StringValue(this, childName), childValue); + } + + return recordMap; + } + + private Sequence buildSequenceContent(final List children, final Map attrs, final Options opts) throws XPathException { + // Build array of child maps/values + final List items = new ArrayList<>(); + for (final Node child : children) { + if (child.getNodeType() == Node.ELEMENT_NODE) { + items.add(convertElement((Element) child, opts)); + } else if (child.getNodeType() == Node.TEXT_NODE || child.getNodeType() == Node.CDATA_SECTION_NODE) { + final String text = child.getTextContent(); + if (!text.trim().isEmpty()) { + items.add(new StringValue(this, text)); + } + } else if (child.getNodeType() == Node.COMMENT_NODE) { + MapType commentMap = new MapType(this, context); + commentMap = (MapType) commentMap.put( + new StringValue(this, opts.commentKey), + new StringValue(this, child.getTextContent())); + items.add(commentMap); + } + } + final ArrayType array = new ArrayType(this, context, items); + + if (attrs.isEmpty()) { + return array; + } else { + MapType attrMap = new MapType(this, context); + for (final Map.Entry a : attrs.entrySet()) { + attrMap = (MapType) attrMap.put(new StringValue(this, a.getKey()), new StringValue(this, a.getValue())); + } + attrMap = (MapType) attrMap.put(new StringValue(this, opts.contentKey), array); + return attrMap; + } + } + + private Sequence buildMixedContent(final List children, final Map attrs, final Options opts) throws XPathException { + final List items = new ArrayList<>(); + for (final Node child : children) { + switch (child.getNodeType()) { + case Node.ELEMENT_NODE: + items.add(convertElement((Element) child, opts)); + break; + case Node.TEXT_NODE: + case Node.CDATA_SECTION_NODE: + final String text = child.getTextContent(); + if (!text.isEmpty()) { + items.add(new StringValue(this, text)); + } + break; + case Node.COMMENT_NODE: + MapType commentMap = new MapType(this, context); + commentMap = (MapType) commentMap.put( + new StringValue(this, opts.commentKey), + new StringValue(this, child.getTextContent())); + items.add(commentMap); + break; + default: + break; + } + } + final ArrayType array = new ArrayType(this, context, items); + + if (attrs.isEmpty()) { + return array; + } else { + MapType attrMap = new MapType(this, context); + for (final Map.Entry a : attrs.entrySet()) { + attrMap = (MapType) attrMap.put(new StringValue(this, a.getKey()), new StringValue(this, a.getValue())); + } + attrMap = (MapType) attrMap.put(new StringValue(this, opts.contentKey), array); + return attrMap; + } + } + + private String formatName(final Element elem, final Options opts) { + final String ns = elem.getNamespaceURI(); + final String local = elem.getLocalName() != null ? elem.getLocalName() : elem.getTagName(); + + switch (opts.nameFormat) { + case "eqname": + if (ns != null && !ns.isEmpty()) { + return "Q{" + ns + "}" + local; + } + return local; + case "lexical": + final String prefix = elem.getPrefix(); + if (prefix != null && !prefix.isEmpty()) { + return prefix + ":" + local; + } + return local; + case "local": + return local; + default: + // Default to eqname + if (ns != null && !ns.isEmpty()) { + return "Q{" + ns + "}" + local; + } + return local; + } + } + + private String formatAttrName(final Attr attr, final Options opts) { + final String ns = attr.getNamespaceURI(); + final String local = attr.getLocalName() != null ? attr.getLocalName() : attr.getName(); + + switch (opts.nameFormat) { + case "eqname": + if (ns != null && !ns.isEmpty()) { + return "Q{" + ns + "}" + local; + } + return local; + case "lexical": + final String prefix = attr.getPrefix(); + if (prefix != null && !prefix.isEmpty()) { + return prefix + ":" + local; + } + return local; + case "local": + return local; + default: + if (ns != null && !ns.isEmpty()) { + return "Q{" + ns + "}" + local; + } + return local; + } + } + + private static String getTextContent(final List children) { + final StringBuilder sb = new StringBuilder(); + for (final Node child : children) { + if (child.getNodeType() == Node.TEXT_NODE || child.getNodeType() == Node.CDATA_SECTION_NODE) { + sb.append(child.getTextContent()); + } + } + return sb.toString(); + } + + private static class Options { + final String nameFormat; + final String attrMarker; + final String contentKey; + final String commentKey; + + Options(final String nameFormat, final String attrMarker, final String contentKey, final String commentKey) { + this.nameFormat = nameFormat; + this.attrMarker = attrMarker; + this.contentKey = contentKey; + this.commentKey = commentKey; + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnElementToMapPlan.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnElementToMapPlan.java new file mode 100644 index 00000000000..57afc1dc54e --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnElementToMapPlan.java @@ -0,0 +1,263 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.value.*; +import org.w3c.dom.*; + +import java.util.*; + +/** + * fn:element-to-map-plan($input as node()*) as map(*) + * + * Analyzes the structure of input elements and returns a plan map + * describing the layout of each element type encountered. + * + * Layout values: empty, empty-plus, simple, simple-plus, list, list-plus, + * record, mixed. + */ +public class FnElementToMapPlan extends BasicFunction { + + public static final FunctionSignature FN_ELEMENT_TO_MAP_PLAN = new FunctionSignature( + new QName("element-to-map-plan", Function.BUILTIN_FUNCTION_NS), + "Analyzes the structure of input elements and returns a plan map.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.NODE, + Cardinality.ZERO_OR_MORE, "The input nodes to analyze") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.EXACTLY_ONE, + "A map describing the element layouts")); + + public FnElementToMapPlan(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (args[0].isEmpty()) { + return new MapType(this, context); + } + + final MapType plan = new MapType(this, context); + final Set processed = new HashSet<>(); + + // Analyze each input node + for (final SequenceIterator iter = args[0].iterate(); iter.hasNext(); ) { + final Item item = iter.nextItem(); + if (item.getType() == Type.DOCUMENT) { + // For document nodes, analyze the document element + final Node docNode = ((NodeValue) item).getNode(); + analyzeNode(docNode, plan, processed); + } else if (Type.subTypeOf(item.getType(), Type.ELEMENT)) { + final Node elemNode = ((NodeValue) item).getNode(); + analyzeElement(elemNode, plan, processed); + } + } + + return plan; + } + + private void analyzeNode(final Node node, final MapType plan, final Set processed) throws XPathException { + final NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + final Node child = children.item(i); + if (child.getNodeType() == Node.ELEMENT_NODE) { + analyzeElement(child, plan, processed); + } + } + } + + private void analyzeElement(final Node elem, final MapType plan, final Set processed) throws XPathException { + final String elemKey = getElementKey(elem); + if (processed.contains(elemKey)) { + return; // Already analyzed this element type + } + processed.add(elemKey); + + final MapType layoutMap = new MapType(this, context); + + // Determine layout + final boolean hasAttributes = hasSignificantAttributes(elem); + final List childElements = getChildElements(elem); + final boolean hasTextContent = hasSignificantTextContent(elem); + + if (childElements.isEmpty() && !hasTextContent) { + // Empty element + layoutMap.add(new StringValue("layout"), + new StringValue(hasAttributes ? "empty-plus" : "empty")); + } else if (childElements.isEmpty() && hasTextContent) { + // Simple content (text only) + final String type = detectContentType(elem); + layoutMap.add(new StringValue("layout"), + new StringValue(hasAttributes ? "simple-plus" : "simple")); + if (type != null) { + layoutMap.add(new StringValue("type"), new StringValue(type)); + } + } else if (!hasTextContent && allChildrenSameName(childElements)) { + // List of same-named elements + final String childName = getElementKey(childElements.get(0)); + layoutMap.add(new StringValue("layout"), + new StringValue(hasAttributes ? "list-plus" : "list")); + layoutMap.add(new StringValue("child"), new StringValue(childName)); + } else if (hasTextContent || hasMixedContent(elem)) { + // Mixed content + layoutMap.add(new StringValue("layout"), new StringValue("mixed")); + } else { + // Record (distinct child element names) + layoutMap.add(new StringValue("layout"), new StringValue("record")); + } + + plan.add(new StringValue(elemKey), layoutMap); + + // Analyze attribute types + if (hasAttributes) { + final NamedNodeMap attrs = elem.getAttributes(); + for (int i = 0; i < attrs.getLength(); i++) { + final Node attr = attrs.item(i); + final String attrName = attr.getLocalName() != null ? attr.getLocalName() : attr.getNodeName(); + final String ns = attr.getNamespaceURI(); + // Skip xmlns declarations + if ("http://www.w3.org/2000/xmlns/".equals(ns) || attrName.startsWith("xmlns")) { + continue; + } + final String attrKey = "@" + (ns != null && !ns.isEmpty() ? + "Q{" + ns + "}" + attrName : attrName); + if (!processed.contains(attrKey)) { + processed.add(attrKey); + final MapType attrMap = new MapType(this, context); + final String type = detectValueType(attr.getNodeValue()); + if (type != null) { + attrMap.add(new StringValue("type"), new StringValue(type)); + } + plan.add(new StringValue(attrKey), attrMap); + } + } + } + + // Recursively analyze child elements + for (final Node child : childElements) { + analyzeElement(child, plan, processed); + } + } + + private String getElementKey(final Node elem) { + final String ns = elem.getNamespaceURI(); + final String local = elem.getLocalName() != null ? elem.getLocalName() : elem.getNodeName(); + if (ns != null && !ns.isEmpty()) { + return "Q{" + ns + "}" + local; + } + return local; + } + + private boolean hasSignificantAttributes(final Node elem) { + final NamedNodeMap attrs = elem.getAttributes(); + if (attrs == null) return false; + for (int i = 0; i < attrs.getLength(); i++) { + final Node attr = attrs.item(i); + final String name = attr.getNodeName(); + if (!name.startsWith("xmlns")) { + return true; + } + } + return false; + } + + private List getChildElements(final Node elem) { + final List result = new ArrayList<>(); + final NodeList children = elem.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + if (children.item(i).getNodeType() == Node.ELEMENT_NODE) { + result.add(children.item(i)); + } + } + return result; + } + + private boolean hasSignificantTextContent(final Node elem) { + final NodeList children = elem.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + final Node child = children.item(i); + if (child.getNodeType() == Node.TEXT_NODE) { + final String text = child.getNodeValue(); + if (text != null && !text.trim().isEmpty()) { + return true; + } + } + } + return false; + } + + private boolean hasMixedContent(final Node elem) { + boolean hasElements = false; + boolean hasText = false; + final NodeList children = elem.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + final Node child = children.item(i); + if (child.getNodeType() == Node.ELEMENT_NODE) { + hasElements = true; + } else if (child.getNodeType() == Node.TEXT_NODE) { + if (child.getNodeValue() != null && !child.getNodeValue().trim().isEmpty()) { + hasText = true; + } + } + } + return hasElements && hasText; + } + + private boolean allChildrenSameName(final List children) { + if (children.isEmpty()) return false; + final String firstName = getElementKey(children.get(0)); + for (int i = 1; i < children.size(); i++) { + if (!firstName.equals(getElementKey(children.get(i)))) { + return false; + } + } + return true; + } + + private String detectContentType(final Node elem) { + final String text = elem.getTextContent(); + if (text == null || text.trim().isEmpty()) { + return null; + } + return detectValueType(text.trim()); + } + + private String detectValueType(final String value) { + if (value == null || value.isEmpty()) { + return null; + } + try { + Double.parseDouble(value); + return "numeric"; + } catch (final NumberFormatException e) { + // Not numeric + } + if ("true".equals(value) || "false".equals(value)) { + return "boolean"; + } + return null; // default: string (not annotated) + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnEverySome.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnEverySome.java new file mode 100644 index 00000000000..ee18e143012 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnEverySome.java @@ -0,0 +1,177 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.AnalyzeContextInfo; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReference; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.IntegerValue; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceIterator; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * Implements XQuery 4.0 fn:every and fn:some. + */ +public class FnEverySome extends BasicFunction { + + public static final FunctionSignature[] FN_EVERY = { + new FunctionSignature( + new QName("every", Function.BUILTIN_FUNCTION_NS), + "Returns true if every item in the input sequence matches the supplied predicate.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("predicate", Type.FUNCTION, Cardinality.ZERO_OR_ONE, "The predicate function (defaults to fn:boolean#1)") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if all items match")), + new FunctionSignature( + new QName("every", Function.BUILTIN_FUNCTION_NS), + "Returns true if every item in the input sequence has an effective boolean value of true.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if all items are truthy")) + }; + + public static final FunctionSignature[] FN_SOME = { + new FunctionSignature( + new QName("some", Function.BUILTIN_FUNCTION_NS), + "Returns true if at least one item in the input sequence matches the supplied predicate.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("predicate", Type.FUNCTION, Cardinality.ZERO_OR_ONE, "The predicate function (defaults to fn:boolean#1)") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if any item matches")), + new FunctionSignature( + new QName("some", Function.BUILTIN_FUNCTION_NS), + "Returns true if at least one item in the input sequence has an effective boolean value of true.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if any item is truthy")) + }; + + private AnalyzeContextInfo cachedContextInfo; + + public FnEverySome(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + cachedContextInfo = new AnalyzeContextInfo(contextInfo); + super.analyze(cachedContextInfo); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence input = args[0]; + final boolean isEvery = isCalledAs("every"); + + // 1-arg overload: use effective boolean value + if (args.length == 1) { + return evalWithEBV(input, isEvery); + } + + // 2-arg overload: use predicate function (empty predicate = use EBV) + if (args[1].isEmpty()) { + return evalWithEBV(input, isEvery); + } + + if (input.isEmpty()) { + return BooleanValue.valueOf(isEvery); + } + + try (final FunctionReference ref = (FunctionReference) args[1].itemAt(0)) { + ref.analyze(cachedContextInfo); + final int arity = ref.getSignature().getArgumentCount(); + + // Validate arity: predicate must accept 0, 1, or 2 arguments + if (arity > 2) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Predicate function must accept 0, 1, or 2 arguments, but has arity " + arity); + } + + int pos = 1; + for (final SequenceIterator i = input.iterate(); i.hasNext(); pos++) { + final Item item = i.nextItem(); + final Sequence r = callPredicate(ref, item, pos, arity); + // XQ4: predicate must return xs:boolean (xs:untypedAtomic is coercible) + if (!r.isEmpty()) { + final int rType = r.itemAt(0).getType(); + if (rType != Type.BOOLEAN && rType != Type.UNTYPED_ATOMIC) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Predicate function must return xs:boolean, but returned " + + Type.getTypeName(rType)); + } + } + final boolean matches = !r.isEmpty() && r.effectiveBooleanValue(); + if (isEvery && !matches) { + return BooleanValue.FALSE; + } + if (!isEvery && matches) { + return BooleanValue.TRUE; + } + } + return BooleanValue.valueOf(isEvery); + } + } + + private Sequence evalWithEBV(final Sequence input, final boolean isEvery) throws XPathException { + if (input.isEmpty()) { + return BooleanValue.valueOf(isEvery); + } + for (final SequenceIterator i = input.iterate(); i.hasNext(); ) { + final Item item = i.nextItem(); + final boolean ebv = item.toSequence().effectiveBooleanValue(); + if (isEvery && !ebv) { + return BooleanValue.FALSE; + } + if (!isEvery && ebv) { + return BooleanValue.TRUE; + } + } + return BooleanValue.valueOf(isEvery); + } + + private Sequence callPredicate(final FunctionReference ref, final Item item, final int pos, final int arity) throws XPathException { + if (arity == 0) { + return ref.evalFunction(null, null, new Sequence[0]); + } else if (arity == 1) { + return ref.evalFunction(null, null, new Sequence[]{item.toSequence()}); + } else { + return ref.evalFunction(null, null, new Sequence[]{item.toSequence(), new IntegerValue(this, pos)}); + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnExpandedQName.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnExpandedQName.java new file mode 100644 index 00000000000..7dc2190314a --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnExpandedQName.java @@ -0,0 +1,74 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.QNameValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; + +/** + * Implements fn:expanded-QName (XQuery 4.0). + * + * Returns a string in Q{uri}local format for a QName value. + */ +public class FnExpandedQName extends BasicFunction { + + public static final FunctionSignature FN_EXPANDED_QNAME = new FunctionSignature( + new QName("expanded-QName", Function.BUILTIN_FUNCTION_NS), + "Returns the expanded QName in Q{uri}local notation.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.QNAME, Cardinality.ZERO_OR_ONE, + "The QName value") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_ONE, + "the expanded QName string in Q{uri}local format")); + + public FnExpandedQName(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + final QNameValue qnameVal = (QNameValue) args[0].itemAt(0); + final QName qname = qnameVal.getQName(); + + final String ns = qname.getNamespaceURI() != null ? qname.getNamespaceURI() : ""; + final String local = qname.getLocalPart(); + + return new StringValue(this, "Q{" + ns + "}" + local); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFormatDates.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFormatDates.java index 2ade21d3117..c39b28f29a6 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFormatDates.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFormatDates.java @@ -21,12 +21,17 @@ */ package org.exist.xquery.functions.fn; +import com.ibm.icu.text.MessageFormat; +import com.ibm.icu.text.RuleBasedNumberFormat; +import org.apache.commons.lang3.StringUtils; import org.exist.dom.QName; import org.exist.xquery.*; import org.exist.xquery.util.NumberFormatter; import org.exist.xquery.value.*; +import java.util.ArrayList; import java.util.Calendar; +import java.util.List; import java.util.Locale; import java.util.Optional; import java.util.TimeZone; @@ -152,6 +157,7 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce final String picture = args[1].getStringValue(); final String language; final Optional place; + String calendar = null; if (getArgumentCount() == 5) { if (args[2].hasOne()) { language = args[2].getStringValue(); @@ -159,6 +165,10 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce language = context.getDefaultLanguage(); } + if (args[3].hasOne()) { + calendar = args[3].getStringValue(); + } + if(args[4].hasOne()) { place = Optional.of(args[4].getStringValue()); } else { @@ -169,6 +179,32 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce place = Optional.empty(); } + // Validate calendar parameter + if (calendar != null) { + if (calendar.startsWith(":")) { + throw new XPathException(this, ErrorCodes.FOFD1340, + "Invalid calendar name: " + calendar); + } + if (calendar.startsWith("Q{}")) { + final String localPart = calendar.substring(3); + if (localPart.isEmpty() || !Character.isLetter(localPart.charAt(0))) { + throw new XPathException(this, ErrorCodes.FOFD1340, + "Invalid calendar name: " + calendar); + } + if (!isKnownCalendar(localPart)) { + throw new XPathException(this, ErrorCodes.FOFD1340, + "Unknown calendar: " + calendar); + } + } else if (calendar.startsWith("Q{") && calendar.contains("}")) { + // EQName with non-empty namespace: accept with fallback + } else if (calendar.contains(":")) { + // Prefixed QName: accept with fallback + } else if (!isKnownCalendar(calendar)) { + throw new XPathException(this, ErrorCodes.FOFD1340, + "Unknown calendar: " + calendar); + } + } + return new StringValue(this, formatDate(picture, value, language, place)); } @@ -214,6 +250,8 @@ private String formatDate(String pic, AbstractDateTimeValue dt, final String lan private void formatComponent(String component, AbstractDateTimeValue dt, final String language, final Optional place, final boolean tzHMZNPictureHint, final StringBuilder sb) throws XPathException { + // Per spec, whitespace within a variable marker is insignificant + component = component.replaceAll("\\s+", ""); final Matcher matcher = componentPattern.matcher(component); if (!matcher.matches()) { throw new XPathException(this, ErrorCodes.FOFD1340, "Unrecognized date/time component: " + component); @@ -349,8 +387,8 @@ private void formatComponent(String component, AbstractDateTimeValue dt, final S break; case 'f': if (allowTime) { - final int fraction = dt.getPart(AbstractDateTimeValue.MILLISECOND); - formatNumber(specifier, picture, width, fraction, language, sb); + final int millis = dt.getPart(AbstractDateTimeValue.MILLISECOND); + formatFractionalSeconds(millis, picture, width, sb); } else { throw new XPathException(this, ErrorCodes.FOFD1350, "format-date does not support a fractional seconds component"); @@ -384,85 +422,255 @@ private void formatComponent(String component, AbstractDateTimeValue dt, final S sb.append(formatTimeZone(picture, dtv.getPart(DurationValue.HOUR), minute, cal.getTimeZone(), language, place)); + } else if ("Z".equals(picture)) { + // Military timezone: J = local time (no timezone specified) + sb.append("J"); } break; + case 'E': + if (allowDate) { + final int year = dt.getPart(AbstractDateTimeValue.YEAR); + sb.append(year >= 0 ? "AD" : "BC"); + } else { + throw new XPathException(this, ErrorCodes.FOFD1350, + "format-time does not support an era component"); + } + break; + case 'C': + sb.append("AD"); + break; default: throw new XPathException(this, ErrorCodes.FOFD1340, "Unrecognized date/time component: " + component); } } - private String formatTimeZone(final String timezonePicture, final int hour, final int minute, + private String formatTimeZone(String timezonePicture, final int hour, final int minute, final TimeZone timeZone, final String language, final Optional place) { - final Locale locale = new Locale(language); + // Military timezone letter + if ("Z".equals(timezonePicture)) { + return formatMilitaryTimeZone(hour, minute); + } - final String format; - switch(timezonePicture) { - case "0": - if(minute != 0) { - format = "%+d:%02d"; + // Named timezone + if ("N".equals(timezonePicture)) { + final Locale locale = new Locale(language); + final TimeZone tz = place.map(TimeZone::getTimeZone).orElse(timeZone); + return tz.getDisplayName(timeZone.useDaylightTime(), TimeZone.SHORT, locale); + } + + // Check for 't' modifier (use "Z" for UTC) + final boolean useZForUTC = timezonePicture.endsWith("t"); + if (useZForUTC) { + timezonePicture = timezonePicture.substring(0, timezonePicture.length() - 1); + } + if (useZForUTC && hour == 0 && minute == 0) { + return "Z"; + } + + // Parse the picture: find digit family, separator, hour/minute digit counts + int zero = '0'; + boolean zeroFound = false; + int hourDigits = 0; + int minuteDigits = 0; + String separator = null; + + for (int i = 0; i < timezonePicture.length(); i++) { + final int ch = timezonePicture.codePointAt(i); + final int family = net.sf.saxon.expr.number.Alphanumeric.getDigitFamily(ch); + if (family >= 0) { + if (!zeroFound) { zero = family; zeroFound = true; } + if (separator == null) { hourDigits++; } else { minuteDigits++; } + } else if (ch == '#') { + if (separator == null) { hourDigits++; } else { minuteDigits++; } + } else if (separator == null && hourDigits > 0) { + separator = new String(Character.toChars(ch)); + } + if (Character.isSupplementaryCodePoint(ch)) { i++; } + } + + final int absHour = Math.abs(hour); + final String sign = (hour < 0) ? "-" : "+"; + final StringBuilder result = new StringBuilder(sign); + + if (separator != null && minuteDigits > 0) { + result.append(padWithDigitFamily(absHour, hourDigits, zero)); + result.append(separator); + result.append(padWithDigitFamily(minute, minuteDigits, zero)); + } else if (hourDigits >= 3) { + result.append(padWithDigitFamily(absHour * 100 + minute, hourDigits, zero)); + } else { + result.append(padWithDigitFamily(absHour, hourDigits, zero)); + if (minute != 0) { + result.append(":"); + result.append(padWithDigitFamily(minute, 2, zero)); + } + } + + return result.toString(); + } + + private static String padWithDigitFamily(int value, int minDigits, int zero) { + String s = Integer.toString(value); + while (s.length() < minDigits) { s = "0" + s; } + if (zero != '0') { + final StringBuilder converted = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + final char ch = s.charAt(i); + if (ch >= '0' && ch <= '9') { + converted.appendCodePoint(zero + (ch - '0')); } else { - format = "%+d"; + converted.append(ch); } - break; + } + return converted.toString(); + } + return s; + } - case "0000": - format = "%+03d%02d"; - break; + // Military timezone: Z(0), A-I(+1 to +9), K-M(+10 to +12), N-Y(-1 to -12) + // J is reserved for local time (no timezone) and is NOT in this array + private final static char[] MILITARY_TZ_CHARS = {'Z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'L', + 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y' }; - case "0:00": - format = "%+d:%02d"; - break; + private String formatMilitaryTimeZone(final int hour, final int minute) { + if (minute == 0 && hour >= -12 && hour <= 12) { + final int offset = (hour < 0) ? 12 + (hour * -1) : hour; + return String.valueOf(MILITARY_TZ_CHARS[offset]); + } else { + return String.format("%+03d:%02d", hour, minute); + } + } + + /** + * Format fractional seconds as left-aligned digits. + * Unlike regular integer formatting, fractional seconds treat the value + * as a fraction (0.456) where digits are extracted left-to-right. + */ + private void formatFractionalSeconds(int millis, String picture, String width, + StringBuilder sb) throws XPathException { + // Build the fractional digit string, left-aligned, padded to 3 digits + String fracDigits = String.format("%03d", millis); + + // Count actual digit positions in picture (ignoring separators and modifiers) + int picMin = 0; + int picMax = 0; + for (int i = 0; i < picture.length(); i++) { + final char ch = picture.charAt(i); + if ((ch == 'o' || ch == 'c') && i == picture.length() - 1) { break; } + final int family = net.sf.saxon.expr.number.Alphanumeric.getDigitFamily(ch); + if (family >= 0) { + picMin++; + picMax++; + } else if (ch == '#') { + picMax++; + } + } + + int min = picMin; + // A multi-digit picture constrains max precision; single-digit is unbounded + final boolean pictureSetsMax = (picMax > 1); + int max = pictureSetsMax ? picMax : Integer.MAX_VALUE; - case "00:00t": - if(hour == 0 && minute == 0) { - format = "Z"; + // Width specifier + final int[] widths = getWidths(width); + if (widths != null) { + if (widths[0] > 0) { min = Math.max(picMin, widths[0]); } + if (widths[1] > 0) { + if (pictureSetsMax) { + max = Math.max(picMax, widths[1]); } else { - format = "%+03d:%02d"; + max = widths[1]; } - break; + } + } + if (max < min) { max = min; } - case "N": - final TimeZone tz = place.map(TimeZone::getTimeZone).orElse(timeZone); - return tz.getDisplayName(timeZone.useDaylightTime(), TimeZone.SHORT, locale); + // Pad to min with trailing zeros + while (fracDigits.length() < min) { + fracDigits += "0"; + } - case "Z": - return formatMilitaryTimeZone(hour, minute); + // Truncate to max precision + if (fracDigits.length() > max) { + fracDigits = fracDigits.substring(0, max); + } - case "00:00": - default: - format = "%+03d:%02d"; + // Remove trailing zeros beyond min (variable-width output) + while (fracDigits.length() > min && fracDigits.endsWith("0")) { + fracDigits = fracDigits.substring(0, fracDigits.length() - 1); + } + + // Apply digit family from picture (e.g., Arabic-Indic digits) + final int digitSign = getFirstDigitInPicture(picture); + if (digitSign >= 0) { + final int zero = net.sf.saxon.expr.number.Alphanumeric.getDigitFamily(digitSign); + if (zero != '0') { + final StringBuilder converted = new StringBuilder(); + for (int i = 0; i < fracDigits.length(); i++) { + final char ch = fracDigits.charAt(i); + if (ch >= '0' && ch <= '9') { + converted.append((char)(zero + (ch - '0'))); + } else { + converted.append(ch); + } + } + fracDigits = converted.toString(); + } + } + + // Insert grouping separators from picture if present + if (hasGroupingSeparators(picture)) { + fracDigits = applyGroupingSeparators(fracDigits, picture); } - return String.format(locale, format, hour, minute); + sb.append(fracDigits); } - private final static char[] MILITARY_TZ_CHARS = {'Z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', - 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y' }; + private static int getFirstDigitInPicture(String picture) { + for (int i = 0; i < picture.length(); i++) { + final char ch = picture.charAt(i); + if (ch != '#' && ch != 'o' && ch != 'c') { + final int family = net.sf.saxon.expr.number.Alphanumeric.getDigitFamily(ch); + if (family >= 0) { + return ch; + } + } + } + return -1; + } - /** - * Military time zone - * - * Z = +00:00, A = +01:00, B = +02:00, ..., M = +12:00, N = -01:00, O = -02:00, ... Y = -12:00. - * - * The letter J (meaning local time) is used in the case of a value that does not specify a timezone - * offset. - * - * Timezone offsets that have no representation in this system (for example Indian Standard Time, +05:30) - * are output as if the format 01:01 had been requested. - */ - private String formatMilitaryTimeZone(final int hour, final int minute) { - if(minute == 0 && hour > -12 && hour < 12) { - final int offset; - if(hour < 0) { - offset = 13 + (hour * -1); + private static boolean hasGroupingSeparators(String picture) { + for (int i = 0; i < picture.length(); i++) { + final char ch = picture.charAt(i); + if ((ch == 'o' || ch == 'c') && i == picture.length() - 1) { break; } + final int family = net.sf.saxon.expr.number.Alphanumeric.getDigitFamily(ch); + if (family < 0 && ch != '#') { + return true; + } + } + return false; + } + + private static String applyGroupingSeparators(String digits, String picture) { + final StringBuilder result = new StringBuilder(); + int digitIdx = 0; + for (int i = 0; i < picture.length() && digitIdx < digits.length(); i++) { + final char ch = picture.charAt(i); + if ((ch == 'o' || ch == 'c') && i == picture.length() - 1) { break; } + final int family = net.sf.saxon.expr.number.Alphanumeric.getDigitFamily(ch); + if (family >= 0 || ch == '#') { + result.append(digits.charAt(digitIdx)); + digitIdx++; } else { - offset = hour; + result.append(ch); } - return String.valueOf(MILITARY_TZ_CHARS[offset]); - } else { - return String.format("%+03d:%02d", hour, minute); } + while (digitIdx < digits.length()) { + result.append(digits.charAt(digitIdx)); + digitIdx++; + } + return result.toString(); } private String getDefaultFormat(char specifier) { @@ -512,6 +720,80 @@ private void formatNumber(char specifier, String picture, String width, int num, return; } + // Word formatting: W (uppercase), w (lowercase), Ww (title case) + // With optional ordinal modifier: Wo, wo, Wwo + final String basePicture = picture.endsWith("o") ? picture.substring(0, picture.length() - 1) : picture; + final boolean ordinalWords = picture.endsWith("o") && (basePicture.equals("W") || basePicture.equals("w") || basePicture.equals("Ww")); + if ("W".equals(basePicture) || "w".equals(basePicture) || "Ww".equals(basePicture)) { + final Locale locale = new Locale(language); + final String spelloutRule = ordinalWords ? "%spellout-ordinal" : "%spellout-cardinal"; + + // Check if the rule exists, fall back to cardinal if ordinal not available + final RuleBasedNumberFormat rbnf = new RuleBasedNumberFormat(locale, RuleBasedNumberFormat.SPELLOUT); + String ruleToUse = spelloutRule; + boolean ruleFound = false; + for (final String ruleName : rbnf.getRuleSetNames()) { + if (ruleName.equals(ruleToUse)) { + ruleFound = true; + break; + } + } + if (!ruleFound) { + ruleToUse = "%spellout-cardinal"; + } + + final MessageFormat fmt = new MessageFormat("{0,spellout," + ruleToUse + "}", locale); + String word = fmt.format(new Object[]{num}); + + if ("W".equals(basePicture)) { + word = word.toUpperCase(locale); + } else if ("Ww".equals(basePicture)) { + // Title case: capitalize each word + final String[] parts = word.split("((?<=[ -])|(?=[ -]))"); + final StringBuilder titled = new StringBuilder(); + for (final String part : parts) { + titled.append(StringUtils.capitalize(part)); + } + word = titled.toString(); + } + // "w" is already lowercase from ICU4J + + sb.append(word); + return; + } + + // Roman numeral formatting: I (uppercase), i (lowercase) + if ("I".equals(picture) || "i".equals(picture)) { + String roman = toRoman(Math.abs(num)); + if ("i".equals(picture)) { + roman = roman.toLowerCase(); + } + sb.append(roman); + return; + } + + // Handle grouping separators in numeric pictures (e.g., [Y9;999], [Y9,999,*]) + if (hasGroupingSeparators(picture)) { + sb.append(formatWithGroupingSeparators(num, picture)); + return; + } + + // Validate optional digit placement: # must precede mandatory digits, not follow + boolean seenMandatory = false; + for (int i = 0; i < picture.length(); i++) { + final char ch = picture.charAt(i); + if ((ch == 'o' || ch == 'c') && i == picture.length() - 1) { break; } + if (ch == '#') { + if (seenMandatory) { + throw new XPathException(this, ErrorCodes.FOFD1340, + "Optional digit '#' must not appear after mandatory digits in: " + picture); + } + } else { + final int family = net.sf.saxon.expr.number.Alphanumeric.getDigitFamily(ch); + if (family >= 0) { seenMandatory = true; } + } + } + // determine min and max width int min = NumberFormatter.getMinDigits(picture); int max = NumberFormatter.getMaxDigits(picture); @@ -531,6 +813,83 @@ private void formatNumber(char specifier, String picture, String width, int num, } } + private static final int[] ROMAN_VALUES = {1000, 900, 500, 400, 100, 90, 50, 40, 10, 9, 5, 4, 1}; + private static final String[] ROMAN_SYMBOLS = {"M", "CM", "D", "CD", "C", "XC", "L", "XL", "X", "IX", "V", "IV", "I"}; + + private static String toRoman(int num) { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < ROMAN_VALUES.length; i++) { + while (num >= ROMAN_VALUES[i]) { + sb.append(ROMAN_SYMBOLS[i]); + num -= ROMAN_VALUES[i]; + } + } + return sb.toString(); + } + + private static String formatWithGroupingSeparators(int num, String picture) { + String pic = picture; + if (pic.endsWith("o") || pic.endsWith("c")) { pic = pic.substring(0, pic.length() - 1); } + if (pic.endsWith(",*")) { pic = pic.substring(0, pic.length() - 2); } + + int zero = '0'; + for (int i = 0; i < pic.length(); i++) { + final int family = net.sf.saxon.expr.number.Alphanumeric.getDigitFamily(pic.charAt(i)); + if (family >= 0) { zero = family; break; } + } + + // Map separator positions (counted from the right) + final List sepPositions = new ArrayList<>(); + final List sepChars = new ArrayList<>(); + int digitCount = 0; + for (int i = pic.length() - 1; i >= 0; i--) { + final char ch = pic.charAt(i); + final int family = net.sf.saxon.expr.number.Alphanumeric.getDigitFamily(ch); + if (family >= 0 || ch == '#') { + digitCount++; + } else { + sepPositions.add(digitCount); + sepChars.add(ch); + } + } + + final String digits = Integer.toString(num); + final StringBuilder result = new StringBuilder(); + int digitIdx = digits.length() - 1; + int pos = 0; + while (digitIdx >= 0) { + for (int s = 0; s < sepPositions.size(); s++) { + if (sepPositions.get(s) == pos && pos > 0) { + result.insert(0, sepChars.get(s)); + } + } + result.insert(0, digits.charAt(digitIdx)); + digitIdx--; + pos++; + } + + if (zero != '0') { + final StringBuilder converted = new StringBuilder(); + for (int i = 0; i < result.length(); i++) { + final char ch = result.charAt(i); + if (ch >= '0' && ch <= '9') { + converted.append((char)(zero + (ch - '0'))); + } else { + converted.append(ch); + } + } + return converted.toString(); + } + return result.toString(); + } + + private static boolean isKnownCalendar(final String calendar) { + return switch (calendar.toUpperCase()) { + case "AD", "ISO", "OS", "NS" -> true; + default -> false; + }; + } + private int[] getWidths(String width) throws XPathException { if (width == null || width.isEmpty()) {return null;} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFormatNumbers.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFormatNumbers.java index 3633d2c71fc..7be81fbf44d 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFormatNumbers.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFormatNumbers.java @@ -83,6 +83,7 @@ import org.exist.dom.QName; import org.exist.util.CodePointString; import org.exist.xquery.*; +import org.exist.xquery.functions.map.MapType; import org.exist.xquery.value.*; import javax.annotation.Nullable; @@ -125,7 +126,7 @@ public class FnFormatNumbers extends BasicFunction { arity( FS_PARAM_VALUE, FS_PARAM_PICTURE, - optParam("decimal-format-name", Type.STRING, "The name (as an EQName) of a decimal format to use.") + optParam("options", Type.ITEM, "The name (as an EQName) of a decimal format, or a map of formatting options (XQuery 4.0).") ) ) ); @@ -138,22 +139,8 @@ public FnFormatNumbers(final XQueryContext context, final FunctionSignature sign public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { - // get the decimal format - final QName qnDecimalFormat; - if (args.length == 3 && !args[2].isEmpty()) { - final String decimalFormatName = args[2].itemAt(0).getStringValue().trim(); - try { - qnDecimalFormat = QName.parse(context, decimalFormatName); - } catch (final QName.IllegalQNameException e) { - throw new XPathException(this, ErrorCodes.FODF1280, "Invalid decimal format QName.", args[2], e); - } - } else { - qnDecimalFormat = null; - } - final DecimalFormat decimalFormat = context.getStaticDecimalFormat(qnDecimalFormat); - if (decimalFormat == null) { - throw new XPathException(this, ErrorCodes.FODF1280, "No known decimal format of that name.", args[2]); - } + // Resolve decimal format from the options argument (XQ4: string or map) + final DecimalFormat decimalFormat = resolveDecimalFormat(args); final NumericValue number; if (args[0].isEmpty()) { @@ -171,6 +158,145 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) return new StringValue(this, value); } + /** + * Resolves the decimal format from the 3rd argument. + * XQ3.1: absent or xs:string (decimal format name). + * XQ4: map(*) with formatting properties and optional format-name. + */ + private DecimalFormat resolveDecimalFormat(final Sequence[] args) throws XPathException { + if (args.length < 3 || args[2].isEmpty()) { + // No options — use unnamed default + final DecimalFormat df = context.getStaticDecimalFormat(null); + if (df == null) { + throw new XPathException(this, ErrorCodes.FODF1280, "No unnamed decimal format in static context."); + } + return df; + } + + final Item optionsItem = args[2].itemAt(0); + + if (optionsItem instanceof MapType) { + // XQ4 map overload + return resolveDecimalFormatFromMap((MapType) optionsItem); + } + + // XQ3.1 string overload (decimal format name) + final String decimalFormatName = optionsItem.getStringValue().trim(); + final QName qnDecimalFormat; + try { + qnDecimalFormat = QName.parse(context, decimalFormatName); + } catch (final QName.IllegalQNameException e) { + throw new XPathException(this, ErrorCodes.FODF1280, "Invalid decimal format QName.", args[2], e); + } + final DecimalFormat df = context.getStaticDecimalFormat(qnDecimalFormat); + if (df == null) { + throw new XPathException(this, ErrorCodes.FODF1280, "No known decimal format of that name.", args[2]); + } + return df; + } + + /** + * Resolves a decimal format from an XQ4 options map. + * The map can contain format-name (to select a base format) and + * individual property overrides (decimal-separator, grouping-separator, etc.). + * + * Properties use the char:rendition pattern — a single character is both + * marker and rendition; "char:string" splits marker from rendition. + * For this implementation, only the marker (first character) is used for + * picture string analysis; the rendition is used for output formatting. + */ + private DecimalFormat resolveDecimalFormatFromMap(final MapType map) throws XPathException { + // Start with the named or unnamed base format + final Sequence formatNameSeq = map.get(new StringValue(this, "format-name")); + DecimalFormat base; + if (formatNameSeq != null && !formatNameSeq.isEmpty()) { + final String formatName = formatNameSeq.itemAt(0).getStringValue().trim(); + final QName qn; + try { + qn = QName.parse(context, formatName); + } catch (final QName.IllegalQNameException e) { + throw new XPathException(this, ErrorCodes.FODF1280, "Invalid format-name in options map.", formatNameSeq, e); + } + base = context.getStaticDecimalFormat(qn); + if (base == null) { + throw new XPathException(this, ErrorCodes.FODF1280, "No known decimal format: " + formatName); + } + } else { + base = context.getStaticDecimalFormat(null); + if (base == null) { + base = DecimalFormat.UNNAMED; + } + } + + // Override individual properties from the map, extracting char:rendition + final CharRendition decSep = getCharRenditionProperty(map, "decimal-separator", base.decimalSeparator); + final CharRendition grpSep = getCharRenditionProperty(map, "grouping-separator", base.groupingSeparator); + final CharRendition expSep = getCharRenditionProperty(map, "exponent-separator", base.exponentSeparator); + final CharRendition pct = getCharRenditionProperty(map, "percent", base.percent); + final CharRendition pml = getCharRenditionProperty(map, "per-mille", base.perMille); + final int zeroDigit = getCharProperty(map, "zero-digit", base.zeroDigit); + final int digit = getCharProperty(map, "digit", base.digit); + final int patternSeparator = getCharProperty(map, "pattern-separator", base.patternSeparator); + final int minusSign = getCharProperty(map, "minus-sign", base.minusSign); + final String infinity = getStringProperty(map, "infinity", base.infinity); + final String nan = getStringProperty(map, "NaN", base.NaN); + + return new DecimalFormat(decSep.marker(), expSep.marker(), grpSep.marker(), + pct.marker(), pml.marker(), zeroDigit, digit, patternSeparator, infinity, nan, minusSign, + decSep.rendition(), expSep.rendition(), grpSep.rendition(), + pct.rendition(), pml.rendition()); + } + + /** + * Result of parsing a char:rendition property value. + * Marker is used for picture string parsing; rendition for output. + */ + private record CharRendition(int marker, String rendition) {} + + /** + * Extracts a single-character property from the map, handling the + * char:rendition pattern. Returns marker (first char) and rendition. + * If the property is absent, returns the default marker with null rendition. + */ + private CharRendition getCharRenditionProperty(final MapType map, final String key, final int defaultValue) throws XPathException { + final Sequence seq = map.get(new StringValue(this, key)); + if (seq == null || seq.isEmpty()) { + return new CharRendition(defaultValue, null); + } + final String value = seq.itemAt(0).getStringValue(); + if (value.isEmpty()) { + throw new XPathException(this, ErrorCodes.FODF1280, + "Decimal format property '" + key + "' must not be empty."); + } + final int marker = value.codePointAt(0); + final int markerLen = Character.charCount(marker); + // char:rendition pattern: "X:rendition" where X is the marker + if (value.length() > markerLen && value.charAt(markerLen) == ':') { + final String rendition = value.substring(markerLen + 1); + return new CharRendition(marker, rendition); + } + return new CharRendition(marker, null); + } + + /** + * Extracts a single-character property (no rendition support). + */ + private int getCharProperty(final MapType map, final String key, final int defaultValue) throws XPathException { + return getCharRenditionProperty(map, key, defaultValue).marker(); + } + + /** + * Extracts a string property from the map. + * If absent, returns the default. + */ + private String getStringProperty(final MapType map, final String key, final String defaultValue) throws XPathException { + final Sequence seq = map.get(new StringValue(this, key)); + if (seq == null || seq.isEmpty()) { + return defaultValue; + } + return seq.itemAt(0).getStringValue(); + } + enum AnalyzeState { MANTISSA_PART, INTEGER_PART, @@ -715,19 +841,58 @@ private String format(final NumericValue number, final DecimalFormat decimalForm if (minimumExponentSize > 0) { formatted.append(decimalFormat.exponentSeparator); - final CodePointString expStr = new CodePointString(String.valueOf(exp)); + // Handle negative exponents: pad the absolute value, then prepend sign + final boolean negativeExp = exp < 0; + final CodePointString expStr = new CodePointString(String.valueOf(Math.abs(exp))); final int expPadLen = subPicture.getMinimumExponentSize() - expStr.length(); if (expPadLen > 0) { expStr.leftPad(decimalFormat.zeroDigit, expPadLen); } + if (negativeExp) { + expStr.insert(0, decimalFormat.minusSign); + } + formatted.append(expStr); } // Rule 14 - concatenate prefix, formatted number, and suffix - final String result = subPicture.getPrefixString() + formatted + subPicture.getSuffixString(); + String result = subPicture.getPrefixString() + formatted + subPicture.getSuffixString(); + + // XQ4: Apply char:rendition substitutions — replace marker characters with + // their rendition strings in the final output + result = applyRenditions(result, decimalFormat); + + return result; + } + /** + * XQ4 char:rendition: replace marker characters with their rendition strings + * in the formatted output. Only applies when a rendition differs from the + * marker (i.e., the property was specified as "marker:rendition"). + */ + private static String applyRenditions(String result, final DecimalFormat df) { + final String decMarker = new String(Character.toChars(df.decimalSeparator)); + if (!decMarker.equals(df.decimalSeparatorRendition)) { + result = result.replace(decMarker, df.decimalSeparatorRendition); + } + final String grpMarker = new String(Character.toChars(df.groupingSeparator)); + if (!grpMarker.equals(df.groupingSeparatorRendition)) { + result = result.replace(grpMarker, df.groupingSeparatorRendition); + } + final String expMarker = new String(Character.toChars(df.exponentSeparator)); + if (!expMarker.equals(df.exponentSeparatorRendition)) { + result = result.replace(expMarker, df.exponentSeparatorRendition); + } + final String pctMarker = new String(Character.toChars(df.percent)); + if (!pctMarker.equals(df.percentRendition)) { + result = result.replace(pctMarker, df.percentRendition); + } + final String pmlMarker = new String(Character.toChars(df.perMille)); + if (!pmlMarker.equals(df.perMilleRendition)) { + result = result.replace(pmlMarker, df.perMilleRendition); + } return result; } diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFunctionAnnotations.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFunctionAnnotations.java new file mode 100644 index 00000000000..6bdfd09ceeb --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFunctionAnnotations.java @@ -0,0 +1,84 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.value.*; + +/** + * Implements fn:function-annotations (XQuery 4.0). + * + * Returns annotations on a function item as a sequence of single-entry maps, + * where each map has the annotation QName as key and annotation values as value. + */ +public class FnFunctionAnnotations extends BasicFunction { + + public static final FunctionSignature FN_FUNCTION_ANNOTATIONS = new FunctionSignature( + new QName("function-annotations", Function.BUILTIN_FUNCTION_NS), + "Returns the annotations of a function item as a sequence of single-entry maps.", + new SequenceType[]{ + new FunctionParameterSequenceType("function", Type.FUNCTION, + Cardinality.EXACTLY_ONE, "The function item to inspect") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.ZERO_OR_MORE, + "A sequence of single-entry maps, one per annotation")); + + public FnFunctionAnnotations(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Item funcItem = args[0].itemAt(0); + if (!(funcItem instanceof FunctionReference ref)) { + return Sequence.EMPTY_SEQUENCE; + } + + final FunctionSignature sig = ref.getSignature(); + final Annotation[] annotations = sig.getAnnotations(); + if (annotations == null || annotations.length == 0) { + return Sequence.EMPTY_SEQUENCE; + } + + final ValueSequence result = new ValueSequence(annotations.length); + for (final Annotation ann : annotations) { + final MapType map = new MapType(this, context); + final QNameValue qnameKey = new QNameValue(this, context, ann.getName()); + + // Build annotation values sequence + final LiteralValue[] values = ann.getValue(); + if (values == null || values.length == 0) { + map.add(qnameKey, Sequence.EMPTY_SEQUENCE); + } else { + final ValueSequence valSeq = new ValueSequence(values.length); + for (final LiteralValue lv : values) { + valSeq.add(lv.getValue()); + } + map.add(qnameKey, valSeq); + } + result.add(map); + } + return result; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFunctionIdentity.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFunctionIdentity.java new file mode 100644 index 00000000000..e4c1fdadf07 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnFunctionIdentity.java @@ -0,0 +1,100 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.functions.array.ArrayType; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.value.*; + +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Implements fn:function-identity (XQuery 4.0). + * + * Returns a string that uniquely identifies a function item. Two calls with + * the same function return codepoint-equal strings; calls with different + * functions return different strings. + * + * For named functions, identity is based on QName + arity. + * For anonymous functions, maps, and arrays, identity is based on object identity. + */ +public class FnFunctionIdentity extends BasicFunction { + + /** Counter for assigning unique IDs to anonymous function items, maps, and arrays. */ + private static final AtomicLong ID_COUNTER = new AtomicLong(1); + + /** Identity-based map to ensure the same object always gets the same ID. + * Uses reference equality (==), not equals(), so structurally equal but + * distinct maps/arrays get different IDs per the spec. */ + private static final Map IDENTITY_MAP = new IdentityHashMap<>(); + + private static synchronized long getOrAssignId(final Object obj) { + return IDENTITY_MAP.computeIfAbsent(obj, k -> ID_COUNTER.getAndIncrement()); + } + + public static final FunctionSignature FN_FUNCTION_IDENTITY = new FunctionSignature( + new QName("function-identity", Function.BUILTIN_FUNCTION_NS), + "Returns a string that uniquely identifies a function item.", + new SequenceType[]{ + new FunctionParameterSequenceType("function", Type.ITEM, + Cardinality.EXACTLY_ONE, "The function item to identify") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, + "A string uniquely identifying the function")); + + public FnFunctionIdentity(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Item funcItem = args[0].itemAt(0); + return new StringValue(this, computeIdentity(funcItem)); + } + + private static String computeIdentity(final Item item) throws XPathException { + if (item instanceof FunctionReference ref) { + final FunctionSignature sig = ref.getSignature(); + final QName name = sig.getName(); + if (name != null && name != InlineFunction.INLINE_FUNCTION_QNAME) { + // Named function: identity based on expanded QName + arity + return "Q{" + (name.getNamespaceURI() != null ? name.getNamespaceURI() : "") + + "}" + name.getLocalPart() + "#" + sig.getArgumentCount(); + } + // Anonymous function: use counter-based identity + return "anon@" + getOrAssignId(ref); + } + if (item instanceof AbstractMapType) { + // Each distinct map object gets a unique ID + return "map@" + getOrAssignId(item); + } + if (item instanceof ArrayType) { + return "array@" + getOrAssignId(item); + } + // Fallback for other function types + return "func@" + getOrAssignId(item); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnGet.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnGet.java new file mode 100644 index 00000000000..a5c1b7d57cc --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnGet.java @@ -0,0 +1,91 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.functions.array.ArrayType; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.value.*; + +/** + * fn:get($key as xs:anyAtomicType) as item()* + * + * XQuery 4.0 context-dependent lookup function. Looks up a value from + * the context item: + * - For arrays: returns the member at the given position + * - For maps: returns the value for the given key + * - For atomic values: returns the value itself (identity) + */ +public class FnGet extends BasicFunction { + + public static final FunctionSignature FN_GET = new FunctionSignature( + new QName("get", Function.BUILTIN_FUNCTION_NS), + "Looks up a value from the context item. For arrays, returns the member " + + "at the given position. For maps, returns the value for the given key.", + new SequenceType[] { + new FunctionParameterSequenceType("key", Type.ANY_ATOMIC_TYPE, + Cardinality.EXACTLY_ONE, "The lookup key or index") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, + "The looked-up value")); + + public FnGet(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + // Get the context item + Sequence ctxSeq = contextSequence; + if (ctxSeq == null || ctxSeq.isEmpty()) { + throw new XPathException(this, ErrorCodes.XPDY0002, + "fn:get requires a context item"); + } + + final Item contextItem = ctxSeq.itemAt(0); + final AtomicValue key = (AtomicValue) args[0].itemAt(0); + + if (contextItem instanceof ArrayType) { + // Array lookup by position + final ArrayType array = (ArrayType) contextItem; + final int index = ((IntegerValue) key.convertTo(Type.INTEGER)).getInt(); + if (index < 1 || index > array.getSize()) { + throw new XPathException(this, ErrorCodes.FOAY0001, + "Array index " + index + " out of bounds (1.." + array.getSize() + ")"); + } + return array.get(index - 1); + } else if (contextItem instanceof AbstractMapType) { + // Map lookup by key + final AbstractMapType map = (AbstractMapType) contextItem; + final Sequence value = map.get(key); + return value != null ? value : Sequence.EMPTY_SEQUENCE; + } else if (contextItem instanceof FunctionReference) { + // Function application + final FunctionReference funcRef = (FunctionReference) contextItem; + return funcRef.evalFunction(null, null, new Sequence[]{key.toSequence()}); + } else { + // Atomic value: return the context item itself + return contextItem.toSequence(); + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnGraphemes.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnGraphemes.java new file mode 100644 index 00000000000..45701961288 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnGraphemes.java @@ -0,0 +1,86 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import com.ibm.icu.text.BreakIterator; +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +/** + * Implements fn:graphemes (XQuery 4.0). + * + * Splits the supplied string into a sequence of strings, each containing + * one Unicode extended grapheme cluster. + * + * Uses ICU4J's BreakIterator for Unicode grapheme cluster boundary detection, + * which handles combining marks, emoji sequences, regional indicators, etc. + */ +public class FnGraphemes extends BasicFunction { + + public static final FunctionSignature FN_GRAPHEMES = new FunctionSignature( + new QName("graphemes", Function.BUILTIN_FUNCTION_NS), + "Splits the supplied string into a sequence of strings, each containing " + + "one Unicode extended grapheme cluster.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.STRING, Cardinality.ZERO_OR_ONE, + "The string to split into grapheme clusters") + }, + new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, + "a sequence of strings, each containing one grapheme cluster")); + + public FnGraphemes(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + final String str = args[0].getStringValue(); + if (str.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + final BreakIterator bi = BreakIterator.getCharacterInstance(); + bi.setText(str); + + final ValueSequence result = new ValueSequence(); + int start = bi.first(); + for (int end = bi.next(); end != BreakIterator.DONE; start = end, end = bi.next()) { + result.add(new StringValue(this, str.substring(start, end))); + } + return result; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHash.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHash.java new file mode 100644 index 00000000000..47a02cb8b9e --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHash.java @@ -0,0 +1,177 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.zip.CRC32; + +import org.bouncycastle.crypto.digests.Blake3Digest; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BinaryValue; +import org.exist.xquery.value.BinaryValueFromBinaryString; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.HexBinaryValueType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * Implements fn:hash (XQuery 4.0). + * + * Returns the result of a hash/checksum function applied to the input. + * Supports MD5, SHA-1, SHA-256, CRC-32. + */ +public class FnHash extends BasicFunction { + + public static final ErrorCodes.ErrorCode FOHA0001 = new ErrorCodes.ErrorCode("FOHA0001", + "Unsupported hash algorithm"); + + public static final FunctionSignature[] FN_HASH = { + new FunctionSignature( + new QName("hash", Function.BUILTIN_FUNCTION_NS), + "Returns the hash of the input value using the default algorithm (MD5).", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.ITEM, Cardinality.ZERO_OR_ONE, "The value to hash (string, hexBinary, or base64Binary)") + }, + new FunctionReturnSequenceType(Type.HEX_BINARY, Cardinality.ZERO_OR_ONE, "the hash value")), + new FunctionSignature( + new QName("hash", Function.BUILTIN_FUNCTION_NS), + "Returns the hash of the input value using the specified algorithm.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.ITEM, Cardinality.ZERO_OR_ONE, "The value to hash (string, hexBinary, or base64Binary)"), + new FunctionParameterSequenceType("algorithm", Type.STRING, Cardinality.ZERO_OR_ONE, "The hash algorithm (MD5, SHA-1, SHA-256, CRC-32)") + }, + new FunctionReturnSequenceType(Type.HEX_BINARY, Cardinality.ZERO_OR_ONE, "the hash value")), + new FunctionSignature( + new QName("hash", Function.BUILTIN_FUNCTION_NS), + "Returns the hash of the input value using the specified algorithm and options.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.ITEM, Cardinality.ZERO_OR_ONE, "The value to hash (string, hexBinary, or base64Binary)"), + new FunctionParameterSequenceType("algorithm", Type.STRING, Cardinality.ZERO_OR_ONE, "The hash algorithm (MD5, SHA-1, SHA-256, CRC-32)"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, Cardinality.ZERO_OR_ONE, "Options map (reserved for future use)") + }, + new FunctionReturnSequenceType(Type.HEX_BINARY, Cardinality.ZERO_OR_ONE, "the hash value")) + }; + + public FnHash(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + // Get the input bytes + final byte[] inputBytes = getInputBytes(args[0]); + + // Get the algorithm + String algorithm = "MD5"; + if (args.length > 1 && !args[1].isEmpty()) { + algorithm = args[1].getStringValue().trim().toUpperCase(); + } + + // Compute hash + final byte[] hashBytes; + if ("CRC-32".equals(algorithm) || "CRC32".equals(algorithm)) { + final CRC32 crc32 = new CRC32(); + crc32.update(inputBytes); + final long crcValue = crc32.getValue(); + // Return as 4-byte big-endian hexBinary + hashBytes = ByteBuffer.allocate(4).putInt((int) crcValue).array(); + } else if ("BLAKE3".equals(algorithm)) { + final Blake3Digest blake3 = new Blake3Digest(32); + blake3.update(inputBytes, 0, inputBytes.length); + hashBytes = new byte[32]; + blake3.doFinal(hashBytes, 0); + } else { + // Map algorithm names to Java MessageDigest names + final String javaAlgorithm; + switch (algorithm) { + case "MD5": + javaAlgorithm = "MD5"; + break; + case "SHA-1": + case "SHA1": + javaAlgorithm = "SHA-1"; + break; + case "SHA-256": + case "SHA256": + javaAlgorithm = "SHA-256"; + break; + case "SHA-384": + case "SHA384": + javaAlgorithm = "SHA-384"; + break; + case "SHA-512": + case "SHA512": + javaAlgorithm = "SHA-512"; + break; + default: + throw new XPathException(this, FOHA0001, + "Unsupported hash algorithm: " + algorithm); + } + try { + final MessageDigest digest = MessageDigest.getInstance(javaAlgorithm); + hashBytes = digest.digest(inputBytes); + } catch (final NoSuchAlgorithmException e) { + throw new XPathException(this, FOHA0001, + "Hash algorithm not available: " + javaAlgorithm); + } + } + + // Return as hexBinary — use BinaryValueFromBinaryString to avoid + // stream registration with the XQuery context (prevents deadlock + // in concurrent test execution environments) + final StringBuilder hex = new StringBuilder(hashBytes.length * 2); + for (final byte b : hashBytes) { + hex.append(String.format("%02X", b & 0xFF)); + } + return new BinaryValueFromBinaryString(this, new HexBinaryValueType(), hex.toString()); + } + + private byte[] getInputBytes(final Sequence value) throws XPathException { + final int type = value.itemAt(0).getType(); + if (Type.subTypeOf(type, Type.STRING) || Type.subTypeOf(type, Type.ANY_URI) || Type.subTypeOf(type, Type.UNTYPED_ATOMIC)) { + return value.getStringValue().getBytes(StandardCharsets.UTF_8); + } else if (Type.subTypeOf(type, Type.BASE64_BINARY) || Type.subTypeOf(type, Type.HEX_BINARY)) { + final BinaryValue binaryValue = (BinaryValue) value.itemAt(0); + return binaryValue.toJavaObject(byte[].class); + } else { + throw new XPathException(this, ErrorCodes.XPTY0004, + "fn:hash expects string, hexBinary, or base64Binary, got: " + Type.getTypeName(type)); + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHigherOrderFun40.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHigherOrderFun40.java new file mode 100644 index 00000000000..bbd77a86d8e --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHigherOrderFun40.java @@ -0,0 +1,361 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.exist.dom.QName; +import org.exist.xquery.AnalyzeContextInfo; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReference; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.IntegerValue; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceIterator; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +import org.exist.xquery.functions.array.ArrayType; + +/** + * Implements XQuery 4.0 higher-order functions: + * fn:index-where, fn:take-while, fn:do-until, fn:while-do, fn:sort-with, + * fn:scan-left, fn:scan-right. + */ +public class FnHigherOrderFun40 extends BasicFunction { + + public static final FunctionSignature FN_INDEX_WHERE = new FunctionSignature( + new QName("index-where", Function.BUILTIN_FUNCTION_NS), + "Returns the positions of items that match the supplied predicate.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("predicate", Type.FUNCTION, Cardinality.EXACTLY_ONE, "The predicate function") + }, + new FunctionReturnSequenceType(Type.INTEGER, Cardinality.ZERO_OR_MORE, "positions where the predicate is true")); + + public static final FunctionSignature FN_TAKE_WHILE = new FunctionSignature( + new QName("take-while", Function.BUILTIN_FUNCTION_NS), + "Returns items from the input sequence prior to the first one that fails to match a supplied predicate.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("predicate", Type.FUNCTION, Cardinality.EXACTLY_ONE, "The predicate function") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "the leading items matching the predicate")); + + public static final FunctionSignature FN_WHILE_DO = new FunctionSignature( + new QName("while-do", Function.BUILTIN_FUNCTION_NS), + "Processes a supplied value repeatedly, continuing while a condition is true.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The initial input"), + new FunctionParameterSequenceType("predicate", Type.FUNCTION, Cardinality.EXACTLY_ONE, "The condition to test"), + new FunctionParameterSequenceType("action", Type.FUNCTION, Cardinality.EXACTLY_ONE, "The action to apply") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "the first value that fails the predicate")); + + public static final FunctionSignature FN_DO_UNTIL = new FunctionSignature( + new QName("do-until", Function.BUILTIN_FUNCTION_NS), + "Processes a supplied value repeatedly, continuing until a condition becomes true.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The initial input"), + new FunctionParameterSequenceType("action", Type.FUNCTION, Cardinality.EXACTLY_ONE, "The action to apply"), + new FunctionParameterSequenceType("predicate", Type.FUNCTION, Cardinality.EXACTLY_ONE, "The condition to test") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "the first value that satisfies the predicate")); + + public static final FunctionSignature FN_SORT_WITH = new FunctionSignature( + new QName("sort-with", Function.BUILTIN_FUNCTION_NS), + "Sorts a sequence according to a supplied comparator function.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The sequence to sort"), + new FunctionParameterSequenceType("comparators", Type.FUNCTION, Cardinality.ONE_OR_MORE, "The comparator function(s)") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "the sorted sequence")); + + public static final FunctionSignature FN_SCAN_LEFT = new FunctionSignature( + new QName("scan-left", Function.BUILTIN_FUNCTION_NS), + "Returns successive partial results of fold-left.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("init", Type.ITEM, Cardinality.ZERO_OR_MORE, "The initial value"), + new FunctionParameterSequenceType("action", Type.FUNCTION, Cardinality.EXACTLY_ONE, + "The accumulation function: fn(accumulator, item) as item()*") + }, + new FunctionReturnSequenceType(Type.ARRAY_ITEM, Cardinality.ZERO_OR_MORE, + "sequence of single-member arrays with successive fold results")); + + public static final FunctionSignature FN_SCAN_RIGHT = new FunctionSignature( + new QName("scan-right", Function.BUILTIN_FUNCTION_NS), + "Returns successive partial results of fold-right.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("init", Type.ITEM, Cardinality.ZERO_OR_MORE, "The initial value"), + new FunctionParameterSequenceType("action", Type.FUNCTION, Cardinality.EXACTLY_ONE, + "The accumulation function: fn(item, accumulator) as item()*") + }, + new FunctionReturnSequenceType(Type.ARRAY_ITEM, Cardinality.ZERO_OR_MORE, + "sequence of single-member arrays with successive fold results")); + + private AnalyzeContextInfo cachedContextInfo; + + public FnHigherOrderFun40(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + cachedContextInfo = new AnalyzeContextInfo(contextInfo); + super.analyze(cachedContextInfo); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (isCalledAs("while-do")) { + return whileDo(args); + } else if (isCalledAs("do-until")) { + return doUntil(args); + } else if (isCalledAs("sort-with")) { + return sortWith(args); + } else if (isCalledAs("scan-left")) { + return scanLeft(args); + } else if (isCalledAs("scan-right")) { + return scanRight(args); + } + + final Sequence input = args[0]; + if (input.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + try (final FunctionReference ref = (FunctionReference) args[1].itemAt(0)) { + ref.analyze(cachedContextInfo); + final int arity = ref.getSignature().getArgumentCount(); + + if (isCalledAs("index-where")) { + return indexWhere(input, ref, arity); + } else { + return takeWhile(input, ref, arity); + } + } + } + + private Sequence indexWhere(final Sequence input, final FunctionReference ref, final int arity) throws XPathException { + final ValueSequence result = new ValueSequence(); + int pos = 1; + for (final SequenceIterator i = input.iterate(); i.hasNext(); pos++) { + final Item item = i.nextItem(); + final Sequence r = callPredicate(ref, item, pos, arity); + if (!r.isEmpty() && r.effectiveBooleanValue()) { + result.add(new IntegerValue(this, pos)); + } + } + return result; + } + + private Sequence takeWhile(final Sequence input, final FunctionReference ref, final int arity) throws XPathException { + final ValueSequence result = new ValueSequence(); + int pos = 1; + for (final SequenceIterator i = input.iterate(); i.hasNext(); pos++) { + final Item item = i.nextItem(); + final Sequence r = callPredicate(ref, item, pos, arity); + if (r.isEmpty() || !r.effectiveBooleanValue()) { + break; + } + result.add(item); + } + return result; + } + + private Sequence callPredicate(final FunctionReference ref, final Item item, final int pos, final int arity) throws XPathException { + if (arity == 1) { + return ref.evalFunction(null, null, new Sequence[]{item.toSequence()}); + } else { + return ref.evalFunction(null, null, new Sequence[]{item.toSequence(), new IntegerValue(this, pos)}); + } + } + + private Sequence callWithSeqAndPos(final FunctionReference ref, final Sequence input, final int pos, final int arity) throws XPathException { + if (arity == 1) { + return ref.evalFunction(null, null, new Sequence[]{input}); + } else { + return ref.evalFunction(null, null, new Sequence[]{input, new IntegerValue(this, pos)}); + } + } + + private Sequence whileDo(final Sequence[] args) throws XPathException { + Sequence input = args[0]; + try (final FunctionReference predicate = (FunctionReference) args[1].itemAt(0); + final FunctionReference action = (FunctionReference) args[2].itemAt(0)) { + predicate.analyze(cachedContextInfo); + action.analyze(cachedContextInfo); + final int predArity = predicate.getSignature().getArgumentCount(); + final int actArity = action.getSignature().getArgumentCount(); + int pos = 1; + while (true) { + final Sequence test = callWithSeqAndPos(predicate, input, pos, predArity); + if (test.isEmpty() || !test.effectiveBooleanValue()) { + return input; + } + input = callWithSeqAndPos(action, input, pos, actArity); + pos++; + } + } + } + + private Sequence doUntil(final Sequence[] args) throws XPathException { + Sequence input = args[0]; + try (final FunctionReference action = (FunctionReference) args[1].itemAt(0); + final FunctionReference predicate = (FunctionReference) args[2].itemAt(0)) { + action.analyze(cachedContextInfo); + predicate.analyze(cachedContextInfo); + final int actArity = action.getSignature().getArgumentCount(); + final int predArity = predicate.getSignature().getArgumentCount(); + int pos = 1; + while (true) { + input = callWithSeqAndPos(action, input, pos, actArity); + final Sequence test = callWithSeqAndPos(predicate, input, pos, predArity); + if (!test.isEmpty() && test.effectiveBooleanValue()) { + return input; + } + pos++; + } + } + } + + private Sequence sortWith(final Sequence[] args) throws XPathException { + final Sequence input = args[0]; + if (input.getItemCount() <= 1) { + return input; + } + final Sequence comparators = args[1]; + + // Collect all items into a list + final List items = new ArrayList<>(input.getItemCount()); + for (final SequenceIterator i = input.iterate(); i.hasNext(); ) { + items.add(i.nextItem()); + } + + // Get the first comparator (most test cases use a single one) + final FunctionReference[] comparatorRefs = new FunctionReference[comparators.getItemCount()]; + for (int c = 0; c < comparators.getItemCount(); c++) { + comparatorRefs[c] = (FunctionReference) comparators.itemAt(c); + comparatorRefs[c].analyze(cachedContextInfo); + } + + // Sort using the comparator(s) + try { + items.sort((a, b) -> { + try { + for (final FunctionReference comp : comparatorRefs) { + final Sequence result = comp.evalFunction(null, null, + new Sequence[]{a.toSequence(), b.toSequence()}); + final long cmp = ((IntegerValue) result.itemAt(0)).getLong(); + if (cmp != 0) { + return Long.compare(cmp, 0); + } + } + return 0; + } catch (final XPathException e) { + throw new RuntimeException(e); + } + }); + } catch (final RuntimeException e) { + if (e.getCause() instanceof XPathException) { + throw (XPathException) e.getCause(); + } + throw e; + } + + final ValueSequence result = new ValueSequence(items.size()); + for (final Item item : items) { + result.add(item); + } + return result; + } + + private Sequence scanLeft(final Sequence[] args) throws XPathException { + final Sequence input = args[0]; + Sequence accumulator = args[1]; + try (final FunctionReference action = (FunctionReference) args[2].itemAt(0)) { + action.analyze(cachedContextInfo); + + final int count = input.getItemCount(); + final ValueSequence result = new ValueSequence(count + 1); + + // First element: [init] + result.add(new ArrayType(this, context, Collections.singletonList(accumulator))); + + // For each input item, apply action and wrap result + for (final SequenceIterator i = input.iterate(); i.hasNext(); ) { + final Item item = i.nextItem(); + accumulator = action.evalFunction(null, null, + new Sequence[]{accumulator, item.toSequence()}); + result.add(new ArrayType(this, context, Collections.singletonList(accumulator))); + } + + return result; + } + } + + private Sequence scanRight(final Sequence[] args) throws XPathException { + final Sequence input = args[0]; + final Sequence init = args[1]; + try (final FunctionReference action = (FunctionReference) args[2].itemAt(0)) { + action.analyze(cachedContextInfo); + + // Collect items into a list for reverse iteration + final List items = new ArrayList<>(input.getItemCount()); + for (final SequenceIterator i = input.iterate(); i.hasNext(); ) { + items.add(i.nextItem()); + } + + // Build results from right to left + final List results = new ArrayList<>(items.size() + 1); + Sequence accumulator = init; + results.add(accumulator); + + for (int idx = items.size() - 1; idx >= 0; idx--) { + accumulator = action.evalFunction(null, null, + new Sequence[]{items.get(idx).toSequence(), accumulator}); + results.add(accumulator); + } + + // Reverse so first result is fold-right of entire sequence + Collections.reverse(results); + + final ValueSequence result = new ValueSequence(results.size()); + for (final Sequence s : results) { + result.add(new ArrayType(this, context, Collections.singletonList(s))); + } + return result; + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHighestLowest.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHighestLowest.java new file mode 100644 index 00000000000..a2abad4fd94 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHighestLowest.java @@ -0,0 +1,226 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import java.util.ArrayList; +import java.util.List; + +import com.ibm.icu.text.Collator; +import org.exist.dom.QName; +import org.exist.xquery.AnalyzeContextInfo; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.AtomicValue; +import org.exist.xquery.value.DoubleValue; +import org.exist.xquery.value.FloatValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReference; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.NumericValue; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceIterator; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +/** + * Implements fn:highest and fn:lowest (XQuery 4.0). + * + * Returns items from the input having the highest/lowest key values. + */ +public class FnHighestLowest extends BasicFunction { + + public static final FunctionSignature[] FN_HIGHEST = { + new FunctionSignature( + new QName("highest", Function.BUILTIN_FUNCTION_NS), + "Returns items with the highest key value.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "items with highest key")), + new FunctionSignature( + new QName("highest", Function.BUILTIN_FUNCTION_NS), + "Returns items with the highest key value.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("collation", Type.STRING, Cardinality.ZERO_OR_ONE, "The collation URI") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "items with highest key")), + new FunctionSignature( + new QName("highest", Function.BUILTIN_FUNCTION_NS), + "Returns items with the highest key value.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("collation", Type.STRING, Cardinality.ZERO_OR_ONE, "The collation URI"), + new FunctionParameterSequenceType("key", Type.FUNCTION, Cardinality.ZERO_OR_ONE, "Key function") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "items with highest key")) + }; + + public static final FunctionSignature[] FN_LOWEST = { + new FunctionSignature( + new QName("lowest", Function.BUILTIN_FUNCTION_NS), + "Returns items with the lowest key value.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "items with lowest key")), + new FunctionSignature( + new QName("lowest", Function.BUILTIN_FUNCTION_NS), + "Returns items with the lowest key value.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("collation", Type.STRING, Cardinality.ZERO_OR_ONE, "The collation URI") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "items with lowest key")), + new FunctionSignature( + new QName("lowest", Function.BUILTIN_FUNCTION_NS), + "Returns items with the lowest key value.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("collation", Type.STRING, Cardinality.ZERO_OR_ONE, "The collation URI"), + new FunctionParameterSequenceType("key", Type.FUNCTION, Cardinality.ZERO_OR_ONE, "Key function") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "items with lowest key")) + }; + + private AnalyzeContextInfo cachedContextInfo; + + public FnHighestLowest(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + cachedContextInfo = new AnalyzeContextInfo(contextInfo); + super.analyze(cachedContextInfo); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence input = args[0]; + if (input.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + // Resolve collation + final Collator collator; + if (args.length >= 2 && !args[1].isEmpty()) { + collator = context.getCollator(args[1].getStringValue()); + } else { + collator = context.getDefaultCollator(); + } + + // Resolve key function (default is data#1) + FunctionReference keyRef = null; + if (args.length >= 3 && !args[2].isEmpty()) { + keyRef = (FunctionReference) args[2].itemAt(0); + keyRef.analyze(cachedContextInfo); + } + + final boolean findHighest = isCalledAs("highest"); + + // Compute keys for all items + final List items = new ArrayList<>(input.getItemCount()); + final List keys = new ArrayList<>(input.getItemCount()); + + for (final SequenceIterator i = input.iterate(); i.hasNext(); ) { + final Item item = i.nextItem(); + items.add(item); + + // Compute key: apply key function or default atomization (fn:data) + final AtomicValue keyVal; + if (keyRef != null) { + final Sequence keyResult = keyRef.evalFunction(null, null, new Sequence[]{item.toSequence()}); + if (keyResult.isEmpty()) { + keyVal = null; + } else { + AtomicValue kv = keyResult.itemAt(0).atomize(); + if (kv.getType() == Type.UNTYPED_ATOMIC) { + kv = kv.convertTo(Type.DOUBLE); + } + keyVal = kv; + } + } else { + // Default key is fn:data() — atomize the item directly + final AtomicValue atomized = item.atomize(); + if (atomized.getType() == Type.UNTYPED_ATOMIC) { + keyVal = atomized.convertTo(Type.DOUBLE); + } else { + keyVal = atomized; + } + } + keys.add(keyVal); + } + + // Find the extreme value + AtomicValue extremeKey = null; + for (final AtomicValue key : keys) { + if (key == null || isNaN(key)) { + continue; + } + if (extremeKey == null) { + extremeKey = key; + } else { + final int cmp = key.compareTo(collator, extremeKey); + if (findHighest ? cmp > 0 : cmp < 0) { + extremeKey = key; + } + } + } + + if (extremeKey == null) { + return Sequence.EMPTY_SEQUENCE; + } + + // Collect all items with the extreme key value + final ValueSequence result = new ValueSequence(); + for (int i = 0; i < items.size(); i++) { + final AtomicValue key = keys.get(i); + if (key != null && !isNaN(key) && key.compareTo(collator, extremeKey) == 0) { + result.add(items.get(i)); + } + } + + if (keyRef != null) { + keyRef.close(); + } + + return result; + } + + private static boolean isNaN(final AtomicValue v) { + if (v instanceof DoubleValue) { + return Double.isNaN(((DoubleValue) v).getDouble()); + } + if (v instanceof FloatValue) { + return Float.isNaN(((FloatValue) v).getValue()); + } + return false; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHtmlDoc.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHtmlDoc.java new file mode 100644 index 00000000000..bade6cf1717 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnHtmlDoc.java @@ -0,0 +1,71 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.value.*; + +/** + * fn:html-doc($uri) — Like fn:doc but for HTML. + * Loads HTML from a URI, parses it through fn:parse-html, returns XHTML document. + */ +public class FnHtmlDoc extends BasicFunction { + + public static final FunctionSignature FN_HTML_DOC = new FunctionSignature( + new QName("html-doc", Function.BUILTIN_FUNCTION_NS), + "Loads an HTML resource from a URI and returns the parsed XHTML document.", + new SequenceType[] { + new FunctionParameterSequenceType("uri", Type.STRING, + Cardinality.ZERO_OR_ONE, "The URI of the HTML resource") + }, + new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.ZERO_OR_ONE, + "The parsed XHTML document")); + + public FnHtmlDoc(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + final String uri = args[0].getStringValue(); + + // Load text content using unparsed-text logic + final FunUnparsedText unparsedText = new FunUnparsedText(context, + FunUnparsedText.FS_UNPARSED_TEXT[0]); + final Sequence textResult = unparsedText.eval( + new Sequence[]{new StringValue(this, uri)}, contextSequence); + + if (textResult.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + // Parse through fn:parse-html + final FnParseHtml parseHtml = new FnParseHtml(context, + FnParseHtml.FN_PARSE_HTML[0]); + return parseHtml.eval(new Sequence[]{textResult}, contextSequence); + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnIdentityVoid.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnIdentityVoid.java new file mode 100644 index 00000000000..777b717788d --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnIdentityVoid.java @@ -0,0 +1,78 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * Implements fn:identity and fn:void (XQuery 4.0). + */ +public class FnIdentityVoid extends BasicFunction { + + public static final FunctionSignature FN_IDENTITY = new FunctionSignature( + new QName("identity", Function.BUILTIN_FUNCTION_NS), + "Returns its argument value unchanged.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input value") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "the input value unchanged")); + + public static final FunctionSignature[] FN_VOID = { + new FunctionSignature( + new QName("void", Function.BUILTIN_FUNCTION_NS), + "Absorbs the argument and returns the empty sequence.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input to discard") + }, + new FunctionReturnSequenceType(Type.EMPTY_SEQUENCE, Cardinality.EMPTY_SEQUENCE, "the empty sequence")), + new FunctionSignature( + new QName("void", Function.BUILTIN_FUNCTION_NS), + "Returns the empty sequence.", + new SequenceType[] {}, + new FunctionReturnSequenceType(Type.EMPTY_SEQUENCE, Cardinality.EMPTY_SEQUENCE, "the empty sequence")) + }; + + public FnIdentityVoid(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (isCalledAs("identity")) { + return args[0]; + } else { + // void: discard input, return empty sequence + return Sequence.EMPTY_SEQUENCE; + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInScopeNamespaces.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInScopeNamespaces.java new file mode 100644 index 00000000000..b75d1d508bf --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInScopeNamespaces.java @@ -0,0 +1,152 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.Namespaces; +import org.exist.dom.QName; +import org.exist.xquery.*; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.value.*; +import org.w3c.dom.Element; +import org.w3c.dom.Node; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.HashSet; + +/** + * Implements XQuery 4.0 fn:in-scope-namespaces. + * + * Returns a map(xs:string, xs:string) where keys are namespace prefixes + * (empty string for the default namespace) and values are namespace URIs. + * + * Uses nearest-ancestor-wins semantics: for each prefix, the declaration on + * the nearest ancestor (or the element itself) takes precedence. + */ +public class FnInScopeNamespaces extends BasicFunction { + + public static final FunctionSignature FN_IN_SCOPE_NAMESPACES = new FunctionSignature( + new QName("in-scope-namespaces", Function.BUILTIN_FUNCTION_NS), + "Returns a map from namespace prefixes to namespace URIs for all in-scope namespaces of the given element.", + new SequenceType[]{ + new FunctionParameterSequenceType("element", Type.ELEMENT, Cardinality.EXACTLY_ONE, "The element node") + }, + new FunctionReturnSequenceType(Type.MAP_ITEM, Cardinality.EXACTLY_ONE, "A map of prefix to URI")); + + public FnInScopeNamespaces(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final NodeValue nodeValue = (NodeValue) args[0].itemAt(0); + + // Collect all in-scope namespaces with nearest-ancestor-wins semantics + final Map nsMap = new LinkedHashMap<>(); + nsMap.put("xml", Namespaces.XML_NS); + + // Start with static context namespaces (lowest priority) + final Map inScopePrefixes = context.getInScopePrefixes(); + if (inScopePrefixes != null) { + nsMap.putAll(inScopePrefixes); + } + + // Walk from element up to root, collecting namespace declarations. + // Track which prefixes we've already seen from closer ancestors + // so that nearer declarations override farther ones. + final Set seen = new HashSet<>(); + final Map elementNs = new LinkedHashMap<>(); + Node node = nodeValue.getNode(); + + if (context.preserveNamespaces()) { + while (node != null && node.getNodeType() == Node.ELEMENT_NODE) { + if (context.inheritNamespaces() || node == nodeValue.getNode()) { + collectElementNamespaces((Element) node, elementNs, seen); + } + node = node.getParentNode(); + } + } + + // Element declarations override static context (merge on top) + nsMap.putAll(elementNs); + + // Clean up: remove entries where both key and value are empty + nsMap.entrySet().removeIf(entry -> + (entry.getKey() == null || entry.getKey().isEmpty()) && + (entry.getValue() == null || entry.getValue().isEmpty())); + + // Build the result map + MapType result = new MapType(this, context); + for (final Map.Entry entry : nsMap.entrySet()) { + result = (MapType) result.put( + new StringValue(this, entry.getKey()), + new StringValue(this, entry.getValue())); + } + + return result; + } + + /** + * Collect namespace declarations from a single element, respecting nearest-wins. + * Only adds prefixes not already in the {@code seen} set. + */ + private static void collectElementNamespaces(final Element element, final Map nsMap, final Set seen) { + // Element's own namespace + final String namespaceURI = element.getNamespaceURI(); + if (namespaceURI != null && !namespaceURI.isEmpty()) { + final String prefix = element.getPrefix(); + final String key = prefix == null ? "" : prefix; + if (seen.add(key)) { + nsMap.put(key, namespaceURI); + } + } + + // Namespace declarations from the element + if (element instanceof org.exist.dom.memtree.ElementImpl) { + final Map elemNs = new LinkedHashMap<>(); + ((org.exist.dom.memtree.ElementImpl) element).getNamespaceMap(elemNs); + for (final Map.Entry entry : elemNs.entrySet()) { + if (seen.add(entry.getKey())) { + nsMap.put(entry.getKey(), entry.getValue()); + } + } + } else if (element instanceof org.exist.dom.persistent.ElementImpl) { + final org.exist.dom.persistent.ElementImpl elemImpl = (org.exist.dom.persistent.ElementImpl) element; + if (elemImpl.declaresNamespacePrefixes()) { + for (final java.util.Iterator i = elemImpl.getPrefixes(); i.hasNext(); ) { + final String prefix = i.next(); + if (seen.add(prefix)) { + nsMap.put(prefix, elemImpl.getNamespaceForPrefix(prefix)); + } + } + } + } + + // Handle undeclaration: if namespace URI is explicitly empty, remove the prefix + if (namespaceURI != null && namespaceURI.isEmpty()) { + final String prefix = element.getPrefix(); + final String key = prefix == null ? "" : prefix; + nsMap.remove(key); + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInsertSeparator.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInsertSeparator.java new file mode 100644 index 00000000000..ffe3729a0d3 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInsertSeparator.java @@ -0,0 +1,74 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +/** + * Implements fn:insert-separator (XQuery 4.0). + * + * Inserts a separator between adjacent items in a sequence. + */ +public class FnInsertSeparator extends BasicFunction { + + public static final FunctionSignature FN_INSERT_SEPARATOR = new FunctionSignature( + new QName("insert-separator", Function.BUILTIN_FUNCTION_NS), + "Inserts a separator between adjacent items in a sequence.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("separator", Type.ITEM, Cardinality.ZERO_OR_MORE, "The separator to insert") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "the sequence with separators inserted")); + + public FnInsertSeparator(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence input = args[0]; + final Sequence separator = args[1]; + final int inputSize = input.getItemCount(); + if (inputSize <= 1 || separator.isEmpty()) { + return input; + } + final ValueSequence result = new ValueSequence(inputSize + (inputSize - 1) * separator.getItemCount()); + result.add(input.itemAt(0)); + for (int i = 1; i < inputSize; i++) { + result.addAll(separator); + result.add(input.itemAt(i)); + } + return result; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInvisibleXml.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInvisibleXml.java new file mode 100644 index 00000000000..3599bf77dac --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnInvisibleXml.java @@ -0,0 +1,308 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import de.bottlecaps.markup.Blitz; +import de.bottlecaps.markup.BlitzException; +import de.bottlecaps.markup.BlitzParseException; + +import org.exist.Namespaces; +import org.exist.dom.QName; +import org.exist.dom.memtree.DocumentImpl; +import org.exist.dom.memtree.SAXAdapter; +import org.exist.util.XMLReaderPool; +import org.exist.xquery.value.NodeValue; +import org.exist.xquery.AbstractExpression; +import org.exist.xquery.AnalyzeContextInfo; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionCall; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.UserDefinedFunction; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.functions.map.AbstractMapType; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReference; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; +import org.xml.sax.InputSource; +import org.xml.sax.XMLReader; + +import javax.xml.XMLConstants; +import java.io.StringReader; + +/** + * Implements fn:invisible-xml() (XQuery 4.0). + * + * Compiles an Invisible XML grammar and returns a function that parses input + * strings into XML documents. + * + * Uses the Markup Blitz library for ixml grammar compilation and parsing. + * Integration pattern informed by BaseX's implementation. + */ +public class FnInvisibleXml extends BasicFunction { + + // Blitz.generateFromXml() is not thread-safe — synchronize XML grammar compilation + private static final Object BLITZ_XML_LOCK = new Object(); + + private static final FunctionParameterSequenceType PARAM_GRAMMAR = + new FunctionParameterSequenceType("grammar", Type.ITEM, + Cardinality.ZERO_OR_ONE, "The ixml grammar (string or element node)"); + private static final FunctionParameterSequenceType PARAM_OPTIONS = + new FunctionParameterSequenceType("options", Type.MAP_ITEM, + Cardinality.ZERO_OR_ONE, "Options map (fail-on-error: xs:boolean)"); + private static final FunctionReturnSequenceType RETURN_TYPE = + new FunctionReturnSequenceType(Type.FUNCTION, Cardinality.EXACTLY_ONE, + "a function that parses strings according to the grammar"); + + public static final FunctionSignature[] SIGNATURES = { + new FunctionSignature( + new QName("invisible-xml", Function.BUILTIN_FUNCTION_NS), + "Compiles an Invisible XML grammar and returns a parsing function.", + new SequenceType[] { PARAM_GRAMMAR }, + RETURN_TYPE), + new FunctionSignature( + new QName("invisible-xml", Function.BUILTIN_FUNCTION_NS), + "Compiles an Invisible XML grammar and returns a parsing function.", + new SequenceType[] { PARAM_GRAMMAR, PARAM_OPTIONS }, + RETURN_TYPE) + }; + + private AnalyzeContextInfo cachedContextInfo; + + public FnInvisibleXml(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + cachedContextInfo = new AnalyzeContextInfo(contextInfo); + super.analyze(cachedContextInfo); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence grammarArg = args[0]; + + // Parse options — default fail-on-error is false per spec + boolean failOnError = false; + if (args.length > 1 && !args[1].isEmpty()) { + final AbstractMapType options = (AbstractMapType) args[1].itemAt(0); + final Sequence failOpt = options.get(new StringValue(this, "fail-on-error")); + if (!failOpt.isEmpty()) { + final Item failItem = failOpt.itemAt(0); + if (failItem.getType() != Type.BOOLEAN) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option 'fail-on-error' must be xs:boolean, got: " + + Type.getTypeName(failItem.getType())); + } + failOnError = ((BooleanValue) failItem).getValue(); + } else if (options.contains(new StringValue(this, "fail-on-error"))) { + // Key exists but value is empty sequence + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option 'fail-on-error' must be xs:boolean, got empty sequence"); + } + // else: key not present, use default (false) + } + + // Compile the grammar + final de.bottlecaps.markup.blitz.Parser parser; + try { + if (grammarArg.isEmpty()) { + // Empty sequence = use default ixml grammar + parser = failOnError + ? Blitz.generate(Blitz.ixmlGrammar(), Blitz.Option.FAIL_ON_ERROR) + : Blitz.generate(Blitz.ixmlGrammar()); + } else { + final Item grammarItem = grammarArg.itemAt(0); + final int grammarType = grammarItem.getType(); + + if (Type.subTypeOf(grammarType, Type.ELEMENT)) { + // Element node — serialize to XML string and use generateFromXml + // Synchronized: Blitz.generateFromXml() is not thread-safe + final String xmlGrammar = serializeItem(grammarItem); + synchronized (BLITZ_XML_LOCK) { + parser = failOnError + ? Blitz.generateFromXml(xmlGrammar, Blitz.Option.FAIL_ON_ERROR) + : Blitz.generateFromXml(xmlGrammar); + } + } else if (Type.subTypeOf(grammarType, Type.STRING) || + grammarType == Type.UNTYPED_ATOMIC) { + // String grammar + final String grammarStr = grammarItem.getStringValue(); + parser = failOnError + ? Blitz.generate(grammarStr, Blitz.Option.FAIL_ON_ERROR) + : Blitz.generate(grammarStr); + } else if (Type.subTypeOf(grammarType, Type.NODE)) { + // Other node types (document, etc.) — not valid + throw new XPathException(this, ErrorCodes.FOIX0001, + "Grammar must be an element node or string, got: " + + Type.getTypeName(grammarType)); + } else { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Grammar must be a string or element node, got: " + + Type.getTypeName(grammarType)); + } + } + } catch (final BlitzParseException ex) { + throw new XPathException(this, ErrorCodes.FOIX0001, + "Invalid ixml grammar at line " + ex.getLine() + ", column " + ex.getColumn() + + ": " + ex.getOffendingToken()); + } catch (final BlitzException ex) { + throw new XPathException(this, ErrorCodes.FOIX0001, + "Invalid ixml grammar: " + ex.getMessage()); + } + + // Create a function item that parses input strings using the compiled grammar + final QName inputParam = new QName("input", XMLConstants.NULL_NS_URI); + + final FunctionSignature parseSig = new FunctionSignature( + new QName("invisible-xml-parser", Function.BUILTIN_FUNCTION_NS), + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.STRING, + Cardinality.EXACTLY_ONE, "The string to parse") + }, + new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.EXACTLY_ONE, + "the parsed XML document")); + + final UserDefinedFunction func = new UserDefinedFunction(context, parseSig); + func.addVariable(inputParam); + func.setFunctionBody(new ParseExpression(context, parser, inputParam, failOnError)); + + final FunctionCall call = new FunctionCall(context, func); + call.setLocation(getLine(), getColumn()); + + return new FunctionReference(this, call); + } + + private String serializeItem(final Item item) throws XPathException { + try { + final org.exist.storage.serializers.Serializer serializer = + context.getBroker().borrowSerializer(); + try { + serializer.setProperty(javax.xml.transform.OutputKeys.OMIT_XML_DECLARATION, "yes"); + serializer.setProperty(javax.xml.transform.OutputKeys.INDENT, "no"); + return serializer.serialize((NodeValue) item); + } finally { + context.getBroker().returnSerializer(serializer); + } + } catch (final Exception ex) { + throw new XPathException(this, ErrorCodes.FOIX0001, + "Failed to serialize grammar node: " + ex.getMessage()); + } + } + + /** + * Expression that parses an input string using a compiled ixml parser. + */ + private static class ParseExpression extends AbstractExpression { + + private final de.bottlecaps.markup.blitz.Parser parser; + private final QName inputVar; + private final boolean failOnError; + + ParseExpression(final XQueryContext context, final de.bottlecaps.markup.blitz.Parser parser, + final QName inputVar, final boolean failOnError) { + super(context); + this.parser = parser; + this.inputVar = inputVar; + this.failOnError = failOnError; + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) throws XPathException { + final String input = context.resolveVariable(inputVar).getValue().getStringValue(); + + // Parse the input using the compiled ixml parser + final String xmlResult; + try { + xmlResult = parser.parse(input); + } catch (final BlitzParseException ex) { + if (failOnError) { + throw new XPathException(this, ErrorCodes.FOIX0002, + "ixml parse error at line " + ex.getLine() + ", column " + ex.getColumn() + + ": " + ex.getOffendingToken()); + } + // Should not happen when FAIL_ON_ERROR is not set, but handle gracefully + throw new XPathException(this, ErrorCodes.FOIX0002, + "ixml parse error: " + ex.getMessage()); + } catch (final BlitzException ex) { + throw new XPathException(this, ErrorCodes.FOIX0002, + "ixml parse error: " + ex.getMessage()); + } + + // Check for ixml:state="failed" on the root element when fail-on-error is true + if (failOnError && xmlResult.contains("ixml:state=\"failed\"")) { + throw new XPathException(this, ErrorCodes.FOIX0002, + "ixml parse failed: input is ambiguous or does not match the grammar"); + } + + // Parse the XML string into an in-memory document + return parseXmlString(xmlResult); + } + + private DocumentImpl parseXmlString(final String xml) throws XPathException { + final XMLReaderPool parserPool = context.getBroker().getBrokerPool().getXmlReaderPool(); + XMLReader xr = null; + try { + xr = parserPool.borrowXMLReader(); + final InputSource src = new InputSource(new StringReader(xml)); + final SAXAdapter adapter = new SAXAdapter(this, context); + xr.setContentHandler(adapter); + xr.setProperty(Namespaces.SAX_LEXICAL_HANDLER, adapter); + xr.parse(src); + return adapter.getDocument(); + } catch (final Exception ex) { + throw new XPathException(this, ErrorCodes.FOIX0002, + "Failed to parse ixml output as XML: " + ex.getMessage()); + } finally { + if (xr != null) { + parserPool.returnXMLReader(xr); + } + } + } + + @Override + public int returnsType() { + return Type.DOCUMENT; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + // nothing to analyze + } + + @Override + public void dump(final ExpressionDumper dumper) { + dumper.display("invisible-xml-parser(...)"); + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnIsNaN.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnIsNaN.java new file mode 100644 index 00000000000..5e3e8b1754b --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnIsNaN.java @@ -0,0 +1,71 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.DoubleValue; +import org.exist.xquery.value.FloatValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * Implements fn:is-NaN (XQuery 4.0). + * + * Returns true if the argument is the xs:float or xs:double value NaN. + */ +public class FnIsNaN extends BasicFunction { + + public static final FunctionSignature FN_IS_NAN = new FunctionSignature( + new QName("is-NaN", Function.BUILTIN_FUNCTION_NS), + "Returns true if the argument is the xs:float or xs:double value NaN.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.ANY_ATOMIC_TYPE, Cardinality.EXACTLY_ONE, "The value to test") + }, + new FunctionReturnSequenceType(Type.BOOLEAN, Cardinality.EXACTLY_ONE, "true if the value is NaN")); + + public FnIsNaN(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Item item = args[0].itemAt(0); + final int type = item.getType(); + if (type == Type.DOUBLE) { + return BooleanValue.valueOf(Double.isNaN(((DoubleValue) item).getValue())); + } else if (type == Type.FLOAT) { + return BooleanValue.valueOf(Float.isNaN(((FloatValue) item).getValue())); + } + return BooleanValue.FALSE; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnItemsAt.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnItemsAt.java new file mode 100644 index 00000000000..55c9bf64d74 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnItemsAt.java @@ -0,0 +1,79 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.IntegerValue; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceIterator; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +/** + * Implements fn:items-at (XQuery 4.0). + * + * Returns items from the input at the positions specified by the second argument. + */ +public class FnItemsAt extends BasicFunction { + + public static final FunctionSignature FN_ITEMS_AT = new FunctionSignature( + new QName("items-at", Function.BUILTIN_FUNCTION_NS), + "Returns the items at the specified positions in the input sequence.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The input sequence"), + new FunctionParameterSequenceType("at", Type.INTEGER, Cardinality.ZERO_OR_MORE, "The positions to select") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "items at the specified positions")); + + public FnItemsAt(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence input = args[0]; + final Sequence at = args[1]; + if (input.isEmpty() || at.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + final int inputSize = input.getItemCount(); + final ValueSequence result = new ValueSequence(); + for (final SequenceIterator i = at.iterate(); i.hasNext(); ) { + final Item posItem = i.nextItem(); + final int pos = (int) ((IntegerValue) posItem).getLong(); + if (pos >= 1 && pos <= inputSize) { + result.add(input.itemAt(pos - 1)); + } + } + return result; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnMessage.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnMessage.java new file mode 100644 index 00000000000..db50f1319b4 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnMessage.java @@ -0,0 +1,85 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.exist.dom.QName; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.Type; + +/** + * Implements XQuery 4.0 fn:message. + * + * Similar to fn:trace but returns empty-sequence() instead of passing through values. + * Outputs the input values (and optional label) to the log. + */ +public class FnMessage extends BasicFunction { + + private static final Logger LOG = LogManager.getLogger(FnMessage.class); + + public static final FunctionSignature[] FN_MESSAGE = { + new FunctionSignature( + new QName("message", Function.BUILTIN_FUNCTION_NS), + "Outputs values to the log and returns empty sequence.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The values to output") + }, + new FunctionReturnSequenceType(Type.EMPTY_SEQUENCE, Cardinality.EMPTY_SEQUENCE, "empty sequence")), + new FunctionSignature( + new QName("message", Function.BUILTIN_FUNCTION_NS), + "Outputs values to the log with a label and returns empty sequence.", + new SequenceType[] { + new FunctionParameterSequenceType("input", Type.ITEM, Cardinality.ZERO_OR_MORE, "The values to output"), + new FunctionParameterSequenceType("label", Type.STRING, Cardinality.ZERO_OR_ONE, "Optional label for the output") + }, + new FunctionReturnSequenceType(Type.EMPTY_SEQUENCE, Cardinality.EMPTY_SEQUENCE, "empty sequence")) + }; + + public FnMessage(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final Sequence input = args[0]; + final String label = (args.length > 1 && !args[1].isEmpty()) ? args[1].getStringValue() : null; + + final String value = input.getStringValue(); + if (label != null && !label.isEmpty()) { + LOG.info("{}: {}", label, value); + } else { + LOG.info("{}", value); + } + + return Sequence.EMPTY_SEQUENCE; + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java index 413c58b5f3d..9aeade4aab4 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnModule.java @@ -67,13 +67,15 @@ public class FnModule extends AbstractInternalModule { new FunctionDef(FunData.signatures[1], FunData.class), new FunctionDef(FunDateTime.signature, FunDateTime.class), new FunctionDef(FunDeepEqual.signatures[0], FunDeepEqual.class), - new FunctionDef(FunDeepEqual.signatures[1], FunDeepEqual.class), + new FunctionDef(FnDeepEqualOptions.FN_DEEP_EQUAL_OPTIONS, FnDeepEqualOptions.class), new FunctionDef(FunDefaultCollation.signature, FunDefaultCollation.class), new FunctionDef(FnDefaultLanguage.FS_DEFAULT_LANGUAGE, FnDefaultLanguage.class), new FunctionDef(FunDistinctValues.signatures[0], FunDistinctValues.class), new FunctionDef(FunDistinctValues.signatures[1], FunDistinctValues.class), new FunctionDef(FunDoc.signature, FunDoc.class), + new FunctionDef(FunDoc.signatureWithOptions, FunDoc.class), new FunctionDef(FunDocAvailable.signature, FunDocAvailable.class), + new FunctionDef(FunDocAvailable.signatureWithOptions, FunDocAvailable.class), new FunctionDef(FunDocumentURI.FS_DOCUMENT_URI_0, FunDocumentURI.class), new FunctionDef(FunDocumentURI.FS_DOCUMENT_URI_1, FunDocumentURI.class), new FunctionDef(FunElementWithId.FS_ELEMENT_WITH_ID_SIGNATURES[0], FunElementWithId.class), @@ -178,6 +180,7 @@ public class FnModule extends AbstractInternalModule { new FunctionDef(FnOuterMost.FNS_OUTERMOST, FnOuterMost.class), new FunctionDef(FunPath.FS_PATH_SIGNATURES[0], FunPath.class), new FunctionDef(FunPath.FS_PATH_SIGNATURES[1], FunPath.class), + new FunctionDef(FunPath.FS_PATH_SIGNATURES[2], FunPath.class), new FunctionDef(FunPosition.signature, FunPosition.class), new FunctionDef(FunQName.signature, FunQName.class), new FunctionDef(FunRemove.signature, FunRemove.class), @@ -190,6 +193,7 @@ public class FnModule extends AbstractInternalModule { new FunctionDef(FunRoot.signatures[1], FunRoot.class), new FunctionDef(FunRound.FN_ROUND_SIGNATURES[0], FunRound.class), new FunctionDef(FunRound.FN_ROUND_SIGNATURES[1], FunRound.class), + new FunctionDef(FunRound.FN_ROUND_SIGNATURES[2], FunRound.class), new FunctionDef(FunRoundHalfToEven.FN_ROUND_HALF_TO_EVEN_SIGNATURES[0], FunRoundHalfToEven.class), new FunctionDef(FunRoundHalfToEven.FN_ROUND_HALF_TO_EVEN_SIGNATURES[1], FunRoundHalfToEven.class), new FunctionDef(FunSerialize.signatures[0], FunSerialize.class), @@ -240,8 +244,10 @@ public class FnModule extends AbstractInternalModule { new FunctionDef(FunEquals.signatures[1], FunEquals.class), new FunctionDef(FunAnalyzeString.signatures[0], FunAnalyzeString.class), new FunctionDef(FunAnalyzeString.signatures[1], FunAnalyzeString.class), - new FunctionDef(FunHeadTail.signatures[0], FunHeadTail.class), - new FunctionDef(FunHeadTail.signatures[1], FunHeadTail.class), + new FunctionDef(FunHeadTail.FN_HEAD, FunHeadTail.class), + new FunctionDef(FunHeadTail.FN_TAIL, FunHeadTail.class), + new FunctionDef(FunHeadTail.FN_FOOT, FunHeadTail.class), + new FunctionDef(FunHeadTail.FN_TRUNK, FunHeadTail.class), new FunctionDef(FunHigherOrderFun.FN_FOR_EACH, FunHigherOrderFun.class), new FunctionDef(FunHigherOrderFun.FN_FOR_EACH_PAIR, FunHigherOrderFun.class), new FunctionDef(FunHigherOrderFun.FN_FILTER, FunHigherOrderFun.class), @@ -252,6 +258,8 @@ public class FnModule extends AbstractInternalModule { new FunctionDef(FunEnvironment.signature[1], FunEnvironment.class), new FunctionDef(ParsingFunctions.signatures[0], ParsingFunctions.class), new FunctionDef(ParsingFunctions.signatures[1], ParsingFunctions.class), + new FunctionDef(ParsingFunctions.signatures[2], ParsingFunctions.class), + new FunctionDef(ParsingFunctions.signatures[3], ParsingFunctions.class), new FunctionDef(JSON.FS_PARSE_JSON[0], JSON.class), new FunctionDef(JSON.FS_PARSE_JSON[1], JSON.class), new FunctionDef(JSON.FS_JSON_DOC[0], JSON.class), @@ -272,7 +280,126 @@ public class FnModule extends AbstractInternalModule { new FunctionDef(FnRandomNumberGenerator.FS_RANDOM_NUMBER_GENERATOR[0], FnRandomNumberGenerator.class), new FunctionDef(FnRandomNumberGenerator.FS_RANDOM_NUMBER_GENERATOR[1], FnRandomNumberGenerator.class), new FunctionDef(FunContainsToken.FS_CONTAINS_TOKEN[0], FunContainsToken.class), - new FunctionDef(FunContainsToken.FS_CONTAINS_TOKEN[1], FunContainsToken.class) + new FunctionDef(FunContainsToken.FS_CONTAINS_TOKEN[1], FunContainsToken.class), + // XQuery 4.0 functions + new FunctionDef(FnIdentityVoid.FN_IDENTITY, FnIdentityVoid.class), + new FunctionDef(FnIdentityVoid.FN_VOID[0], FnIdentityVoid.class), + new FunctionDef(FnIdentityVoid.FN_VOID[1], FnIdentityVoid.class), + new FunctionDef(FnIsNaN.FN_IS_NAN, FnIsNaN.class), + new FunctionDef(FnCharacters.FN_CHARACTERS, FnCharacters.class), + new FunctionDef(FnGraphemes.FN_GRAPHEMES, FnGraphemes.class), + new FunctionDef(FnParseHtml.FN_PARSE_HTML[0], FnParseHtml.class), + new FunctionDef(FnParseHtml.FN_PARSE_HTML[1], FnParseHtml.class), + new FunctionDef(FnCollation.FN_COLLATION[0], FnCollation.class), + new FunctionDef(FnCollation.FN_COLLATION[1], FnCollation.class), + new FunctionDef(FnCollation.FN_COLLATION_AVAILABLE, FnCollation.class), + new FunctionDef(FnHtmlDoc.FN_HTML_DOC, FnHtmlDoc.class), + new FunctionDef(FnUnparsedBinary.FN_UNPARSED_BINARY, FnUnparsedBinary.class), + new FunctionDef(FnSchemaType.FN_SCHEMA_TYPE, FnSchemaType.class), + new FunctionDef(FnElementToMapPlan.FN_ELEMENT_TO_MAP_PLAN, FnElementToMapPlan.class), + new FunctionDef(FnGet.FN_GET, FnGet.class), + new FunctionDef(FnFunctionAnnotations.FN_FUNCTION_ANNOTATIONS, FnFunctionAnnotations.class), + new FunctionDef(FnFunctionIdentity.FN_FUNCTION_IDENTITY, FnFunctionIdentity.class), + new FunctionDef(FnDateTimeParts.FN_BUILD_DATETIME, FnDateTimeParts.class), + new FunctionDef(FnDateTimeParts.FN_PARTS_OF_DATETIME, FnDateTimeParts.class), + new FunctionDef(FnReplicate.FN_REPLICATE, FnReplicate.class), + new FunctionDef(FnInsertSeparator.FN_INSERT_SEPARATOR, FnInsertSeparator.class), + new FunctionDef(FnAllEqualDifferent.FN_ALL_EQUAL[0], FnAllEqualDifferent.class), + new FunctionDef(FnAllEqualDifferent.FN_ALL_EQUAL[1], FnAllEqualDifferent.class), + new FunctionDef(FnAllEqualDifferent.FN_ALL_DIFFERENT[0], FnAllEqualDifferent.class), + new FunctionDef(FnAllEqualDifferent.FN_ALL_DIFFERENT[1], FnAllEqualDifferent.class), + new FunctionDef(FnItemsAt.FN_ITEMS_AT, FnItemsAt.class), + new FunctionDef(FnHigherOrderFun40.FN_INDEX_WHERE, FnHigherOrderFun40.class), + new FunctionDef(FnHigherOrderFun40.FN_TAKE_WHILE, FnHigherOrderFun40.class), + new FunctionDef(FnHigherOrderFun40.FN_WHILE_DO, FnHigherOrderFun40.class), + new FunctionDef(FnHigherOrderFun40.FN_DO_UNTIL, FnHigherOrderFun40.class), + new FunctionDef(FnHigherOrderFun40.FN_SORT_WITH, FnHigherOrderFun40.class), + new FunctionDef(FnSlice.FN_SLICE[0], FnSlice.class), + new FunctionDef(FnSlice.FN_SLICE[1], FnSlice.class), + new FunctionDef(FnSlice.FN_SLICE[2], FnSlice.class), + new FunctionDef(FnSlice.FN_SLICE[3], FnSlice.class), + new FunctionDef(FnDuplicateValues.FN_DUPLICATE_VALUES[0], FnDuplicateValues.class), + new FunctionDef(FnDuplicateValues.FN_DUPLICATE_VALUES[1], FnDuplicateValues.class), + new FunctionDef(FnHash.FN_HASH[0], FnHash.class), + new FunctionDef(FnHash.FN_HASH[1], FnHash.class), + new FunctionDef(FnHash.FN_HASH[2], FnHash.class), + new FunctionDef(FnOp.FN_OP, FnOp.class), + new FunctionDef(FnChar.FN_CHAR, FnChar.class), + new FunctionDef(FnAtomicEqual.FN_ATOMIC_EQUAL, FnAtomicEqual.class), + new FunctionDef(FnExpandedQName.FN_EXPANDED_QNAME, FnExpandedQName.class), + new FunctionDef(FnHighestLowest.FN_HIGHEST[0], FnHighestLowest.class), + new FunctionDef(FnHighestLowest.FN_HIGHEST[1], FnHighestLowest.class), + new FunctionDef(FnHighestLowest.FN_HIGHEST[2], FnHighestLowest.class), + new FunctionDef(FnHighestLowest.FN_LOWEST[0], FnHighestLowest.class), + new FunctionDef(FnHighestLowest.FN_LOWEST[1], FnHighestLowest.class), + new FunctionDef(FnHighestLowest.FN_LOWEST[2], FnHighestLowest.class), + new FunctionDef(FnPartition.FN_PARTITION, FnPartition.class), + new FunctionDef(FnParseUri.FN_PARSE_URI[0], FnParseUri.class), + new FunctionDef(FnParseUri.FN_PARSE_URI[1], FnParseUri.class), + new FunctionDef(FnBuildUri.FN_BUILD_URI[0], FnBuildUri.class), + new FunctionDef(FnBuildUri.FN_BUILD_URI[1], FnBuildUri.class), + new FunctionDef(FnHigherOrderFun40.FN_SCAN_LEFT, FnHigherOrderFun40.class), + new FunctionDef(FnHigherOrderFun40.FN_SCAN_RIGHT, FnHigherOrderFun40.class), + // XQuery 4.0 functions — batch 1: HOFs and subsequence matching + new FunctionDef(FnEverySome.FN_EVERY[0], FnEverySome.class), + new FunctionDef(FnEverySome.FN_EVERY[1], FnEverySome.class), + new FunctionDef(FnEverySome.FN_SOME[0], FnEverySome.class), + new FunctionDef(FnEverySome.FN_SOME[1], FnEverySome.class), + new FunctionDef(FnSortBy.FN_SORT_BY, FnSortBy.class), + new FunctionDef(FnPartialApply.FN_PARTIAL_APPLY, FnPartialApply.class), + new FunctionDef(FnSubsequenceMatching.FN_CONTAINS_SUBSEQUENCE[0], FnSubsequenceMatching.class), + new FunctionDef(FnSubsequenceMatching.FN_CONTAINS_SUBSEQUENCE[1], FnSubsequenceMatching.class), + new FunctionDef(FnSubsequenceMatching.FN_STARTS_WITH_SUBSEQUENCE[0], FnSubsequenceMatching.class), + new FunctionDef(FnSubsequenceMatching.FN_STARTS_WITH_SUBSEQUENCE[1], FnSubsequenceMatching.class), + new FunctionDef(FnSubsequenceMatching.FN_ENDS_WITH_SUBSEQUENCE[0], FnSubsequenceMatching.class), + new FunctionDef(FnSubsequenceMatching.FN_ENDS_WITH_SUBSEQUENCE[1], FnSubsequenceMatching.class), + // XQuery 4.0 functions — batch 2: string/number/URI + new FunctionDef(FnDecodeFromUri.FN_DECODE_FROM_URI, FnDecodeFromUri.class), + new FunctionDef(FnParseInteger.FN_PARSE_INTEGER[0], FnParseInteger.class), + new FunctionDef(FnParseInteger.FN_PARSE_INTEGER[1], FnParseInteger.class), + new FunctionDef(FnDivideDecimals.FN_DIVIDE_DECIMALS[0], FnDivideDecimals.class), + new FunctionDef(FnDivideDecimals.FN_DIVIDE_DECIMALS[1], FnDivideDecimals.class), + // XQuery 4.0 functions — batch 3: node and type + new FunctionDef(FnDistinctOrderedNodes.FN_DISTINCT_ORDERED_NODES, FnDistinctOrderedNodes.class), + new FunctionDef(FnSiblings.FN_SIBLINGS[0], FnSiblings.class), + new FunctionDef(FnSiblings.FN_SIBLINGS[1], FnSiblings.class), + new FunctionDef(FnTypeOf.FN_TYPE_OF, FnTypeOf.class), + // XQuery 4.0 functions — batch 4: date/time and misc + new FunctionDef(FnUnixDateTime.FN_UNIX_DATETIME[0], FnUnixDateTime.class), + new FunctionDef(FnUnixDateTime.FN_UNIX_DATETIME[1], FnUnixDateTime.class), + new FunctionDef(FnMessage.FN_MESSAGE[0], FnMessage.class), + new FunctionDef(FnMessage.FN_MESSAGE[1], FnMessage.class), + // XQuery 4.0 functions — batch 2 (continued): parse-QName + new FunctionDef(FnParseQName.FN_PARSE_QNAME, FnParseQName.class), + // XQuery 4.0 functions — batch 3 (continued): type annotation + new FunctionDef(FnTypeAnnotation.FN_ATOMIC_TYPE_ANNOTATION, FnTypeAnnotation.class), + new FunctionDef(FnTypeAnnotation.FN_NODE_TYPE_ANNOTATION, FnTypeAnnotation.class), + // XQuery 4.0 functions — batch 4 (continued): civil-timezone + new FunctionDef(FnCivilTimezone.FN_CIVIL_TIMEZONE[0], FnCivilTimezone.class), + new FunctionDef(FnCivilTimezone.FN_CIVIL_TIMEZONE[1], FnCivilTimezone.class), + // XQuery 4.0 functions — batch 5: CSV functions + new FunctionDef(CsvFunctions.FN_CSV_TO_ARRAYS[0], CsvFunctions.class), + new FunctionDef(CsvFunctions.FN_CSV_TO_ARRAYS[1], CsvFunctions.class), + new FunctionDef(CsvFunctions.FN_PARSE_CSV[0], CsvFunctions.class), + new FunctionDef(CsvFunctions.FN_PARSE_CSV[1], CsvFunctions.class), + new FunctionDef(CsvFunctions.FN_CSV_TO_XML[0], CsvFunctions.class), + new FunctionDef(CsvFunctions.FN_CSV_TO_XML[1], CsvFunctions.class), + new FunctionDef(CsvFunctions.FN_CSV_DOC[0], CsvFunctions.class), + new FunctionDef(CsvFunctions.FN_CSV_DOC[1], CsvFunctions.class), + // XQuery 4.0 functions — batch 6: subsequence-where, seconds, in-scope-namespaces + new FunctionDef(FnSubsequenceWhere.FN_SUBSEQUENCE_WHERE[0], FnSubsequenceWhere.class), + new FunctionDef(FnSubsequenceWhere.FN_SUBSEQUENCE_WHERE[1], FnSubsequenceWhere.class), + new FunctionDef(FnSeconds.FN_SECONDS, FnSeconds.class), + new FunctionDef(FnInScopeNamespaces.FN_IN_SCOPE_NAMESPACES, FnInScopeNamespaces.class), + // XQuery 4.0 functions — batch 7: transitive-closure, element-to-map + new FunctionDef(FnTransitiveClosure.FN_TRANSITIVE_CLOSURE, FnTransitiveClosure.class), + new FunctionDef(FnElementToMap.FN_ELEMENT_TO_MAP[0], FnElementToMap.class), + new FunctionDef(FnElementToMap.FN_ELEMENT_TO_MAP[1], FnElementToMap.class), + + // --- Invisible XML (feature/fn-invisible-xml) --- + new FunctionDef(FnInvisibleXml.SIGNATURES[0], FnInvisibleXml.class), + new FunctionDef(FnInvisibleXml.SIGNATURES[1], FnInvisibleXml.class) + // --- End Invisible XML --- }; static { diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnOp.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnOp.java new file mode 100644 index 00000000000..f0785604f57 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnOp.java @@ -0,0 +1,404 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import org.exist.dom.QName; +import org.exist.xquery.AbstractExpression; +import org.exist.xquery.AnalyzeContextInfo; +import org.exist.xquery.BasicFunction; +import org.exist.xquery.Cardinality; +import org.exist.xquery.ErrorCodes; +import org.exist.xquery.Function; +import org.exist.xquery.FunctionCall; +import org.exist.xquery.FunctionSignature; +import org.exist.xquery.UserDefinedFunction; +import org.exist.xquery.ValueComparison; +import org.exist.xquery.XPathException; +import org.exist.xquery.XQueryContext; +import org.exist.xquery.util.ExpressionDumper; +import org.exist.xquery.value.BooleanValue; +import org.exist.xquery.value.ComputableValue; +import org.exist.xquery.value.FunctionParameterSequenceType; +import org.exist.xquery.value.FunctionReference; +import org.exist.xquery.value.FunctionReturnSequenceType; +import org.exist.xquery.value.IntegerValue; +import org.exist.xquery.value.Item; +import org.exist.xquery.value.Sequence; +import org.exist.xquery.value.SequenceType; +import org.exist.xquery.value.StringValue; +import org.exist.xquery.value.Type; +import org.exist.xquery.value.ValueSequence; + +/** + * Implements fn:op (XQuery 4.0). + * + * Returns a function reference for a named operator. + */ +public class FnOp extends BasicFunction { + + private static final QName PARAM_A = new QName("a", javax.xml.XMLConstants.NULL_NS_URI); + private static final QName PARAM_B = new QName("b", javax.xml.XMLConstants.NULL_NS_URI); + + public static final FunctionSignature FN_OP = new FunctionSignature( + new QName("op", Function.BUILTIN_FUNCTION_NS), + "Returns a function that applies a given operator.", + new SequenceType[] { + new FunctionParameterSequenceType("operator", Type.STRING, Cardinality.EXACTLY_ONE, "The operator name") + }, + new FunctionReturnSequenceType(Type.FUNCTION, Cardinality.EXACTLY_ONE, "a function implementing the operator")); + + public FnOp(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + private static final ErrorCodes.ErrorCode FOAP0001 = new ErrorCodes.ErrorCode( + "FOAP0001", "Invalid operator name"); + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + final String operator = args[0].getStringValue(); + + // Validate operator name + if (!isValidOperator(operator)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Unknown operator: " + operator); + } + + // Create a UserDefinedFunction with 2 parameters ($a, $b) + final FunctionSignature opSig = new FunctionSignature( + new QName("op#" + operator, Function.BUILTIN_FUNCTION_NS), + new SequenceType[] { + new FunctionParameterSequenceType("a", Type.ITEM, Cardinality.ZERO_OR_MORE, "left operand"), + new FunctionParameterSequenceType("b", Type.ITEM, Cardinality.ZERO_OR_MORE, "right operand") + }, + new FunctionReturnSequenceType(Type.ITEM, Cardinality.ZERO_OR_MORE, "result")); + + final UserDefinedFunction func = new UserDefinedFunction(context, opSig); + func.addVariable(PARAM_A); + func.addVariable(PARAM_B); + + // Set the body to an expression that evaluates the operator + func.setFunctionBody(new OperatorExpression(context, operator)); + + final FunctionCall call = new FunctionCall(context, func); + call.setLocation(getLine(), getColumn()); + + return new FunctionReference(this, call); + } + + private boolean isValidOperator(final String op) { + switch (op) { + case ",": case "and": case "or": + case "+": case "-": case "*": case "div": case "idiv": case "mod": + case "=": case "<": case "<=": case ">": case ">=": case "!=": + case "eq": case "lt": case "le": case "gt": case "ge": case "ne": + case "<<": case ">>": case "precedes": case "follows": + case "precedes-or-is": case "follows-or-is": + case "is": case "is-not": + case "||": case "|": case "union": case "except": case "intersect": + case "to": case "otherwise": + return true; + default: + return false; + } + } + + /** + * Expression that evaluates an operator on two variables $a and $b + * from the local variable context. + */ + private static class OperatorExpression extends AbstractExpression { + + private final String operator; + + OperatorExpression(final XQueryContext context, final String operator) { + super(context); + this.operator = operator; + } + + @Override + public Sequence eval(final Sequence contextSequence, final Item contextItem) throws XPathException { + final Sequence a = context.resolveVariable(PARAM_A).getValue(); + final Sequence b = context.resolveVariable(PARAM_B).getValue(); + + switch (operator) { + // Arithmetic + case "+": return arithmetic(a, b, "plus"); + case "-": return arithmetic(a, b, "minus"); + case "*": return arithmetic(a, b, "mult"); + case "div": return arithmetic(a, b, "div"); + case "idiv": return arithmetic(a, b, "idiv"); + case "mod": return arithmetic(a, b, "mod"); + + // General comparison + case "=": return generalCompare(a, b, org.exist.xquery.Constants.Comparison.EQ); + case "!=": return generalCompare(a, b, org.exist.xquery.Constants.Comparison.NEQ); + case "<": return generalCompare(a, b, org.exist.xquery.Constants.Comparison.LT); + case "<=": return generalCompare(a, b, org.exist.xquery.Constants.Comparison.LTEQ); + case ">": return generalCompare(a, b, org.exist.xquery.Constants.Comparison.GT); + case ">=": return generalCompare(a, b, org.exist.xquery.Constants.Comparison.GTEQ); + + // Value comparison + case "eq": return valueCompare(a, b, org.exist.xquery.Constants.Comparison.EQ); + case "ne": return valueCompare(a, b, org.exist.xquery.Constants.Comparison.NEQ); + case "lt": return valueCompare(a, b, org.exist.xquery.Constants.Comparison.LT); + case "le": return valueCompare(a, b, org.exist.xquery.Constants.Comparison.LTEQ); + case "gt": return valueCompare(a, b, org.exist.xquery.Constants.Comparison.GT); + case "ge": return valueCompare(a, b, org.exist.xquery.Constants.Comparison.GTEQ); + + // Boolean + case "and": return BooleanValue.valueOf(a.effectiveBooleanValue() && b.effectiveBooleanValue()); + case "or": return BooleanValue.valueOf(a.effectiveBooleanValue() || b.effectiveBooleanValue()); + + // String concatenation + case "||": return new StringValue(this, a.getStringValue() + b.getStringValue()); + + // Sequence + case ",": return opComma(a, b); + case "|": + case "union": return opVenn(a, b, "union"); + case "except": return opVenn(a, b, "except"); + case "intersect": return opVenn(a, b, "intersect"); + case "to": return opTo(a, b); + case "otherwise": return a.isEmpty() ? b : a; + + // Node comparison + case "is": return nodeIs(a, b); + case "is-not": return nodeIsNot(a, b); + case "<<": + case "precedes": return nodePrecedes(a, b); + case ">>": + case "follows": return nodeFollows(a, b); + case "precedes-or-is": return nodePrecedesOrIs(a, b); + case "follows-or-is": return nodeFollowsOrIs(a, b); + + default: + throw new XPathException(this, ErrorCodes.FOJS0005, "Unknown operator: " + operator); + } + } + + private Sequence arithmetic(final Sequence a, final Sequence b, final String op) throws XPathException { + if (a.isEmpty() || b.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + final ComputableValue left = toComputable(a.itemAt(0).atomize()); + final ComputableValue right = toComputable(b.itemAt(0).atomize()); + switch (op) { + case "plus": return left.plus(right); + case "minus": return left.minus(right); + case "mult": return left.mult(right); + case "div": return left.div(right); + case "idiv": return ((org.exist.xquery.value.NumericValue) left).idiv((org.exist.xquery.value.NumericValue) right); + case "mod": return ((org.exist.xquery.value.NumericValue) left).mod((org.exist.xquery.value.NumericValue) right); + default: throw new IllegalStateException(); + } + } + + private Sequence generalCompare(final Sequence a, final Sequence b, + final org.exist.xquery.Constants.Comparison comp) throws XPathException { + // General comparison: existential semantics — true if any pair matches + if (a.isEmpty() || b.isEmpty()) { + return BooleanValue.FALSE; + } + final com.ibm.icu.text.Collator collator = context.getDefaultCollator(); + for (int i = 0; i < a.getItemCount(); i++) { + final org.exist.xquery.value.AtomicValue lv = a.itemAt(i).atomize(); + for (int j = 0; j < b.getItemCount(); j++) { + final org.exist.xquery.value.AtomicValue rv = b.itemAt(j).atomize(); + if (ValueComparison.compareAtomic(collator, lv, rv, + org.exist.xquery.Constants.StringTruncationOperator.NONE, comp)) { + return BooleanValue.TRUE; + } + } + } + return BooleanValue.FALSE; + } + + private Sequence valueCompare(final Sequence a, final Sequence b, + final org.exist.xquery.Constants.Comparison comp) throws XPathException { + if (a.isEmpty() || b.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + if (a.getItemCount() > 1 || b.getItemCount() > 1) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Value comparison requires singleton operands"); + } + final org.exist.xquery.value.AtomicValue lv = a.itemAt(0).atomize(); + final org.exist.xquery.value.AtomicValue rv = b.itemAt(0).atomize(); + final com.ibm.icu.text.Collator collator = context.getDefaultCollator(); + return BooleanValue.valueOf(ValueComparison.compareAtomic(collator, lv, rv, + org.exist.xquery.Constants.StringTruncationOperator.NONE, comp)); + } + + private Sequence opComma(final Sequence a, final Sequence b) throws XPathException { + final ValueSequence result = new ValueSequence(a.getItemCount() + b.getItemCount()); + result.addAll(a); + result.addAll(b); + return result; + } + + private Sequence opVenn(final Sequence a, final Sequence b, final String op) throws XPathException { + // Check that operands are nodes + for (int i = 0; i < a.getItemCount(); i++) { + if (!(a.itemAt(i) instanceof org.w3c.dom.Node)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Set operation requires node operands, got " + Type.getTypeName(a.itemAt(i).getType())); + } + } + for (int i = 0; i < b.getItemCount(); i++) { + if (!(b.itemAt(i) instanceof org.w3c.dom.Node)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Set operation requires node operands, got " + Type.getTypeName(b.itemAt(i).getType())); + } + } + try { + switch (op) { + case "union": return a.toNodeSet().union(b.toNodeSet()); + case "except": return a.toNodeSet().except(b.toNodeSet()); + case "intersect": return a.toNodeSet().intersection(b.toNodeSet()); + default: throw new IllegalStateException(); + } + } catch (final XPathException e) { + throw new XPathException(this, ErrorCodes.XPTY0004, e.getMessage()); + } + } + + private Sequence opTo(final Sequence a, final Sequence b) throws XPathException { + if (a.isEmpty() || b.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + final long start = ((IntegerValue) a.itemAt(0)).getLong(); + final long end = ((IntegerValue) b.itemAt(0)).getLong(); + if (start > end) { + return Sequence.EMPTY_SEQUENCE; + } + final ValueSequence result = new ValueSequence((int) (end - start + 1)); + for (long i = start; i <= end; i++) { + result.add(new IntegerValue(this, i)); + } + return result; + } + + private void checkNodeOperands(final Sequence a, final Sequence b) throws XPathException { + if (!a.isEmpty() && !(a.itemAt(0) instanceof org.w3c.dom.Node)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Node comparison requires node operands, got " + Type.getTypeName(a.itemAt(0).getType())); + } + if (!b.isEmpty() && !(b.itemAt(0) instanceof org.w3c.dom.Node)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Node comparison requires node operands, got " + Type.getTypeName(b.itemAt(0).getType())); + } + } + + private Sequence nodeIs(final Sequence a, final Sequence b) throws XPathException { + checkNodeOperands(a, b); + if (a.isEmpty() || b.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + return BooleanValue.valueOf(a.itemAt(0).equals(b.itemAt(0))); + } + + private Sequence nodeIsNot(final Sequence a, final Sequence b) throws XPathException { + checkNodeOperands(a, b); + if (a.isEmpty() || b.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + return BooleanValue.valueOf(!a.itemAt(0).equals(b.itemAt(0))); + } + + private ComputableValue toComputable(final org.exist.xquery.value.AtomicValue v) throws XPathException { + if (v instanceof ComputableValue) { + return (ComputableValue) v; + } + // Untyped atomic → promote to xs:double for arithmetic + if (v.getType() == Type.UNTYPED_ATOMIC) { + return (ComputableValue) v.convertTo(Type.DOUBLE); + } + throw new XPathException(this, ErrorCodes.XPTY0004, + "Cannot use " + Type.getTypeName(v.getType()) + " in arithmetic"); + } + + private int nodeCompare(final Sequence a, final Sequence b) throws XPathException { + checkNodeOperands(a, b); + final Item left = a.itemAt(0); + final Item right = b.itemAt(0); + if (left instanceof org.exist.dom.persistent.NodeProxy && right instanceof org.exist.dom.persistent.NodeProxy) { + return ((org.exist.dom.persistent.NodeProxy) left).compareTo((org.exist.dom.persistent.NodeProxy) right); + } + // For in-memory nodes, compare using NodeId if available + if (left instanceof org.exist.dom.memtree.NodeImpl && right instanceof org.exist.dom.memtree.NodeImpl) { + final org.exist.dom.memtree.NodeImpl leftNode = (org.exist.dom.memtree.NodeImpl) left; + final org.exist.dom.memtree.NodeImpl rightNode = (org.exist.dom.memtree.NodeImpl) right; + return Integer.compare(leftNode.getNodeNumber(), rightNode.getNodeNumber()); + } + throw new XPathException(this, ErrorCodes.XPTY0004, + "Node comparison requires node operands"); + } + + private Sequence nodePrecedes(final Sequence a, final Sequence b) throws XPathException { + checkNodeOperands(a, b); + if (a.isEmpty() || b.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + return BooleanValue.valueOf(nodeCompare(a, b) < 0); + } + + private Sequence nodeFollows(final Sequence a, final Sequence b) throws XPathException { + checkNodeOperands(a, b); + if (a.isEmpty() || b.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + return BooleanValue.valueOf(nodeCompare(a, b) > 0); + } + + private Sequence nodePrecedesOrIs(final Sequence a, final Sequence b) throws XPathException { + checkNodeOperands(a, b); + if (a.isEmpty() || b.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + return BooleanValue.valueOf(nodeCompare(a, b) <= 0); + } + + private Sequence nodeFollowsOrIs(final Sequence a, final Sequence b) throws XPathException { + checkNodeOperands(a, b); + if (a.isEmpty() || b.isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + return BooleanValue.valueOf(nodeCompare(a, b) >= 0); + } + + @Override + public int returnsType() { + return Type.ITEM; + } + + @Override + public void analyze(final AnalyzeContextInfo contextInfo) throws XPathException { + // nothing to analyze + } + + @Override + public void dump(final ExpressionDumper dumper) { + dumper.display("op(\"" + operator + "\")"); + } + } +} diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/FnParseHtml.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnParseHtml.java new file mode 100644 index 00000000000..b7c0325aba9 --- /dev/null +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/FnParseHtml.java @@ -0,0 +1,177 @@ +/* + * eXist-db Open Source Native XML Database + * Copyright (C) 2001 The eXist-db Authors + * + * info@exist-db.org + * http://www.exist-db.org + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +package org.exist.xquery.functions.fn; + +import nu.validator.htmlparser.common.XmlViolationPolicy; +import nu.validator.htmlparser.sax.HtmlParser; +import org.exist.Namespaces; +import org.exist.dom.QName; +import org.exist.dom.memtree.SAXAdapter; +import org.exist.xquery.*; +import org.exist.xquery.functions.map.MapType; +import org.exist.xquery.value.*; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.Charset; + +/** + * Implements fn:parse-html (XQuery 4.0). + * + * Parses an HTML string (which may be malformed) into an XDM document node + * with all elements in the XHTML namespace. + */ +public class FnParseHtml extends BasicFunction { + + public static final FunctionSignature[] FN_PARSE_HTML = { + new FunctionSignature( + new QName("parse-html", Function.BUILTIN_FUNCTION_NS), + "Parses the supplied HTML string into an XDM document node. " + + "The input need not be well-formed; it is processed by an HTML parser " + + "that corrects errors and produces well-formed XHTML output.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.ITEM, + Cardinality.ZERO_OR_ONE, "The HTML to parse (string or binary)") + }, + new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.ZERO_OR_ONE, + "The parsed XHTML document")), + new FunctionSignature( + new QName("parse-html", Function.BUILTIN_FUNCTION_NS), + "Parses the supplied HTML string into an XDM document node with options. " + + "The input need not be well-formed; it is processed by an HTML parser " + + "that corrects errors and produces well-formed XHTML output.", + new SequenceType[] { + new FunctionParameterSequenceType("value", Type.ITEM, + Cardinality.ZERO_OR_ONE, "The HTML to parse (string or binary)"), + new FunctionParameterSequenceType("options", Type.MAP_ITEM, + Cardinality.EXACTLY_ONE, "Options map") + }, + new FunctionReturnSequenceType(Type.DOCUMENT, Cardinality.ZERO_OR_ONE, + "The parsed XHTML document")) + }; + + public FnParseHtml(final XQueryContext context, final FunctionSignature signature) { + super(context, signature); + } + + @Override + public Sequence eval(final Sequence[] args, final Sequence contextSequence) throws XPathException { + if (args[0].isEmpty()) { + return Sequence.EMPTY_SEQUENCE; + } + + // Extract options if present + boolean failOnError = false; + String encoding = "UTF-8"; + if (getArgumentCount() == 2 && !args[1].isEmpty()) { + final MapType options = (MapType) args[1].itemAt(0); + failOnError = getBooleanOption(options, "fail-on-error", false); + encoding = getStringOption(options, "encoding", "UTF-8"); + + // Validate option types per spec — unknown options with wrong types raise XPTY0004 + validateOptionType(options, "include-template-content"); + validateOptionType(options, "exclude-template-content"); + } + + // Get the HTML content as a string + final String htmlContent = getHtmlContent(args[0].itemAt(0), encoding); + + // Parse with the configured HTML-to-XML parser + return parseHtml(htmlContent, failOnError); + } + + private String getHtmlContent(final Item item, final String encoding) throws XPathException { + if (item instanceof BinaryValue) { + final BinaryValue binary = (BinaryValue) item; + try (final java.io.InputStream is = binary.getInputStream()) { + final Charset charset = Charset.forName(encoding); + return new String(is.readAllBytes(), charset); + } catch (final Exception e) { + throw new XPathException(this, ErrorCodes.FODC0006, + "Error decoding binary value: " + e.getMessage()); + } + } + return item.getStringValue(); + } + + private Sequence parseHtml(final String htmlContent, final boolean failOnError) throws XPathException { + final SAXAdapter adapter = new SAXAdapter(this, context); + + try { + // Use Validator.nu HTML5 parser — SAX-based, same pipeline as NekoHTML + // but follows the WHATWG HTML5 parsing algorithm. Outputs XHTML namespace + // by default, handles