From e3a2da8781d82f9ac3346cdf16810337286c26d5 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Mon, 11 May 2026 16:34:26 -0400 Subject: [PATCH 1/3] [bugfix] fn:json-to-xml: enforce option-parameter type and permitted-value checks Per F&O 3.1 section 2.4 (option parameter conventions) plus section 17.5.3: * Wrong-typed option values now raise XPTY0004 (was: FOJS0005 for 'escape', silently accepted for others). * 'duplicates' values are checked against the permitted set {reject, use-first, retain}; non-permitted values raise FOJS0005. * 'validate' + 'duplicates: retain' is rejected as an incompatible combination (FOJS0005). * Unknown option keys on fn:json-to-xml raise FOJS0005. * Empty-sequence option values trigger XPTY0004 (cardinality 0). * 'fallback' arity is validated against the declared function signature. Behavior under recognized, valid options is unchanged. Implementing the 'validate', 'escape', and 'fallback' semantics themselves is out of scope and tracked in the parent triage report. Targets ~11 fn-json-to-xml XQTS HEAD failures (json-to-xml-error-018/019/020/021/022/023/024/025/026/040/041). Refs F&O 3.1: https://www.w3.org/TR/xpath-functions-31/#options Co-Authored-By: Claude Opus 4.7 (1M context) --- .../org/exist/xquery/functions/fn/JSON.java | 177 +++++++++++++++--- .../src/test/xquery/xquery3/json-to-xml.xql | 60 +++++- 2 files changed, 213 insertions(+), 24 deletions(-) diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java index 5dddbb7ecca..8dc3eb38801 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; +import io.lacuna.bifurcan.IEntry; import org.exist.Namespaces; import org.exist.dom.QName; import org.exist.dom.memtree.MemTreeBuilder; @@ -40,6 +41,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.Set; import static java.nio.file.Files.isReadable; import static java.nio.file.Files.newInputStream; @@ -111,11 +113,20 @@ public class JSON extends BasicFunction { public static final String OPTION_DUPLICATES_REJECT = "reject"; public static final String OPTION_DUPLICATES_USE_FIRST = "use-first"; public static final String OPTION_DUPLICATES_USE_LAST = "use-last"; + public static final String OPTION_DUPLICATES_RETAIN = "retain"; public static final String OPTION_LIBERAL = "liberal"; public static final String OPTION_ESCAPE = "escape"; public static final String OPTION_UNESCAPE = "unescape"; + public static final String OPTION_VALIDATE = "validate"; + public static final String OPTION_FALLBACK = "fallback"; public static final QName KEY = new QName("key",null); + private static final Set PERMITTED_DUPLICATES = Set.of( + OPTION_DUPLICATES_REJECT, OPTION_DUPLICATES_USE_FIRST, OPTION_DUPLICATES_RETAIN); + + private static final Set JSON_TO_XML_KNOWN_OPTIONS = Set.of( + OPTION_LIBERAL, OPTION_DUPLICATES, OPTION_VALIDATE, OPTION_ESCAPE, OPTION_FALLBACK); + public JSON(XQueryContext context, FunctionSignature signature) { super(context, signature); } @@ -128,30 +139,12 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce } // process options if present // TODO: jackson does not allow access to raw string, so option "unescape" is not supported - boolean liberal = false; - String handleDuplicates = OPTION_DUPLICATES_USE_LAST; - if (getArgumentCount() == 2) { - final MapType options = (MapType)args[1].itemAt(0); - final Sequence liberalOpt = options.get(new StringValue(OPTION_LIBERAL)); - if (liberalOpt.hasOne()) { - liberal = liberalOpt.itemAt(0).convertTo(Type.BOOLEAN).effectiveBooleanValue(); - } - final Sequence duplicateOpt = options.get(new StringValue(OPTION_DUPLICATES)); - if (duplicateOpt.hasOne()) { - handleDuplicates = duplicateOpt.itemAt(0).getStringValue(); - } - final Sequence escapeOpt = options.get(new StringValue(OPTION_ESCAPE)); - if (escapeOpt.hasOne()) { - try { - escapeOpt.itemAt(0).convertTo(Type.BOOLEAN); - } catch (final XPathException e) { - throw new XPathException(this, ErrorCodes.FOJS0005, - "Value of option 'escape' is not a valid xs:boolean: " + escapeOpt.itemAt(0).getStringValue()); - } - } - } + final ParsedOptions opts = getArgumentCount() == 2 + ? parseOptions((MapType) args[1].itemAt(0), isCalledAs(FS_JSON_TO_XML_NAME)) + : ParsedOptions.DEFAULTS; - JsonFactory factory = createJsonFactory(liberal); + JsonFactory factory = createJsonFactory(opts.liberal); + final String handleDuplicates = opts.handleDuplicates; if (isCalledAs(FS_PARSE_JSON_NAME)) { return parse(args[0], handleDuplicates, factory); @@ -162,6 +155,144 @@ public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathExce } } + private record ParsedOptions(boolean liberal, String handleDuplicates) { + static final ParsedOptions DEFAULTS = new ParsedOptions(false, OPTION_DUPLICATES_USE_LAST); + } + + /** + * Validate and extract options per F&O 3.1 §2.4 (option-parameter conventions) and §17.5.3. + * Wrong-typed values raise XPTY0004; bad permitted values raise FOJS0005. + */ + private ParsedOptions parseOptions(final MapType options, final boolean isJsonToXml) throws XPathException { + final Boolean liberalOpt = requireBooleanOpt(options, OPTION_LIBERAL); + final String duplicatesOpt = requireStringOpt(options, OPTION_DUPLICATES); + final Boolean validateOpt = requireBooleanOpt(options, OPTION_VALIDATE); + requireBooleanOpt(options, OPTION_ESCAPE); + requireFunctionOpt(options, OPTION_FALLBACK, 1); + + String handleDuplicates = OPTION_DUPLICATES_USE_LAST; + if (duplicatesOpt != null) { + if (!PERMITTED_DUPLICATES.contains(duplicatesOpt)) { + throw new XPathException(this, ErrorCodes.FOJS0005, + "Value of option 'duplicates' is not permitted: " + duplicatesOpt); + } + handleDuplicates = duplicatesOpt; + } else if (isJsonToXml) { + handleDuplicates = Boolean.TRUE.equals(validateOpt) + ? OPTION_DUPLICATES_REJECT + : OPTION_DUPLICATES_RETAIN; + } + + if (isJsonToXml) { + if (Boolean.TRUE.equals(validateOpt) && OPTION_DUPLICATES_RETAIN.equals(duplicatesOpt)) { + throw new XPathException(this, ErrorCodes.FOJS0005, + "Option 'duplicates' value 'retain' is not permitted when 'validate' is true"); + } + rejectUnknownJsonToXmlOptions(options); + } + + return new ParsedOptions(Boolean.TRUE.equals(liberalOpt), handleDuplicates); + } + + private void rejectUnknownJsonToXmlOptions(final MapType options) throws XPathException { + for (final IEntry entry : options) { + final String key = entry.key().getStringValue(); + if (!JSON_TO_XML_KNOWN_OPTIONS.contains(key)) { + throw new XPathException(this, ErrorCodes.FOJS0005, + "Unknown option key for fn:json-to-xml: " + key); + } + } + } + + /** + * Return the value of an option as an xs:boolean, or null if the option is absent. + * Raises XPTY0004 if the value is the wrong type or has cardinality other than 1. + * An xs:untypedAtomic value is cast to xs:boolean per F&O 3.1 §2.4. + */ + private Boolean requireBooleanOpt(final MapType options, final String key) throws XPathException { + final Sequence value = options.get(new StringValue(key)); + if (value == null) { + return null; + } + if (value.isEmpty()) { + if (!options.contains(new StringValue(key))) { + return null; + } + // Key present with empty-sequence value: cardinality mismatch. + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option '" + key + "' must be a single value, got empty sequence"); + } + if (!value.hasOne()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option '" + key + "' must be a single xs:boolean, got cardinality " + value.getItemCount()); + } + final Item item = value.itemAt(0); + final int t = item.getType(); + if (Type.subTypeOf(t, Type.BOOLEAN)) { + return ((BooleanValue) item).getValue(); + } + if (t == Type.UNTYPED_ATOMIC) { + return ((AtomicValue) item).convertTo(Type.BOOLEAN).effectiveBooleanValue(); + } + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option '" + key + "' must be an xs:boolean, got " + Type.getTypeName(t)); + } + + /** + * Return the value of an option as an xs:string, or null if the option is absent. + * Raises XPTY0004 if the value is the wrong type or has cardinality other than 1. + */ + private String requireStringOpt(final MapType options, final String key) throws XPathException { + final Sequence value = options.get(new StringValue(key)); + if (value == null) { + return null; + } + if (value.isEmpty()) { + if (!options.contains(new StringValue(key))) { + return null; + } + // Key present with empty-sequence value: cardinality mismatch. + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option '" + key + "' must be a single value, got empty sequence"); + } + if (!value.hasOne()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option '" + key + "' must be a single xs:string, got cardinality " + value.getItemCount()); + } + final Item item = value.itemAt(0); + final int t = item.getType(); + if (Type.subTypeOf(t, Type.STRING) || t == Type.UNTYPED_ATOMIC) { + return item.getStringValue(); + } + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option '" + key + "' must be an xs:string, got " + Type.getTypeName(t)); + } + + /** + * Validate that an option, if present, is a single function reference of the given arity. + * Raises XPTY0004 on type or arity mismatch (or wrong cardinality). + */ + private void requireFunctionOpt(final MapType options, final String key, final int arity) throws XPathException { + final Sequence value = options.get(new StringValue(key)); + if (value == null || value.isEmpty()) { + return; + } + if (!value.hasOne()) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option '" + key + "' must be a single function, got cardinality " + value.getItemCount()); + } + final Item item = value.itemAt(0); + if (!(item instanceof final FunctionReference fr)) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option '" + key + "' must be a function, got " + Type.getTypeName(item.getType())); + } + final int actualArity = fr.getSignature().getArgumentCount(); + if (actualArity != arity) { + throw new XPathException(this, ErrorCodes.XPTY0004, + "Option '" + key + "' must be a function with arity " + arity + ", got arity " + actualArity); + } + } + /** * Create and initialize JSON factory. * diff --git a/exist-core/src/test/xquery/xquery3/json-to-xml.xql b/exist-core/src/test/xquery/xquery3/json-to-xml.xql index c4249436e8f..4f0b0d4abac 100644 --- a/exist-core/src/test/xquery/xquery3/json-to-xml.xql +++ b/exist-core/src/test/xquery/xquery3/json-to-xml.xql @@ -58,7 +58,7 @@ function jsonxml:json-to-xml-error-1() { declare - %test:assertError("err:FOJS0005") + %test:assertError("err:XPTY0004") function jsonxml:json-to-xml-error-2() { json-to-xml('{"x": "\\", "y": "\u0025"}', map{'escape': 'invalid-value'}) }; @@ -67,3 +67,61 @@ function jsonxml:json-to-xml-error-2() { + +(: F&O 3.1 section 2.4 option-parameter conventions: wrong-typed option value -> XPTY0004. :) + +declare + %test:assertError("err:XPTY0004") +function jsonxml:json-to-xml-liberal-wrong-type() { + json-to-xml('[1]', map{'liberal': 'x'}) +}; + +declare + %test:assertError("err:XPTY0004") +function jsonxml:json-to-xml-validate-empty-seq() { + json-to-xml('[1]', map{'validate': ()}) +}; + +declare + %test:assertError("err:XPTY0004") +function jsonxml:json-to-xml-validate-wrong-type() { + json-to-xml('[1]', map{'validate': 'EMCA-262'}) +}; + +declare + %test:assertError("err:XPTY0004") +function jsonxml:json-to-xml-escape-empty-seq() { + json-to-xml('[1]', map{'escape': ()}) +}; + +declare + %test:assertError("err:XPTY0004") +function jsonxml:json-to-xml-fallback-not-function() { + json-to-xml('[1]', map{'fallback': 'dummy'}) +}; + +declare + %test:assertError("err:XPTY0004") +function jsonxml:json-to-xml-fallback-wrong-arity() { + json-to-xml('[1]', map{'fallback': concat#2}) +}; + +(: F&O 3.1 section 17.5.3: permitted-value and consistency checks -> FOJS0005. :) + +declare + %test:assertError("err:FOJS0005") +function jsonxml:json-to-xml-duplicates-bad-value() { + json-to-xml('{"a":1,"a":2}', map{'duplicates': 'use-fish'}) +}; + +declare + %test:assertError("err:FOJS0005") +function jsonxml:json-to-xml-validate-retain-incompat() { + json-to-xml('{}', map{'validate': true(), 'duplicates': 'retain'}) +}; + +declare + %test:assertError("err:FOJS0005") +function jsonxml:json-to-xml-unknown-option() { + json-to-xml('[1]', map{'unknown-opt': 'x'}) +}; From 873f28de31b0fe19f5bc3e5971db66ddf0d68f02 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Mon, 11 May 2026 16:53:55 -0400 Subject: [PATCH 2/3] [bugfix] fn:json-to-xml: atomize option values before type-check Per F&O 3.1 section 2.4, option values undergo function-call-style coercion: nodes are atomized to xs:untypedAtomic before the type check, which is then cast to the declared type. Calling .atomize() on the item picks up both element() and attribute() values that the previous strict-type check was wrongly rejecting with XPTY0004. Closes fn-parse-json-946 (XQTS HEAD) which uses map { 'duplicates': reject }. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/org/exist/xquery/functions/fn/JSON.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java index 8dc3eb38801..79da689ab3e 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java @@ -226,13 +226,13 @@ private Boolean requireBooleanOpt(final MapType options, final String key) throw throw new XPathException(this, ErrorCodes.XPTY0004, "Option '" + key + "' must be a single xs:boolean, got cardinality " + value.getItemCount()); } - final Item item = value.itemAt(0); - final int t = item.getType(); + final AtomicValue atom = value.itemAt(0).atomize(); + final int t = atom.getType(); if (Type.subTypeOf(t, Type.BOOLEAN)) { - return ((BooleanValue) item).getValue(); + return ((BooleanValue) atom).getValue(); } if (t == Type.UNTYPED_ATOMIC) { - return ((AtomicValue) item).convertTo(Type.BOOLEAN).effectiveBooleanValue(); + return atom.convertTo(Type.BOOLEAN).effectiveBooleanValue(); } throw new XPathException(this, ErrorCodes.XPTY0004, "Option '" + key + "' must be an xs:boolean, got " + Type.getTypeName(t)); @@ -259,10 +259,10 @@ private String requireStringOpt(final MapType options, final String key) throws throw new XPathException(this, ErrorCodes.XPTY0004, "Option '" + key + "' must be a single xs:string, got cardinality " + value.getItemCount()); } - final Item item = value.itemAt(0); - final int t = item.getType(); + final AtomicValue atom = value.itemAt(0).atomize(); + final int t = atom.getType(); if (Type.subTypeOf(t, Type.STRING) || t == Type.UNTYPED_ATOMIC) { - return item.getStringValue(); + return atom.getStringValue(); } throw new XPathException(this, ErrorCodes.XPTY0004, "Option '" + key + "' must be an xs:string, got " + Type.getTypeName(t)); From 7c4245e40e1bc70ee8a8df6b67af6570557df986 Mon Sep 17 00:00:00 2001 From: Joe Wicentowski Date: Tue, 12 May 2026 01:33:40 -0400 Subject: [PATCH 3/3] [bugfix] fn:parse-json: split permitted 'duplicates' values per function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit e3a2da8781 introduced a single PERMITTED_DUPLICATES set {reject, use-first, retain} that was applied to both fn:parse-json and fn:json-to-xml. That regressed fn:parse-json calls using "use-last" (the XQSuite test arr:parse-json-duplicates use-last case) with FOJS0005 "value not permitted". Per F&O 3.1 the permitted values differ: * §17.4.1 fn:parse-json: "reject", "use-first", "use-last" https://www.w3.org/TR/xpath-functions-31/#func-parse-json * §17.5.1 fn:json-to-xml: "reject", "use-first", "retain" https://www.w3.org/TR/xpath-functions-31/#func-json-to-xml Both share "reject" and "use-first"; parse-json adds "use-last" (preserve last value), json-to-xml adds "retain" (keep duplicates in the XML output, incompatible with validate=true). Splits PERMITTED_DUPLICATES into per-function sets selected by the existing isJsonToXml flag. fn:json-doc shares fn:parse-json's option semantics (§17.4.4) and so picks up the parse-json permitted set, which is correct. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/org/exist/xquery/functions/fn/JSON.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java b/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java index 79da689ab3e..f5af4fcc5b9 100644 --- a/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java +++ b/exist-core/src/main/java/org/exist/xquery/functions/fn/JSON.java @@ -121,7 +121,14 @@ public class JSON extends BasicFunction { public static final String OPTION_FALLBACK = "fallback"; public static final QName KEY = new QName("key",null); - private static final Set PERMITTED_DUPLICATES = Set.of( + // Per F&O 3.1 §17.4.1 (fn:parse-json) and §17.5.1 (fn:json-to-xml): the permitted + // 'duplicates' values differ between the two functions. parse-json permits use-last + // (preserving the last value); json-to-xml permits retain (keeping all duplicates in + // the XML output, which is incompatible with validate=true). + private static final Set PERMITTED_DUPLICATES_PARSE_JSON = Set.of( + OPTION_DUPLICATES_REJECT, OPTION_DUPLICATES_USE_FIRST, OPTION_DUPLICATES_USE_LAST); + + private static final Set PERMITTED_DUPLICATES_JSON_TO_XML = Set.of( OPTION_DUPLICATES_REJECT, OPTION_DUPLICATES_USE_FIRST, OPTION_DUPLICATES_RETAIN); private static final Set JSON_TO_XML_KNOWN_OPTIONS = Set.of( @@ -172,7 +179,10 @@ private ParsedOptions parseOptions(final MapType options, final boolean isJsonTo String handleDuplicates = OPTION_DUPLICATES_USE_LAST; if (duplicatesOpt != null) { - if (!PERMITTED_DUPLICATES.contains(duplicatesOpt)) { + final Set permitted = isJsonToXml + ? PERMITTED_DUPLICATES_JSON_TO_XML + : PERMITTED_DUPLICATES_PARSE_JSON; + if (!permitted.contains(duplicatesOpt)) { throw new XPathException(this, ErrorCodes.FOJS0005, "Value of option 'duplicates' is not permitted: " + duplicatesOpt); }