Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ object FlexiDecimalSupport {
lit(normalizedValue.scale()).as("scale")
)
} else {
lit(null)
lit(null).cast(FlexiDecimal.DATA_TYPE)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -253,17 +253,22 @@ private static Ucum.ValueWithUnit canonicalOf(@Nonnull final FhirPathQuantity qu
public static Column encodeLiteral(@Nonnull final FhirPathQuantity quantity) {
final BigDecimal value = quantity.getValue();
@Nullable final Ucum.ValueWithUnit canonical = canonicalOf(quantity);
// Cast the struct to the canonical Quantity schema so that fields set to lit(null) carry their
// declared types instead of Spark's NullType (VOID). VOID-typed fields prevent the struct from
// being converted to VARIANT, which is required by variantTransformTree in
// repeat()/repeatAll().
return toStruct(
lit(null),
lit(value),
lit(value.scale()),
lit(null),
lit(quantity.getUnitName()),
lit(quantity.getSystem()),
lit(quantity.getCode()),
FlexiDecimalSupport.toLiteral(canonical != null ? canonical.value() : null),
lit(canonical != null ? canonical.unit() : null),
lit(null));
lit(null),
lit(value),
lit(value.scale()),
lit(null),
lit(quantity.getUnitName()),
lit(quantity.getSystem()),
lit(quantity.getCode()),
FlexiDecimalSupport.toLiteral(canonical != null ? canonical.value() : null),
lit(canonical != null ? canonical.unit() : null),
lit(null))
.cast(dataType());
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
* Copyright © 2018-2026 Commonwealth Scientific and Industrial Research
* Organisation (CSIRO) ABN 41 687 119 230.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package au.csiro.pathling.fhirpath.operator;

import au.csiro.pathling.fhirpath.collection.Collection;
import jakarta.annotation.Nonnull;
import org.apache.spark.sql.Column;

/**
* Provides the functionality of the FHIRPath {@code combine(other)} function, which merges two
* collections into a single collection without eliminating duplicate values. Combining an empty
* collection with a non-empty collection returns the non-empty collection. There is no expectation
* of order in the resulting collection.
*
* <p>Unlike {@link UnionOperator}, {@code combine} does not deduplicate and does not need to
* consult the collection's equality comparator. The array-level merge primitive is shared with
* {@link UnionOperator} via {@link CombiningLogic}.
*
* @author Piotr Szul
* @see <a href="https://hl7.org/fhirpath/#combineother-collection-collection">combine</a>
*/
public class CombineOperator extends SameTypeBinaryOperator {

@Nonnull
@Override
protected Collection handleOneEmpty(
@Nonnull final Collection nonEmpty, @Nonnull final BinaryOperatorInput input) {
// Combine preserves duplicates, so no deduplication is required against an empty peer.
return nonEmpty;
}

@Nonnull
@Override
protected Collection handleEquivalentTypes(
@Nonnull final Collection left,
@Nonnull final Collection right,
@Nonnull final BinaryOperatorInput input) {
final Column leftArray = CombiningLogic.prepareArray(left);
final Column rightArray = CombiningLogic.prepareArray(right);
final Column combined = CombiningLogic.combineArrays(leftArray, rightArray);
return left.copyWithColumn(combined);
}

@Nonnull
@Override
public String getOperatorName() {
return "combine";
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*
* Copyright © 2018-2026 Commonwealth Scientific and Industrial Research
* Organisation (CSIRO) ABN 41 687 119 230.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package au.csiro.pathling.fhirpath.operator;

import static org.apache.spark.sql.functions.array_distinct;
import static org.apache.spark.sql.functions.array_union;
import static org.apache.spark.sql.functions.concat;

import au.csiro.pathling.fhirpath.collection.Collection;
import au.csiro.pathling.fhirpath.collection.DecimalCollection;
import au.csiro.pathling.fhirpath.comparison.ColumnEquality;
import au.csiro.pathling.sql.SqlFunctions;
import jakarta.annotation.Nonnull;
import lombok.experimental.UtilityClass;
import org.apache.spark.sql.Column;

/**
* Shared array-level primitives used by the FHIRPath combining operators. These helpers are used by
* {@link UnionOperator} (which deduplicates) and {@link CombineOperator} (which concatenates
* without deduplication) to share type reconciliation, Decimal normalization, and comparator-aware
* merging.
*
* <p>The helpers operate on already type-reconciled, non-empty {@link Collection} instances. Empty
* operand handling and type reconciliation remain the responsibility of the enclosing operator's
* {@link SameTypeBinaryOperator} template methods.
*
* @author Piotr Szul
*/
@UtilityClass
public class CombiningLogic {

/**
* Extracts the array column from a collection in a form suitable for combining. For {@link
* DecimalCollection}, normalizes the Decimal representation to {@code DECIMAL(32,6)} so that two
* operands with different precisions can be merged without schema mismatch.
*
* @param collection the collection to extract the array column from
* @return the array column ready for combining
*/
@Nonnull
public static Column prepareArray(@Nonnull final Collection collection) {
if (collection instanceof final DecimalCollection decimalCollection) {
return decimalCollection.normalizeDecimalType().getColumn().plural().getValue();
}
return collection.getColumn().plural().getValue();
}

/**
* Deduplicates the values in an array using the appropriate equality strategy. Types that use
* default SQL equality leverage Spark's {@code array_distinct}, while types with custom equality
* (Quantity, Coding, temporal types) use element-wise comparison via {@link
* SqlFunctions#arrayDistinctWithEquality}.
*
* @param arrayColumn the array column to deduplicate
* @param comparator the equality comparator that defines element equality
* @return the deduplicated array column
*/
@Nonnull
public static Column dedupeArray(
@Nonnull final Column arrayColumn, @Nonnull final ColumnEquality comparator) {
if (comparator.usesDefaultSqlEquality()) {
return array_distinct(arrayColumn);
}
return SqlFunctions.arrayDistinctWithEquality(arrayColumn, comparator::equalsTo);
}

/**
* Merges two arrays and deduplicates the result using the appropriate equality strategy. Types
* that use default SQL equality leverage Spark's {@code array_union}, while types with custom
* equality use element-wise comparison via {@link SqlFunctions#arrayUnionWithEquality}.
*
* @param leftArray the left array column
* @param rightArray the right array column
* @param comparator the equality comparator that defines element equality
* @return the merged, deduplicated array column
*/
@Nonnull
public static Column unionArrays(
@Nonnull final Column leftArray,
@Nonnull final Column rightArray,
@Nonnull final ColumnEquality comparator) {
if (comparator.usesDefaultSqlEquality()) {
return array_union(leftArray, rightArray);
}
return SqlFunctions.arrayUnionWithEquality(leftArray, rightArray, comparator::equalsTo);
}

/**
* Concatenates two arrays without deduplication, preserving all duplicate values from both
* operands. Used by the FHIRPath {@code combine(other)} function.
*
* @param leftArray the left array column
* @param rightArray the right array column
* @return the concatenated array column
*/
@Nonnull
public static Column combineArrays(
@Nonnull final Column leftArray, @Nonnull final Column rightArray) {
return concat(leftArray, rightArray);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,7 @@

package au.csiro.pathling.fhirpath.operator;

import static org.apache.spark.sql.functions.array_distinct;
import static org.apache.spark.sql.functions.array_union;

import au.csiro.pathling.fhirpath.collection.Collection;
import au.csiro.pathling.fhirpath.collection.DecimalCollection;
import au.csiro.pathling.fhirpath.comparison.ColumnEquality;
import au.csiro.pathling.sql.SqlFunctions;
import jakarta.annotation.Nonnull;
import org.apache.spark.sql.Column;

Expand All @@ -38,7 +32,8 @@
*
* <p>Equality semantics are determined by the collection's comparator. Types using default SQL
* equality leverage Spark's native array operations, while types with custom equality (Quantity,
* Coding, temporal types) use element-wise comparison.
* Coding, temporal types) use element-wise comparison. The array-level merge primitives are shared
* with {@link CombineOperator} via {@link CombiningLogic}.
*
* @author Piotr Szul
* @see <a href="https://hl7.org/fhirpath/#union-collections">union</a>
Expand All @@ -49,8 +44,8 @@ public class UnionOperator extends SameTypeBinaryOperator {
@Override
protected Collection handleOneEmpty(
@Nonnull final Collection nonEmpty, @Nonnull final BinaryOperatorInput input) {
final Column array = getArrayForUnion(nonEmpty);
final Column deduplicatedArray = deduplicateArray(array, nonEmpty.getComparator());
final Column array = CombiningLogic.prepareArray(nonEmpty);
final Column deduplicatedArray = CombiningLogic.dedupeArray(array, nonEmpty.getComparator());
return nonEmpty.copyWithColumn(deduplicatedArray);
}

Expand All @@ -60,66 +55,13 @@ protected Collection handleEquivalentTypes(
@Nonnull final Collection left,
@Nonnull final Collection right,
@Nonnull final BinaryOperatorInput input) {

final Column leftArray = getArrayForUnion(left);
final Column rightArray = getArrayForUnion(right);
final Column unionResult = unionArrays(leftArray, rightArray, left.getComparator());

final Column leftArray = CombiningLogic.prepareArray(left);
final Column rightArray = CombiningLogic.prepareArray(right);
final Column unionResult =
CombiningLogic.unionArrays(leftArray, rightArray, left.getComparator());
return left.copyWithColumn(unionResult);
}

/**
* Extracts and prepares an array column for union operations. For DecimalCollection, normalizes
* to DECIMAL(32,6) to ensure type compatibility.
*
* @param collection the collection to extract array from
* @return the array column ready for union operation
*/
@Nonnull
private Column getArrayForUnion(@Nonnull final Collection collection) {
if (collection instanceof DecimalCollection decimalCollection) {
return decimalCollection.normalizeDecimalType().getColumn().plural().getValue();
}
return collection.getColumn().plural().getValue();
}

/**
* Deduplicates an array using the appropriate strategy based on comparator type.
*
* @param arrayColumn the array column to deduplicate
* @param comparator the equality comparator to use
* @return deduplicated array column
*/
@Nonnull
private Column deduplicateArray(
@Nonnull final Column arrayColumn, @Nonnull final ColumnEquality comparator) {
if (comparator.usesDefaultSqlEquality()) {
return array_distinct(arrayColumn);
} else {
return SqlFunctions.arrayDistinctWithEquality(arrayColumn, comparator::equalsTo);
}
}

/**
* Merges and deduplicates two arrays using the appropriate strategy.
*
* @param leftArray the left array column
* @param rightArray the right array column
* @param comparator the equality comparator to use
* @return merged and deduplicated array column
*/
@Nonnull
private Column unionArrays(
@Nonnull final Column leftArray,
@Nonnull final Column rightArray,
@Nonnull final ColumnEquality comparator) {
if (comparator.usesDefaultSqlEquality()) {
return array_union(leftArray, rightArray);
} else {
return SqlFunctions.arrayUnionWithEquality(leftArray, rightArray, comparator::equalsTo);
}
}

@Nonnull
@Override
public String getOperatorName() {
Expand Down
Loading
Loading