Support query filters matching list values in generic and mapped tables

openidm-repo-jdbc/pom.xml

@@ -13,7 +13,7 @@
     information: "Portions copyright [year] [name of copyright owner]".
 
     Copyright (c) 2011-2013 ForgeRock AS. All Rights Reserved
-    Portions Copyright 2017-2024 Wren Security.
+    Portions Copyright 2017-2026 Wren Security.
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
@@ -35,10 +35,17 @@
         <openidm.osgi.import.before.defaults>
             !org.testng.annotations,!sun.misc,!org.w3c.dom,jakarta.inject;resolution:=optional
         </openidm.osgi.import.before.defaults>
-        <testcontainers.version>1.19.3</testcontainers.version>
+        <testcontainers.version>2.0.5</testcontainers.version>
+        <testcontainers-module.version>2.0.1</testcontainers-module.version>
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.wrensecurity.wrenidm</groupId>
             <artifactId>openidm-enhanced-config</artifactId>
@@ -118,8 +125,8 @@
 
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>db2</artifactId>
-            <version>${testcontainers.version}</version>
+            <artifactId>testcontainers-db2</artifactId>
+            <version>${testcontainers-module.version}</version>
             <scope>test</scope>
         </dependency>
 
@@ -132,8 +139,8 @@
 
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>mssqlserver</artifactId>
-            <version>${testcontainers.version}</version>
+            <artifactId>testcontainers-mssqlserver</artifactId>
+            <version>${testcontainers-module.version}</version>
             <scope>test</scope>
         </dependency>
 
@@ -146,8 +153,8 @@
 
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>oracle-xe</artifactId>
-            <version>${testcontainers.version}</version>
+            <artifactId>testcontainers-oracle-xe</artifactId>
+            <version>${testcontainers-module.version}</version>
             <scope>test</scope>
         </dependency>
 
@@ -160,8 +167,8 @@
 
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>mysql</artifactId>
-            <version>${testcontainers.version}</version>
+            <artifactId>testcontainers-mysql</artifactId>
+            <version>${testcontainers-module.version}</version>
             <scope>test</scope>
         </dependency>
 
@@ -174,8 +181,8 @@
 
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>postgresql</artifactId>
-            <version>${testcontainers.version}</version>
+            <artifactId>testcontainers-postgresql</artifactId>
+            <version>${testcontainers-module.version}</version>
             <scope>test</scope>
         </dependency>
 

openidm-repo-jdbc/src/main/java/org/forgerock/openidm/datasource/jdbc/impl/JDBCDataSourceService.java

@@ -111,7 +111,7 @@ public DataSourceConfig deserialize(JsonParser jsonParser, DeserializationContex
             final Iterable<Map.Entry<String, JsonNode>> fields = new Iterable<Map.Entry<String, JsonNode>>() {
                 @Override
                 public Iterator<Map.Entry<String, JsonNode>> iterator() {
-                    return node.fields();
+                    return node.properties().iterator();
                 }
             };
             for (Map.Entry<String, JsonNode> element : fields) {

openidm-repo-jdbc/src/main/java/org/forgerock/openidm/repo/jdbc/impl/handler/GenericTableHandler.java

@@ -13,7 +13,7 @@
  * information: "Portions copyright [year] [name of copyright owner]".
  *
  * Copyright 2011-2016 ForgeRock AS.
- * Portions Copyright 2024 Wren Security
+ * Portions Copyright 2024-2026 Wren Security
  */
 package org.forgerock.openidm.repo.jdbc.impl.handler;
 
@@ -26,13 +26,14 @@
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.AbstractMap.SimpleEntry;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumMap;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
@@ -63,7 +64,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Generic table handler that supports objects stored as JSON string with a separate properies table used for
+ * Generic table handler that supports objects stored as JSON string with a separate properties table used for
  * indexing and querying objects by property value filters.
  *
  * <p>
@@ -347,13 +348,13 @@ protected void writeValueProperties(String fullId, long databaseId, JsonValue va
             return; // no searchable properties, no need to index
         }
 
-        Map<JsonPointer, Object> pairs = new LinkedHashMap<JsonPointer, Object>();
-        extractValueProperties(value, pairs::put);
+        List<Entry<JsonPointer, Object>> pairs = new ArrayList<>();
+        extractValueProperties(value, (p, v) -> pairs.add(new SimpleEntry<>(p, v)), false);
 
         try (var createStatement = resolveImplicitStatement(ImplicitSqlType.PROPCREATE, false, connection)) {
             int batchingCount = 0;
 
-            for (var pair : pairs.entrySet()) {
+            for (var pair : pairs) {
                 // prepare index properties
                 var object = pair.getValue();
                 var idxkey = pair.getKey().toString();
@@ -401,26 +402,30 @@ protected void writeValueProperties(String fullId, long databaseId, JsonValue va
      *
      * @param value JSON value (array or object)
      * @param collector callback for collecting extracted property values
+     * @param unwrap whether to collect primitive values under the parent pointer
      */
-    private void extractValueProperties(JsonValue json, BiConsumer<JsonPointer, Object> collector) {
+    private void extractValueProperties(JsonValue json, BiConsumer<JsonPointer, Object> collector, boolean unwrap) {
         for (JsonValue entry : json) {
             JsonPointer pointer = entry.getPointer();
             if (!tableConfig.isSearchable(pointer)) {
                 continue;
             }
             if (entry.isMap() || entry.isList()) {
-                extractValueProperties(entry, collector);
-                continue;
+                extractValueProperties(entry, collector, entry.isList());
+            } else {
+                collector.accept(pointer, entry.getObject());
+            }
+            if (unwrap) {
+                collector.accept(pointer.parent(), entry.getObject());
             }
-            collector.accept(pointer, entry.getObject());
         }
     }
 
     /**
      * Remove properties of a resource stored under the specified database identifier from the properties table.
      *
-     * @param databaseId the identifier that link the properties table with the main table (foreign key)
      * @param fullId the qualified identifier of the owner object
+     * @param databaseId the identifier that link the properties table with the main table (foreign key)
      * @param connection the DB connection
      * @throws SQLException if the insert failed
      */
@@ -640,7 +645,7 @@ protected int getSearchableLength() {
      * @return new GenericSQLQueryFilterVisitor instance
      */
     protected GenericSQLQueryFilterVisitor createFilterVisitor(SQLBuilder builder) {
-        return new GenericSQLQueryFilterVisitor(getSearchableLength(), builder);
+        return new GenericSQLQueryFilterVisitor(getSearchableLength());
     }
 
     @Override

openidm-repo-jdbc/src/main/java/org/forgerock/openidm/repo/jdbc/impl/handler/MappedTableHandler.java

@@ -12,7 +12,7 @@
  * information: "Portions copyright [year] [name of copyright owner]".
  *
  * Copyright 2011-2016 ForgeRock AS.
- * Portions Copyright 2018-2025 Wren Security.
+ * Portions Copyright 2018-2026 Wren Security.
  */
 package org.forgerock.openidm.repo.jdbc.impl.handler;
 
@@ -45,10 +45,13 @@
 import org.forgerock.openidm.repo.jdbc.Constants;
 import org.forgerock.openidm.repo.jdbc.SQLExceptionHandler;
 import org.forgerock.openidm.repo.jdbc.impl.SQLBuilder;
+import org.forgerock.openidm.repo.jdbc.impl.handler.MappedColumnConfig.ValueType;
 import org.forgerock.openidm.repo.jdbc.impl.mapper.MappedResultMapper;
 import org.forgerock.openidm.repo.jdbc.impl.mapper.ResultMapper;
 import org.forgerock.openidm.repo.jdbc.impl.mapper.ResultMappers;
-import org.forgerock.openidm.repo.jdbc.impl.query.MappedSQLQueryFilterVisitor;
+import org.forgerock.openidm.repo.jdbc.impl.query.SQLRendererFieldFilterVisitor;
+import org.forgerock.openidm.repo.jdbc.impl.query.SQLRendererQueryFilterVisitor;
+import org.forgerock.openidm.repo.jdbc.impl.query.SimpleFieldFilterVisitor;
 import org.forgerock.openidm.repo.jdbc.impl.query.TableQueryHandler;
 import org.forgerock.openidm.repo.jdbc.impl.statement.NamedParameterCollector;
 import org.forgerock.openidm.repo.jdbc.impl.statement.NamedParameterSql;
@@ -471,14 +474,21 @@ protected SQLBuilder resolveQueryFilter(QueryFilter<JsonPointer> queryFilter, Li
      * @return new configuration resolver instance
      */
     protected MappedConfigResolver createConfigResolver() {
-        Map<JsonPointer, MappedColumnConfig> columnConfig = columnMapping.values().stream()
+        Map<JsonPointer, MappedColumnConfig> columnConfigs = columnMapping.values().stream()
                 .collect(Collectors.toMap(value -> value.propertyName, value -> value));
         return field -> {
-            var config = columnConfig.get(field);
-            if (config == null) {
-                throw new IllegalArgumentException("Unknown object field: " + field.toString());
-            }
-            return config;
+            var columnPath = field;
+            do {
+                var config = columnConfigs.get(columnPath);
+                if (config == null) {
+                    continue;
+                }
+                if (columnPath != field && config.valueType != ValueType.JSON_MAP) {
+                    break; // only JSON_MAP can be mapped to parent path
+                }
+                return config;
+            } while ((columnPath = columnPath.parent()) != null);
+            throw new IllegalArgumentException("Unknown object field: " + field.toString());
         };
     }
 
@@ -503,13 +513,29 @@ public String toSQL() {
     }
 
     /**
-     * Create new {@link MappedSQLQueryFilterVisitor} to render query filter queries.
+     * Create new {@link SQLRendererQueryFilterVisitor} to render query filter queries.
      *
      * @param configResolver column configuration resolver
-     * @return new MappedSQLQueryFilterVisitor instance
+     * @return new query filter visitor instance
      */
-    protected MappedSQLQueryFilterVisitor createFilterVisitor(MappedConfigResolver configResolver) {
-        return new MappedSQLQueryFilterVisitor(configResolver, objectMapper);
+    protected SQLRendererQueryFilterVisitor createFilterVisitor(MappedConfigResolver configResolver) {
+        return new SQLRendererQueryFilterVisitor(field -> {
+            var columnConfig = configResolver.resolve(field);
+            switch (columnConfig.valueType) {
+            case JSON_LIST, JSON_MAP:
+                return createJsonFieldVisitor(columnConfig);
+            default:
+                return createSimpleFieldVisitor(columnConfig);
+            }
+        });
+    }
+
+    protected SQLRendererFieldFilterVisitor createSimpleFieldVisitor(MappedColumnConfig columnConfig) {
+        return new SimpleFieldFilterVisitor(columnConfig, objectMapper);
+    }
+
+    protected SQLRendererFieldFilterVisitor createJsonFieldVisitor(MappedColumnConfig columnConfig) {
+        throw new UnsupportedOperationException("JSON based field filtering not supported");
     }
 
     @Override

openidm-repo-jdbc/src/main/java/org/forgerock/openidm/repo/jdbc/impl/query/FieldFilterVisitorResolver.java

@@ -0,0 +1,34 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.1.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.1.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 Wren Security
+ */
+package org.forgerock.openidm.repo.jdbc.impl.query;
+
+import org.forgerock.json.JsonPointer;
+
+/**
+ * Resolver for field based filter rendering visitors.
+ */
+@FunctionalInterface
+public interface FieldFilterVisitorResolver {
+
+    /**
+     * Resolve field filter rendering visitor.
+     *
+     * @param field field being visited
+     * @return visitor for the field
+     */
+    SQLRendererFieldFilterVisitor resolve(JsonPointer field);
+
+}

openidm-repo-jdbc/src/main/java/org/forgerock/openidm/repo/jdbc/impl/query/GenericSQLQueryFilterVisitor.java

@@ -2,7 +2,7 @@
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
  * Copyright 2015 ForgeRock AS. All rights reserved.
- * Portions Copyright 2024 Wren Security.
+ * Portions Copyright 2024-2026 Wren Security.
  *
  * The contents of this file are subject to the terms
  * of the Common Development and Distribution License
@@ -33,7 +33,6 @@
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
 import org.forgerock.json.JsonPointer;
-import org.forgerock.openidm.repo.jdbc.impl.SQLBuilder;
 import org.forgerock.openidm.repo.jdbc.impl.statement.NamedParameterCollector;
 import org.forgerock.openidm.repo.util.AbstractSQLQueryFilterVisitor;
 import org.forgerock.openidm.repo.util.Clause;
@@ -48,22 +47,17 @@
  * Filter visitor does not support <i>contains</i> filters for collection members. Only simple <code>string</code>
  * based <i>contains</i> is supported.
  */
-// TODO support collection based assertions
 public class GenericSQLQueryFilterVisitor extends AbstractSQLQueryFilterVisitor<Clause, NamedParameterCollector> {
 
     private final int searchableLength;
 
-    private final SQLBuilder builder;
-
     /**
      * Construct a QueryFilterVisitor to produce SQL for managed objects using the generic table structure.
      *
      * @param searchableLength the searchable length; properties longer than this will be trimmed to this length
-     * @param builder the {@link SQLBuilder} to use to keep track of the select columns, table joins, and order by lists
      */
-    public GenericSQLQueryFilterVisitor(final int searchableLength, SQLBuilder builder) {
+    public GenericSQLQueryFilterVisitor(final int searchableLength) {
         this.searchableLength = searchableLength;
-        this.builder = builder;
     }
 
     private boolean isNumeric(final Object valueAssertion) {
@@ -136,8 +130,8 @@ public Clause visitValueAssertion(NamedParameterCollector collector, String oper
             return where("obj.objectid " + operand + " ${" + valueParam + "}");
         }
 
-        String propParam = collector.register("k", field.toString());
         String joinAlias = collector.generate("p");
+        String propParam = collector.register("k", field.toString());
         final Clause valueClause;
         if (isNumeric(valueAssertion)) {
             valueClause = buildNumericValueClause(joinAlias, operand, valueParam);
@@ -146,10 +140,11 @@ public Clause visitValueAssertion(NamedParameterCollector collector, String oper
         } else {
             valueClause = buildStringValueClause(joinAlias, operand, valueParam);
         }
-        builder.leftJoin("${_dbSchema}.${_propTable}", joinAlias)
-                .on(where(joinAlias + ".${_mainTable}_id = obj.id")
-                        .and(where(joinAlias + ".propkey = ${" + propParam + "}")));
-        return valueClause;
+
+        return where("EXISTS (SELECT 1 FROM ${_dbSchema}.${_propTable} " + joinAlias + " WHERE "
+                + joinAlias + ".${_mainTable}_id = obj.id"
+                + " AND " + joinAlias + ".propkey = ${" + propParam + "}"
+                + " AND " + valueClause.toSQL() + ")");
     }
 
     /**
@@ -182,10 +177,9 @@ public Clause visitPresentFilter(NamedParameterCollector collector, JsonPointer
         } else {
             var propParam = collector.register("k", field.toString());
             var joinAlias = collector.generate("p");
-            builder.leftJoin("${_dbSchema}.${_propTable}", joinAlias)
-                    .on(where(joinAlias + ".${_mainTable}_id = obj.id")
-                            .and(joinAlias + ".propkey = ${" + propParam + "}"));
-            return where(joinAlias + ".propvalue IS NOT NULL");
+            return where("EXISTS (SELECT 1 FROM ${_dbSchema}.${_propTable} " + joinAlias + " WHERE "
+                    + joinAlias + ".${_mainTable}_id = obj.id"
+                    + " AND " + joinAlias + ".propkey = ${" + propParam + "})");
         }
     }