Python: Normalize constant-on-left and chained comparisons in VectorSearch._build_filter

python/semantic_kernel/data/vector.py

@@ -4,7 +4,30 @@
 import logging
 import operator
 from abc import abstractmethod
-from ast import AST, Constant, Lambda, NodeTransformer, NodeVisitor, UnaryOp, USub, copy_location, expr, parse
+from ast import (
+    AST,
+    And,
+    Attribute,
+    BoolOp,
+    Compare,
+    Constant,
+    Eq,
+    Gt,
+    GtE,
+    Lambda,
+    Lt,
+    LtE,
+    Name,
+    NodeTransformer,
+    NodeVisitor,
+    NotEq,
+    UnaryOp,
+    USub,
+    cmpop,
+    copy_location,
+    expr,
+    parse,
+)
 from collections.abc import AsyncIterable, Callable, Mapping, Sequence
 from copy import deepcopy
 from dataclasses import dataclass
@@ -742,6 +765,66 @@ def visit_UnaryOp(self, node: UnaryOp) -> AST:
         return node
 
 
+_OP_INVERSION: Final[dict[type[cmpop], type[cmpop]]] = {
+    Lt: Gt,
+    LtE: GtE,
+    Gt: Lt,
+    GtE: LtE,
+    Eq: Eq,
+    NotEq: NotEq,
+}
+
+
+class _NormalizeComparisonConstants(NodeTransformer):
+    """Normalize constant-on-left comparisons and chained comparisons in filter lambdas.
+
+    Vector-store ``_lambda_parser`` implementations assume comparison operations have the
+    record property on the left and the constant value on the right (e.g. ``x.price > 5``).
+    When a user writes a comparison with the constant on the left (e.g. ``5 < x.price`` or
+    ``10 == x.id``) or writes a chained range comparison (e.g. ``0 <= x.price < 5``),
+    connectors either produce malformed query filters (e.g. Chroma/MongoDB using the constant
+    as field key), fail schema validation (e.g. Qdrant FieldCondition), or raise
+    NotImplementedError (e.g. Weaviate).
+
+    Normalizing these comparisons into canonical ``property op constant`` comparisons and
+    decomposing chained comparisons into logical ANDs ensures consistent behavior across
+    all vector database connectors.
+    """
+
+    def _normalize_single(self, node: Compare) -> Compare:
+        """Invert a single comparison if the constant is on the left and property on the right."""
+        if (
+            len(node.ops) == 1
+            and type(node.ops[0]) in _OP_INVERSION
+            and isinstance(node.left, Constant)
+            and isinstance(node.comparators[0], (Attribute, Name))
+        ):
+            inv_op_cls = _OP_INVERSION[type(node.ops[0])]
+            inv_op = copy_location(inv_op_cls(), node.ops[0])
+            return copy_location(
+                Compare(left=node.comparators[0], ops=[inv_op], comparators=[node.left]),
+                node,
+            )
+        return node
+
+    def visit_Compare(self, node: Compare) -> AST:
+        """Normalize chained comparisons into BoolOp(And) and flip constant-on-left operands."""
+        self.generic_visit(node)
+        if len(node.ops) > 1:
+            values: list[expr] = []
+            for idx in range(len(node.ops)):
+                left = node.left if idx == 0 else node.comparators[idx - 1]
+                right = node.comparators[idx]
+                op = node.ops[idx]
+                sub = copy_location(
+                    Compare(left=left, ops=[op], comparators=[right]),
+                    node,
+                )
+                values.append(self._normalize_single(sub))
+            return copy_location(BoolOp(op=And(), values=values), node)
+        return self._normalize_single(node)
+
+
 class LambdaVisitor(NodeVisitor, Generic[TFilters]):
     """Visitor class to visit the AST nodes."""
 
@@ -2016,11 +2099,14 @@ def _build_filter(self, search_filter: OptionalOneOrMany[Callable | str] | None)
 
         visitor = LambdaVisitor(self._lambda_parser)
         folder = _FoldUnaryNumericConstants()
+        normalizer = _NormalizeComparisonConstants()
         for filter_ in filters:
             # parse lambda expression with AST
             tree = parse(filter_ if isinstance(filter_, str) else getsource(filter_).strip())
             # Fold negative literals so connector parsers see a numeric Constant (#14571).
             tree = folder.visit(tree)
+            # Normalize constant-on-left comparisons and decompose chained comparisons.
+            tree = normalizer.visit(tree)
             visitor.visit(tree)
         created_filters = visitor.output_filters
         if len(created_filters) == 0:

python/tests/unit/data/test_vector_comparison_normalization.py

@@ -0,0 +1,215 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Unit and integration tests for comparison normalization in vector store filter lambdas."""
+
+import ast
+from ast import Attribute, BoolOp, Compare, Constant, Eq, Gt, GtE, Lt, LtE, Name, NotEq, parse
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from chromadb.api import ClientAPI
+from qdrant_client.async_qdrant_client import AsyncQdrantClient
+from qdrant_client.models import FieldCondition, Range
+from qdrant_client.models import Filter as QdrantFilter
+from weaviate import WeaviateAsyncClient
+from weaviate.classes.query import Filter as WeaviateFilter
+
+from semantic_kernel.connectors.chroma import ChromaCollection
+from semantic_kernel.connectors.qdrant import QdrantCollection
+from semantic_kernel.connectors.weaviate import WeaviateCollection
+from semantic_kernel.data.vector import (
+    VectorStoreCollectionDefinition,
+    VectorStoreField,
+    _FoldUnaryNumericConstants,
+    _NormalizeComparisonConstants,
+)
+
+
+@pytest.fixture
+def numeric_definition() -> VectorStoreCollectionDefinition:
+    return VectorStoreCollectionDefinition(
+        fields=[
+            VectorStoreField("key", name="id", type="str"),
+            VectorStoreField("data", name="price", type="float"),
+            VectorStoreField("data", name="temperature", type="float"),
+            VectorStoreField("vector", name="vector", dimensions=2, type="float"),
+        ]
+    )
+
+
+@pytest.mark.parametrize(
+    "expr_str,expected_op,expected_constant",
+    [
+        ("lambda x: 5 < x.price", Gt, 5),
+        ("lambda x: 5 <= x.price", GtE, 5),
+        ("lambda x: 5 > x.price", Lt, 5),
+        ("lambda x: 5 >= x.price", LtE, 5),
+        ('lambda x: "active" == x.status', Eq, "active"),
+        ("lambda x: 10 != x.id", NotEq, 10),
+    ],
+)
+def test_normalize_single_comparison_flips_constant_on_left(expr_str, expected_op, expected_constant):
+    tree = parse(expr_str)
+    transformed = _NormalizeComparisonConstants().visit(tree)
+    compare = transformed.body[0].value.body
+
+    assert isinstance(compare, Compare)
+    assert len(compare.ops) == 1
+    assert isinstance(compare.ops[0], expected_op)
+    assert isinstance(compare.left, Attribute)
+    assert isinstance(compare.comparators[0], Constant)
+    assert compare.comparators[0].value == expected_constant
+
+
+def test_normalize_single_comparison_works_with_bare_name():
+    tree = parse("lambda price: 5 > price")
+    transformed = _NormalizeComparisonConstants().visit(tree)
+    compare = transformed.body[0].value.body
+
+    assert isinstance(compare, Compare)
+    assert isinstance(compare.ops[0], Lt)
+    assert isinstance(compare.left, Name)
+    assert compare.left.id == "price"
+    assert isinstance(compare.comparators[0], Constant)
+    assert compare.comparators[0].value == 5
+
+
+def test_normalize_folded_negative_constant_on_left():
+    tree = parse("lambda x: -5 > x.price")
+    tree = _FoldUnaryNumericConstants().visit(tree)
+    transformed = _NormalizeComparisonConstants().visit(tree)
+    compare = transformed.body[0].value.body
+
+    assert isinstance(compare, Compare)
+    assert isinstance(compare.ops[0], Lt)
+    assert isinstance(compare.left, Attribute)
+    assert isinstance(compare.comparators[0], Constant)
+    assert compare.comparators[0].value == -5
+
+
+@pytest.mark.parametrize(
+    "expr_str",
+    [
+        "lambda x: x.price > 5",
+        "lambda x: x.price <= 5",
+        'lambda x: x.status == "active"',
+        "lambda x: x.id != 10",
+    ],
+)
+def test_normalize_preserves_constant_on_right(expr_str):
+    tree = parse(expr_str)
+    original_dump = ast.dump(tree)
+    transformed = _NormalizeComparisonConstants().visit(tree)
+    assert ast.dump(transformed) == original_dump
+
+
+@pytest.mark.parametrize(
+    "expr_str",
+    [
+        'lambda x: x.tag in ["a", "b"]',
+        'lambda x: "a" in x.tags',
+        "lambda x: x.price == x.discount",
+        "lambda x: 5 == 5",
+    ],
+)
+def test_normalize_preserves_non_invertible_comparisons(expr_str):
+    tree = parse(expr_str)
+    original_dump = ast.dump(tree)
+    transformed = _NormalizeComparisonConstants().visit(tree)
+    assert ast.dump(transformed) == original_dump
+
+
+def test_normalize_chained_comparisons_decomposes_into_and():
+    tree = parse("lambda x: 0 <= x.price < 5")
+    transformed = _NormalizeComparisonConstants().visit(tree)
+    body = transformed.body[0].value.body
+
+    assert isinstance(body, BoolOp)
+    assert isinstance(body.op, ast.And)
+    assert len(body.values) == 2
+
+    first = body.values[0]
+    assert isinstance(first, Compare)
+    assert isinstance(first.ops[0], GtE)
+    assert isinstance(first.left, Attribute)
+    assert first.left.attr == "price"
+    assert isinstance(first.comparators[0], Constant)
+    assert first.comparators[0].value == 0
+
+    second = body.values[1]
+    assert isinstance(second, Compare)
+    assert isinstance(second.ops[0], Lt)
+    assert isinstance(second.left, Attribute)
+    assert second.left.attr == "price"
+    assert isinstance(second.comparators[0], Constant)
+    assert second.comparators[0].value == 5
+
+
+def test_normalize_chained_comparisons_with_folded_negative_literal():
+    tree = parse("lambda x: -10 < x.temperature <= 20")
+    tree = _FoldUnaryNumericConstants().visit(tree)
+    transformed = _NormalizeComparisonConstants().visit(tree)
+    body = transformed.body[0].value.body
+
+    assert isinstance(body, BoolOp)
+    assert isinstance(body.op, ast.And)
+    assert len(body.values) == 2
+
+    first = body.values[0]
+    assert isinstance(first, Compare)
+    assert isinstance(first.ops[0], Gt)
+    assert isinstance(first.left, Attribute)
+    assert first.left.attr == "temperature"
+    assert first.comparators[0].value == -10
+
+    second = body.values[1]
+    assert isinstance(second, Compare)
+    assert isinstance(second.ops[0], LtE)
+    assert isinstance(second.left, Attribute)
+    assert second.left.attr == "temperature"
+    assert second.comparators[0].value == 20
+
+
+def test_chroma_build_filter_normalized_comparisons(numeric_definition):
+    collection = ChromaCollection(
+        collection_name="temps",
+        record_type=dict,
+        definition=numeric_definition,
+        client=MagicMock(spec=ClientAPI),
+    )
+    assert collection._build_filter("lambda x: 5 > x.price") == {"price": {"$lt": 5}}
+    assert collection._build_filter("lambda x: 0 <= x.price < 5") == {
+        "$and": [{"price": {"$gte": 0}}, {"price": {"$lt": 5}}]
+    }
+
+
+def test_qdrant_build_filter_normalized_comparisons(numeric_definition):
+    collection = QdrantCollection(
+        collection_name="temps",
+        record_type=dict,
+        definition=numeric_definition,
+        client=MagicMock(spec=AsyncQdrantClient),
+    )
+    assert collection._build_filter("lambda x: 5 > x.price") == FieldCondition(key="price", range=Range(lt=5.0))
+    assert collection._build_filter("lambda x: 0 <= x.price < 5") == QdrantFilter(
+        must=[
+            FieldCondition(key="price", range=Range(gte=0.0)),
+            FieldCondition(key="price", range=Range(lt=5.0)),
+        ]
+    )
+
+
+def test_weaviate_build_filter_normalized_comparisons(numeric_definition):
+    collection = WeaviateCollection(
+        collection_name="temps",
+        record_type=dict,
+        definition=numeric_definition,
+        async_client=AsyncMock(spec=WeaviateAsyncClient),
+    )
+    assert collection._build_filter("lambda x: 5 > x.price") == WeaviateFilter.by_property("price").less_than(5)
+    parsed = collection._build_filter("lambda x: 0 <= x.price < 5")
+    expected = WeaviateFilter.all_of([
+        WeaviateFilter.by_property("price").greater_or_equal(0),
+        WeaviateFilter.by_property("price").less_than(5),
+    ])
+    assert parsed.filters == expected.filters