.Net: Improve Milvus filtering (#14498)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

Improve Milvus filtering to allow more id types to be supported

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [ ] The code builds clean without any errors or warnings
- [ ] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [ ] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄

Author: westey-m

dotnet/MEVD.slnx

@@ -26,5 +26,6 @@
   <Folder Name="/test/" />
   <Folder Name="/test/VectorData/">
     <Project Path="test/VectorData/Chroma.UnitTests/Chroma.UnitTests.csproj" />
+    <Project Path="test/VectorData/Milvus.UnitTests/Milvus.UnitTests.csproj" />
   </Folder>
 </Solution>

dotnet/src/VectorData/Milvus/Milvus.csproj

@@ -28,6 +28,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\SemanticKernel.Core\SemanticKernel.Core.csproj" />
+    <InternalsVisibleTo Include="SemanticKernel.Connectors.Milvus.UnitTests" />
   </ItemGroup>
 
 </Project>

dotnet/src/VectorData/Milvus/MilvusMemoryStore.cs

@@ -247,6 +247,7 @@ public Task DeleteCollectionAsync(string collectionName, CancellationToken cance
     /// <inheritdoc />
     public async Task<string> UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default)
     {
+        await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false);
         MilvusCollection collection = this.Client.GetCollection(collectionName);
 
         var metadata = record.Metadata;
@@ -276,8 +277,7 @@ public async IAsyncEnumerable<string> UpsertBatchAsync(
         IEnumerable<MemoryRecord> records,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        StringBuilder idString = new();
-
+        await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false);
         List<bool> isReferenceData = [];
         List<string> externalSourceNameData = [];
         List<string> idData = [];
@@ -292,13 +292,6 @@ public async IAsyncEnumerable<string> UpsertBatchAsync(
         {
             var metadata = record.Metadata;
 
-            if (idString.Length > 0)
-            {
-                idString.Append(',');
-            }
-
-            idString.Append('"').Append(metadata.Id).Append('"');
-
             isReferenceData.Add(metadata.IsReference);
             externalSourceNameData.Add(metadata.ExternalSourceName);
             idData.Add(record.Metadata.Id);
@@ -356,22 +349,11 @@ public async IAsyncEnumerable<MemoryRecord> GetBatchAsync(
         bool withEmbeddings = false,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        StringBuilder idString = new();
-
-        foreach (string key in keys)
-        {
-            if (idString.Length > 0)
-            {
-                idString.Append(',');
-            }
-
-            idString.Append('"').Append(key).Append('"');
-        }
-
+        await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false);
         IReadOnlyList<FieldData> fields = await this.Client
             .GetCollection(collectionName)
             .QueryAsync(
-                $"{IdFieldName} in [{idString}]",
+                BuildIdFilter(keys),
                 withEmbeddings ? this._queryParametersWithEmbedding : this._queryParametersWithoutEmbedding,
                 cancellationToken: cancellationToken)
             .ConfigureAwait(false);
@@ -386,36 +368,16 @@ public async IAsyncEnumerable<MemoryRecord> GetBatchAsync(
 
     /// <inheritdoc />
     public Task RemoveAsync(string collectionName, string key, CancellationToken cancellationToken = default)
-        => this.Client.GetCollection(collectionName)
-            .DeleteAsync($@"{IdFieldName} in [""{key}""]", cancellationToken: cancellationToken);
+        => this.RemoveBatchAsync(collectionName, [key], cancellationToken);
 
     /// <inheritdoc />
-    public Task RemoveBatchAsync(string collectionName, IEnumerable<string> keys, CancellationToken cancellationToken = default)
+    public async Task RemoveBatchAsync(string collectionName, IEnumerable<string> keys, CancellationToken cancellationToken = default)
     {
-        StringBuilder idString = new();
-
-        idString.Append(IdFieldName).Append(" in [");
-
-        bool first = true;
-        foreach (string id in keys)
-        {
-            if (first)
-            {
-                first = false;
-            }
-            else
-            {
-                idString.Append(',');
-            }
-
-            idString.Append('"').Append(id).Append('"');
-        }
-
-        idString.Append(']');
-
-        return this.Client
+        await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false);
+        await this.Client
             .GetCollection(collectionName)
-            .DeleteAsync(idString.ToString(), cancellationToken: cancellationToken);
+            .DeleteAsync(BuildIdFilter(keys), cancellationToken: cancellationToken)
+            .ConfigureAwait(false);
     }
 
     /// <inheritdoc />
@@ -443,6 +405,7 @@ public Task RemoveBatchAsync(string collectionName, IEnumerable<string> keys, Ca
         bool withEmbeddings = false,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
+        await this.EnsureSupportedServerAsync(cancellationToken).ConfigureAwait(false);
         MilvusCollection collection = this.Client.GetCollection(collectionName);
 
         SearchResults results = await collection
@@ -459,23 +422,8 @@ public Task RemoveBatchAsync(string collectionName, IEnumerable<string> keys, Ca
         Dictionary<string, ReadOnlyMemory<float>>? embeddingMap = null;
         if (withEmbeddings)
         {
-            StringBuilder filter = new();
-            filter.Append(IdFieldName).Append(" in [");
-
-            for (int rowNum = 0; rowNum < ids.Count; rowNum++)
-            {
-                if (rowNum > 0)
-                {
-                    filter.Append(',');
-                }
-
-                filter.Append('"').Append(ids[rowNum]).Append('"');
-            }
-
-            filter.Append(']');
-
             IReadOnlyList<FieldData> fieldData = await collection.QueryAsync(
-                    filter.ToString(),
+                    BuildIdFilter(ids),
                     new() { OutputFields = { EmbeddingFieldName } },
                     cancellationToken: cancellationToken)
                 .ConfigureAwait(false);
@@ -502,6 +450,83 @@ public Task RemoveBatchAsync(string collectionName, IEnumerable<string> keys, Ca
         }
     }
 
+    private async Task EnsureSupportedServerAsync(CancellationToken cancellationToken)
+    {
+        string version = await this.Client.GetVersionAsync(cancellationToken).ConfigureAwait(false);
+        ValidateServerVersion(version);
+    }
+
+    internal static void ValidateServerVersion(string version)
+    {
+        string numericVersion = version.StartsWith("v", StringComparison.Ordinal) ? version.Substring(1) : version;
+        // The official 2.3.0 release image reports "v2.3.0-dev".
+        if (numericVersion.EndsWith("-dev", StringComparison.Ordinal))
+        {
+            numericVersion = numericVersion.Substring(0, numericVersion.Length - 4);
+        }
+
+        // Older servers accept escaped strings but retain the escapes as part of the ID.
+        if (!Version.TryParse(numericVersion, out Version? parsedVersion) ||
+            parsedVersion < new Version(2, 3, 0))
+        {
+            throw new NotSupportedException($"Milvus 2.3.0 or newer is required to handle record IDs safely. The server reported version '{version}'.");
+        }
+    }
+
+    internal static string BuildIdFilter(IEnumerable<string> keys)
+    {
+        StringBuilder filter = new();
+        filter.Append(IdFieldName).Append(" in [");
+
+        bool first = true;
+        foreach (string key in keys)
+        {
+            if (!first)
+            {
+                filter.Append(',');
+            }
+
+            first = false;
+            filter.Append('"');
+            foreach (char character in key)
+            {
+                switch (character)
+                {
+                    case '\\':
+                        filter.Append(@"\\");
+                        break;
+                    case '"':
+                        filter.Append("\\\"");
+                        break;
+                    case '\r':
+                        filter.Append(@"\r");
+                        break;
+                    case '\n':
+                        filter.Append(@"\n");
+                        break;
+                    case '\t':
+                        filter.Append(@"\t");
+                        break;
+                    default:
+                        if (char.IsControl(character))
+                        {
+                            filter.Append(@"\u").Append(((int)character).ToString("x4", CultureInfo.InvariantCulture));
+                        }
+                        else
+                        {
+                            filter.Append(character);
+                        }
+
+                        break;
+                }
+            }
+
+            filter.Append('"');
+        }
+
+        return filter.Append(']').ToString();
+    }
+
     private MemoryRecord ReadMemoryRecord(IReadOnlyList<FieldData> data, int rowNum, ReadOnlyMemory<float>? externalEmbedding = null)
     {
         bool isReference = false;

dotnet/src/VectorData/Milvus/README.md

@@ -2,14 +2,16 @@
 
 This is an implementation of the Semantic Kernel Memory Store abstraction for the [Milvus vector database](https://milvus.io).
 
-**Note:** Currently, only Milvus v2.2 is supported. v2.3 is coming soon, older versions are untested.
+**Server requirement:** Record operations require Milvus v2.3.0 or newer. The connector checks the server version before saving, reading, removing, or searching records and throws `NotSupportedException` for older or unrecognized versions. Collection management and direct calls through `Client` are not covered by this check.
+
+Record IDs are treated as literal values, including quotes, backslashes, brackets, and line breaks. IDs are not renamed or normalized. Milvus v2.2.14 does not decode escaped IDs correctly and could otherwise target a different record. Upgrade the server before using this connector with existing collections.
 
 ## Quickstart using a standalone Milvus installation
 
 1. Download the Milvus docker-compose.yml:
 
 ```bash
-wget https://github.com/milvus-io/milvus/releases/download/v2.2.14/milvus-standalone-docker-compose.yml -O docker-compose.yml
+wget https://github.com/milvus-io/milvus/releases/download/v2.3.0/milvus-standalone-docker-compose.yml -O docker-compose.yml
 ```
 
 2. Start Milvus:
@@ -30,4 +32,4 @@ SemanticTextMemory textMemory = new(memoryStore, embeddingGenerator);
 var memoryPlugin = kernel.ImportPluginFromObject(new TextMemoryPlugin(textMemory));
 ```
 
-More information on setting up Milvus can be found [here](https://milvus.io/docs/v2.2.x/install_standalone-docker.md). The `MilvusMemoryStore` constructor provides additional configuration options, such as the vector size, the similarity metric type, etc.
+More information on setting up Milvus can be found [here](https://milvus.io/docs/v2.3.x/install_standalone-docker.md). The `MilvusMemoryStore` constructor provides additional configuration options, such as the vector size, the similarity metric type, etc.

dotnet/test/VectorData/Milvus.UnitTests/Milvus.UnitTests.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <AssemblyName>SemanticKernel.Connectors.Milvus.UnitTests</AssemblyName>
+    <RootNamespace>SemanticKernel.Connectors.Milvus.UnitTests</RootNamespace>
+    <TargetFramework>net10.0</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+    <IsPackable>false</IsPackable>
+    <NoWarn>$(NoWarn);SKEXP0001;SKEXP0020</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="Testcontainers.Milvus" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\VectorData\Milvus\Milvus.csproj" />
+  </ItemGroup>
+</Project>

dotnet/test/VectorData/Milvus.UnitTests/MilvusFilterTests.cs

@@ -0,0 +1,84 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.SemanticKernel.Connectors.Milvus;
+using Xunit;
+
+namespace SemanticKernel.Connectors.Milvus.UnitTests;
+
+public sealed class MilvusFilterTests
+{
+    [Theory]
+    [InlineData("v2.3.0")]
+    [InlineData("v2.3.0-dev")]
+    [InlineData("2.3.0")]
+    [InlineData("v2.6.1")]
+    [InlineData("3.0.0")]
+    public void AcceptsSupportedServerVersions(string version)
+    {
+        MilvusMemoryStore.ValidateServerVersion(version);
+    }
+
+    [Theory]
+    [InlineData("v2.2.14")]
+    [InlineData("v2.2.14-dev")]
+    [InlineData("v2.2.16")]
+    [InlineData("v2.3.0-beta")]
+    [InlineData("unknown")]
+    [InlineData("")]
+    public void RejectsOlderOrUnrecognizedServerVersions(string version)
+    {
+        Assert.Throws<NotSupportedException>(() => MilvusMemoryStore.ValidateServerVersion(version));
+    }
+
+    [Theory]
+    [InlineData("", "")]
+    [InlineData("ordinary-id", "ordinary-id")]
+    [InlineData("[],'", "[],'")]
+    [InlineData("\"", "\\\"")]
+    [InlineData("\\", "\\\\")]
+    [InlineData("\\\"", "\\\\\\\"")]
+    [InlineData("trailing\\", "trailing\\\\")]
+    [InlineData("\r\n\t", "\\r\\n\\t")]
+    [InlineData("\\r\\n\\t", "\\\\r\\\\n\\\\t")]
+    [InlineData("\0\b\f\v\u001f\u007f\u0085", "\\u0000\\u0008\\u000c\\u000b\\u001f\\u007f\\u0085")]
+    [InlineData("\u00e9\u4e2d\ud83d\ude00", "\u00e9\u4e2d\ud83d\ude00")]
+    [InlineData("a\"] || true || id in [\"", "a\\\"] || true || id in [\\\"")]
+    [InlineData("z\"] && text like \"%secret%\" || id in [\"", "z\\\"] && text like \\\"%secret%\\\" || id in [\\\"")]
+    [InlineData("x\" || id != \"", "x\\\" || id != \\\"")]
+    [InlineData("a\",\"victim", "a\\\",\\\"victim")]
+    public void EncodesEachIdAsOneString(string key, string expected)
+    {
+        Assert.Equal("id in [\"" + expected + "\"]", MilvusMemoryStore.BuildIdFilter([key]));
+    }
+
+    [Fact]
+    public void PreservesEmptyLists()
+    {
+        Assert.Equal("id in []", MilvusMemoryStore.BuildIdFilter([]));
+    }
+
+    [Fact]
+    public void PreservesOrderDuplicatesAndEmptyIds()
+    {
+        Assert.Equal("id in [\"a\",\"\",\"a\",\"b\"]", MilvusMemoryStore.BuildIdFilter(["a", "", "a", "b"]));
+    }
+
+    [Fact]
+    public void EnumeratesKeysOnlyOnce()
+    {
+        bool enumerated = false;
+
+        IEnumerable<string> Keys()
+        {
+            Assert.False(enumerated);
+            enumerated = true;
+            yield return "first";
+            yield return "second";
+        }
+
+        Assert.Equal("id in [\"first\",\"second\"]", MilvusMemoryStore.BuildIdFilter(Keys()));
+        Assert.True(enumerated);
+    }
+}