diff --git a/Directory.Packages.props b/Directory.Packages.props index 33422b7..3cccf33 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -18,6 +18,9 @@ + + + diff --git a/Wallaby.slnx b/Wallaby.slnx index 4ca334d..7061628 100644 --- a/Wallaby.slnx +++ b/Wallaby.slnx @@ -15,6 +15,7 @@ + @@ -23,6 +24,7 @@ + diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 9818787..a163e6a 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -51,6 +51,7 @@ export default defineConfig({ { text: 'Getting Started', link: '/getting-started' }, { text: 'Mappings', link: '/mappings' }, { text: 'Backfill', link: '/backfill' }, + { text: 'RAG & Embeddings', link: '/rag' }, { text: 'External Slots', link: '/external-slots' }, { text: 'Configuration', link: '/configuration' }, { text: 'Testing', link: '/testing' }, @@ -82,6 +83,7 @@ export default defineConfig({ { text: 'Kafka', link: '/sinks/kafka' }, { text: 'Elasticsearch', link: '/sinks/elasticsearch' }, { text: 'OpenSearch', link: '/sinks/opensearch' }, + { text: 'Pgvector', link: '/sinks/pgvector' }, { text: 'Custom', link: '/sinks/custom' }, ] }, @@ -101,6 +103,11 @@ export default defineConfig({ { icon: 'github', link: 'https://github.com/Hawxy/Wallaby' } ], + footer: { + message: 'Released under the Apache 2.0 License.', + copyright: 'Copyright © 2026-present JT' + }, + }, markdown: { theme: { light: 'github-light-high-contrast', dark: 'ayu-dark' }, diff --git a/docs/.vitepress/theme/ConfigPicker.vue b/docs/.vitepress/theme/ConfigPicker.vue index 4d2fb4f..adf353d 100644 --- a/docs/.vitepress/theme/ConfigPicker.vue +++ b/docs/.vitepress/theme/ConfigPicker.vue @@ -59,7 +59,7 @@ const sinks = [ }, { title: 'custom', - sub: 'your own delivery target', + sub: 'your own target', label: 'Custom Sinks →', link: '/sinks/custom', }, diff --git a/docs/backfill.md b/docs/backfill.md index 56d010e..a5ad016 100644 --- a/docs/backfill.md +++ b/docs/backfill.md @@ -21,7 +21,8 @@ sink.Map() ``` Each entity is versioned and backfilled independently, so reindexing one doesn't disturb others or the -live stream. +live stream. Version bumps are also how [embedding-model migrations](/rag#model-migrations) re-embed +a corpus: encode the model in the version string and bump it with `purgeOnChange: true`. When an entity is mapped to **several sinks**, backfill state is still per table: bumping *any* mapping's version re-snapshots the table, and the snapshot flows through every sink mapped to it. diff --git a/docs/index.md b/docs/index.md index 99053bd..4d4ebe2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,7 +20,7 @@ features: - title: Transform + Enrich details: Convert, enhance & flatten materialized changes into the required shape for your output destination. Use your existing EF & Marten tooling or drop down to manual SQL. - title: Pluggable Sinks - details: Ship your transformed data to anywhere it needs to go, be it a search index, vector DB or just a plain HTTP endpoint. At-least-once delivery ensures you data never goes missing. + details: Ship your transformed data to anywhere it needs to go, be it a search index, vector DB or just a plain HTTP endpoint. At-least-once delivery ensures your data never goes missing. - title: Versioned Backfilling details: Automatically run backfill operations as output shape is changed. Ensure your destination is always up to date. --- diff --git a/docs/mappings.md b/docs/mappings.md index cd19592..07d6520 100644 --- a/docs/mappings.md +++ b/docs/mappings.md @@ -52,6 +52,9 @@ interface as a class - [`IWallabyEfTransform`](/providers/entity-framework-co (EF Core) or [`IWallabyMartenTransform`](/providers/marten/#class-based-transforms) (Marten) - and register it with `UsingTransform()`. This class is registered & resolved from the container. +Transforms can also enrich documents with vector embeddings for semantic search - see +[RAG & Embeddings](/rag). + ## Mapping classes Inline mappings grow the `AddWallaby` callback and can make your `Program.cs` unwieldy. Move each mapping into a diff --git a/docs/rag.md b/docs/rag.md new file mode 100644 index 0000000..4ed6c86 --- /dev/null +++ b/docs/rag.md @@ -0,0 +1,126 @@ +--- +description: "Keeping embeddings and RAG corpora continuously in sync with Postgres using CDC: destination-side embedding, the pgvector sink, and model migrations." +--- + +# RAG & Embeddings + +A RAG corpus or semantic search index is only as good as its freshness, and keeping vectors in sync +with a database is exactly the shape of problem Wallaby already solves: changes stream through your +transform, deletes propagate, [backfill](/backfill) seeds and re-seeds destinations, and +`WithBackfillVersion` gives embedding-model migrations a one-line answer. + +The guiding principle: **let the destination own embedding**. When the party that stores the vector +also computes it, no vectors transit the pipeline, there is no cache to build or invalidate, and +the destination can skip re-embedding text it has already seen. Every path below follows that +shape; computing vectors inside a transform is the fallback, not the default. + +## Search sinks: the destination embeds + +Each search destination has a native way to embed the text Wallaby syncs: + +- **Meilisearch** - declare a [server-side embedder](/sinks/meilisearch#embedders-vector-search) + (`OpenAi`, `HuggingFace`, `Ollama`, or `Rest`) with a `documentTemplate`; the index becomes + hybrid-searchable with zero embedding code: + +```csharp +cdc.AddMeilisearchSink("meili", m => +{ + m.Host = "http://localhost:7700"; + m.ConfigureIndex("products", s => + { + s.SearchableAttributes = ["name", "description"]; + s.Embedders = new Dictionary + { + ["default"] = new Embedder + { + Source = EmbedderSource.OpenAi, + Model = "text-embedding-3-small", + ApiKey = openAiKey, + DocumentTemplate = "{{doc.name}}: {{doc.description}}", + }, + }; + }); +}); +``` + +- **Elasticsearch** - map the field as + [`semantic_text`](/sinks/elasticsearch#vector-search) backed by an inference endpoint; the + cluster chunks and embeds at index time. The inference API needs an appropriate Elastic + subscription, and the default ELSER endpoint needs ML nodes. +- **OpenSearch** - attach a [neural-search ingest pipeline](/sinks/opensearch#vector-search) + (a `text_embedding` processor over a deployed model) to the index. + +In all three, Wallaby delivers plain text and every insert, update, delete, and backfill keeps the +index converged. They differ on re-embedding cost: Meilisearch caches embeddings and calls the +embedder only for documents whose rendered `documentTemplate` changed, while Elasticsearch and +OpenSearch run inference on every indexed document - fine for the live change stream (Wallaby only +delivers rows that changed), but a [backfill](/backfill) re-embeds the whole corpus. + +## Postgres as the vector store: the pgvector sink + +For "my RAG corpus is just Postgres", the [pgvector sink](/sinks/pgvector) plays the +destination-embeds role itself, since Postgres has no native embedder: + +```csharp +cdc.AddPgvectorSink("vectors", v => +{ + v.ConnectionString = vectorDbConn; + v.Dimensions = 1536; + v.EmbeddingGenerator = generator; // any Microsoft.Extensions.AI IEmbeddingGenerator + v.EmbedText = d => $"{d["name"]}\n{d["description"]}"; + v.EmbeddingVersion = "text-embedding-3-small/1"; +}) +.WithMappings(sink => sink + .Map() + .ToDestination("products") + .WithBackfillVersion("v1", purgeOnChange: true) + .UsingTransform(/* emit name + description as plain text */)); +``` + +The sink embeds at delivery time and stores a content hash next to each vector, so it +[re-embeds only rows whose text changed](/sinks/pgvector#how-embedding-is-gated) - across restarts, +failovers, and re-backfills, with the destination table itself as the durable cache. Embedding-API +throttling surfaces as a retryable delivery, riding the dispatcher's normal backoff. + +## Hand-rolled: embedding in a transform + +For destinations that can't embed and can't be read back - Kafka topics, HTTP receivers, or a +Meilisearch `UserProvided` embedder - compute the vector in the transform and emit it as a document +field. A `float[]` or `ReadOnlyMemory` value is written as a plain JSON number array by the +Elasticsearch, OpenSearch, HTTP, and Kafka sinks. + +Two rules make this safe and affordable: + +- **Batch and retry inside the transform.** Transforms receive whole batches (deduplicated to one + change per row) - make one provider call per batch, never per row. And a transform exception + **halts the pipeline** (retry classification exists only at sink delivery), so wrap the embedding + call in your own retry (e.g. a Polly `ResiliencePipeline` with exponential backoff and jitter) + and throw only when retries are exhausted; at that point halting is correct backpressure. +- **Skip unchanged text.** On updates, `ChangeEvent.Changes` holds the previous values of changed + columns - when none of the embedded columns appear in it, skip the API call. Combine with the + mapping's [column selection](/providers/entity-framework-core/#declaring-consumed-columns) so + updates to irrelevant columns never reach the transform at all. Note `Changes` is `null` on + inserts and backfill rows, so a re-backfill re-embeds everything on this path - one more reason + to prefer the destination-side options above. + +## Model migrations + +Changing the embedding model (or the prompt template baked into the text) makes every stored vector +stale. Encode the model in the [backfill version](/backfill#automatic-backfill) and bump it: + +```csharp +.WithBackfillVersion("text-embedding-3-large/1", purgeOnChange: true) +``` + +The bump triggers a full re-backfill of the entity, and `purgeOnChange: true` +[purges the destination first](/backfill#purging-before-a-backfill) so no old-model vectors survive +alongside new ones. For the pgvector sink, change `EmbeddingVersion` in the same deploy (it feeds +the stored hash); for destination-side embedders, update the embedder/endpoint configuration. +Dimension changes (e.g. 1536 → 3072) also need the index or column recreated - purge handles the +documents, not the schema. + +## Delete propagation + +Nothing extra to do: a source row's delete removes its document (and vector) from the destination, +and a transform returning `null` for a key does the same. Stale-vector cleanup is the pipeline's +normal delete path, not a special case. diff --git a/docs/sinks/elasticsearch.md b/docs/sinks/elasticsearch.md index a78b653..d5b2e48 100644 --- a/docs/sinks/elasticsearch.md +++ b/docs/sinks/elasticsearch.md @@ -59,6 +59,33 @@ default). For explicit settings or mappings (analyzers, `dense_vector` fields, s create the index up front — via Kibana Dev Tools, your infrastructure tooling, or a deployment script. In-sink index bootstrapping is planned. +## Vector search + +A transform can emit an embedding as a `float[]` or `ReadOnlyMemory` field - the sink writes +either as a plain JSON number array, with no `SerializerOptions` needed. Dynamic mapping would infer +an ordinary `float` field, so pre-create the index with an explicit `dense_vector` mapping: + +```json +PUT /products +{ + "mappings": { + "properties": { + "embedding": { "type": "dense_vector", "dims": 1536, "index": true, "similarity": "cosine" } + } + } +} +``` + +Don't pass a quantized vector as `byte[]` - byte arrays serialize as base64 strings, not arrays. + +Elasticsearch can also embed for you: map the field as +[`semantic_text`](https://www.elastic.co/docs/solutions/search/semantic-search/semantic-search-semantic-text) +(backed by an inference endpoint) and sync plain text - the cluster chunks and embeds at index +time, with no vectors in your pipeline at all. Weigh the prerequisites and cost profile first: the +inference API needs an appropriate Elastic subscription (and the default ELSER endpoint needs ML +nodes), and inference runs on every indexed document - live changes embed incrementally, but a +[backfill](/backfill) re-runs inference over the whole corpus. See [RAG & Embeddings](/rag). + ## Authentication `ApiKey` or `Username`/`Password` cover the common schemes. Everything else — Elastic Cloud ids, diff --git a/docs/sinks/http.md b/docs/sinks/http.md index 3e745f4..423081d 100644 --- a/docs/sinks/http.md +++ b/docs/sinks/http.md @@ -239,7 +239,7 @@ against the body your endpoint reads after the middleware has decompressed it. ## NativeAOT The envelope structure and common scalar document values (strings, numbers, booleans, `Guid`, date/time -types, byte arrays, nested dictionaries, and sequences of these) are written without reflection. Any other +types, byte arrays, nested dictionaries, `ReadOnlyMemory` vectors, and sequences of these) are written without reflection. Any other value type is serialized through `SerializerOptions`; on trimmed/NativeAOT hosts, point it at a source-generated context covering the types your transforms emit: diff --git a/docs/sinks/kafka.md b/docs/sinks/kafka.md index 72faedc..0128ad2 100644 --- a/docs/sinks/kafka.md +++ b/docs/sinks/kafka.md @@ -152,7 +152,7 @@ Leave `Topics` empty when your platform pre-provisions topics or the broker has ## NativeAOT The envelope structure and common scalar document values (strings, numbers, booleans, `Guid`, date/time -types, byte arrays, nested dictionaries, and sequences of these) are written without reflection. Any other +types, byte arrays, nested dictionaries, `ReadOnlyMemory` vectors, and sequences of these) are written without reflection. Any other value type is serialized through `SerializerOptions`; on trimmed/NativeAOT hosts, point it at a source-generated context covering the types your transforms emit: diff --git a/docs/sinks/meilisearch.md b/docs/sinks/meilisearch.md index 18ab1a6..f9cceaf 100644 --- a/docs/sinks/meilisearch.md +++ b/docs/sinks/meilisearch.md @@ -85,6 +85,42 @@ cdc.AddMeilisearchSink("meili", m => `Settings` is Meilisearch's own settings type, so you have full control (ranking rules, stop words, synonyms, faceting, …). Setup is idempotent and re-applied on each leadership acquisition. +### Embedders (vector search) + +[AI-powered search](https://www.meilisearch.com/docs/learn/ai_powered_search/getting_started_with_ai_search) can be setup via the +index configuration: + +```csharp +m.ConfigureIndex("products", s => +{ + s.SearchableAttributes = ["name", "description"]; + s.Embedders = new Dictionary + { + ["default"] = new Embedder + { + Source = EmbedderSource.OpenAi, + Model = "text-embedding-3-small", + ApiKey = openAiKey, + DocumentTemplate = "{{doc.name}}: {{doc.description}}", + }, + }; +}); +``` + +With a server-side source (`OpenAi`, `HuggingFace`, `Ollama`, `Rest`), Meilisearch computes vectors +itself from the synced documents. Optionally, with `EmbedderSource.UserProvided`, the +transform carries the vector in the document's `_vectors` field instead: + +```csharp +new WallabyDocument +{ + ["name"] = p.Name, + ["_vectors"] = new Dictionary { ["default"] = embedding }, // float[] +}; +``` + +See [RAG & Embeddings](/rag) for the full pattern, including re-embedding on model changes. + ### Attribute validation By default (`ValidateConfiguredAttributes = true`), every upsert routed to a `ConfigureIndex`-declared index @@ -92,7 +128,6 @@ is checked against that index's configured **searchable**, **filterable**, and * document is missing a key for any of them, delivery fails **permanently** with a `MeilisearchDocumentValidationException` (which halts the pipeline), rather than silently indexing a document that has a mismatched configuration. -A few details: - A key whose value is `null` counts as present, only an **absent** key is a failure. - The sink's `PrimaryKey` and Meilisearch's `*` wildcard are exempt. diff --git a/docs/sinks/opensearch.md b/docs/sinks/opensearch.md index 90433bd..d1ffe5f 100644 --- a/docs/sinks/opensearch.md +++ b/docs/sinks/opensearch.md @@ -59,6 +59,33 @@ default). For explicit settings or mappings (analyzers, `knn_vector` fields, sha create the index up front — via Dev Tools, your infrastructure tooling, or a deployment script. In-sink index bootstrapping is planned. +## Vector search + +A transform can emit an embedding as a `float[]` or `ReadOnlyMemory` field - the sink writes +either as a plain JSON number array, with no `SerializerOptions` needed. Dynamic mapping would infer +an ordinary `float` field, so pre-create the index with an explicit `knn_vector` mapping: + +```json +PUT /products +{ + "settings": { "index.knn": true }, + "mappings": { + "properties": { + "embedding": { "type": "knn_vector", "dimension": 1536 } + } + } +} +``` + +Don't pass a quantized vector as `byte[]` - byte arrays serialize as base64 strings, not arrays. + +OpenSearch can also embed for you: attach a +[neural-search ingest pipeline](https://docs.opensearch.org/latest/vector-search/ai-search/semantic-search/) +(a `text_embedding` processor over a deployed model) to the index and sync plain text - the cluster +computes vectors at index time, with none in your pipeline at all. Note the pipeline runs on every +indexed document, so live changes embed incrementally but a [backfill](/backfill) re-runs the model +over the whole corpus. See [RAG & Embeddings](/rag). + ## Authentication `Username`/`Password` cover basic auth. Everything else — AWS SigV4, client certificates, diff --git a/docs/sinks/pgvector.md b/docs/sinks/pgvector.md new file mode 100644 index 0000000..858e4db --- /dev/null +++ b/docs/sinks/pgvector.md @@ -0,0 +1,150 @@ +--- +description: "Syncing Postgres tables into pgvector vector tables, with optional sink-side embedding that re-embeds only changed text." +--- + +# Pgvector Sink + +The `Wallaby.Sinks.Pgvector` package keeps [pgvector](https://github.com/pgvector/pgvector) tables +continuously in sync with your source tables - the "my RAG corpus is just Postgres" setup, with no +extra search infrastructure. Upserts are idempotent by id and deletes remove by id, so redelivery +converges. Its headline feature is **sink-side embedding**: configure an embedding generator and the +sink embeds documents at delivery time, re-embedding only rows whose text actually changed. The +destination table doubles as the durable embedding cache, so restarts, failovers, and re-backfills +never re-embed unchanged text. + +## Install + +```bash +dotnet add package Wallaby.Sinks.Pgvector +``` + +## Register + +```csharp +cdc.AddPgvectorSink("vectors", v => +{ + v.ConnectionString = vectorDbConn; // often a different database than the CDC source + v.Dimensions = 1536; + v.EmbeddingGenerator = generator; // any Microsoft.Extensions.AI IEmbeddingGenerator + v.EmbedText = d => $"{d["name"]}\n{d["description"]}"; + v.EmbeddingVersion = "text-embedding-3-small/1"; +}) +.WithMappings(sink => sink + .Map() + .ToDestination("products") // the destination table + .WithBackfillVersion("v1", purgeOnChange: true) + .UsingTransform(/* emit name + description as plain text */)); +``` + +`generator` is any +[`Microsoft.Extensions.AI`](https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai) +`IEmbeddingGenerator>` - OpenAI, Azure, Ollama, Bedrock, ONNX, or your own. +Leave the three embedding options unset and the sink instead stores a vector your transform supplies +in the document's `VectorField` (default `embedding`, as `float[]` or `ReadOnlyMemory`). + +## The table + +One table per destination, created on initialization (or first delivery) when `CreateTable` is on: + +```sql +CREATE TABLE IF NOT EXISTS "public"."products" ( + id text PRIMARY KEY, + text_hash text, + embedding vector(1536), + document jsonb NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); +``` + +- `id` is the record's document id (source primary key or your `KeyedBy(...)` rule). +- `document` is the transform's field bag as jsonb (in pass-through mode, minus the vector field). +- `text_hash` is SHA-256 over `EmbeddingVersion` + the embedded text; `embedding`/`text_hash` are + null when `EmbedText` returned nothing for the row. +- **No vector index is created.** Build the HNSW index yourself *after* the initial backfill - + index maintenance during a large bulk load is the slow way around: + +```sql +CREATE INDEX ON "public"."products" USING hnsw (embedding vector_cosine_ops); +``` + +Query it like any pgvector table (`ORDER BY embedding <=> $1 LIMIT 10`), joining fields out of +`document` as needed. + +## Options + +| Option | Default | Purpose | +| --- | --- | --- | +| `ConnectionString` | *(required)* | Destination database. | +| `ConfigureDataSource` | `null` | Extra `NpgsqlDataSourceBuilder` configuration (TLS callbacks, loggers, ...). | +| `Schema` | `public` | Schema holding the destination tables. | +| `DefaultTable` | `null` | Table used when a routed record has no destination. | +| `Dimensions` | *(required)* | The `vector(N)` dimension; every stored vector must match, a mismatch fails permanently. | +| `CreateTable` | `true` | Create missing destination tables on initialization / first delivery. | +| `CreateExtension` | `true` | `CREATE EXTENSION IF NOT EXISTS vector` on initialization (fails with guidance when the role lacks the privilege). | +| `EmbeddingGenerator` | `null` | Embeds at delivery time; configure together with `EmbedText` and `EmbeddingVersion`. | +| `EmbedText` | `null` | Selects the text to embed from a document's field bag; null/empty stores a null vector. | +| `EmbeddingVersion` | `null` | Model/prompt identity folded into `text_hash`; change it (and bump the mapping's backfill version) to re-embed. | +| `MaxEmbeddingBatchSize` | `96` | Texts per embedding call. | +| `MaxEmbeddingConcurrency` | `1` | Embedding calls in flight at once; raise it to overlap calls on large backfills when the provider's rate limits allow. | +| `IsTransientEmbeddingError` | `null` | Classifies embedding exceptions retryable vs permanent; the default retries everything except `ArgumentException`/`NotSupportedException`. | +| `VectorField` | `embedding` | Without a generator: the document field carrying the transform-supplied vector. | +| `MaxRowsPerBatch` | `500` | Rows per database round-trip. | +| `SerializerOptions` | `null` | Serializer for document values beyond the natively written scalar types (required for such values on NativeAOT hosts). | + +## How embedding is gated + +Per delivered batch (after last-write-wins dedup per id), the sink reads the stored `text_hash` for +the affected ids from the destination table and embeds **only** rows whose hash is missing or +different - one `SELECT`, then one embedding call per `MaxEmbeddingBatchSize` changed texts. Rows +whose hash matches update `document`/`updated_at` and keep their stored vector untouched - and when +the document is unchanged too, the upsert skips the row entirely, so a re-backfill of an unchanged +corpus costs no embedding calls *and* no row rewrites (no WAL or vacuum churn). + +Because the gate is the destination row itself, it needs no cache infrastructure and survives +everything the process doesn't: restarts, leader failover, and re-backfills all skip unchanged text. +The two ways to force re-embedding are the ones that should: changing `EmbeddingVersion` (new hash) +and [purging](/backfill#purging-before-a-backfill) (deletes the rows, hashes included) - which is +exactly what `WithBackfillVersion(..., purgeOnChange: true)` does on a model migration. + +## Delivery semantics + +Delivery is **at-least-once** and idempotent by id. Within a batch the last write per id wins; +writes for a table run in one transaction. Failures classify for the dispatcher: + +| Error | Outcome | +| --- | --- | +| Transient database failures (Npgsql transient errors, timeouts, connection loss) | **Retryable** - the dispatcher retries with backoff. | +| Embedding-provider failures classified transient by `IsTransientEmbeddingError` (default: nearly all, e.g. 429/5xx) | **Retryable** - already-stored hashes keep the retry cheap. | +| Non-transient Postgres rejections, dimension mismatches, non-transient embedding errors, a record with no destination and no `DefaultTable`, an invalid destination table name | **Permanent** - the pipeline halts. | + +Destination table names (including [`ScopedDestination`](/providers/entity-framework-core/multi-tenancy) +results) must be 1-63 characters of `[a-zA-Z0-9_]`; anything else fails permanently rather than being +quoted into DDL. + +The sink implements purge-then-backfill: a [purge](/backfill#purging-before-a-backfill) issues +`DELETE FROM` on the destination table (the table and any indexes survive), so the following +backfill rebuilds - and re-embeds - from scratch. `DELETE` works under ordinary grants but writes +WAL proportional to the corpus; for a very large table you can `TRUNCATE` it manually before the +versioned re-backfill and let the purge find it already empty. + +## Performance + +Vectors travel in pgvector's binary wire format, and identical redeliveries skip the row write +server-side, so the remaining per-row cost is statement parsing: the sink pipelines its upserts in +batches, and enabling Npgsql's automatic preparation makes the server parse each statement shape +once instead of once per row: + +```text +Host=...;Database=...;Max Auto Prepare Statements=16 +``` + +(Or set `MaxAutoPrepare` via `ConfigureDataSource`.) When connecting through a transaction-pooling +PgBouncer, this needs PgBouncer 1.21+ with `max_prepared_statements` set; on older versions leave +it off. + +## Per-tenant tables + +Route each tenant to its own table with `ScopedDestination` - see multi-tenancy for +[EF Core](/providers/entity-framework-core/multi-tenancy) or [Marten](/providers/marten/multi-tenancy). +Runtime tables are created on first delivery with the same shape (when `CreateTable` is on) and +subject to the identifier rule above. diff --git a/package-lock.json b/package-lock.json index e4cc1ae..c3e3ff5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,8 +5,22 @@ "packages": { "": { "devDependencies": { - "vitepress": "^2.0.0-alpha.18", - "vitepress-plugin-llms": "^1.13.3" + "vitepress": "^2.0.0-alpha.19", + "vitepress-plugin-llms": "^1.13.5" + } + }, + "node_modules/@11ty/gray-matter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@11ty/gray-matter/-/gray-matter-2.1.0.tgz", + "integrity": "sha512-fNdBOb3MgDz/1UCIoeY4wDuGbp+/3s5y6UrsyfMabECbh/CVycj7Er33JXqSxRwxrRKXcGE1Jipuq/1mODInsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-yaml": "^4.2.0", + "section-matter": "^1.0.0" + }, + "engines": { + "node": ">=11" } }, "node_modules/@babel/helper-string-parser": { @@ -60,64 +74,30 @@ } }, "node_modules/@docsearch/css": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-4.6.3.tgz", - "integrity": "sha512-nlOwcXcsNAptQl4vlL4MA78qNJKO0Qlds5GuBjCoePgkebTXLSf8Qt1oyZ3YBshYupKXG9VRGEsk1zr23d+bzQ==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-4.7.0.tgz", + "integrity": "sha512-Sk5xkdRFeE7PeWjG9l4AfTwdvMfr9wHiwNNCpHXT4v4SNyNMKdHGvEILc31BgaVFGDDNbv5u/a73tofRiwbEZw==", "dev": true, "license": "MIT" }, "node_modules/@docsearch/js": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/@docsearch/js/-/js-4.6.3.tgz", - "integrity": "sha512-qUIX2b4Apew3tv4F0qhmgShsl/Lfw4m6mqv/5/5dWNxwTcDdLMp2s3YwZ+NMGh3IKCg0pBaXm7Q5VdyU5Rj+cQ==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@docsearch/js/-/js-4.7.0.tgz", + "integrity": "sha512-x5lCqu1tetgsJFkjQ6VSocbHldsRkGEgwg5N98Vx21sq/V5wcmj4u226PY9k+TEpIgQ772zlYbPLTPicWyGnpA==", "dev": true, "license": "MIT" }, "node_modules/@docsearch/sidepanel-js": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/@docsearch/sidepanel-js/-/sidepanel-js-4.6.3.tgz", - "integrity": "sha512-grGSmvXzG0if+mrzdIKykvpIAuEQ9u0sEJ2eLRRCaQfJvsWqh2C2/aY04bIzWvDh7myi5rvl8D+tUNsVrjYQ3A==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@docsearch/sidepanel-js/-/sidepanel-js-4.7.0.tgz", + "integrity": "sha512-A8r34jCU8kcIk2viECEn2msA28ojUF1BLi/3v5OWWc5G2N3jOuuumBXoeYjfr8dA0UxgFSy5R2bt12dnFJQSyA==", "dev": true, "license": "MIT" }, - "node_modules/@emnapi/core": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", - "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.2", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", - "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", - "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@iconify-json/simple-icons": { - "version": "1.2.90", - "resolved": "https://registry.npmjs.org/@iconify-json/simple-icons/-/simple-icons-1.2.90.tgz", - "integrity": "sha512-zt2o2ZvQpHVvZJARIkZ51RnaHY2oqcPJMvHE+mVnxkSr+c33fnX4gciiXu+wyX5ei+s0qbVX1wD0DWBbaGBYMA==", + "version": "1.2.94", + "resolved": "https://registry.npmjs.org/@iconify-json/simple-icons/-/simple-icons-1.2.94.tgz", + "integrity": "sha512-l8UWzVxKaqZd9ABsE/M/9p6NyGkQnmCnOoZyhQmjlXCtY5PuL2rcWxOFk2l9pk7ux3ERMPkTLE4jl6kQpTkwxA==", "dev": true, "license": "CC0-1.0", "dependencies": { @@ -138,39 +118,37 @@ "dev": true, "license": "MIT" }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", - "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "node_modules/@oxc-project/types": { + "version": "0.147.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.147.0.tgz", + "integrity": "sha512-IJ3s6ltHLp45S0bh7phkX+gJO7A1Wuz2EaqpAhb8WjqDwbzMiWKHhyyT42tskaWjEYXtHtVCPpnBJVT9+dcRLg==", "dev": true, "license": "MIT", - "optional": true, - "dependencies": { - "@tybys/wasm-util": "^0.10.3" - }, "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "peerDependencies": { - "@emnapi/core": "^1.7.1", - "@emnapi/runtime": "^1.7.1" + "url": "https://github.com/sponsors/Boshen" } }, - "node_modules/@oxc-project/types": { - "version": "0.139.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", - "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.6.tgz", + "integrity": "sha512-b+jTcARdTiFLI6jB4a5XjTm0RWd6KcRfQj/I2356fxUZemiho9zQLxo0RtCuMDAyKcLo6cEltkgbQp6d1+sjjQ==", + "cpu": [ + "arm" + ], "dev": true, "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" } }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", - "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.6.tgz", + "integrity": "sha512-lkWU8ZJaRk9q3CIEY1Tc7vIFALp3Xw5NfGJo2hQg5oIqNgxWi1zI+IiDEK3r70BF5Dzol1tcXsnzsRc8NLhG+Q==", "cpu": [ "arm64" ], @@ -185,9 +163,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", - "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.6.tgz", + "integrity": "sha512-dgR56NYnvAszm7Ob1B2/Vn0e8bUQYZH2UjVaMMtMVOCKFSfjhfLmuA/9+O+F+ajUdG6B/bSssrKW6JJYASa8jA==", "cpu": [ "arm64" ], @@ -202,9 +180,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", - "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.6.tgz", + "integrity": "sha512-vpVxFvUCFioJqug7OTvqptkc4yb8UX0AwfDmJpaR/0sWz+BUmqSVAf7c8JkUgnN8YLspb4a/N6NhTyMAmdyQ7Q==", "cpu": [ "x64" ], @@ -219,9 +197,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", - "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.6.tgz", + "integrity": "sha512-h1wG6Y6K3JlRswxsI64qQJqBAy4vrLuHgRbc8CZMGSWTOFRY6ghMApM1NKzB2I0n5xV1fjkE18SuVl2QpLeNpA==", "cpu": [ "x64" ], @@ -236,9 +214,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", - "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.6.tgz", + "integrity": "sha512-tbCiqub0q2MVWJKgF5PoAlNWCtQydiOYSLIkd8sByqK/6MMYLJRcSXSYodqYtd0O+Fw7QaVmKKlS4oL94YRZ0w==", "cpu": [ "arm" ], @@ -253,9 +231,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", - "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.6.tgz", + "integrity": "sha512-oxK9+baEBPhZG5HB4URY+uU04zJWeZlH6Tb9rB5DK4DF9XR1uXNLXt5Q5ZsugTKayNCNLhkcwz/ye74hRI98dg==", "cpu": [ "arm64" ], @@ -270,9 +248,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", - "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.6.tgz", + "integrity": "sha512-muWCk27FVBEZtv0MsK8gnfSmgczA8KQ0uRVJbTABKhkRfQc38aUrcb7fhi3BNiyseFmgcRsoMfQsSNJ+DbZdSw==", "cpu": [ "arm64" ], @@ -287,9 +265,9 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", - "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.6.tgz", + "integrity": "sha512-eWDoSfU7Co2qj3vgB3Dt4lj1mG6CoWbcJQkRMP3XJplyCMtuaq3LHvPFjS9QIPvMGWVadJC04Xiy0IdcVPtnwQ==", "cpu": [ "ppc64" ], @@ -304,9 +282,9 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", - "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.6.tgz", + "integrity": "sha512-2bWNjRSIayvupRKxXUY2tWG9fYdoUlTqWywHRvE8Eq3GvuQ+f2HeIkve697fIt+IQs/PV8yFsdWuhp1aJ1PdnA==", "cpu": [ "s390x" ], @@ -321,9 +299,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", - "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.6.tgz", + "integrity": "sha512-KekI0gS0wLxe1UBSQSjenBVwou/JkcQPDzBPICGZjxUv9k3RteHDPBQaiOicZUFKRIH2wKEimGwVpnJsbPzu7w==", "cpu": [ "x64" ], @@ -338,9 +316,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", - "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.6.tgz", + "integrity": "sha512-TvtPnfVr+HtyGiDmPK4VWmlNm7QhNNAcK5Q9A7aOXsI8545yCyaoMaicXrFZ72JzeYjaUVk7yT243zT0jzjFKQ==", "cpu": [ "x64" ], @@ -355,9 +333,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", - "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.6.tgz", + "integrity": "sha512-iOo0VEay2XFhaCcH0sps5XIimkSuOnNaZrf6+ZkoSOQBJPKNU48RkmJv0/lSpipexu5P+ouFgafe5IGr/DiQfg==", "cpu": [ "arm64" ], @@ -371,29 +349,10 @@ "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", - "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "1.11.1", - "@emnapi/runtime": "1.11.1", - "@napi-rs/wasm-runtime": "^1.1.6" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", - "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.6.tgz", + "integrity": "sha512-y5NTmmasMS455JlOCO4ZM9krIchv3Mvm1crL1iUPGOPgEzSkves9n0SdC5Sjz6+qWDFhd8/JpfWMH8NSWNHe+A==", "cpu": [ "arm64" ], @@ -408,9 +367,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", - "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.6.tgz", + "integrity": "sha512-np8iZSLfXlAD4kWhiyq/u0Yt8oZDtRQ8lGhQaCXo2rl37KNjeU0GjJuwr4P3oeZ++ROfofsKNBqR5LTO8aXyWQ==", "cpu": [ "x64" ], @@ -432,16 +391,16 @@ "license": "MIT" }, "node_modules/@shikijs/core": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-4.3.1.tgz", - "integrity": "sha512-ANMDxuaPsNMdDC1m4vfvhlDmJweMwkE5XitTwrq2rWHx5jM+dlm4MmHt2PP6t0uejfR77SuhrhJ0zEijIF/uhA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-4.4.3.tgz", + "integrity": "sha512-QCR4q2ZO/ILJEuwiBMel4wdcTDb1JGwfjKTxPDF6x8ixOaluPrVqIn06C99AcRPhmYlBR56d/Fb+GN58GzExpg==", "dev": true, "license": "MIT", "dependencies": { - "@shikijs/primitive": "4.3.1", - "@shikijs/types": "4.3.1", + "@shikijs/primitive": "4.4.3", + "@shikijs/types": "4.4.3", "@shikijs/vscode-textmate": "^10.0.2", - "@types/hast": "^3.0.4", + "@types/hast": "^3.0.5", "hast-util-to-html": "^9.0.5" }, "engines": { @@ -449,13 +408,13 @@ } }, "node_modules/@shikijs/engine-javascript": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-4.3.1.tgz", - "integrity": "sha512-JBItcnPuYq7jVJdZo/vMj94r+szT7XEjHFX+mvFDGSEIbVAXAGyHAHzhbWzpGOwYidCZrErJLLgn2PVeiokHnQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-4.4.3.tgz", + "integrity": "sha512-FbOjFJp9VLdo1Wevs10BBtVxiTWwNLqZh5Gkhjgda/ioL15YOgeSl9n+6XMa3qRlPQzfhFNe641SrynFHYG0nQ==", "dev": true, "license": "MIT", "dependencies": { - "@shikijs/types": "4.3.1", + "@shikijs/types": "4.4.3", "@shikijs/vscode-textmate": "^10.0.2", "oniguruma-to-es": "^4.3.6" }, @@ -464,13 +423,13 @@ } }, "node_modules/@shikijs/engine-oniguruma": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-4.3.1.tgz", - "integrity": "sha512-OXyNMzg0pews+msMj4cHeqT4xiYKKvbnn6VbdAXxfoFl3SSx4fJTc8FadECuc5/H9p3BzhNAoAUXKwAu9rWYhg==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-4.4.3.tgz", + "integrity": "sha512-EcOQkxdxGQrc1Row/cC2c96/v1dbZqGnEVu1qTuT/MJmp6+cXCvQussowVmCv5Tqr3KuY3c7IbM6HTW3LJ1k9w==", "dev": true, "license": "MIT", "dependencies": { - "@shikijs/types": "4.3.1", + "@shikijs/types": "4.4.3", "@shikijs/vscode-textmate": "^10.0.2" }, "engines": { @@ -478,69 +437,69 @@ } }, "node_modules/@shikijs/langs": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-4.3.1.tgz", - "integrity": "sha512-m0l9nsDqgBHvbZbk7A0/kXz/impK3uB/c6rAn6Gpg/uPtdZRQ+alsN/17MU5thb68XTj/4DxkZAotrM0GGSpDQ==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-4.4.3.tgz", + "integrity": "sha512-ePic0yfAJGOF83D5wBHK/00EjK65oahBYxFk5epgq33WRv7X9UuxLEV8PtR0szC0z8dl7INIpIodB99JRFlR+A==", "dev": true, "license": "MIT", "dependencies": { - "@shikijs/types": "4.3.1" + "@shikijs/types": "4.4.3" }, "engines": { "node": ">=20" } }, "node_modules/@shikijs/primitive": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@shikijs/primitive/-/primitive-4.3.1.tgz", - "integrity": "sha512-CXQRQOYy1leqQ8ceTeJdmXv/bsUY++6QyLpXJ94LZAAYj5X2SKRdc5ipguv4NPyGVKItB2PPwUpRNe0Sjh5S1A==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/primitive/-/primitive-4.4.3.tgz", + "integrity": "sha512-m0wBeLDQDeIxRdUmrCPdQqfuUamDwRL5isCfYbguKD6NiaKpVbsv+3J81DyIKgNW5h4WAIIr8T4EkgQrBBxvaQ==", "dev": true, "license": "MIT", "dependencies": { - "@shikijs/types": "4.3.1", + "@shikijs/types": "4.4.3", "@shikijs/vscode-textmate": "^10.0.2", - "@types/hast": "^3.0.4" + "@types/hast": "^3.0.5" }, "engines": { "node": ">=20" } }, "node_modules/@shikijs/themes": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-4.3.1.tgz", - "integrity": "sha512-dgpoJ4WqNi2yTmizQHBJ5zcX6j2lE6icN/0yt4l1kkf16jrY/pwPLoTb1ETsWMz0OBLf9ZNvwmxft+cH+N9qSA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-4.4.3.tgz", + "integrity": "sha512-w8UHjeUnIR965KMWJHUPXOc2mNJUnK3vpVLYLvw5IYU2mnTTJ89E24OrJDBNiJDQ0qzb0tc4l7mrIXx5cFeIyw==", "dev": true, "license": "MIT", "dependencies": { - "@shikijs/types": "4.3.1" + "@shikijs/types": "4.4.3" }, "engines": { "node": ">=20" } }, "node_modules/@shikijs/transformers": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@shikijs/transformers/-/transformers-4.3.1.tgz", - "integrity": "sha512-z6ir0bGDgWcF2FduktEfPgIsdOtIlDiLAjFBgBzE42Q9xHbkkIXZtORHzlLVB71iZP9elEcqKg6keajvOUwE2A==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/transformers/-/transformers-4.4.3.tgz", + "integrity": "sha512-oJSARV6NaWd+rnNJbtnpAdj3Zg0ZVyzsnMgb3vi3HA+35y8lBWUCpOnWsmyiXZIikY+x1BDqrQUgmxfzWh7Jvw==", "dev": true, "license": "MIT", "dependencies": { - "@shikijs/core": "4.3.1", - "@shikijs/types": "4.3.1" + "@shikijs/core": "4.4.3", + "@shikijs/types": "4.4.3" }, "engines": { "node": ">=20" } }, "node_modules/@shikijs/types": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-4.3.1.tgz", - "integrity": "sha512-CHFxE0jztBIZRHH6gxXE7DXUCFXjReEGxZ/j0rfSLGKZuwp2xBYycEP14875DSa9KLL/6700oxIq6oO6ef9K2g==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-4.4.3.tgz", + "integrity": "sha512-UEJxmRR++MAGR6hugn0vgVS2W/6lWAts84FFSrnlH9sP0LNol7E5+NQ792pH8liWUhyMyjhTgSUH3k7iD7tc5g==", "dev": true, "license": "MIT", "dependencies": { "@shikijs/vscode-textmate": "^10.0.2", - "@types/hast": "^3.0.4" + "@types/hast": "^3.0.5" }, "engines": { "node": ">=20" @@ -553,17 +512,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", - "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -641,9 +589,9 @@ "license": "MIT" }, "node_modules/@ungap/structured-clone": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.3.tgz", - "integrity": "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", + "integrity": "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ==", "dev": true, "license": "ISC" }, @@ -719,32 +667,32 @@ } }, "node_modules/@vue/devtools-api": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-8.1.5.tgz", - "integrity": "sha512-YJipMVAKe5wT5CWf5kTYCaNV7NMNjFVxJkIkJaJ4W/nCxEBzlZzrOsYKeCymdCrFZmBS/+wTWFoUs3Jf/Q6XSQ==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-8.2.1.tgz", + "integrity": "sha512-6u4vXBlIBAC1wMplIZgpyPn7uh/s4Bf6F5bMzvLv+EdJ0aHs/+4B7Ygv864EStQSjRbsRzTko/kUG1A1IejQ3A==", "dev": true, "license": "MIT", "dependencies": { - "@vue/devtools-kit": "^8.1.5" + "@vue/devtools-kit": "^8.2.1" } }, "node_modules/@vue/devtools-kit": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@vue/devtools-kit/-/devtools-kit-8.1.5.tgz", - "integrity": "sha512-FcSAxsi4eWuXLCB7Rv9lj0aIVHHPNVQ2BazGf4RJTc2JCqb4BQg0hk87ZFhminCfl+mD5OUI0rX2cgyu4kJOGA==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/@vue/devtools-kit/-/devtools-kit-8.2.1.tgz", + "integrity": "sha512-FIGIuq3AWReEpbAHY/cRGeHDfI0qOb8OCQ3YjbEAX04uaxIDbGc9rhkbVcG7rnfHPXE3RsU5KrWOu9V/okd8AQ==", "dev": true, "license": "MIT", "dependencies": { - "@vue/devtools-shared": "^8.1.5", + "@vue/devtools-shared": "^8.2.1", "birpc": "^2.6.1", "hookable": "^5.5.3", "perfect-debounce": "^2.0.0" } }, "node_modules/@vue/devtools-shared": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@vue/devtools-shared/-/devtools-shared-8.1.5.tgz", - "integrity": "sha512-mhT4zcPFhF+Xk1O4BfhhrbXzpmfqY03fS6xGpcllbQG7lDjhQf8pQHcTIhqQIYx1hfwtHmk/6jM96ele0UxPqQ==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/@vue/devtools-shared/-/devtools-shared-8.2.1.tgz", + "integrity": "sha512-Fkac7lUdGReh6pVOi3AYPRGe82LQqRmAfThW7RRligOAP0ZA/Z1z9XLHDM9dv34pV2HRc79DK8uKPeG2fLnA/g==", "dev": true, "license": "MIT" }, @@ -802,15 +750,15 @@ "license": "MIT" }, "node_modules/@vueuse/core": { - "version": "14.3.0", - "resolved": "https://registry.npmjs.org/@vueuse/core/-/core-14.3.0.tgz", - "integrity": "sha512-aHfz47g0ZhMtTVHmIzMVpJy8ePhhOy68GY5bv110+5DVtZ+W7BsOx+m61UNQqfrWyPztIHIanWa3E2tib3NFIw==", + "version": "14.4.0", + "resolved": "https://registry.npmjs.org/@vueuse/core/-/core-14.4.0.tgz", + "integrity": "sha512-X4WHz1HlCzCBoYXesUkifzzWBAcZgXG8Fi5iNPQg/epdzOB3gu8Fawj3hvuwYR1nGcXGnvxwYYcUC/71++svtQ==", "dev": true, "license": "MIT", "dependencies": { "@types/web-bluetooth": "^0.0.21", - "@vueuse/metadata": "14.3.0", - "@vueuse/shared": "14.3.0" + "@vueuse/metadata": "14.4.0", + "@vueuse/shared": "14.4.0" }, "funding": { "url": "https://github.com/sponsors/antfu" @@ -820,14 +768,14 @@ } }, "node_modules/@vueuse/integrations": { - "version": "14.3.0", - "resolved": "https://registry.npmjs.org/@vueuse/integrations/-/integrations-14.3.0.tgz", - "integrity": "sha512-76I5FT2ESvCmCaSwapI+a/u/CFtNXmzl9f9lNp1hRtx8vKB8hfiokJr8IvQqcQG5ckGXElyXK516b54ozV3MvA==", + "version": "14.4.0", + "resolved": "https://registry.npmjs.org/@vueuse/integrations/-/integrations-14.4.0.tgz", + "integrity": "sha512-oJz9qTgczvA7L1nXQFRU7h8tQbOCoiceqvMMhT9XYMyOGTqLJ2rEa09PON+nD2t48sZUfeOmg4eaWJXV4sZb/w==", "dev": true, "license": "MIT", "dependencies": { - "@vueuse/core": "14.3.0", - "@vueuse/shared": "14.3.0" + "@vueuse/core": "14.4.0", + "@vueuse/shared": "14.4.0" }, "funding": { "url": "https://github.com/sponsors/antfu" @@ -887,9 +835,9 @@ } }, "node_modules/@vueuse/metadata": { - "version": "14.3.0", - "resolved": "https://registry.npmjs.org/@vueuse/metadata/-/metadata-14.3.0.tgz", - "integrity": "sha512-BwxmbAzwAVF50+MW57GXOUEV61nFBGnlBvrTqj49PqWJu3uw7hdu72ztXeZ33RdZtDY6kO+bfCAE1PCn88Tktw==", + "version": "14.4.0", + "resolved": "https://registry.npmjs.org/@vueuse/metadata/-/metadata-14.4.0.tgz", + "integrity": "sha512-swx/255R6JyHZFJhx845iz5CRWDZdCfvkZOpACWc5+c5WHcG24mv8gUT1WIdFQaHt6dq79rvILd9QnCWiyVm9g==", "dev": true, "license": "MIT", "funding": { @@ -897,9 +845,9 @@ } }, "node_modules/@vueuse/shared": { - "version": "14.3.0", - "resolved": "https://registry.npmjs.org/@vueuse/shared/-/shared-14.3.0.tgz", - "integrity": "sha512-bZpge9eSXwa4ToSiqJ7j6KRwhAsneMFoSz3LMWKQDkqimm3D/tbFlrklrs/IOqC8tEcYmXQZJ6N0UrjhBirVCg==", + "version": "14.4.0", + "resolved": "https://registry.npmjs.org/@vueuse/shared/-/shared-14.4.0.tgz", + "integrity": "sha512-JRgY90Sz8DDtPMsaDflvPMp9xYk69JZAmbuDvAquUVXKr2gEjqtzGNTTthLfckH0BzBqvnu31gb4a8TGLRe79g==", "dev": true, "license": "MIT", "funding": { @@ -936,14 +884,11 @@ } }, "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } + "license": "Python-2.0" }, "node_modules/bail": { "version": "2.0.2", @@ -977,16 +922,16 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/ccount": { @@ -1195,20 +1140,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/estree-walker": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", @@ -1312,22 +1243,6 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/gray-matter": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", - "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "js-yaml": "^3.13.1", - "kind-of": "^6.0.2", - "section-matter": "^1.0.0", - "strip-bom-string": "^1.0.0" - }, - "engines": { - "node": ">=6.0" - } - }, "node_modules/hast-util-to-html": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz", @@ -1418,14 +1333,23 @@ } }, "node_modules/js-yaml": { - "version": "3.15.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.0.tgz", - "integrity": "sha512-ttBQIIQPDeLjpPOohtUdXuXUVoA2uIB6fEH9HyJ7234s5mBJ5wTx20njxplLZQgLaOfpmPQA7X2t5AX6tIPbog==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -1778,13 +1702,6 @@ "markdown-it": "bin/markdown-it.mjs" } }, - "node_modules/markdown-it/node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, "node_modules/markdown-it/node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -2426,13 +2343,13 @@ } }, "node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.5" + "brace-expansion": "^5.0.8" }, "engines": { "node": "18 || 20 || >=22" @@ -2456,9 +2373,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -2528,9 +2445,9 @@ } }, "node_modules/postcss": { - "version": "8.5.22", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.22.tgz", - "integrity": "sha512-KBDEIpLrvpv16pp3K0Fw+UCoZfopFjjgeB+0tA/aaThfEE74kKDLrgg603YvOWJyg3+WYtyq3xYsQWsIyZlPqQ==", + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", "dev": true, "funding": [ { @@ -2548,7 +2465,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.16", + "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -2557,9 +2474,9 @@ } }, "node_modules/pretty-bytes": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-7.1.0.tgz", - "integrity": "sha512-nODzvTiYVRGRqAOvE84Vk5JDPyyxsVk0/fbA/bq7RqlnhksGpset09XTxbpvLTIjoaF7K8Z8DG8yHtKGTPSYRw==", + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-7.1.2.tgz", + "integrity": "sha512-spZIOSORH9joSPGN0L/tYfvINH+xbOpnI3kdcEkq0mzGMjFvDZzyRVbqEOV0wACF0gggdKE9RWO2mosGsBh/Rg==", "dev": true, "license": "MIT", "engines": { @@ -2695,13 +2612,13 @@ } }, "node_modules/rolldown": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", - "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.6.tgz", + "integrity": "sha512-vMM4q3aixf46GiF1Kok8jDPFsEpXgFWGjUHXNkNHNm+Y2adXAG2dbX91jkti3i0ZRsOlcmbuzAz1poObSHCmUA==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.139.0", + "@oxc-project/types": "=0.147.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -2711,21 +2628,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.1.5", - "@rolldown/binding-darwin-arm64": "1.1.5", - "@rolldown/binding-darwin-x64": "1.1.5", - "@rolldown/binding-freebsd-x64": "1.1.5", - "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", - "@rolldown/binding-linux-arm64-gnu": "1.1.5", - "@rolldown/binding-linux-arm64-musl": "1.1.5", - "@rolldown/binding-linux-ppc64-gnu": "1.1.5", - "@rolldown/binding-linux-s390x-gnu": "1.1.5", - "@rolldown/binding-linux-x64-gnu": "1.1.5", - "@rolldown/binding-linux-x64-musl": "1.1.5", - "@rolldown/binding-openharmony-arm64": "1.1.5", - "@rolldown/binding-wasm32-wasi": "1.1.5", - "@rolldown/binding-win32-arm64-msvc": "1.1.5", - "@rolldown/binding-win32-x64-msvc": "1.1.5" + "@rolldown/binding-android-arm-eabi": "1.2.6", + "@rolldown/binding-android-arm64": "1.2.6", + "@rolldown/binding-darwin-arm64": "1.2.6", + "@rolldown/binding-darwin-x64": "1.2.6", + "@rolldown/binding-freebsd-x64": "1.2.6", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.6", + "@rolldown/binding-linux-arm64-gnu": "1.2.6", + "@rolldown/binding-linux-arm64-musl": "1.2.6", + "@rolldown/binding-linux-ppc64-gnu": "1.2.6", + "@rolldown/binding-linux-s390x-gnu": "1.2.6", + "@rolldown/binding-linux-x64-gnu": "1.2.6", + "@rolldown/binding-linux-x64-musl": "1.2.6", + "@rolldown/binding-openharmony-arm64": "1.2.6", + "@rolldown/binding-win32-arm64-msvc": "1.2.6", + "@rolldown/binding-win32-x64-msvc": "1.2.6" } }, "node_modules/section-matter": { @@ -2743,20 +2660,20 @@ } }, "node_modules/shiki": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/shiki/-/shiki-4.3.1.tgz", - "integrity": "sha512-oR+qDVi2OjX1tmDpyv+3KviX01KzO6Af+0NNnKnsp9491UEGz2YpxTuJboS/6VhYpTdqzmuJBuiTlrAWWJAssw==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-4.4.3.tgz", + "integrity": "sha512-Mb/GvXPHBAXdgGIcnfU5L3ldpn1XcxrGkPHwqgRx17/I2XRfqlFKk2vGkHWINn1kdXvzJZeuO3is6I9KLPFm0g==", "dev": true, "license": "MIT", "dependencies": { - "@shikijs/core": "4.3.1", - "@shikijs/engine-javascript": "4.3.1", - "@shikijs/engine-oniguruma": "4.3.1", - "@shikijs/langs": "4.3.1", - "@shikijs/themes": "4.3.1", - "@shikijs/types": "4.3.1", + "@shikijs/core": "4.4.3", + "@shikijs/engine-javascript": "4.4.3", + "@shikijs/engine-oniguruma": "4.4.3", + "@shikijs/langs": "4.4.3", + "@shikijs/themes": "4.4.3", + "@shikijs/types": "4.4.3", "@shikijs/vscode-textmate": "^10.0.2", - "@types/hast": "^3.0.4" + "@types/hast": "^3.0.5" }, "engines": { "node": ">=20" @@ -2783,13 +2700,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -2833,16 +2743,6 @@ "node": ">=8" } }, - "node_modules/strip-bom-string": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", - "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/tabbable": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.5.0.tgz", @@ -2868,9 +2768,9 @@ } }, "node_modules/tokenx": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/tokenx/-/tokenx-1.3.0.tgz", - "integrity": "sha512-NLdXTEZkKiO0gZuLtMoZKjCXTREXeZZt8nnnNeyoXtNZAfG/GKGSbQtLU5STspc0rMSwcA+UJfWZkbNU01iKmQ==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/tokenx/-/tokenx-1.6.0.tgz", + "integrity": "sha512-CKTjk345ajvBAUp5xUI9a5KKN0zU0lBueVHQbCskH1Hp6WkUKsPW2qGCYNs0pxNyfzxfo+IIjdt2W4sMbw/qBw==", "dev": true, "license": "MIT" }, @@ -2896,14 +2796,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, "node_modules/uc.micro": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", @@ -3051,16 +2943,16 @@ } }, "node_modules/vite": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz", - "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", + "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", "dev": true, "license": "MIT", "dependencies": { - "lightningcss": "^1.32.0", + "lightningcss": "^1.33.0", "picomatch": "^4.0.5", - "postcss": "^8.5.17", - "rolldown": "~1.1.5", + "postcss": "^8.5.26", + "rolldown": "~1.2.4", "tinyglobby": "^0.2.17" }, "bin": { @@ -3077,7 +2969,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.3.0", + "@vitejs/devtools": "^0.4.0 || ^0.5.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", @@ -3129,31 +3021,31 @@ } }, "node_modules/vitepress": { - "version": "2.0.0-alpha.18", - "resolved": "https://registry.npmjs.org/vitepress/-/vitepress-2.0.0-alpha.18.tgz", - "integrity": "sha512-Lk1G2/QqSf+MwNLICl9fmzWOtoCEwjZoWgaQy41QvWTfcGpLE9XwJDbcCyES/9rj6R2g1zFqFvLVkYKLLDALFw==", + "version": "2.0.0-alpha.19", + "resolved": "https://registry.npmjs.org/vitepress/-/vitepress-2.0.0-alpha.19.tgz", + "integrity": "sha512-WnBsb0Bwr43kXKyiis+lld/7ri3hnMbthS8N3hpFtjjwsdLO4IRmiAE08D7aud4q6oMDf9uwRowxzNqRFe/amw==", "dev": true, "license": "MIT", "dependencies": { - "@docsearch/css": "^4.6.3", - "@docsearch/js": "^4.6.3", - "@docsearch/sidepanel-js": "^4.6.3", - "@iconify-json/simple-icons": "^1.2.87", - "@shikijs/core": "^4.3.0", - "@shikijs/transformers": "^4.3.0", - "@shikijs/types": "^4.3.0", + "@docsearch/css": "^4.7.0", + "@docsearch/js": "^4.7.0", + "@docsearch/sidepanel-js": "^4.7.0", + "@iconify-json/simple-icons": "^1.2.92", + "@shikijs/core": "^4.4.1", + "@shikijs/transformers": "^4.4.1", + "@shikijs/types": "^4.4.1", "@types/markdown-it": "^14.1.2", - "@vitejs/plugin-vue": "^6.0.7", - "@vue/devtools-api": "^8.1.4", - "@vue/shared": "^3.5.39", - "@vueuse/core": "^14.3.0", - "@vueuse/integrations": "^14.3.0", + "@vitejs/plugin-vue": "^6.0.8", + "@vue/devtools-api": "^8.2.1", + "@vue/shared": "^3.5.40", + "@vueuse/core": "^14.4.0", + "@vueuse/integrations": "^14.4.0", "focus-trap": "^8.2.2", "mark.js": "8.11.1", "minisearch": "^7.2.0", - "shiki": "^4.3.0", - "vite": "^8.1.3", - "vue": "^3.5.39" + "shiki": "^4.4.1", + "vite": "^8.2.0", + "vue": "^3.5.40" }, "bin": { "vitepress": "bin/vitepress.js" @@ -3172,24 +3064,24 @@ } }, "node_modules/vitepress-plugin-llms": { - "version": "1.13.3", - "resolved": "https://registry.npmjs.org/vitepress-plugin-llms/-/vitepress-plugin-llms-1.13.3.tgz", - "integrity": "sha512-C6bygRzuVp54kveM6pngVNg4H9sKMx7K67iwoO4PUzBK22NiXF9SoFm/WdjKp2fJrMcOabKtQMuyNFF20c2Anw==", + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/vitepress-plugin-llms/-/vitepress-plugin-llms-1.13.5.tgz", + "integrity": "sha512-CTiDkKRs0h64J5rdiICdHmXH4pM1gBu2S/MJ+OxSAm+vM72srBkHr86ZubzfMJbGJh4AMVKff4R3oCu8etI1ww==", "dev": true, "license": "MIT", "dependencies": { - "gray-matter": "^4.0.3", + "@11ty/gray-matter": "^2.1.0", "markdown-it": "^14.1.0", "markdown-title": "^1.0.2", "mdast-util-from-markdown": "^2.0.3", "millify": "^6.1.0", - "minimatch": "^10.2.5", + "minimatch": "^10.2.6", "path-to-regexp": "^6.3.0", "picocolors": "^1.1.1", - "pretty-bytes": "^7.1.0", + "pretty-bytes": "^7.1.1", "remark": "^15.0.1", "remark-frontmatter": "^5.0.0", - "tokenx": "^1.3.0", + "tokenx": "^1.6.0", "unist-util-remove": "^4.0.0", "unist-util-visit": "^5.1.0" }, diff --git a/package.json b/package.json index a7d4c00..82e81bc 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "docs:preview": "vitepress preview docs" }, "devDependencies": { - "vitepress": "^2.0.0-alpha.18", - "vitepress-plugin-llms": "^1.13.3" + "vitepress": "^2.0.0-alpha.19", + "vitepress-plugin-llms": "^1.13.5" } } diff --git a/src/Wallaby.Sinks.Pgvector/Internal/DeliveryExceptions.cs b/src/Wallaby.Sinks.Pgvector/Internal/DeliveryExceptions.cs new file mode 100644 index 0000000..b35ad38 --- /dev/null +++ b/src/Wallaby.Sinks.Pgvector/Internal/DeliveryExceptions.cs @@ -0,0 +1,11 @@ +namespace Wallaby.Sinks.Pgvector.Internal; + +/// Delivery failure that must halt the pipeline rather than retry. +internal sealed class PermanentDeliveryException(string message, Exception? inner = null) + : Exception(message, inner); + +/// Embedding-provider failure, carrying its retryability classification. +internal sealed class EmbeddingException(bool transient, Exception inner) : Exception(inner.Message, inner) +{ + public bool Transient { get; } = transient; +} diff --git a/src/Wallaby.Sinks.Pgvector/Internal/PgvectorFormat.cs b/src/Wallaby.Sinks.Pgvector/Internal/PgvectorFormat.cs new file mode 100644 index 0000000..21f723a --- /dev/null +++ b/src/Wallaby.Sinks.Pgvector/Internal/PgvectorFormat.cs @@ -0,0 +1,35 @@ +using System.Security.Cryptography; +using System.Text; + +namespace Wallaby.Sinks.Pgvector.Internal; + +/// Text hashing and vector extraction for the sink's write path. +internal static class PgvectorFormat +{ + /// + /// The stored content hash: SHA-256 over the embedding version and the embedded text, so a model + /// change re-embeds even when the text is unchanged. + /// + public static string TextHash(string version, string text) + => Convert.ToBase64String(SHA256.HashData(Encoding.UTF8.GetBytes($"{version}\n{text}"))); + + /// Extracts a vector value a transform placed in a document field. + public static bool TryGetVector(object? value, out ReadOnlyMemory vector) + { + switch (value) + { + case ReadOnlyMemory memory: + vector = memory; + return true; + case Memory memory: + vector = memory; + return true; + case float[] array: + vector = array; + return true; + default: + vector = default; + return false; + } + } +} diff --git a/src/Wallaby.Sinks.Pgvector/Internal/PgvectorRowBuilder.cs b/src/Wallaby.Sinks.Pgvector/Internal/PgvectorRowBuilder.cs new file mode 100644 index 0000000..792c03b --- /dev/null +++ b/src/Wallaby.Sinks.Pgvector/Internal/PgvectorRowBuilder.cs @@ -0,0 +1,162 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.AI; +using Pgvector; +using Wallaby.Abstractions; + +namespace Wallaby.Sinks.Pgvector.Internal; + +/// One upsert for the write batch. KeepStoredVector marks a hash-gated row whose stored vector stays. +internal sealed record PgvectorRow(string Id, string? Hash, Vector? Vector, bool KeepStoredVector, string DocumentJson); + +/// +/// Builds the rows a delivery writes: pass-through vector extraction, or hash-gated embedding where +/// only rows whose stored hash is missing or different reach the generator. +/// +internal sealed class PgvectorRowBuilder(PgvectorSinkOptions options, PgvectorTables tables) +{ + public Task> BuildAsync(string table, List upserts, CancellationToken ct) + => options.EmbeddingGenerator is null + ? Task.FromResult(BuildPassThrough(upserts)) + : BuildEmbeddedAsync(table, upserts, ct); + + private List BuildPassThrough(List upserts) + { + var rows = new List(upserts.Count); + using var buffer = new MemoryStream(); + using var writer = new Utf8JsonWriter(buffer); + foreach (var record in upserts) + { + Vector? stored = null; + if (record.Document!.TryGetValue(options.VectorField, out var value) && value is not null) + { + if (!PgvectorFormat.TryGetVector(value, out var vector)) + { + throw new PermanentDeliveryException( + $"Document '{record.DocumentId}' field '{options.VectorField}' has type " + + $"'{value.GetType()}'; expected ReadOnlyMemory or float[]."); + } + stored = RequireDimensions(vector, record.DocumentId); + } + rows.Add(new PgvectorRow(record.DocumentId, Hash: null, stored, KeepStoredVector: false, + BuildDocumentJson(record, options.VectorField, buffer, writer))); + } + return rows; + } + + private async Task> BuildEmbeddedAsync(string table, List upserts, CancellationToken ct) + { + var rows = new List(upserts.Count); + if (upserts.Count == 0) + { + return rows; + } + + // The destination is the cache: compare stored hashes and embed only what changed. + var storedHashes = await tables.LoadStoredHashesAsync( + table, upserts.Select(u => u.DocumentId).ToArray(), ct); + var pendingTexts = new List(); + var pendingRows = new List(); + using (var buffer = new MemoryStream()) + using (var writer = new Utf8JsonWriter(buffer)) + { + foreach (var record in upserts) + { + var text = options.EmbedText!(record.Document!); + var json = BuildDocumentJson(record, excludeField: null, buffer, writer); + if (string.IsNullOrEmpty(text)) + { + rows.Add(new PgvectorRow(record.DocumentId, Hash: null, Vector: null, KeepStoredVector: false, json)); + continue; + } + + var hash = PgvectorFormat.TextHash(options.EmbeddingVersion!, text); + if (storedHashes.TryGetValue(record.DocumentId, out var stored) && stored == hash) + { + rows.Add(new PgvectorRow(record.DocumentId, hash, Vector: null, KeepStoredVector: true, json)); + continue; + } + + pendingRows.Add(rows.Count); + pendingTexts.Add(text); + rows.Add(new PgvectorRow(record.DocumentId, hash, Vector: null, KeepStoredVector: false, json)); + } + } + + var vectors = await EmbedAsync(pendingTexts, i => rows[pendingRows[i]].Id, ct); + for (var i = 0; i < pendingRows.Count; i++) + { + rows[pendingRows[i]] = rows[pendingRows[i]] with { Vector = vectors[i] }; + } + return rows; + } + + // Sub-batches fill disjoint slices of the result array, so they can run concurrently up to + // MaxEmbeddingConcurrency (default 1: sequential). + private async Task EmbedAsync(List texts, Func documentIdAt, CancellationToken ct) + { + var vectors = new Vector[texts.Count]; + var subBatches = new List<(int Offset, int Count)>(); + for (var offset = 0; offset < texts.Count; offset += options.MaxEmbeddingBatchSize) + { + subBatches.Add((offset, Math.Min(options.MaxEmbeddingBatchSize, texts.Count - offset))); + } + await Parallel.ForEachAsync(subBatches, + new ParallelOptions { MaxDegreeOfParallelism = options.MaxEmbeddingConcurrency, CancellationToken = ct }, + async (subBatch, token) => + { + GeneratedEmbeddings> embeddings; + try + { + embeddings = await options.EmbeddingGenerator!.GenerateAsync( + texts.GetRange(subBatch.Offset, subBatch.Count), options: null, token); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + throw new EmbeddingException(IsTransient(ex), ex); + } + if (embeddings.Count != subBatch.Count) + { + throw new PermanentDeliveryException( + $"The embedding generator returned {embeddings.Count} embeddings for {subBatch.Count} inputs; " + + "counts must match one-to-one."); + } + for (var i = 0; i < subBatch.Count; i++) + { + var index = subBatch.Offset + i; + vectors[index] = RequireDimensions(embeddings[i].Vector, documentIdAt(index)); + } + }); + return vectors; + } + + private Vector RequireDimensions(ReadOnlyMemory vector, string documentId) + => vector.Length == options.Dimensions + ? new Vector(vector) + : throw new PermanentDeliveryException( + $"Document '{documentId}' has a {vector.Length}-dimensional vector; the sink is configured " + + $"for vector({options.Dimensions})."); + + // Callers own buffer and writer so one pair serves a whole batch; both are reset per document. + private string BuildDocumentJson(SinkRecord record, string? excludeField, MemoryStream buffer, Utf8JsonWriter writer) + { + var document = record.Document!; + if (excludeField is not null && document.ContainsKey(excludeField)) + { + document = document.Where(f => f.Key != excludeField).ToDictionary(f => f.Key, f => f.Value); + } + buffer.SetLength(0); + writer.Reset(); + SinkEnvelopeJson.WriteDocument(writer, document, record.DocumentId, options.SerializerOptions); + writer.Flush(); + return Encoding.UTF8.GetString(buffer.GetBuffer(), 0, (int)buffer.Length); + } + + private bool IsTransient(Exception ex) + => options.IsTransientEmbeddingError?.Invoke(ex) + ?? ex is not (ArgumentException or NotSupportedException); +} diff --git a/src/Wallaby.Sinks.Pgvector/Internal/PgvectorTables.cs b/src/Wallaby.Sinks.Pgvector/Internal/PgvectorTables.cs new file mode 100644 index 0000000..e15a8b0 --- /dev/null +++ b/src/Wallaby.Sinks.Pgvector/Internal/PgvectorTables.cs @@ -0,0 +1,166 @@ +using Npgsql; +using NpgsqlTypes; + +namespace Wallaby.Sinks.Pgvector.Internal; + +/// +/// The sink's SQL surface: identifier policy, extension and table creation, stored-hash reads, and +/// the transactional upsert/delete write. +/// +internal sealed class PgvectorTables(string sinkName, NpgsqlDataSource dataSource, PgvectorSinkOptions options) +{ + private readonly HashSet _ensured = []; + + public static bool IsValidIdentifier(string? value) + => value is { Length: > 0 and <= 63 } && value.All(c => char.IsAsciiLetterOrDigit(c) || c == '_'); + + public void RequireValidTable(string table) + { + if (!IsValidIdentifier(table)) + { + throw new PermanentDeliveryException( + $"Destination '{table}' is not a valid pgvector table name for sink '{sinkName}': use 1-63 " + + "characters of [a-zA-Z0-9_]."); + } + } + + public async Task EnsureExtensionAsync(CancellationToken ct) + { + try + { + await using var cmd = dataSource.CreateCommand("CREATE EXTENSION IF NOT EXISTS vector"); + await cmd.ExecuteNonQueryAsync(ct); + } + catch (PostgresException ex) when (ex.SqlState is PostgresErrorCodes.InsufficientPrivilege) + { + throw new WallabyConfigurationException( + $"Pgvector sink '{sinkName}' cannot create the 'vector' extension (insufficient privilege). " + + "Have a superuser run CREATE EXTENSION vector; on the destination database, or set " + + "CreateExtension = false once it exists.", ex); + } + catch (PostgresException ex) when ( + ex.SqlState is PostgresErrorCodes.UniqueViolation or PostgresErrorCodes.DuplicateObject) + { + // IF NOT EXISTS is not concurrency-safe; a concurrent creator winning means it exists. + } + // The data source may have loaded its type catalog before the extension existed (its first + // connection is this CREATE EXTENSION); reload so the 'vector' type resolves for parameters. + await dataSource.ReloadTypesAsync(ct); + } + + public async Task EnsureTableAsync(string table, CancellationToken ct) + { + lock (_ensured) + { + if (_ensured.Contains(table)) + { + return; + } + } + try + { + await using var cmd = dataSource.CreateCommand( + $""" + CREATE TABLE IF NOT EXISTS {Qualified(table)} ( + id text PRIMARY KEY, + text_hash text, + embedding vector({options.Dimensions}), + document jsonb NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() + ) + """); + await cmd.ExecuteNonQueryAsync(ct); + } + catch (PostgresException ex) when ( + ex.SqlState is PostgresErrorCodes.UniqueViolation or PostgresErrorCodes.DuplicateTable) + { + // IF NOT EXISTS is not concurrency-safe; a concurrent creator winning means it exists. + } + lock (_ensured) + { + _ensured.Add(table); + } + } + + public async Task> LoadStoredHashesAsync( + string table, string[] ids, CancellationToken ct) + { + var hashes = new Dictionary(StringComparer.Ordinal); + // A hash only counts alongside a stored vector; a hash next to a null embedding (however it + // arose) must re-embed rather than gate. + await using var cmd = dataSource.CreateCommand( + $"SELECT id, text_hash FROM {Qualified(table)} " + + "WHERE id = ANY($1) AND text_hash IS NOT NULL AND embedding IS NOT NULL"); + cmd.Parameters.Add(new NpgsqlParameter { Value = ids }); + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + hashes[reader.GetString(0)] = reader.GetString(1); + } + return hashes; + } + + public async Task WriteAsync( + string table, IReadOnlyList rows, IReadOnlyList deletes, CancellationToken ct) + { + await using var connection = await dataSource.OpenConnectionAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + foreach (var chunk in rows.Chunk(options.MaxRowsPerBatch)) + { + await using var writeBatch = new NpgsqlBatch(connection, transaction); + foreach (var row in chunk) + { + // A KeepStoredVector row normally exists (its stored hash matched) and takes the update + // arm leaving embedding and text_hash untouched - but only after re-verifying the hash + // and vector under the row lock, so a row changed between the hash read and this write + // (e.g. by a concurrent deliverer) is left intact rather than paired with a foreign + // vector. If the row vanished instead, the insert arm writes the hash with a null + // vector, which the hash read's embedding filter treats as absent, so the next delivery + // re-embeds. Both arms guard with IS DISTINCT FROM so an identical redelivery does not + // rewrite the tuple (no WAL or dead-tuple churn on re-backfills of unchanged rows). + var update = row.KeepStoredVector + ? "document = EXCLUDED.document, updated_at = now() " + + "WHERE t.text_hash = EXCLUDED.text_hash AND t.embedding IS NOT NULL " + + "AND t.document IS DISTINCT FROM EXCLUDED.document" + : "text_hash = EXCLUDED.text_hash, embedding = EXCLUDED.embedding, " + + "document = EXCLUDED.document, updated_at = now() " + + "WHERE (t.text_hash, t.embedding, t.document) IS DISTINCT FROM " + + "(EXCLUDED.text_hash, EXCLUDED.embedding, EXCLUDED.document)"; + var cmd = new NpgsqlBatchCommand( + $"INSERT INTO {Qualified(table)} AS t (id, text_hash, embedding, document) " + + $"VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO UPDATE SET {update}"); + cmd.Parameters.Add(new NpgsqlParameter { Value = row.Id }); + cmd.Parameters.Add(new NpgsqlParameter { Value = (object?)row.Hash ?? DBNull.Value, NpgsqlDbType = NpgsqlDbType.Text }); + // A Vector value infers the vector type via the plugin; a bare null is sent untyped and + // the server infers it from the target column. + cmd.Parameters.Add(new NpgsqlParameter { Value = (object?)row.Vector ?? DBNull.Value }); + cmd.Parameters.Add(new NpgsqlParameter { Value = row.DocumentJson, NpgsqlDbType = NpgsqlDbType.Jsonb }); + writeBatch.BatchCommands.Add(cmd); + } + await writeBatch.ExecuteNonQueryAsync(ct); + } + if (deletes.Count > 0) + { + await using var delete = new NpgsqlCommand( + $"DELETE FROM {Qualified(table)} WHERE id = ANY($1)", connection, transaction); + delete.Parameters.Add(new NpgsqlParameter { Value = deletes.ToArray() }); + await delete.ExecuteNonQueryAsync(ct); + } + await transaction.CommitAsync(ct); + } + + public async Task PurgeAsync(string table, CancellationToken ct) + { + try + { + await using var cmd = dataSource.CreateCommand($"DELETE FROM {Qualified(table)}"); + await cmd.ExecuteNonQueryAsync(ct); + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.UndefinedTable) + { + // Nothing to purge; the table is created on initialization or first delivery. + } + } + + private string Qualified(string table) => $"\"{options.Schema}\".\"{table}\""; +} diff --git a/src/Wallaby.Sinks.Pgvector/PgvectorBuilderExtensions.cs b/src/Wallaby.Sinks.Pgvector/PgvectorBuilderExtensions.cs new file mode 100644 index 0000000..0d77ed2 --- /dev/null +++ b/src/Wallaby.Sinks.Pgvector/PgvectorBuilderExtensions.cs @@ -0,0 +1,95 @@ +using Wallaby.DependencyInjection; +using Wallaby.Sinks.Pgvector.Internal; + +namespace Wallaby.Sinks.Pgvector; + +/// Fluent helpers for registering a pgvector sink on a . +public static class PgvectorBuilderExtensions +{ + /// + /// Register a pgvector sink under . Attach the entities it stores via + /// on the returned builder; each mapping's + /// destination is the table name. + /// + public static WallabySinkBuilder AddPgvectorSink(this WallabyBuilder builder, string name, Action configure) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(configure); + + var options = new PgvectorSinkOptions { ConnectionString = "", Dimensions = 0 }; + configure(options); + Validate(options); + + return builder.AddSink(name, _ => new PgvectorSink(name, options)); + } + + /// + /// Provider-aware overload: runs on first resolution, so option values + /// can come from services (e.g. IConfiguration) while the registration itself stays eager. + /// Validation failures surface at host start rather than at registration. + /// + public static WallabySinkBuilder AddPgvectorSink(this WallabyBuilder builder, string name, Action configure) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(configure); + + return builder.AddSink(name, sp => + { + var options = new PgvectorSinkOptions { ConnectionString = "", Dimensions = 0 }; + configure(sp, options); + Validate(options); + return new PgvectorSink(name, options); + }); + } + + internal static void Validate(PgvectorSinkOptions options) + { + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException("PgvectorSinkOptions.ConnectionString is required.", nameof(options)); + } + if (options.Dimensions <= 0) + { + throw new ArgumentException("PgvectorSinkOptions.Dimensions must be positive.", nameof(options)); + } + if (!PgvectorTables.IsValidIdentifier(options.Schema)) + { + throw new ArgumentException( + "PgvectorSinkOptions.Schema must be 1-63 characters of [a-zA-Z0-9_].", nameof(options)); + } + if (options.DefaultTable is { } table && !PgvectorTables.IsValidIdentifier(table)) + { + throw new ArgumentException( + "PgvectorSinkOptions.DefaultTable must be 1-63 characters of [a-zA-Z0-9_].", nameof(options)); + } + if (options.MaxRowsPerBatch <= 0) + { + throw new ArgumentException("PgvectorSinkOptions.MaxRowsPerBatch must be positive.", nameof(options)); + } + if (options.MaxEmbeddingBatchSize <= 0) + { + throw new ArgumentException("PgvectorSinkOptions.MaxEmbeddingBatchSize must be positive.", nameof(options)); + } + if (options.MaxEmbeddingConcurrency <= 0) + { + throw new ArgumentException("PgvectorSinkOptions.MaxEmbeddingConcurrency must be positive.", nameof(options)); + } + + var embedParts = (options.EmbeddingGenerator is not null, options.EmbedText is not null, + !string.IsNullOrWhiteSpace(options.EmbeddingVersion)); + if (embedParts is not ((true, true, true) or (false, false, false))) + { + throw new ArgumentException( + "PgvectorSinkOptions embedding requires EmbeddingGenerator, EmbedText, and EmbeddingVersion " + + "together (or none of them, for transform-provided vectors via VectorField).", nameof(options)); + } + if (options.EmbeddingGenerator is null && string.IsNullOrWhiteSpace(options.VectorField)) + { + throw new ArgumentException( + "PgvectorSinkOptions.VectorField must be a non-empty field name when no EmbeddingGenerator is set.", + nameof(options)); + } + } +} diff --git a/src/Wallaby.Sinks.Pgvector/PgvectorSink.cs b/src/Wallaby.Sinks.Pgvector/PgvectorSink.cs new file mode 100644 index 0000000..c27cb7d --- /dev/null +++ b/src/Wallaby.Sinks.Pgvector/PgvectorSink.cs @@ -0,0 +1,158 @@ +using Npgsql; +using Pgvector.Npgsql; +using Wallaby.Abstractions; +using Wallaby.Sinks.Pgvector.Internal; + +namespace Wallaby.Sinks.Pgvector; + +/// +/// Delivers documents into per-destination pgvector tables shaped (id text primary key, text_hash +/// text, embedding vector(N), document jsonb, updated_at timestamptz). Upserts are idempotent by +/// id and deletes remove by id, so redelivery converges. With an embedding generator configured the +/// sink embeds at delivery time and re-embeds only rows whose text hash changed - the destination +/// table doubles as the durable embedding cache, so restarts, failovers, and re-backfills never +/// re-embed unchanged text. Embedding and transient database failures surface as retryable delivery +/// results, riding the dispatcher's backoff. +/// +public sealed class PgvectorSink : ISink, ISinkInitializer, ISinkPurger, IAsyncDisposable +{ + private readonly PgvectorSinkOptions _options; + private readonly NpgsqlDataSource _dataSource; + private readonly PgvectorTables _tables; + private readonly PgvectorRowBuilder _rows; + + /// Creates a sink that delivers to the database described by . + /// The sink's registration name (used for routing, telemetry, and test replacement). + /// Connection, table, and embedding settings. + public PgvectorSink(string name, PgvectorSinkOptions options) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(options); + PgvectorBuilderExtensions.Validate(options); + Name = name; + _options = options; + var builder = new NpgsqlDataSourceBuilder(options.ConnectionString); + builder.UseVector(); + options.ConfigureDataSource?.Invoke(builder); + _dataSource = builder.Build(); + _tables = new PgvectorTables(name, _dataSource, options); + _rows = new PgvectorRowBuilder(options, _tables); + } + + /// + public string Name { get; } + + /// + public async Task InitializeAsync(CancellationToken ct) + { + if (_options.CreateExtension) + { + await _tables.EnsureExtensionAsync(ct); + } + if (_options.CreateTable && _options.DefaultTable is { } table) + { + await _tables.EnsureTableAsync(table, ct); + } + } + + /// + public async Task DeliverAsync(SinkBatch batch, CancellationToken ct) + { + try + { + foreach (var (table, records) in GroupByTable(batch.Records)) + { + await DeliverTableAsync(table, records, ct); + } + return DeliveryResult.Success; + } + catch (OperationCanceledException) + { + throw; + } + catch (PermanentDeliveryException ex) + { + return DeliveryResult.Permanent(ex.Message, ex.InnerException); + } + catch (EmbeddingException ex) + { + var reason = $"Embedding failed for sink '{Name}': {ex.InnerException!.Message}"; + return ex.Transient + ? DeliveryResult.Retry(reason, ex.InnerException) + : DeliveryResult.Permanent(reason, ex.InnerException); + } + catch (Exception ex) + { + return Classify(ex); + } + } + + /// + public async Task PurgeAsync(SinkPurgeRequest request, CancellationToken ct) + { + var table = request.Destination ?? _options.DefaultTable + ?? throw new WallabyConfigurationException( + $"Pgvector sink '{Name}' cannot purge for '{request.QualifiedTableName}': the mapping has no " + + "destination and the sink has no DefaultTable."); + _tables.RequireValidTable(table); + await _tables.PurgeAsync(table, ct); + } + + /// + public ValueTask DisposeAsync() => _dataSource.DisposeAsync(); + + private async Task DeliverTableAsync(string table, Dictionary records, CancellationToken ct) + { + _tables.RequireValidTable(table); + if (_options.CreateTable) + { + await _tables.EnsureTableAsync(table, ct); + } + + var deletes = new List(); + var upserts = new List(); + foreach (var record in records.Values) + { + if (record.IsDeletion) + { + deletes.Add(record.DocumentId); + } + else + { + upserts.Add(record); + } + } + + var rows = await _rows.BuildAsync(table, upserts, ct); + await _tables.WriteAsync(table, rows, deletes, ct); + } + + // Last write per id wins within the batch (same as replaying it row-by-row), grouped per table in + // first-seen order. + private Dictionary> GroupByTable(IReadOnlyList records) + { + var byTable = new Dictionary>(StringComparer.Ordinal); + foreach (var record in records) + { + var table = record.Destination ?? _options.DefaultTable + ?? throw new PermanentDeliveryException( + $"A record for sink '{Name}' has no destination and the sink declares no DefaultTable. " + + "Set ToDestination(...) on the mapping or DefaultTable on the sink."); + if (!byTable.TryGetValue(table, out var byId)) + { + byTable[table] = byId = new Dictionary(StringComparer.Ordinal); + } + byId[record.DocumentId] = record; + } + return byTable; + } + + private DeliveryResult Classify(Exception ex) => ex switch + { + PostgresException pg when !pg.IsTransient => + DeliveryResult.Permanent($"Postgres rejected the delivery for sink '{Name}': {pg.MessageText}", pg), + NpgsqlException or TimeoutException or System.IO.IOException or System.Net.Sockets.SocketException => + DeliveryResult.Retry($"Transient database failure for sink '{Name}': {ex.Message}", ex), + _ => DeliveryResult.Permanent($"Delivery failed for sink '{Name}': {ex.Message}", ex), + }; +} diff --git a/src/Wallaby.Sinks.Pgvector/PgvectorSinkOptions.cs b/src/Wallaby.Sinks.Pgvector/PgvectorSinkOptions.cs new file mode 100644 index 0000000..4181952 --- /dev/null +++ b/src/Wallaby.Sinks.Pgvector/PgvectorSinkOptions.cs @@ -0,0 +1,93 @@ +using System.Text.Json; +using Microsoft.Extensions.AI; +using Npgsql; + +namespace Wallaby.Sinks.Pgvector; + +/// +/// Settings for a pgvector sink. The sink writes one row per document into a per-destination table +/// shaped (id text primary key, text_hash text, embedding vector(N), document jsonb, updated_at +/// timestamptz). With an configured the sink embeds at delivery +/// time, re-embedding only rows whose output changed (hash-gated against the +/// destination table itself); without one, the transform supplies the vector via +/// . +/// +public sealed class PgvectorSinkOptions +{ + /// Connection string of the destination database (often not the CDC source). + public required string ConnectionString { get; set; } + + /// Extra data-source configuration (TLS callbacks, loggers, ...). + public Action? ConfigureDataSource { get; set; } + + /// Schema holding the destination tables. + public string Schema { get; set; } = "public"; + + /// Table used when a routed record has no destination. + public string? DefaultTable { get; set; } + + /// + /// Dimension count of the embedding vector(N) column; every stored vector must match. + /// + public required int Dimensions { get; set; } + + /// + /// Create missing destination tables (and, with , the extension) on + /// initialization and on first delivery to a runtime destination. + /// + public bool CreateTable { get; set; } = true; + + /// Run CREATE EXTENSION IF NOT EXISTS vector during initialization. + public bool CreateExtension { get; set; } = true; + + /// + /// Embeds at delivery time. Configure together with and + /// ; leave null to have transforms supply vectors via + /// instead. + /// + public IEmbeddingGenerator>? EmbeddingGenerator { get; set; } + + /// + /// Selects the text to embed from a document's field bag. Null/empty output stores the row with a + /// null vector. Required when is set. + /// + public Func, string?>? EmbedText { get; set; } + + /// + /// Identifies the embedding model and prompt shape, e.g. "text-embedding-3-small/1". Folded + /// into the stored text hash, so changing it re-embeds rows as they re-deliver; pair it with the + /// mapping's WithBackfillVersion(..., purgeOnChange: true) to re-embed the whole corpus at + /// once. Required when is set. + /// + public string? EmbeddingVersion { get; set; } + + /// Max texts per embedding call; larger sets split into multiple calls. + public int MaxEmbeddingBatchSize { get; set; } = 96; + + /// + /// Max embedding calls in flight at once. The default (1) sends calls sequentially; raise it to + /// overlap calls on large backfills when the provider's rate limits allow. + /// + public int MaxEmbeddingConcurrency { get; set; } = 1; + + /// + /// Classifies an embedding-provider exception as retryable (delivery backs off and retries) versus + /// permanent (the pipeline halts). Null uses the default: everything is retryable except + /// and . Narrow it when your + /// provider surfaces typed auth/quota errors that retrying cannot fix. + /// + public Func? IsTransientEmbeddingError { get; set; } + + /// + /// Without an : the document field carrying the vector + /// (ReadOnlyMemory<float> or float[]). The field is stored in the vector + /// column and dropped from the jsonb payload; a document without it stores a null vector. + /// + public string VectorField { get; set; } = "embedding"; + + /// Rows per database round-trip; larger batches split into sequential command batches. + public int MaxRowsPerBatch { get; set; } = 500; + + /// Serializer for document values beyond the natively written scalar types (required for such values on NativeAOT hosts). + public JsonSerializerOptions? SerializerOptions { get; set; } +} diff --git a/src/Wallaby.Sinks.Pgvector/Wallaby.Sinks.Pgvector.csproj b/src/Wallaby.Sinks.Pgvector/Wallaby.Sinks.Pgvector.csproj new file mode 100644 index 0000000..a936d64 --- /dev/null +++ b/src/Wallaby.Sinks.Pgvector/Wallaby.Sinks.Pgvector.csproj @@ -0,0 +1,31 @@ + + + + true + Wallaby.Sinks.Pgvector + true + true + Postgres pgvector destination/sink for Wallaby: sync documents into vector tables, with optional sink-side embedding via Microsoft.Extensions.AI. + pgvector;vector;embeddings;rag;sink + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/src/Wallaby/Sinks/SinkEnvelopeJson.cs b/src/Wallaby/Sinks/SinkEnvelopeJson.cs index 1f10317..0e3cb2c 100644 --- a/src/Wallaby/Sinks/SinkEnvelopeJson.cs +++ b/src/Wallaby/Sinks/SinkEnvelopeJson.cs @@ -108,6 +108,10 @@ private static void WriteValue(Utf8JsonWriter writer, object? value, string key, case char c: writer.WriteStringValue(c.ToString()); break; case byte[] bytes: writer.WriteBase64StringValue(bytes); break; case Uri uri: writer.WriteStringValue(uri.ToString()); break; + // Embedding vectors (Embedding.Vector is ReadOnlyMemory); structs, so the + // IEnumerable arm below never sees them. + case ReadOnlyMemory vector: WriteFloatArray(writer, vector.Span); break; + case Memory vector: WriteFloatArray(writer, vector.Span); break; case IReadOnlyDictionary nested: WriteDocument(writer, nested, documentId, serializerOptions); break; @@ -125,6 +129,16 @@ private static void WriteValue(Utf8JsonWriter writer, object? value, string key, } } + private static void WriteFloatArray(Utf8JsonWriter writer, ReadOnlySpan values) + { + writer.WriteStartArray(); + foreach (var value in values) + { + writer.WriteNumberValue(value); + } + writer.WriteEndArray(); + } + [UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Consumer-supplied SerializerOptions resolve types through their own TypeInfoResolver " + "(source-generated on AOT); the reflection default only runs when IsReflectionEnabledByDefault is true.")] diff --git a/tests/Wallaby.AotSmokeTest/Program.cs b/tests/Wallaby.AotSmokeTest/Program.cs index e0e5f59..a4251e1 100644 --- a/tests/Wallaby.AotSmokeTest/Program.cs +++ b/tests/Wallaby.AotSmokeTest/Program.cs @@ -11,6 +11,8 @@ using NpgsqlTypes; using Wallaby.Abstractions; using Wallaby.AotSmokeTest; +using Wallaby.Sinks; +using Wallaby.Sinks.Pgvector; using Wallaby.Internal.Backfill; using Wallaby.Internal.Replication; using Wallaby.Providers.Marten.Internal; @@ -189,6 +191,30 @@ AssertEqual(doc.Id, row.PrimaryKey[0], "primary key"); }); +Check("vector documents serialize reflection-free and the pgvector sink constructs", () => +{ + var document = new WallabyDocument { ["name"] = "roo", ["embedding"] = new ReadOnlyMemory([3f, 1f]) }; + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream)) + { + SinkEnvelopeJson.WriteDocument(writer, document, "1", serializerOptions: null); + } + var json = System.Text.Encoding.UTF8.GetString(stream.ToArray()); + if (!json.Contains("\"embedding\":[3,1]")) + { + throw new InvalidOperationException($"vector not serialized as a number array: {json}"); + } + + // Construction validates options and builds the data source without connecting. + var sink = new PgvectorSink("smoke", new PgvectorSinkOptions + { + ConnectionString = "Host=localhost;Database=vectors;Username=u;Password=p", + Dimensions = 2, + DefaultTable = "documents", + }); + sink.DisposeAsync().AsTask().GetAwaiter().GetResult(); +}); + Console.WriteLine(failures == 0 ? "AOT smoke: all checks passed." : $"AOT smoke: {failures} check(s) FAILED."); return failures == 0 ? 0 : 1; diff --git a/tests/Wallaby.AotSmokeTest/Wallaby.AotSmokeTest.csproj b/tests/Wallaby.AotSmokeTest/Wallaby.AotSmokeTest.csproj index d401638..1fd0e26 100644 --- a/tests/Wallaby.AotSmokeTest/Wallaby.AotSmokeTest.csproj +++ b/tests/Wallaby.AotSmokeTest/Wallaby.AotSmokeTest.csproj @@ -17,6 +17,7 @@ + diff --git a/tests/Wallaby.Sinks.Meilisearch.Tests/Integration/Infrastructure/MeiliProbe.cs b/tests/Wallaby.Sinks.Meilisearch.Tests/Integration/Infrastructure/MeiliProbe.cs index f28635e..ed83971 100644 --- a/tests/Wallaby.Sinks.Meilisearch.Tests/Integration/Infrastructure/MeiliProbe.cs +++ b/tests/Wallaby.Sinks.Meilisearch.Tests/Integration/Infrastructure/MeiliProbe.cs @@ -57,6 +57,9 @@ public async Task IndexExistsAsync(string index) /// The index's current settings. public Task SettingsAsync(string index) => _client.Index(index).GetSettingsAsync(); + /// The index's configured embedders (empty when none). + public Task> EmbeddersAsync(string index) => _client.Index(index).GetEmbeddersAsync(); + /// Delete the index and wait for the task to finish (used to prove a re-backfill repopulates it). public async Task DropAsync(string index) { diff --git a/tests/Wallaby.Sinks.Meilisearch.Tests/Integration/MeilisearchSinkTests.cs b/tests/Wallaby.Sinks.Meilisearch.Tests/Integration/MeilisearchSinkTests.cs index 68b1a94..ef7a2ca 100644 --- a/tests/Wallaby.Sinks.Meilisearch.Tests/Integration/MeilisearchSinkTests.cs +++ b/tests/Wallaby.Sinks.Meilisearch.Tests/Integration/MeilisearchSinkTests.cs @@ -194,6 +194,38 @@ public async Task Document_with_all_configured_attributes_passes_validation_and_ (await probe.NameAsync(index, id)).ShouldBe("alpha"); } + [Test] + public async Task Configured_embedder_applies_and_user_provided_vectors_index() + { + await using var harness = WallabyTestHarness.ForTestModel(pg.ConnectionString); + var index = harness.Names.Named("products_vec"); + + // Embedders ride the same settings update the initializer already applies. + var options = new MeilisearchSinkOptions { Host = meili.Host, ApiKey = meili.ApiKey }; + options.ConfigureIndex(index, s => s.Embedders = new Dictionary + { + ["default"] = new Embedder { Source = EmbedderSource.UserProvided, Dimensions = 3 }, + }); + harness.AddSink(TestMeilisearchSink.Create("meili", options)) + .Project("meili", index, p => new WallabyDocument + { + ["name"] = p.Name, + ["_vectors"] = new Dictionary { ["default"] = new[] { 0.1f, 0.2f, 0.3f } }, + }); + await harness.SelfConfigureAsync(); + + var probe = new MeiliProbe(meili); + var categoryId = await harness.Db.AddCategoryAsync(); + var id = await harness.Db.AddProductAsync(categoryId, "alpha"); + + // A rejected _vectors payload would fail the enqueue task and nothing would index. + await harness.RunUntilAsync(async () => await probe.NameAsync(index, id) == "alpha"); + + var embedders = await probe.EmbeddersAsync(index); + embedders.ShouldContainKey("default"); + embedders["default"].Source.ShouldBe(EmbedderSource.UserProvided); + } + [Test] public async Task Declared_index_is_created_and_configured_on_start() { diff --git a/tests/Wallaby.Sinks.OpenSearch.Tests/Unit/BulkPayloadTests.cs b/tests/Wallaby.Sinks.OpenSearch.Tests/Unit/BulkPayloadTests.cs index 3d85b61..ed83598 100644 --- a/tests/Wallaby.Sinks.OpenSearch.Tests/Unit/BulkPayloadTests.cs +++ b/tests/Wallaby.Sinks.OpenSearch.Tests/Unit/BulkPayloadTests.cs @@ -121,6 +121,23 @@ public void Scalar_document_values_are_written_natively() root.GetProperty("nil").ValueKind.ShouldBe(JsonValueKind.Null); } + [Test] + public void Vector_values_are_written_as_number_arrays() + { + var document = new Dictionary + { + ["floats"] = new[] { 0.25f, -1f, 3f }, + ["memory"] = new ReadOnlyMemory([0.5f, 2f]), + }; + + var payload = Write([Upsert("1", document)]); + + using var parsed = JsonDocument.Parse(Lines(payload)[1]); + var root = parsed.RootElement; + root.GetProperty("floats").EnumerateArray().Select(e => e.GetSingle()).ShouldBe([0.25f, -1f, 3f]); + root.GetProperty("memory").EnumerateArray().Select(e => e.GetSingle()).ShouldBe([0.5f, 2f]); + } + [Test] public void Mixed_upserts_and_deletes_preserve_record_order() { diff --git a/tests/Wallaby.Sinks.Pgvector.Tests/Integration/EndToEndTests.cs b/tests/Wallaby.Sinks.Pgvector.Tests/Integration/EndToEndTests.cs new file mode 100644 index 0000000..04b8548 --- /dev/null +++ b/tests/Wallaby.Sinks.Pgvector.Tests/Integration/EndToEndTests.cs @@ -0,0 +1,65 @@ +using Npgsql; +using Wallaby.Abstractions; +using Wallaby.Sinks.Pgvector.Tests.Integration.Infrastructure; +using Wallaby.TestInfrastructure; +using Wallaby.TestInfrastructure.EntityFrameworkCore; +using Wallaby.TestModel; + +namespace Wallaby.Sinks.Pgvector.Tests.Integration; + +[NotInParallel] +[ClassDataSource(Shared = new[] { SharedType.PerTestSession, SharedType.PerTestSession })] +public class EndToEndTests(TestModelPostgresFixture source, PgvectorFixture destination) +{ + private async Task VectorAsync(string table, int id) + { + await using var cmd = destination.DataSource.CreateCommand( + $"SELECT embedding::text FROM public.\"{table}\" WHERE id = $1"); + cmd.Parameters.Add(new NpgsqlParameter { Value = id.ToString() }); + return await cmd.ExecuteScalarAsync() as string; + } + + [Test] + public async Task Live_changes_embed_into_pgvector_and_a_re_backfill_reuses_stored_vectors() + { + var table = $"t_{Guid.NewGuid():N}"; + var generator = new StubEmbeddingGenerator(); + var sink = new PgvectorSink("pgv", new PgvectorSinkOptions + { + ConnectionString = destination.ConnectionString, + Dimensions = 2, + DefaultTable = table, + EmbeddingGenerator = generator, + EmbedText = d => (string?)d["name"], + EmbeddingVersion = "m/1", + }); + await sink.InitializeAsync(CancellationToken.None); + + await using var harness = WallabyTestHarness.ForTestModel(source.ConnectionString); + harness.AddSink(sink) + .Project("pgv", table, p => new WallabyDocument { ["name"] = p.Name }, + backfill: true, backfillVersion: "v1"); + await harness.SelfConfigureAsync(); + await harness.StartAsync(); + try + { + var categoryId = await harness.Db.AddCategoryAsync(); + var id = await harness.Db.AddProductAsync(categoryId, "alpha"); + + await harness.WaitUntilAsync(async () => await VectorAsync(table, id) is not null); + (await VectorAsync(table, id)).ShouldBe("[5,1]"); // the stub embeds "alpha" as [length, 1] + var callsAfterLive = generator.Calls; + + // A backfill re-delivers every row; unchanged text is served from the destination's + // stored hash, costing zero embedding calls. + await harness.RunBackfillAsync(version: "v2"); + (await VectorAsync(table, id)).ShouldBe("[5,1]"); + generator.Calls.ShouldBe(callsAfterLive); + } + finally + { + await harness.StopAsync(); + await sink.DisposeAsync(); + } + } +} diff --git a/tests/Wallaby.Sinks.Pgvector.Tests/Integration/Infrastructure/PgvectorFixture.cs b/tests/Wallaby.Sinks.Pgvector.Tests/Integration/Infrastructure/PgvectorFixture.cs new file mode 100644 index 0000000..5ed70fa --- /dev/null +++ b/tests/Wallaby.Sinks.Pgvector.Tests/Integration/Infrastructure/PgvectorFixture.cs @@ -0,0 +1,32 @@ +using Npgsql; +using Testcontainers.PostgreSql; +using TUnit.Core.Interfaces; + +namespace Wallaby.Sinks.Pgvector.Tests.Integration.Infrastructure; + +/// A shared pgvector-enabled Postgres container acting as the sink destination. +public sealed class PgvectorFixture : IAsyncInitializer, IAsyncDisposable +{ + private readonly PostgreSqlContainer _container = new PostgreSqlBuilder("pgvector/pgvector:pg17").Build(); + private NpgsqlDataSource? _dataSource; + + public string ConnectionString => _container.GetConnectionString(); + + public NpgsqlDataSource DataSource => _dataSource + ?? throw new InvalidOperationException("PgvectorFixture has not been initialized."); + + public async Task InitializeAsync() + { + await _container.StartAsync(); + _dataSource = NpgsqlDataSource.Create(ConnectionString); + } + + public async ValueTask DisposeAsync() + { + if (_dataSource is not null) + { + await _dataSource.DisposeAsync(); + } + await _container.DisposeAsync(); + } +} diff --git a/tests/Wallaby.Sinks.Pgvector.Tests/Integration/PgvectorSinkTests.cs b/tests/Wallaby.Sinks.Pgvector.Tests/Integration/PgvectorSinkTests.cs new file mode 100644 index 0000000..a0072b4 --- /dev/null +++ b/tests/Wallaby.Sinks.Pgvector.Tests/Integration/PgvectorSinkTests.cs @@ -0,0 +1,382 @@ +using Npgsql; +using Wallaby.Abstractions; +using Wallaby.Sinks.Pgvector.Tests.Integration.Infrastructure; + +namespace Wallaby.Sinks.Pgvector.Tests.Integration; + +[ClassDataSource(Shared = SharedType.PerTestSession)] +public class PgvectorSinkTests(PgvectorFixture pg) +{ + private static string UniqueTable() => $"t_{Guid.NewGuid():N}"; + + private PgvectorSinkOptions Options(string table, Action? mutate = null) + { + var options = new PgvectorSinkOptions + { + ConnectionString = pg.ConnectionString, + Dimensions = 2, + DefaultTable = table, + }; + mutate?.Invoke(options); + return options; + } + + private PgvectorSinkOptions EmbedOptions(string table, StubEmbeddingGenerator generator, string version = "m/1") + => Options(table, o => + { + o.EmbeddingGenerator = generator; + o.EmbedText = d => (string?)d.GetValueOrDefault("name"); + o.EmbeddingVersion = version; + }); + + private static ChangeMetadata Meta() => + new("public", "products", ChangeAction.Insert, DateTimeOffset.UtcNow, 1, 0, false); + + private static SinkRecord Upsert(string id, WallabyDocument document, string? destination = null) + => new(destination, id, document, IsDeletion: false, Meta()); + + private static SinkRecord Delete(string id, string? destination = null) + => new(destination, id, Document: null, IsDeletion: true, Meta()); + + private static SinkBatch Batch(params SinkRecord[] records) => new("pgv", records); + + private async Task<(string? Hash, string? Vector, string Json)?> RowAsync(string table, string id) + { + await using var cmd = pg.DataSource.CreateCommand( + $"SELECT text_hash, embedding::text, document::text FROM public.\"{table}\" WHERE id = $1"); + cmd.Parameters.Add(new NpgsqlParameter { Value = id }); + await using var reader = await cmd.ExecuteReaderAsync(); + if (!await reader.ReadAsync()) + { + return null; + } + return (reader.IsDBNull(0) ? null : reader.GetString(0), + reader.IsDBNull(1) ? null : reader.GetString(1), + reader.GetString(2)); + } + + [Test] + public async Task Initialize_creates_the_extension_and_default_table() + { + var table = UniqueTable(); + await using var sink = new PgvectorSink("pgv", Options(table)); + + await sink.InitializeAsync(CancellationToken.None); + + await using var cmd = pg.DataSource.CreateCommand($"SELECT to_regclass('public.{table}')::text"); + (await cmd.ExecuteScalarAsync()).ShouldNotBe(DBNull.Value); + } + + [Test] + public async Task Embed_mode_stores_vectors_and_hash_gates_redelivery() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + await using var sink = new PgvectorSink("pgv", EmbedOptions(table, generator)); + await sink.InitializeAsync(CancellationToken.None); + + var first = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" }), + Upsert("2", new WallabyDocument { ["name"] = "cdef" })), CancellationToken.None); + + first.Status.ShouldBe(DeliveryStatus.Success); + generator.Calls.ShouldBe(1); + var row = await RowAsync(table, "1"); + row!.Value.Vector.ShouldBe("[2,1]"); + row.Value.Hash.ShouldNotBeNull(); + row.Value.Json.ShouldContain("\"ab\""); + + // Same text again: the stored hash matches, so no embedding call happens. + var redelivery = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "ab", ["extra"] = 7 })), CancellationToken.None); + redelivery.Status.ShouldBe(DeliveryStatus.Success); + generator.Calls.ShouldBe(1); + (await RowAsync(table, "1"))!.Value.Json.ShouldContain("\"extra\""); + + // Changed text re-embeds. + var changed = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "abc" })), CancellationToken.None); + changed.Status.ShouldBe(DeliveryStatus.Success); + generator.Calls.ShouldBe(2); + (await RowAsync(table, "1"))!.Value.Vector.ShouldBe("[3,1]"); + } + + private async Task RowVersionAsync(string table, string id) + { + // xmin changes on any tuple rewrite, so an unchanged xmin proves the redelivery wrote nothing. + await using var cmd = pg.DataSource.CreateCommand( + $"SELECT xmin::text FROM public.\"{table}\" WHERE id = $1"); + cmd.Parameters.Add(new NpgsqlParameter { Value = id }); + return (string)(await cmd.ExecuteScalarAsync())!; + } + + [Test] + public async Task An_identical_redelivery_does_not_rewrite_the_row() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + await using var sink = new PgvectorSink("pgv", EmbedOptions(table, generator)); + await sink.InitializeAsync(CancellationToken.None); + + await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "ab", ["note"] = "x" })), CancellationToken.None); + var version = await RowVersionAsync(table, "1"); + + var redelivery = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "ab", ["note"] = "x" })), CancellationToken.None); + redelivery.Status.ShouldBe(DeliveryStatus.Success); + (await RowVersionAsync(table, "1")).ShouldBe(version); + + // Same text but a changed document still updates the row (without re-embedding). + await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "ab", ["note"] = "y" })), CancellationToken.None); + (await RowVersionAsync(table, "1")).ShouldNotBe(version); + (await RowAsync(table, "1"))!.Value.Json.ShouldContain("\"y\""); + generator.Calls.ShouldBe(1); + } + + [Test] + public async Task An_identical_pass_through_redelivery_does_not_rewrite_the_row() + { + var table = UniqueTable(); + await using var sink = new PgvectorSink("pgv", Options(table)); + await sink.InitializeAsync(CancellationToken.None); + var batch = Batch(Upsert("1", new WallabyDocument + { + ["name"] = "ab", + ["embedding"] = new[] { 0.5f, -1f }, + })); + + await sink.DeliverAsync(batch, CancellationToken.None); + var version = await RowVersionAsync(table, "1"); + + (await sink.DeliverAsync(batch, CancellationToken.None)).Status.ShouldBe(DeliveryStatus.Success); + (await RowVersionAsync(table, "1")).ShouldBe(version); + } + + [Test] + public async Task Concurrent_embedding_sub_batches_store_every_vector() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + var options = EmbedOptions(table, generator); + options.MaxEmbeddingBatchSize = 1; + options.MaxEmbeddingConcurrency = 4; + await using var sink = new PgvectorSink("pgv", options); + await sink.InitializeAsync(CancellationToken.None); + + var result = await sink.DeliverAsync(Batch( + Upsert("1", new WallabyDocument { ["name"] = "a" }), + Upsert("2", new WallabyDocument { ["name"] = "bb" }), + Upsert("3", new WallabyDocument { ["name"] = "ccc" }), + Upsert("4", new WallabyDocument { ["name"] = "dddd" }), + Upsert("5", new WallabyDocument { ["name"] = "eeeee" })), CancellationToken.None); + + result.Status.ShouldBe(DeliveryStatus.Success); + generator.Calls.ShouldBe(5); + (await RowAsync(table, "1"))!.Value.Vector.ShouldBe("[1,1]"); + (await RowAsync(table, "5"))!.Value.Vector.ShouldBe("[5,1]"); + } + + [Test] + public async Task A_stored_hash_without_a_vector_does_not_gate_re_embedding() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + await using var sink = new PgvectorSink("pgv", EmbedOptions(table, generator)); + await sink.InitializeAsync(CancellationToken.None); + await sink.DeliverAsync(Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })), CancellationToken.None); + generator.Calls.ShouldBe(1); + + // An inconsistent row (hash with no vector, e.g. left by a purge racing a delivery) must + // re-embed instead of matching the hash. + await using (var cmd = pg.DataSource.CreateCommand($"UPDATE public.\"{table}\" SET embedding = NULL")) + { + await cmd.ExecuteNonQueryAsync(); + } + var redelivery = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })), CancellationToken.None); + + redelivery.Status.ShouldBe(DeliveryStatus.Success); + generator.Calls.ShouldBe(2); + (await RowAsync(table, "1"))!.Value.Vector.ShouldBe("[2,1]"); + } + + [Test] + public async Task The_stored_hash_survives_a_new_sink_instance() + { + // A restarted host (or another node) skips re-embedding: the destination is the cache. + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + await using (var sink = new PgvectorSink("pgv", EmbedOptions(table, generator))) + { + await sink.InitializeAsync(CancellationToken.None); + await sink.DeliverAsync(Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })), CancellationToken.None); + } + generator.Calls.ShouldBe(1); + + await using var restarted = new PgvectorSink("pgv", EmbedOptions(table, generator)); + var result = await restarted.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })), CancellationToken.None); + + result.Status.ShouldBe(DeliveryStatus.Success); + generator.Calls.ShouldBe(1); + } + + [Test] + public async Task An_embedding_version_change_re_embeds_the_same_text() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + await using (var sink = new PgvectorSink("pgv", EmbedOptions(table, generator))) + { + await sink.InitializeAsync(CancellationToken.None); + await sink.DeliverAsync(Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })), CancellationToken.None); + } + + await using var bumped = new PgvectorSink("pgv", EmbedOptions(table, generator, version: "m/2")); + await bumped.DeliverAsync(Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })), CancellationToken.None); + + generator.Calls.ShouldBe(2); + } + + [Test] + public async Task Empty_text_stores_a_null_vector() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + await using var sink = new PgvectorSink("pgv", EmbedOptions(table, generator)); + await sink.InitializeAsync(CancellationToken.None); + + var result = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["title"] = "no name field" })), CancellationToken.None); + + result.Status.ShouldBe(DeliveryStatus.Success); + generator.Calls.ShouldBe(0); + var row = await RowAsync(table, "1"); + row!.Value.Vector.ShouldBeNull(); + row.Value.Hash.ShouldBeNull(); + } + + [Test] + public async Task Deletes_remove_rows_and_last_write_wins_within_a_batch() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + await using var sink = new PgvectorSink("pgv", EmbedOptions(table, generator)); + await sink.InitializeAsync(CancellationToken.None); + + await sink.DeliverAsync(Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })), CancellationToken.None); + // An upsert then delete for the same id within one batch nets out to the delete. + var result = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "abc" }), Delete("1")), CancellationToken.None); + + result.Status.ShouldBe(DeliveryStatus.Success); + (await RowAsync(table, "1")).ShouldBeNull(); + } + + [Test] + public async Task Pass_through_mode_stores_the_provided_vector_and_strips_the_field() + { + var table = UniqueTable(); + await using var sink = new PgvectorSink("pgv", Options(table)); + await sink.InitializeAsync(CancellationToken.None); + + var result = await sink.DeliverAsync(Batch(Upsert("1", new WallabyDocument + { + ["name"] = "ab", + ["embedding"] = new ReadOnlyMemory([0.5f, -1f]), + })), CancellationToken.None); + + result.Status.ShouldBe(DeliveryStatus.Success); + var row = await RowAsync(table, "1"); + row!.Value.Vector.ShouldBe("[0.5,-1]"); + row.Value.Json.ShouldNotContain("embedding"); + } + + [Test] + public async Task A_dimension_mismatch_fails_permanently() + { + var table = UniqueTable(); + await using var sink = new PgvectorSink("pgv", Options(table)); + await sink.InitializeAsync(CancellationToken.None); + + var result = await sink.DeliverAsync(Batch(Upsert("1", new WallabyDocument + { + ["embedding"] = new[] { 1f, 2f, 3f }, + })), CancellationToken.None); + + result.Status.ShouldBe(DeliveryStatus.PermanentFailure); + result.Error!.ShouldContain("vector(2)"); + } + + [Test] + public async Task A_transient_embedding_failure_is_retryable_and_the_retry_succeeds() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + generator.Failures.Enqueue(new HttpRequestException("429")); + await using var sink = new PgvectorSink("pgv", EmbedOptions(table, generator)); + await sink.InitializeAsync(CancellationToken.None); + var batch = Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })); + + (await sink.DeliverAsync(batch, CancellationToken.None)).Status.ShouldBe(DeliveryStatus.RetryableFailure); + (await sink.DeliverAsync(batch, CancellationToken.None)).Status.ShouldBe(DeliveryStatus.Success); + } + + [Test] + public async Task A_non_transient_embedding_failure_is_permanent() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + generator.Failures.Enqueue(new InvalidOperationException("bad api key")); + var options = EmbedOptions(table, generator); + options.IsTransientEmbeddingError = ex => ex is HttpRequestException; + await using var sink = new PgvectorSink("pgv", options); + await sink.InitializeAsync(CancellationToken.None); + + var result = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })), CancellationToken.None); + + result.Status.ShouldBe(DeliveryStatus.PermanentFailure); + } + + [Test] + public async Task A_record_without_destination_or_default_table_fails_permanently() + { + await using var sink = new PgvectorSink("pgv", Options(UniqueTable(), o => o.DefaultTable = null)); + + var result = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" }, destination: null)), CancellationToken.None); + + result.Status.ShouldBe(DeliveryStatus.PermanentFailure); + result.Error!.ShouldContain("DefaultTable"); + } + + [Test] + public async Task An_invalid_runtime_destination_fails_permanently() + { + await using var sink = new PgvectorSink("pgv", Options(UniqueTable())); + + var result = await sink.DeliverAsync( + Batch(Upsert("1", new WallabyDocument(), destination: "bad\"; DROP TABLE x;--")), CancellationToken.None); + + result.Status.ShouldBe(DeliveryStatus.PermanentFailure); + } + + [Test] + public async Task Purge_empties_the_table_and_tolerates_a_missing_one() + { + var table = UniqueTable(); + var generator = new StubEmbeddingGenerator(); + await using var sink = new PgvectorSink("pgv", EmbedOptions(table, generator)); + await sink.InitializeAsync(CancellationToken.None); + await sink.DeliverAsync(Batch(Upsert("1", new WallabyDocument { ["name"] = "ab" })), CancellationToken.None); + + await sink.PurgeAsync(new SinkPurgeRequest("public", "products", Destination: null), CancellationToken.None); + (await RowAsync(table, "1")).ShouldBeNull(); + + // A destination whose table was never created purges as a no-op. + await sink.PurgeAsync(new SinkPurgeRequest("public", "products", $"never_{Guid.NewGuid():N}"), CancellationToken.None); + } +} diff --git a/tests/Wallaby.Sinks.Pgvector.Tests/StubEmbeddingGenerator.cs b/tests/Wallaby.Sinks.Pgvector.Tests/StubEmbeddingGenerator.cs new file mode 100644 index 0000000..d2f0b61 --- /dev/null +++ b/tests/Wallaby.Sinks.Pgvector.Tests/StubEmbeddingGenerator.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.AI; + +namespace Wallaby.Sinks.Pgvector.Tests; + +/// +/// Deterministic : records every batch, embeds a +/// text as [length, 1] (override via ), and throws queued +/// first, one per call. Thread-safe, so it works with concurrent sub-batches. +/// +internal sealed class StubEmbeddingGenerator : IEmbeddingGenerator> +{ + private readonly Lock _lock = new(); + + public int Calls { get; private set; } + public List Batches { get; } = []; + public Queue Failures { get; } = new(); + public Func VectorFor { get; set; } = text => [text.Length, 1f]; + + public Task>> GenerateAsync( + IEnumerable values, EmbeddingGenerationOptions? options = null, CancellationToken cancellationToken = default) + { + var texts = values.ToArray(); + lock (_lock) + { + Calls++; + if (Failures.TryDequeue(out var failure)) + { + throw failure; + } + Batches.Add(texts); + } + return Task.FromResult(new GeneratedEmbeddings>( + texts.Select(t => new Embedding(VectorFor(t))))); + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() { } +} diff --git a/tests/Wallaby.Sinks.Pgvector.Tests/Unit/FormatTests.cs b/tests/Wallaby.Sinks.Pgvector.Tests/Unit/FormatTests.cs new file mode 100644 index 0000000..9cb63c6 --- /dev/null +++ b/tests/Wallaby.Sinks.Pgvector.Tests/Unit/FormatTests.cs @@ -0,0 +1,38 @@ +using Wallaby.Sinks.Pgvector.Internal; + +namespace Wallaby.Sinks.Pgvector.Tests.Unit; + +public class FormatTests +{ + [Test] + public void Text_hash_is_stable_and_version_sensitive() + { + PgvectorFormat.TextHash("m/1", "alpha").ShouldBe(PgvectorFormat.TextHash("m/1", "alpha")); + PgvectorFormat.TextHash("m/1", "alpha").ShouldNotBe(PgvectorFormat.TextHash("m/2", "alpha")); + PgvectorFormat.TextHash("m/1", "alpha").ShouldNotBe(PgvectorFormat.TextHash("m/1", "beta")); + } + + [Test] + public void Vectors_extract_from_memory_and_array_values() + { + PgvectorFormat.TryGetVector(new ReadOnlyMemory([1f, 2f]), out var fromMemory).ShouldBeTrue(); + fromMemory.ToArray().ShouldBe([1f, 2f]); + + PgvectorFormat.TryGetVector(new[] { 3f }, out var fromArray).ShouldBeTrue(); + fromArray.ToArray().ShouldBe([3f]); + + PgvectorFormat.TryGetVector("not a vector", out _).ShouldBeFalse(); + PgvectorFormat.TryGetVector(new[] { 1.0 }, out _).ShouldBeFalse(); + } + + [Test] + public void Identifiers_validate_to_safe_table_names() + { + PgvectorTables.IsValidIdentifier("products").ShouldBeTrue(); + PgvectorTables.IsValidIdentifier("tenant_42").ShouldBeTrue(); + PgvectorTables.IsValidIdentifier("").ShouldBeFalse(); + PgvectorTables.IsValidIdentifier("bad-name").ShouldBeFalse(); + PgvectorTables.IsValidIdentifier("x\"; DROP TABLE t;--").ShouldBeFalse(); + PgvectorTables.IsValidIdentifier(new string('x', 64)).ShouldBeFalse(); + } +} diff --git a/tests/Wallaby.Sinks.Pgvector.Tests/Unit/RegistrationTests.cs b/tests/Wallaby.Sinks.Pgvector.Tests/Unit/RegistrationTests.cs new file mode 100644 index 0000000..2033732 --- /dev/null +++ b/tests/Wallaby.Sinks.Pgvector.Tests/Unit/RegistrationTests.cs @@ -0,0 +1,61 @@ +namespace Wallaby.Sinks.Pgvector.Tests.Unit; + +public class RegistrationTests +{ + private static PgvectorSinkOptions Valid(Action? mutate = null) + { + var options = new PgvectorSinkOptions + { + ConnectionString = "Host=localhost;Database=vectors;Username=u;Password=p", + Dimensions = 3, + DefaultTable = "documents", + }; + mutate?.Invoke(options); + return options; + } + + [Test] + public void Valid_options_pass() + { + PgvectorBuilderExtensions.Validate(Valid()); + PgvectorBuilderExtensions.Validate(Valid(o => + { + o.EmbeddingGenerator = new StubEmbeddingGenerator(); + o.EmbedText = d => (string?)d["name"]; + o.EmbeddingVersion = "m/1"; + })); + } + + [Test] + public void Invalid_options_fail() + { + Should.Throw(() => PgvectorBuilderExtensions.Validate(Valid(o => o.ConnectionString = " "))); + Should.Throw(() => PgvectorBuilderExtensions.Validate(Valid(o => o.Dimensions = 0))); + Should.Throw(() => PgvectorBuilderExtensions.Validate(Valid(o => o.Schema = "bad-schema"))); + Should.Throw(() => PgvectorBuilderExtensions.Validate(Valid(o => o.DefaultTable = "bad.table"))); + Should.Throw(() => PgvectorBuilderExtensions.Validate(Valid(o => o.MaxRowsPerBatch = 0))); + Should.Throw(() => PgvectorBuilderExtensions.Validate(Valid(o => o.MaxEmbeddingBatchSize = 0))); + Should.Throw(() => PgvectorBuilderExtensions.Validate(Valid(o => o.MaxEmbeddingConcurrency = 0))); + Should.Throw(() => PgvectorBuilderExtensions.Validate(Valid(o => o.VectorField = " "))); + } + + [Test] + public void Direct_construction_validates_options() + { + // Schema and DefaultTable reach interpolated SQL, so the constructor enforces the identifier + // rule even when the builder's Validate is bypassed. + Should.Throw(() => new PgvectorSink("pgv", Valid(o => o.Schema = "bad\"schema"))); + Should.Throw(() => new PgvectorSink("pgv", Valid(o => o.DefaultTable = "bad.table"))); + } + + [Test] + public void A_partial_embedding_configuration_fails() + { + var ex = Should.Throw(() => PgvectorBuilderExtensions.Validate( + Valid(o => o.EmbeddingGenerator = new StubEmbeddingGenerator()))); + ex.Message.ShouldContain("together"); + + Should.Throw(() => PgvectorBuilderExtensions.Validate( + Valid(o => { o.EmbedText = d => "x"; o.EmbeddingVersion = "m/1"; }))); + } +} diff --git a/tests/Wallaby.Sinks.Pgvector.Tests/Wallaby.Sinks.Pgvector.Tests.csproj b/tests/Wallaby.Sinks.Pgvector.Tests/Wallaby.Sinks.Pgvector.Tests.csproj new file mode 100644 index 0000000..a57cc02 --- /dev/null +++ b/tests/Wallaby.Sinks.Pgvector.Tests/Wallaby.Sinks.Pgvector.Tests.csproj @@ -0,0 +1,26 @@ + + + + Exe + false + true + + + + + + + + + + + + + + + + + + + +