Merge branch 'main' of github.com:adobe/spacecat-shared into oas

Author: dipratap

docs/adr/0002-embedding-client-and-semantic-index-utils.md

@@ -1,3 +1,21 @@
+## [@adobe/spacecat-shared-data-access-v4.40.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v4.39.1...@adobe/spacecat-shared-data-access-v4.40.0) (2026-10-02)
+
+### Features
+
+* **data-access:** semantic claim functions and in-table vector reuse (LLMO-7445) ([#1968](https://github.com/adobe/spacecat-shared/issues/1968)) ([72b7ac0](https://github.com/adobe/spacecat-shared/commit/72b7ac0f4d377c76cf982295dc4530f2ad624abb))
+
+## [@adobe/spacecat-shared-data-access-v4.39.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v4.39.0...@adobe/spacecat-shared-data-access-v4.39.1) (2026-10-02)
+
+### Bug Fixes
+
+* **data-access:** rename MAX_SOURCE_TEXT_LENGTH to MAX_TEXT_LENGTH (LLMO-7445) ([#1967](https://github.com/adobe/spacecat-shared/issues/1967)) ([d20b120](https://github.com/adobe/spacecat-shared/commit/d20b1207c5633598788e96ff8ec9bbea1668ddb2)), closes [#1966](https://github.com/adobe/spacecat-shared/issues/1966) [spacecat-api-service#79](https://github.com/adobe/spacecat-api-service/issues/79) [#79](https://github.com/adobe/spacecat-shared/issues/79) [#79](https://github.com/adobe/spacecat-shared/issues/79)
+
+## [@adobe/spacecat-shared-data-access-v4.39.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v4.38.0...@adobe/spacecat-shared-data-access-v4.39.0) (2026-10-02)
+
+### Features
+
+* **data-access:** batched semantic index writer and suggestion lookup (LLMO-7445) ([#1966](https://github.com/adobe/spacecat-shared/issues/1966)) ([3479136](https://github.com/adobe/spacecat-shared/commit/3479136b0e67923291376b036f9399d8c52f4c0b)), closes [mysticat-data-service#1153](https://github.com/adobe/mysticat-data-service/issues/1153) [mysticat-data-service#1153](https://github.com/adobe/mysticat-data-service/issues/1153) [#1153](https://github.com/adobe/spacecat-shared/issues/1153) [#1153](https://github.com/adobe/spacecat-shared/issues/1153) [#1153](https://github.com/adobe/spacecat-shared/issues/1153)
+
 ## [@adobe/spacecat-shared-data-access-v4.38.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v4.37.0...@adobe/spacecat-shared-data-access-v4.38.0) (2026-09-30)
 
 ### Features

package-lock.json

@@ -142,48 +142,76 @@ See the package `CLAUDE.md` ("URL Index") for the canonicalization, single-write
 
 ## Semantic Index
 
-For the topic vector index (`opportunity_semantic_embedding`) and the global query-embedding cache (`semantic_query_embedding`), use the shared helpers so the writer and reader share one text normalization and one storage path. They store and read pre-embedded vectors; embedding is the caller's job (e.g. `AzureEmbeddingClient` from `@adobe/spacecat-shared-gpt-client`).
+For the vector indexes (`opportunity_semantic_embedding`, `suggestion_semantic_embedding`) and the global query-embedding cache (`semantic_query_embedding`), use the shared helpers so the writer and reader share one text normalization, one embedding generation and one storage path. The helpers take an embedding client with `createEmbeddings(texts)` (e.g. `AzureEmbeddingClient` from `@adobe/spacecat-shared-gpt-client`) and do the embedding themselves.
 
 ```js
 import {
-  syncOpportunitySemantic, copyEntityVectors, lookupOpportunitiesByVectors,
-  getQueryEmbeddings, upsertQueryEmbeddings, touchQueryEmbeddings,
-  OPPORTUNITY_SEMANTIC_SOURCE_TYPES, OPPORTUNITY_SEMANTIC_ENTITY_TYPES, SEMANTIC_MATCHING_CONFIG,
+  indexSemanticTopics, indexSemanticClaims, embedQueries,
+  lookupOpportunitiesByTopic, lookupSuggestionsByTopic, lookupSuggestionsByClaim,
+  SEMANTIC_TARGETS,
 } from '@adobe/spacecat-shared-data-access';
 
 const { postgrestClient } = dataAccess.services;
-const scope = { ...SEMANTIC_MATCHING_CONFIG.embedding }; // { model, dims }
-const { TOPIC } = OPPORTUNITY_SEMANTIC_SOURCE_TYPES;
-const { CITED_ANALYSIS } = OPPORTUNITY_SEMANTIC_ENTITY_TYPES;
 
-// writer (needs the postgrest_writer role): full-replace an opportunity's topic vectors
-await syncOpportunitySemantic(postgrestClient, {
-  siteId, entityId, entityType: CITED_ANALYSIS, sourceType: TOPIC, sources: [{ text, vector, ...scope }],
+// writer (needs the postgrest_writer role): full-replace each (entity, matchFieldType) topic group
+// in one call. Unchanged texts are kept; a new text reuses a vector already stored in the same
+// table for the site, then the query cache, and is embedded otherwise (in batches); stale texts
+// are deleted; fields not listed are left alone. texts: [] clears a field.
+const result = await indexSemanticTopics(postgrestClient, embeddingClient, {
+  target: SEMANTIC_TARGETS.OPPORTUNITY, // or SEMANTIC_TARGETS.SUGGESTION
+  siteId,
+  entities: [{
+    entityId, entityType: 'cited-analysis',
+    fields: [{ matchFieldType: 'topic', texts: ['running shoes'] }],
+  }],
+});
+// result: { entities: [{ entityId, fields: [{ matchFieldType, rejected, inserted, deleted, unchanged, ... }] }],
+//           embedded, indexHits, cacheHits, indexError?, cacheError? }
+
+// claim writer: same parameters and result, stored under the claim match type.
+await indexSemanticClaims(postgrestClient, embeddingClient, {
+  target: SEMANTIC_TARGETS.SUGGESTION,
+  siteId,
+  entities: [{
+    entityId: suggestionId, entityType: 'cited-analysis',
+    fields: [
+      { matchFieldType: 'title', texts: ['Publish a pricing comparison page'] },
+      { matchFieldType: 'topic', texts: ['running shoes'] },
+    ],
+  }],
 });
 
-// writer: copy an opportunity's vector rows to another opportunity in the same site
-await copyEntityVectors(postgrestClient, { siteId, fromEntityId, toEntityId });
-
-// query cache, batched: one entry per text, in input order (null on a miss)
-const cached = await getQueryEmbeddings(postgrestClient, { texts, ...scope });
-await upsertQueryEmbeddings(postgrestClient, { entries: [{ text, vector }], ...scope });
-await touchQueryEmbeddings(postgrestClient, { textHashes, ...scope });
+// query side: validate (at most MAX_QUERY_TEXTS texts, each up to 2048 chars once normalized), read the
+// cache (best-effort), embed the distinct misses in one call within a time budget.
+// cacheWrites (touch hits + store misses) is best-effort; await it or hand it to waitUntil.
+const { vectors, cacheWrites, cacheError } = await embedQueries(postgrestClient, embeddingClient, texts, { log });
 
 // reader, batched: one best-first list of { entityId, entityType, score } per query vector.
-// entityTypes is optional (omitted or [] searches all entity types). statuses is optional too
-// (omitted or [] searches all; values from Opportunity.STATUSES); it is applied before k, so
-// filtered-out opportunities never take result slots. Needs a data-service with p_statuses.
-const matches = await lookupOpportunitiesByVectors(postgrestClient, {
-  siteId, sourceTypes: [TOPIC], entityTypes: [CITED_ANALYSIS], statuses: ['NEW', 'IN_PROGRESS'],
-  vectors, k: 10, minScore: 0.5, ...scope,
+// matchFieldTypes / entityTypes / statuses are optional filters (omitted or [] means no filter).
+// statuses is applied before k, so filtered-out rows never take result slots.
+// Searches run in the SEMANTIC_MATCHING_CONFIG.embedding generation, like the writer.
+const opportunities = await lookupOpportunitiesByTopic(postgrestClient, {
+  siteId, matchFieldTypes: ['topic'], entityTypes: ['cited-analysis'],
+  statuses: ['NEW', 'IN_PROGRESS'], vectors, k: 10, minScore: 0.5,
+});
+// suggestions: entityTypes are parent opportunity types, statuses are Suggestion.STATUSES,
+// opportunityStatuses filter on the parent opportunity.
+const suggestions = await lookupSuggestionsByTopic(postgrestClient, {
+  siteId, opportunityStatuses: ['NEW'], vectors,
+});
+// claim reader: same parameters, over the claim rows.
+const claimMatches = await lookupSuggestionsByClaim(postgrestClient, {
+  siteId, matchFieldTypes: ['title'], vectors,
 });
 ```
 
-`OPPORTUNITY_SEMANTIC_SOURCE_TYPES` (kinds of embedded text) and `OPPORTUNITY_SEMANTIC_ENTITY_TYPES` (opportunity types) are the registries for `source_type` and `entity_type`: the writer and reader reject any value outside them. They are defined in `@adobe/spacecat-shared-utils` and re-exported here. To index a new kind or opportunity type, add it to the utils registry first.
+The match type (`topic`, `claim`) is never a parameter: each dimension has its own writer and readers (`indexSemanticTopics` / `...ByTopic`, `indexSemanticClaims` / `...ByClaim`), which set `match_type` internally, so a topic search never sees claim rows and vice versa. Claim writes accept either target, though only suggestions have a claim reader today. `matchFieldType` (which field the text came from) and `entityType` are free-form, length-bounded strings: there is no registry, so a producer can index a new field or entity type without a shared release.
+
+`SEMANTIC_MATCHING_CONFIG.embedding` (`model`, `dims`) is the embedding generation the writer and reader both use; the writer, `embedQueries` and the lookups read it from there rather than taking it as a parameter. It is a code constant, not env config, so writer and reader agree as long as both run the same data-access version; changing it means a re-embed plus upgrading both consumers. The re-embed is automatic: the writer treats rows from another generation as stale and overwrites them on the entity's next refresh.
 
-`SEMANTIC_MATCHING_CONFIG.embedding` (`model`, `dims`) is the embedding generation the writer and reader both use; spread it into the helpers rather than hardcoding the values. It is a code constant, not env config, so writer and reader agree as long as both run the same data-access version; changing it means a re-embed plus upgrading both consumers.
+The writer reuses vectors it already stored in the same table for the site (any entity, match type or field, current generation only), so indexing a suggestion's claims after its topics embeds nothing twice; it does not read the other target's table. It reads the query cache but never writes it; only `embedQueries` does. The low-level cache helpers are internal, so nothing outside data-access can write vectors that the writer and readers trust.
 
-The batched helpers group their calls internally (search in groups of up to 20 vectors, cache reads/touches in groups of 50 hashes), so callers pass whole lists rather than looping per item.
+Calls are grouped internally (embeddings in batches of 256, writes and searches in groups of 20, id/hash reads and deletes in groups of 50), so callers pass whole lists rather than looping per item.
 
 ## Field Mapping Behavior
 

packages/spacecat-shared-data-access/CHANGELOG.md

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "4.38.0",
+  "version": "4.40.0",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "engines": {

packages/spacecat-shared-data-access/README.md

@@ -52,27 +52,32 @@ export * from '../models/index.js';
 export * from '../util/index.js';
 // Not in util/index.js: semantic-index imports the Opportunity model, which imports util/index.js.
 export {
+  SEMANTIC_TARGETS,
   SEMANTIC_INDEX_TABLES,
+  SEMANTIC_SEARCH_RPCS,
   QUERY_EMBEDDING_TABLE,
-  SEMANTIC_SEARCH_RPC,
-  COPY_VECTORS_RPC,
   SEMANTIC_CHUNK_SIZE,
   QUERY_HASH_CHUNK_SIZE,
-  MAX_SOURCE_TEXT_LENGTH,
-  OPPORTUNITY_SEMANTIC_SOURCE_TYPES,
-  OPPORTUNITY_SEMANTIC_ENTITY_TYPES,
+  EMBEDDING_BATCH_SIZE,
+  MAX_QUERY_TEXTS,
+  MAX_TEXT_LENGTH,
+  MAX_MATCH_FIELD_TYPE_LENGTH,
+  MAX_ENTITY_TYPE_LENGTH,
+  MAX_TYPE_FILTER_ITEMS,
+  EMBEDDING_TIMEOUT_MS,
   SEMANTIC_MATCHING_CONFIG,
+  EmbeddingUnavailableError,
   normalizeText,
   hashText,
   cleanTopicText,
   serializeVector,
   parseVector,
-  syncOpportunitySemantic,
-  copyEntityVectors,
-  lookupOpportunitiesByVectors,
-  getQueryEmbeddings,
-  upsertQueryEmbeddings,
-  touchQueryEmbeddings,
+  indexSemanticTopics,
+  indexSemanticClaims,
+  embedQueries,
+  lookupOpportunitiesByTopic,
+  lookupSuggestionsByTopic,
+  lookupSuggestionsByClaim,
 } from '../util/semantic-index.utils.js';
 
 const createPostgrestService = (config, client = undefined) => {