feat(data-access): filter lookupOpportunitiesByVectors by opportunity status (LLMO-7445)

Author: Tanase Butcaru

packages/spacecat-shared-data-access/README.md

@@ -170,9 +170,11 @@ await upsertQueryEmbeddings(postgrestClient, { entries: [{ text, vector }], ...s
 await touchQueryEmbeddings(postgrestClient, { textHashes, ...scope });
 
 // reader, batched: one best-first list of { entityId, entityType, score } per query vector.
-// entityTypes is optional (omitted or [] searches all entity types).
+// entityTypes is optional (omitted or [] searches all entity types). statuses is optional too
+// (omitted or [] searches all); it is applied before k, so hidden opportunities can't take the slots.
 const matches = await lookupOpportunitiesByVectors(postgrestClient, {
-  siteId, sourceTypes: [TOPIC], entityTypes: [CITED_ANALYSIS], vectors, k: 10, minScore: 0.5, ...scope,
+  siteId, sourceTypes: [TOPIC], entityTypes: [CITED_ANALYSIS], statuses: ['NEW', 'IN_PROGRESS'],
+  vectors, k: 10, minScore: 0.5, ...scope,
 });
 ```
 

packages/spacecat-shared-data-access/src/util/semantic-index.utils.js

@@ -163,6 +163,31 @@ function toAllowedList(values, name, registry, { required }) {
   return [...new Set(values)];
 }
 
+/**
+ * Omitted or empty means no status filter. Not checked against `Opportunity.STATUSES` (importing
+ * the model here is a dependency cycle); the data-service matches the enum, so a typo matches
+ * nothing.
+ */
+function toStatusList(statuses) {
+  if (statuses === undefined || statuses === null
+    || (Array.isArray(statuses) && statuses.length === 0)) {
+    return null;
+  }
+  if (!Array.isArray(statuses)) {
+    throw new ValidationError('statuses must be an array');
+  }
+  const rejected = [];
+  for (const status of statuses) {
+    if (typeof status !== 'string' || status.trim() === '') {
+      rejected.push(status);
+    }
+  }
+  if (rejected.length > 0) {
+    throw new ValidationError(`statuses must only contain non-empty strings (got ${describeValue(rejected)})`);
+  }
+  return [...new Set(statuses)];
+}
+
 /** Only an empty/omitted `sources` clears; a non-empty input reduced to nothing throws. */
 function assertClearable(sources, entityId, sourceType) {
   const explicitClear = sources === undefined || sources === null
@@ -453,6 +478,8 @@ export async function copyEntityVectors(postgrestClient, {
  *   `OPPORTUNITY_SEMANTIC_SOURCE_TYPES` (required, non-empty)
  * @param {string[]} [params.entityTypes] - opportunity types to narrow to, from
  *   `OPPORTUNITY_SEMANTIC_ENTITY_TYPES`; omitted or empty searches all
+ * @param {string[]} [params.statuses] - opportunity statuses (`Opportunity.STATUSES` values) to
+ *   narrow to, applied before `k`; omitted or empty searches all
  * @param {number[][]} params.vectors - the query embeddings
  * @param {string} params.model - the model the queries were embedded with
  * @param {number} params.dims - the query embedding dimension
@@ -462,12 +489,13 @@ export async function copyEntityVectors(postgrestClient, {
  *   best-first list per input vector, in input order
  */
 export async function lookupOpportunitiesByVectors(postgrestClient, {
-  siteId, sourceTypes, entityTypes, vectors, model, dims, k = 10, minScore = 0,
+  siteId, sourceTypes, entityTypes, statuses, vectors, model, dims, k = 10, minScore = 0,
 } = {}) {
   assertClient(postgrestClient);
   assertId(siteId, 'siteId');
   const sourceTypeList = toAllowedList(sourceTypes, 'sourceTypes', OPPORTUNITY_SEMANTIC_SOURCE_TYPES, { required: true });
   const entityTypeList = toAllowedList(entityTypes, 'entityTypes', OPPORTUNITY_SEMANTIC_ENTITY_TYPES, { required: false });
+  const statusList = toStatusList(statuses);
   assertModel(model);
   assertDims(dims);
   if (!Array.isArray(vectors)) {
@@ -495,12 +523,19 @@ export async function lookupOpportunitiesByVectors(postgrestClient, {
       p_dims: dims,
       p_limit: k,
       p_min_score: minScore,
+      // Only when set: a data-service without p_statuses rejects the unknown argument.
+      ...(statusList ? { p_statuses: statusList } : {}),
     });
     if (error) {
       throw rpcError(
         `Failed semantic search for site ${siteId}`,
         SEMANTIC_SEARCH_RPC,
-        { siteId, sourceTypes: sourceTypeList, entityTypes: entityTypeList },
+        {
+          siteId,
+          sourceTypes: sourceTypeList,
+          entityTypes: entityTypeList,
+          ...(statusList ? { statuses: statusList } : {}),
+        },
         error,
       );
     }

packages/spacecat-shared-data-access/test/unit/util/semantic-index.utils.test.js

@@ -671,6 +671,33 @@ describe('semantic-index.utils', () => {
       expect(client.calls.rpc[2].params.p_entity_types).to.equal(null);
     });
 
+    it('sends deduped statuses as p_statuses, and omits p_statuses when none are given', async () => {
+      const client = makeClient({ rpcResult: { data: [], error: null } });
+      await lookupOpportunitiesByVectors(client, {
+        ...base, statuses: ['NEW', 'RESOLVED', 'NEW'], vectors: [[0.1, 0.2]],
+      });
+      await lookupOpportunitiesByVectors(client, { ...base, statuses: [], vectors: [[0.1, 0.2]] });
+      await lookupOpportunitiesByVectors(client, {
+        ...base, statuses: null, vectors: [[0.1, 0.2]],
+      });
+      expect(client.calls.rpc[0].params.p_statuses).to.deep.equal(['NEW', 'RESOLVED']);
+      expect(client.calls.rpc[1].params).to.not.have.property('p_statuses');
+      expect(client.calls.rpc[2].params).to.not.have.property('p_statuses');
+    });
+
+    it('rejects statuses that are not a list of non-empty strings', async () => {
+      const call = (statuses) => lookupOpportunitiesByVectors(makeClient(), {
+        ...base, statuses, vectors: [[0.1, 0.2]],
+      });
+      await expect(call('NEW')).to.be.rejectedWith(ValidationError, 'statuses must be an array');
+      await expect(call(['NEW', ' '])).to.be.rejectedWith(
+        ValidationError,
+        'statuses must only contain non-empty strings (got [" "])',
+      );
+      await expect(call(['NEW', 1])).to.be.rejectedWith(ValidationError, '(got [1])');
+      await expect(call(new Array(1))).to.be.rejectedWith(ValidationError, 'statuses must only contain');
+    });
+
     it('renders rejected values safely: unserializable falls back to String, long ones are truncated', async () => {
       const call = (over) => lookupOpportunitiesByVectors(makeClient(), {
         ...base, vectors: [[0.1, 0.2]], ...over,
@@ -743,6 +770,12 @@ describe('semantic-index.utils', () => {
       expect(err.details).to.deep.equal({
         siteId: SITE_ID, sourceTypes: [SOURCE_TYPE], entityTypes: ['reddit-analysis'],
       });
+      const withStatuses = await lookupOpportunitiesByVectors(client, {
+        ...base, statuses: ['NEW'], vectors: [[0.1, 0.2]],
+      }).catch((e) => e);
+      expect(withStatuses.details).to.deep.equal({
+        siteId: SITE_ID, sourceTypes: [SOURCE_TYPE], entityTypes: null, statuses: ['NEW'],
+      });
     });
 
     it('adds a deploy-ordering hint when PostgREST cannot find the RPC signature', async () => {