diff --git a/.gitignore b/.gitignore index 6f0ca768..8dc53a7c 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ Thumbs.db .superpowers/ output/ + +# Search-eval query cache — paraphrases of real mail; the harness regenerates it diff --git a/README.md b/README.md index d79ebb5a..f78afd2a 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,193 @@ configurable per account, and accounts with GTD off behave exactly as before. --- +## MCP server + +MailFlow exposes a Model Context Protocol endpoint at `POST /mcp`. Authenticate +with an API token in the `Authorization: Bearer ` header. Tokens have one or +more scopes: + +- `read` — search, inspect messages, and list mailbox state +- `write` — change mailbox state; also implies `read` +- `send` — send, queue, cancel, or recall mail; also implies `read` +- `settings` — manage accounts, aliases, and rules; also implies `read` + +`stage_deletion` requires `write`, even though execution is a separate, +session-authorized step. `run_rules` requires both `settings` and `write` because +rule actions can move, archive, or delete messages. + +### Tools + +#### Search and read + +| Tool | Scope | Description | +|---|---|---| +| `ping` | `read` | Check transport and authentication health. | +| `search_metadata` | `read` | Search message metadata with the supported Gmail-style query subset. | +| `search_message_bodies` | `read` | Run keyword full-text search over message bodies. | +| `semantic_search_messages` | `read` | Run vector or hybrid semantic search over indexed messages. | +| `get_message` | `read` | Get message details and a pageable body slice. | +| `list_messages` | `read` | List messages with account, participant, date, attachment, and conversation filters. | +| `get_stats` | `read` | Get archive totals, attachment statistics, and accounts. | +| `aggregate` | `read` | Group message statistics by sender, recipient, domain, label, or year. | +| `search_by_domains` | `read` | Find messages involving any of a set of domains. | +| `find_similar_messages` | `read` | Find messages closest to a seed message in the vector index. | +| `search_in_message` | `read` | Find keyword or semantic matches inside one message. | + +#### Compose sessions + +| Tool | Scope | Description | +|---|---|---| +| `list_compose_sessions` | `read` | List live compose sessions and available slots. | +| `get_compose_session` | `read` | Get a complete live compose session by slot. | +| `create_compose_session` | `write` | Create a live compose session in a requested or available slot. | +| `update_compose_session` | `write` | Update explicitly provided fields on a live compose session. | +| `minimize_compose_session` | `write` | Minimize a live compose session. | +| `restore_compose_session` | `write` | Restore a minimized live compose session. | +| `add_compose_attachment` | `write` | Add a base64-encoded attachment to a live compose session. | +| `remove_compose_attachment` | `write` | Remove an attachment from a live compose session. | +| `close_compose_session` | `write` | Close a compose session, saving meaningful content as an IMAP draft. | +| `discard_compose_session` | `write` | Permanently discard a compose session and free its slot. | +| `send_compose_session` | `send` | Send a compose session and free its slot after acceptance. | + +Compose-session tools control Mailflow's nine live server-owned slots. Draft +tools control messages already stored in the account's IMAP Drafts folder. +Closing a meaningful compose session converts it into an IMAP draft and frees +the slot. + +#### Drafts + +| Tool | Scope | Description | +|---|---|---| +| `create_draft` | `write` | Create a draft without sending it. | +| `update_draft` | `write` | Replace a draft and return its new IMAP UID. | +| `list_drafts` | `read` | List drafts newest-first. | +| `get_draft` | `read` | Get draft bodies, threading headers, and attachment metadata. | +| `delete_draft` | `write` | Permanently delete a draft from IMAP. | + +#### Send, outbox, and undo-send + +| Tool | Scope | Description | +|---|---|---| +| `send_email` | `send` | Send immediately or queue within an undo-send window. | +| `send_draft` | `send` | Send a stored draft and delete it only after delivery or enqueue succeeds. | +| `reply_email` | `send` | Reply to a message sender. | +| `reply_all_email` | `send` | Reply to the sender and stored To/Cc recipients. | +| `forward_email` | `send` | Forward a message, including server-side attachment references. | +| `unsend_email` | `send` | Cancel a queued message before its `send_at` time. | +| `list_outbox` | `read` | List messages still waiting in the undo-send outbox. | +| `recall_email` | `send` | Cancel a queued send, or clean up a Sent copy and prepare a follow-up draft. | + +#### Mailbox and folders + +| Tool | Scope | Description | +|---|---|---| +| `list_folders` | `read` | List scoped folders and live message counts. | +| `create_folder` | `write` | Create a folder in one account. | +| `rename_folder` | `write` | Rename the final component of a folder path. | +| `delete_folder` | `write` | Delete a folder after confirming its live message count. | +| `move_messages` | `write` | Move messages and return replacement IDs. | +| `archive_messages` | `write` | Archive messages and return replacement IDs. | +| `trash_messages` | `write` | Move messages to Trash without silently expunging existing Trash items. | +| `mark_read` / `mark_unread` | `write` | Change read state for explicit message IDs. | +| `star_message` / `unstar_message` | `write` | Change starred state for explicit message IDs. | +| `mark_spam` / `mark_not_spam` | `write` | Move a message into or out of the configured spam folder. | +| `snooze_message` / `unsnooze_message` | `write` | Snooze or restore a message and its reply-chain siblings. | +| `set_category` | `write` | Set the category of one message. | +| `gtd_classify` | `write` | Apply or remove one GTD state label. | +| `gtd_done` | `write` | Remove GTD labels and archive the Inbox copy. | +| `stage_deletion` | `write` | Stage a filtered deletion for separate session-authorized execution. | + +#### Triage + +| Tool | Scope | Description | +|---|---|---| +| `triage_inbox` | `read` | Page through untriaged Inbox messages with sender, thread, category, and optional semantic signals. | +| `get_triage_context` | `read` | Get thread, sender-history, similar-message, and matched-rule context. | +| `mark_triaged` | `write` | Checkpoint messages so later triage runs skip them. | + +#### Accounts and settings + +| Tool | Scope | Description | +|---|---|---| +| `list_accounts` | `read` | List connected accounts and aliases without credentials. | +| `add_account` | `settings` | Stage non-secret account configuration and return a `stage_id`. | +| `update_account_settings` | `settings` | Update non-secret account settings. | +| `test_account_connection` | `settings` | Test stored IMAP and SMTP credentials without sending mail. | +| `create_alias` / `update_alias` / `delete_alias` | `settings` | Manage send-as aliases on scoped accounts. | +| `list_rules` | `settings` | List global and account-specific inbox rules. | +| `create_rule` / `update_rule` / `delete_rule` | `settings` | Manage user-owned inbox rules. | +| `run_rules` | `settings` + `write` | Run enabled rules against current Inbox messages. | + +### Undo-send and recall + +`send_email` and `send_draft` accept an undo window from 0 to 120 seconds. A +zero-second window hands the message to SMTP immediately. A positive window puts +it in the outbox until `send_at`; use `list_outbox` to inspect queued messages and +`unsend_email` to cancel one before handoff. + +Recall is deliberately honest. `recall_email` can withdraw a message only while +it is still queued. SMTP cannot claw back mail already delivered; for an +already-sent message, recall can delete MailFlow's Sent copy and prepare a +"please disregard" follow-up draft, but it never pretends recipients lost their +copy and never sends the follow-up automatically. + +### Adding an account + +`add_account` accepts non-secret connection configuration only. It returns a +`stage_id`; a human then supplies fresh credentials in **Settings → Accounts**, +or through the session-authenticated +`POST /api/mcp-account-stages/:id/execute` endpoint. + +Passwords, OAuth access tokens, and OAuth refresh tokens never cross the MCP +bearer-token channel. + +### Agent cookbook + +A morning triage loop can page oldest-first and checkpoint every disposition: + +```text +07:00 triage_inbox { limit: 25, unread_only: true } + → items, cursor, has_more, counts.untriaged_unread + + Obvious newsletters/promotions: + archive_messages { message_ids: ["m1", "m2", ...] } + → archived: [{ id: "m1", new_id: "n1", ... }, ...] + mark_triaged { + message_ids: ["n1", "n2", ...], + action: "archived" + } + + Ambiguous message with a known sender and active thread: + get_triage_context { message_id: "m17" } + gtd_classify { message_id: "m17", state: "todo" } + star_message { message_ids: ["m17"] } + mark_triaged { + message_ids: ["m17"], + action: "flagged_todo", + note: "Needs a reply" + } + + Snooze a reply chain: + mark_triaged { message_ids: ["m22"], action: "snoozed" } + snooze_message { + message_id: "m22", + until: "2026-07-31T08:00:00Z" + } + → { ok: true, moved_count: 3, sibling_ids: [...], folder: "Snoozed" } + +07:04 triage_inbox { cursor: "", limit: 25 } + → repeat until has_more is false +``` + +**Mark before move:** call `mark_triaged` before archiving, moving, or snoozing, +or use each successful move/archive receipt's `new_id`. Message IDs change during +a move; a stale pre-move ID resolves to `skipped`. Re-running `triage_inbox` +without a cursor is safe: the checkpoint table, not the cursor, is what prevents +already-triaged messages from being offered again. + +--- + ## Screenshots @@ -188,6 +375,14 @@ docker compose up -d To pin to a specific version instead of `latest`, add `MAILFLOW_VERSION=2.7.0` to your `.env`. +> **Upgrading an existing install across the Postgres image change** (`postgres:16-alpine` → `pgvector/pgvector:pg16`): the new image uses a different C library (musl → glibc), which changes text collation order. Reindex once after the switch or text indexes can silently return wrong results — the backend also prints this warning at startup when it detects the mismatch: +> +> ```bash +> docker compose exec postgres psql -U mailflow -d mailflow \ +> -c 'REINDEX DATABASE "mailflow";' \ +> -c 'ALTER DATABASE "mailflow" REFRESH COLLATION VERSION;' +> ``` + --- ## Option B — Build from source diff --git a/backend/migrations/0040_gtd_delegations.sql b/backend/migrations/0040_gtd_delegations.sql new file mode 100644 index 00000000..c637ffbb --- /dev/null +++ b/backend/migrations/0040_gtd_delegations.sql @@ -0,0 +1,15 @@ +CREATE TABLE gtd_delegations ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + thread_key TEXT NOT NULL, + contact_id UUID REFERENCES contacts(id) ON DELETE SET NULL, + contact_display_name_snapshot TEXT NOT NULL, + contact_primary_email_snapshot TEXT, + delegated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, account_id, thread_key) +); + +CREATE INDEX idx_gtd_delegations_contact_id + ON gtd_delegations(contact_id) + WHERE contact_id IS NOT NULL; diff --git a/backend/migrations/0041_search_fts.sql b/backend/migrations/0041_search_fts.sql new file mode 100644 index 00000000..519c213c --- /dev/null +++ b/backend/migrations/0041_search_fts.sql @@ -0,0 +1,55 @@ +-- Weighted full-text search column (search_fts) + version stamp, kept fresh by +-- a BEFORE trigger. Fast, metadata-only DDL only (README D1): the nullable +-- columns force no table rewrite on PG16, and the function/trigger are instant. +-- Pre-existing rows are populated by the resumable drainer (ftsBackfill.js); +-- the GIN index is built CONCURRENTLY in the separate no-transaction migration +-- 0037 (a $$-quoted plpgsql body cannot survive the no-transaction ; splitter, +-- so the trigger and the CONCURRENTLY index MUST live in different files). +-- +-- The setweight(...) expression MUST stay identical to +-- lexicalRepo.searchFtsExpr('NEW'); fts_version = 1 matches lexicalRepo.FTS_VERSION. +-- Idempotent (IF NOT EXISTS / CREATE OR REPLACE / DROP IF EXISTS) because a +-- crash before the schema_migrations INSERT retries this migration. + +ALTER TABLE messages ADD COLUMN IF NOT EXISTS search_fts tsvector; +ALTER TABLE messages ADD COLUMN IF NOT EXISTS fts_version int; + +CREATE OR REPLACE FUNCTION messages_search_fts_refresh() RETURNS trigger AS $$ +BEGIN + -- Skip recompute on an UPDATE that changes none of the indexed source + -- columns (read/star flag flips, snippet-only writes), so the trigger does + -- not tax hot sync UPSERTs; this also lets the backfill's explicit SET win. + IF TG_OP = 'UPDATE' + AND NEW.subject IS NOT DISTINCT FROM OLD.subject + AND NEW.from_name IS NOT DISTINCT FROM OLD.from_name + AND NEW.from_email IS NOT DISTINCT FROM OLD.from_email + AND NEW.to_addresses IS NOT DISTINCT FROM OLD.to_addresses + AND NEW.cc_addresses IS NOT DISTINCT FROM OLD.cc_addresses + AND NEW.body_text IS NOT DISTINCT FROM OLD.body_text + THEN + RETURN NEW; + END IF; + + BEGIN + NEW.search_fts := setweight(to_tsvector('english', coalesce(NEW.subject,'')), 'A') || + setweight(to_tsvector('english', coalesce(NEW.from_name,'') || ' ' || coalesce(NEW.from_email,'')), 'B') || + setweight(to_tsvector('english', coalesce(NEW.to_addresses::text,'') || ' ' || coalesce(NEW.cc_addresses::text,'')), 'C') || + setweight(to_tsvector('english', LEFT(coalesce(NEW.body_text,''), 600000)), 'D'); + NEW.fts_version := 1; + EXCEPTION WHEN program_limit_exceeded THEN + -- Even with the 600k LEFT cap, a pathologically dense/multibyte body can + -- exceed Postgres's ~1MB tsvector limit (SQLSTATE 54000). Never fail the + -- row write: leave search_fts NULL so the message still persists and stays + -- findable via the ILIKE fallback; the backfill's row-by-row skip stamps it. + NEW.search_fts := NULL; + NEW.fts_version := NULL; + END; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_messages_search_fts ON messages; +CREATE TRIGGER trg_messages_search_fts + BEFORE INSERT OR UPDATE ON messages + FOR EACH ROW EXECUTE FUNCTION messages_search_fts_refresh(); diff --git a/backend/migrations/0042_background_jobs.sql b/backend/migrations/0042_background_jobs.sql new file mode 100644 index 00000000..d189933c --- /dev/null +++ b/backend/migrations/0042_background_jobs.sql @@ -0,0 +1,19 @@ +-- Generic progress substrate for observable background drainers (first consumer: +-- the FTS backfill). Plain table, fast DDL. One row per (kind, account); global +-- jobs use a NULL account_id, COALESCE'd to '' in the unique index so the upsert +-- has a single conflict target for both cases. + +CREATE TABLE IF NOT EXISTS background_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + kind VARCHAR(64) NOT NULL, + account_id UUID REFERENCES email_accounts(id) ON DELETE CASCADE, + state VARCHAR(20) NOT NULL DEFAULT 'idle', + processed BIGINT NOT NULL DEFAULT 0, + total BIGINT NOT NULL DEFAULT 0, + last_error TEXT, + started_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_background_jobs_kind_account + ON background_jobs (kind, COALESCE(account_id::text, '')); diff --git a/backend/migrations/0043_search_fts_index.sql b/backend/migrations/0043_search_fts_index.sql new file mode 100644 index 00000000..f82b7dbb --- /dev/null +++ b/backend/migrations/0043_search_fts_index.sql @@ -0,0 +1,24 @@ +-- no-transaction +-- +-- GIN index that serves `search_fts @@ tsquery`, plus a partial btree that lets +-- the backfill drainer (and any "needs backfill" probe) find not-yet-stamped +-- rows without a full seq scan; it self-prunes as fts_version = 1 fills in. +-- CONCURRENTLY must run outside a transaction, so these live apart from 0035's +-- trigger. Idempotent via IF NOT EXISTS (retried if a crash precedes the +-- schema_migrations INSERT). +-- +-- DROP ... IF EXISTS before each CREATE: a cancelled or crashed CREATE INDEX +-- CONCURRENTLY leaves an INVALID index under the target name. On retry, plain +-- IF NOT EXISTS sees that name and silently skips the create, recording the +-- migration as done while the scan stays unindexed forever. This file only re-runs +-- after such a failure — in which case the index is either absent (drop is a no-op) +-- or invalid (drop removes the dead stub) — so dropping first is safe and cheap. + +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_search_fts; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_search_fts + ON messages USING GIN (search_fts); + +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_fts_stale_v1; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_fts_stale_v1 + ON messages (date DESC) + WHERE fts_version IS DISTINCT FROM 1; diff --git a/backend/migrations/0044_embed_watermark.sql b/backend/migrations/0044_embed_watermark.sql new file mode 100644 index 00000000..ecf8214a --- /dev/null +++ b/backend/migrations/0044_embed_watermark.sql @@ -0,0 +1,44 @@ +-- Vector-substrate CAS columns + last_modified trigger (slice 04). +-- Extension-independent, fast DDL only. NO vector-typed DDL here — the +-- embeddings/index_generations/embed_watermark/embed_runs tables and the HNSW +-- index are created by ensureVectorSchema() at startup (README invariant). +-- Transactional migration (NOT -- no-transaction): the no-transaction runner +-- splits on ';' and would break the dollar-quoted function below. + +-- last_modified: content-change CAS token the embed worker compares to detect a +-- late-arriving body invalidating a stale subject-only embedding. NOT NULL DEFAULT +-- now() is metadata-only on PG16 (now() is STABLE → one stored missing-value, no +-- table rewrite). +ALTER TABLE messages ADD COLUMN IF NOT EXISTS last_modified TIMESTAMPTZ NOT NULL DEFAULT now(); + +-- embed_gen: the index generation this row is embedded under. NULL = needs embedding. +-- Plain BIGINT soft-stamp (no FK): a generation can be retired/deleted while stamps linger. +ALTER TABLE messages ADD COLUMN IF NOT EXISTS embed_gen BIGINT; + +-- Bump last_modified AND clear embed_gen when an embedding-input column changes. The +-- WHEN clause filters at the C level so unchanged-content UPDATEs (the hot re-sync path) +-- and stamp-only UPDATEs (the worker setting embed_gen) skip the function entirely. +-- Clearing embed_gen is what re-surfaces a late-arriving body: a row embedded +-- subject-only and stamped, whose body later lands (phase-2 drainer / on-open fetch), +-- has its stamp cleared here so the NULL-only embed scan re-finds it and the idempotent +-- upsert replaces the stale chunks. The CAS only guards the read→stamp window; this +-- trigger covers the post-stamp case (Mailflow's late-arriving bodies — msgvault's rows +-- are immutable after ingest, so it never needed this). Together with createGeneration's +-- stamp-reset (which handles generation rebuilds: unchanged content, new fingerprint), +-- the invariant is exact: embed_gen IS NULL ⟺ the row needs embedding. +CREATE OR REPLACE FUNCTION messages_bump_last_modified() RETURNS trigger AS $$ +BEGIN + NEW.last_modified := now(); + NEW.embed_gen := NULL; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_messages_last_modified ON messages; +CREATE TRIGGER trg_messages_last_modified + BEFORE UPDATE ON messages + FOR EACH ROW + WHEN (NEW.subject IS DISTINCT FROM OLD.subject + OR NEW.body_text IS DISTINCT FROM OLD.body_text + OR NEW.body_html IS DISTINCT FROM OLD.body_html) + EXECUTE FUNCTION messages_bump_last_modified(); diff --git a/backend/migrations/0045_embed_pending_index.sql b/backend/migrations/0045_embed_pending_index.sql new file mode 100644 index 00000000..abf9a874 --- /dev/null +++ b/backend/migrations/0045_embed_pending_index.sql @@ -0,0 +1,17 @@ +-- no-transaction +-- Partial index over the embed-scan's steady-state hot predicate. Once a generation +-- reaches full coverage, the only live rows still needing work are newly-arrived +-- messages with embed_gen IS NULL, so the 60s scheduler scan (scanForEmbedding) should +-- touch O(pending), not O(mailbox). Built CONCURRENTLY (hence -- no-transaction) so it +-- never blocks boot on a large messages table; extension-independent and cheap to +-- maintain (only the sparse NULL set is indexed). Idempotent (IF NOT EXISTS) because a +-- crash before the schema_migrations INSERT retries the migration. +-- +-- DROP ... IF EXISTS before the CREATE: a cancelled or crashed CREATE INDEX +-- CONCURRENTLY leaves an INVALID index under this name, and plain IF NOT EXISTS would +-- then silently skip the create on retry — recording the migration as done while the +-- embed scan stays unindexed forever. This file only re-runs after such a failure, so +-- the index is either absent (drop is a no-op) or invalid (drop clears the dead stub). +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_embed_pending; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_embed_pending + ON messages (id) WHERE embed_gen IS NULL; diff --git a/backend/migrations/0046_api_tokens.sql b/backend/migrations/0046_api_tokens.sql new file mode 100644 index 00000000..df17efa9 --- /dev/null +++ b/backend/migrations/0046_api_tokens.sql @@ -0,0 +1,12 @@ +-- MCP API tokens. We store only a SHA-256 hash of the token; the plaintext is +-- shown to the operator exactly once at mint time and is never recoverable. +CREATE TABLE IF NOT EXISTS api_tokens ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_used_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_api_tokens_user_id ON api_tokens(user_id); diff --git a/backend/migrations/0047_mcp_deletion_batches.sql b/backend/migrations/0047_mcp_deletion_batches.sql new file mode 100644 index 00000000..cf3b77c4 --- /dev/null +++ b/backend/migrations/0047_mcp_deletion_batches.sql @@ -0,0 +1,21 @@ +-- Staged (not executed) MCP deletions. stage_deletion records a batch; a separate +-- session-authenticated execute step flips messages.is_deleted (soft delete). No +-- tool ever hard-deletes. Renumbered to 0041 (0040 is api_tokens) per the README +-- migration-numbering rule. +CREATE TABLE IF NOT EXISTS mcp_deletion_batches ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + description TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'staged', + message_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + executed_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS mcp_deletion_batch_messages ( + batch_id UUID NOT NULL REFERENCES mcp_deletion_batches(id) ON DELETE CASCADE, + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + PRIMARY KEY (batch_id, message_id) +); + +CREATE INDEX IF NOT EXISTS idx_mcp_deletion_batches_user_id ON mcp_deletion_batches(user_id); diff --git a/backend/migrations/0048_api_token_scopes.sql b/backend/migrations/0048_api_token_scopes.sql new file mode 100644 index 00000000..6aca6d7d --- /dev/null +++ b/backend/migrations/0048_api_token_scopes.sql @@ -0,0 +1,13 @@ +-- Scope MCP bearer tokens. Existing tokens were minted when the MCP surface was +-- read-only, so 'read' is both the default and the safe upgrade backfill. +ALTER TABLE api_tokens + ADD COLUMN IF NOT EXISTS scopes TEXT[] NOT NULL DEFAULT ARRAY['read']::TEXT[]; + +-- Keep every storage writer constrained to the scopes the server can enforce. +ALTER TABLE api_tokens + DROP CONSTRAINT IF EXISTS api_tokens_scopes_valid; +ALTER TABLE api_tokens + ADD CONSTRAINT api_tokens_scopes_valid CHECK ( + scopes <@ ARRAY['read','write','send','settings']::TEXT[] + AND COALESCE(array_length(scopes, 1), 0) >= 1 + ); diff --git a/backend/migrations/0049_outbox_messages.sql b/backend/migrations/0049_outbox_messages.sql new file mode 100644 index 00000000..bcd9a1f0 --- /dev/null +++ b/backend/migrations/0049_outbox_messages.sql @@ -0,0 +1,30 @@ +-- Deferred sends. A row is a fully-resolved compose payload waiting out its undo +-- window; the worker claims it at send_at and hands it to sendService. Rows are +-- short-lived by construction (max window is 120s) — payload is NULLed on any +-- terminal status so delivered mail bodies do not accumulate at rest. +CREATE TABLE IF NOT EXISTS outbox_messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + payload JSONB NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','claimed','sent','cancelled','failed')), + send_at TIMESTAMPTZ NOT NULL, + claimed_at TIMESTAMPTZ, + attempts INT NOT NULL DEFAULT 0, + subject TEXT, + to_preview JSONB NOT NULL DEFAULT '[]', + message_id TEXT, + sent_message_id TEXT, + error TEXT, + idempotency_key TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_outbox_due + ON outbox_messages(send_at) WHERE status = 'pending'; +CREATE INDEX IF NOT EXISTS idx_outbox_user + ON outbox_messages(user_id, created_at DESC); +CREATE UNIQUE INDEX IF NOT EXISTS idx_outbox_idem + ON outbox_messages(user_id, idempotency_key) WHERE idempotency_key IS NOT NULL; diff --git a/backend/migrations/0050_message_triage.sql b/backend/migrations/0050_message_triage.sql new file mode 100644 index 00000000..0ba80ce5 --- /dev/null +++ b/backend/migrations/0050_message_triage.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS message_triage ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + account_id UUID NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + message_id_header TEXT NOT NULL, + triaged_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + action TEXT, -- free-form: archived | replied | snoozed | left | … + note TEXT, + source TEXT NOT NULL DEFAULT 'mcp', + token_id UUID REFERENCES api_tokens(id) ON DELETE SET NULL, + UNIQUE (account_id, message_id_header) +); +CREATE INDEX idx_message_triage_user_time ON message_triage (user_id, triaged_at DESC); diff --git a/backend/migrations/0051_mcp_account_stages.sql b/backend/migrations/0051_mcp_account_stages.sql new file mode 100644 index 00000000..dbde0ae5 --- /dev/null +++ b/backend/migrations/0051_mcp_account_stages.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS mcp_account_stages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + status TEXT NOT NULL DEFAULT 'staged', -- staged | completed | discarded + payload JSONB NOT NULL, -- name, sender_name, email_address, color, protocol, + -- imap_host, imap_port, imap_skip_tls_verify, + -- smtp_host, smtp_port, smtp_tls, auth_user, signature + -- NEVER auth_pass / oauth_* tokens + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_account_id UUID REFERENCES email_accounts(id) ON DELETE SET NULL +); +CREATE INDEX IF NOT EXISTS idx_mcp_account_stages_user_id ON mcp_account_stages(user_id); diff --git a/backend/migrations/0052_message_triage_feed_index.sql b/backend/migrations/0052_message_triage_feed_index.sql new file mode 100644 index 00000000..9273f867 --- /dev/null +++ b/backend/migrations/0052_message_triage_feed_index.sql @@ -0,0 +1,12 @@ +-- no-transaction +-- +-- This partial index supports the oldest-first inbox triage feed on the existing +-- messages table. CONCURRENTLY avoids blocking mailbox writes while it is built. +-- +-- DROP before CREATE makes retries safe after a cancelled or crashed concurrent +-- build: PostgreSQL can leave an invalid index with the target name, which plain +-- IF NOT EXISTS would otherwise skip and then record as successfully migrated. +DROP INDEX CONCURRENTLY IF EXISTS idx_messages_triage_feed; +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_triage_feed + ON messages (account_id, date, message_id) + WHERE folder = 'INBOX' AND is_deleted = false; diff --git a/backend/migrations/0053_compose_sessions.sql b/backend/migrations/0053_compose_sessions.sql new file mode 100644 index 00000000..bf7acca4 --- /dev/null +++ b/backend/migrations/0053_compose_sessions.sql @@ -0,0 +1,59 @@ +CREATE TABLE IF NOT EXISTS compose_sessions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + slot SMALLINT NOT NULL CHECK (slot BETWEEN 1 AND 9), + account_id UUID REFERENCES email_accounts(id) ON DELETE SET NULL, + alias_id UUID REFERENCES account_aliases(id) ON DELETE SET NULL, + mode TEXT NOT NULL DEFAULT 'new' + CHECK (mode IN ('new','reply','reply_all','forward')), + to_recipients JSONB NOT NULL DEFAULT '[]', + cc_recipients JSONB NOT NULL DEFAULT '[]', + bcc_recipients JSONB NOT NULL DEFAULT '[]', + subject TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '', + body_is_html BOOLEAN NOT NULL DEFAULT TRUE, + quoted_body TEXT, + quoted_body_html TEXT, + edited_signature TEXT, + forwarded_attachments JSONB NOT NULL DEFAULT '[]', + from_changed BOOLEAN NOT NULL DEFAULT FALSE, + priority TEXT NOT NULL DEFAULT 'normal' + CHECK (priority IN ('low','normal','high')), + in_reply_to TEXT, + thread_references JSONB NOT NULL DEFAULT '[]', + source_draft_account_id UUID REFERENCES email_accounts(id) ON DELETE SET NULL, + source_draft_folder TEXT, + source_draft_uid BIGINT, + source_draft_message_id TEXT, + source_initial_revision JSONB, + presentation_state TEXT NOT NULL DEFAULT 'expanded' + CHECK (presentation_state IN ('expanded','minimized')), + operation_state TEXT NOT NULL DEFAULT 'idle' + CHECK (operation_state IN ('idle','closing','discarding','sending')), + operation_token UUID, + revision BIGINT NOT NULL DEFAULT 1, + field_revisions JSONB NOT NULL DEFAULT '{}', + last_focused_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (user_id, slot) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_compose_sessions_source_draft + ON compose_sessions(user_id, source_draft_account_id, source_draft_folder, source_draft_uid) + WHERE source_draft_account_id IS NOT NULL + AND source_draft_folder IS NOT NULL + AND source_draft_uid IS NOT NULL; + +CREATE TABLE IF NOT EXISTS compose_session_attachments ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + session_id UUID NOT NULL REFERENCES compose_sessions(id) ON DELETE CASCADE, + filename TEXT NOT NULL, + content_type TEXT NOT NULL DEFAULT 'application/octet-stream', + byte_count INTEGER NOT NULL CHECK (byte_count >= 0 AND byte_count <= 26214400), + content BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_compose_session_attachments_session + ON compose_session_attachments(session_id, created_at, id); diff --git a/backend/migrations/0054_compose_restore_reply_all.sql b/backend/migrations/0054_compose_restore_reply_all.sql new file mode 100644 index 00000000..a2d1305c --- /dev/null +++ b/backend/migrations/0054_compose_restore_reply_all.sql @@ -0,0 +1,10 @@ +-- Server-owned source recipients let Reply -> Reply All survive reload without +-- making those recipients editable until the UI explicitly patches to/cc. +ALTER TABLE compose_sessions + ADD COLUMN IF NOT EXISTS reply_all_recipients JSONB NOT NULL DEFAULT '[]'::jsonb; + +-- Records an accepted outbox restore after its private payload is wiped, making +-- owner-scoped restore retries idempotent without retaining message content. +ALTER TABLE outbox_messages + ADD COLUMN IF NOT EXISTS restored_compose_session_id UUID + REFERENCES compose_sessions(id) ON DELETE SET NULL; diff --git a/backend/package-lock.json b/backend/package-lock.json index b5300d5b..0daa576c 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -8,6 +8,7 @@ "name": "mailflow-backend", "version": "2.7.0", "dependencies": { + "@modelcontextprotocol/sdk": "^1.29.0", "archiver": "^7.0.1", "bcryptjs": "^2.4.3", "connect-redis": "^7.1.0", @@ -224,6 +225,18 @@ "node": "^20.19.0 || ^22.13.0 || >=24" } }, + "node_modules/@hono/node-server": { + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -393,6 +406,398 @@ "dev": true, "license": "MIT" }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.29.0.tgz", + "integrity": "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", @@ -1078,6 +1483,45 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -2198,6 +2642,27 @@ "bare-events": "^2.7.0" } }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.0.tgz", + "integrity": "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -2263,6 +2728,24 @@ "express": "^4.16.2" } }, + "node_modules/express-rate-limit": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", + "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/express-session": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.19.0.tgz", @@ -2290,7 +2773,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, "node_modules/fast-fifo": { @@ -2313,6 +2795,22 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-uri": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", + "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fast-xml-builder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", @@ -2652,6 +3150,15 @@ "node": ">= 0.4" } }, + "node_modules/hono": { + "version": "4.12.30", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.30.tgz", + "integrity": "sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, "node_modules/htmlparser2": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", @@ -2898,6 +3405,12 @@ "node": ">=0.10.0" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -2972,6 +3485,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -3642,6 +4161,15 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -3935,6 +4463,15 @@ "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", "license": "MIT" }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/pngjs": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-5.0.0.tgz", @@ -4218,6 +4755,15 @@ "node": ">=0.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -4258,6 +4804,55 @@ "@rolldown/binding-win32-x64-msvc": "1.0.3" } }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/router/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/router/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/router/node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -5108,6 +5703,12 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/ws": { "version": "8.21.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", @@ -5226,6 +5827,24 @@ "engines": { "node": ">= 14" } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } } } } diff --git a/backend/package.json b/backend/package.json index 75388b23..caabb123 100644 --- a/backend/package.json +++ b/backend/package.json @@ -14,6 +14,7 @@ "audit:redos": "npm i --no-save --silent eslint-plugin-redos && eslint -c eslint.redos.config.mjs src" }, "dependencies": { + "@modelcontextprotocol/sdk": "^1.29.0", "archiver": "^7.0.1", "bcryptjs": "^2.4.3", "connect-redis": "^7.1.0", diff --git a/backend/scripts/bench-last-modified-trigger.js b/backend/scripts/bench-last-modified-trigger.js new file mode 100644 index 00000000..4593994b --- /dev/null +++ b/backend/scripts/bench-last-modified-trigger.js @@ -0,0 +1,45 @@ +// Benchmark the last_modified trigger overhead on the hot re-sync UPSERT. +// Usage: node scripts/bench-last-modified-trigger.js [rows=5000] [iters=20000] +// Requires DB_* env pointing at a migrated Postgres (trigger present). +import { pool } from '../src/services/db.js'; +import { seedAccount, cleanupAccount } from '../src/services/embeddings/testSupport.js'; + +const ROWS = Number(process.argv[2]) || 5000; +const ITERS = Number(process.argv[3]) || 20000; + +const { accountId: acctId, userId } = await seedAccount(pool, 'bench'); +const ids = []; +for (let i = 0; i < ROWS; i++) { + const r = await pool.query(`INSERT INTO messages (account_id, uid, folder, subject, is_read) + VALUES ($1, $2, 'INBOX', 'bench', false) RETURNING id`, [acctId, 700000 + i]); + ids.push(r.rows[0].id); +} + +// Flag-only churn: toggles is_read (NOT an embedding-input column) so the trigger's +// WHEN clause should skip the function entirely. +async function churn() { + const t0 = process.hrtime.bigint(); + for (let i = 0; i < ITERS; i++) { + const id = ids[i % ids.length]; + await pool.query('UPDATE messages SET is_read = NOT is_read WHERE id = $1', [id]); + } + return Number(process.hrtime.bigint() - t0) / 1e6; // ms +} + +const withTrig = await churn(); +await pool.query('DROP TRIGGER IF EXISTS trg_messages_last_modified ON messages'); +const withoutTrig = await churn(); +// Restore the trigger. +await pool.query(`CREATE TRIGGER trg_messages_last_modified + BEFORE UPDATE ON messages FOR EACH ROW + WHEN (NEW.subject IS DISTINCT FROM OLD.subject OR NEW.body_text IS DISTINCT FROM OLD.body_text OR NEW.body_html IS DISTINCT FROM OLD.body_html) + EXECUTE FUNCTION messages_bump_last_modified()`); + +await cleanupAccount(pool, userId); +await pool.end(); + +const regression = ((withTrig - withoutTrig) / withoutTrig) * 100; +console.log(`with trigger: ${withTrig.toFixed(0)} ms`); +console.log(`without trigger: ${withoutTrig.toFixed(0)} ms`); +console.log(`regression: ${regression.toFixed(1)}% (gate: < 10%)`); +process.exit(regression < 10 ? 0 : 2); diff --git a/backend/scripts/search-eval.mjs b/backend/scripts/search-eval.mjs new file mode 100644 index 00000000..a4ce1cb9 --- /dev/null +++ b/backend/scripts/search-eval.mjs @@ -0,0 +1,453 @@ +#!/usr/bin/env node +// search-eval.mjs — IR relevance eval harness for Mailflow's lexical/vector/hybrid search. +// +// WHY THIS EXISTS +// A user reported they "can't tell the difference" between search modes on their real +// 18k-message mailbox and suspected hybrid might be *worse* than pure vector. Phase 4's +// rankingQuality.test.js proved hybrid never LOSES a lexical hit on a synthetic 16-doc +// fixture; this harness is the complementary real-corpus measurement: it builds labeled +// query sets FROM the live mailbox, runs them through the deployed REST API in all three +// modes, and reports the standard IR metrics (Recall@1/5/20, MRR@20) plus cross-mode +// result overlap and hybrid explain-score composition — so the "is hybrid working?" +// question is answered with numbers instead of vibes. +// +// DESIGN +// * No new deps. Node >=18 global fetch; psql sampling via `docker exec` (the same +// read-only path the eval brief documents); explain scores via one `docker exec ... +// node` pass against the in-container searchService seam (the deployed REST route does +// not plumb `explain` through, but the seam it calls does). +// * Deterministic. Message sampling is ordered by md5(id || seed); paraphrase queries +// are cached to JSON keyed by {messageId, promptVersion}, so reruns are free & stable. +// * Two query sets, both with a single ground-truth message id: +// KEYWORD — 2 distinctive subject tokens (lexical should win/tie). +// PARAPHRASE — an LLM rewrites the email's topic WITHOUT its distinctive keywords +// (semantic recall test; degrades to hand-written queries if the LLM +// is unavailable). +// +// USAGE +// EVAL_USER='admin@example.com' \ # login username (no default) +// EVAL_PASS='' \ +// OPENAI_API_KEY="$(cat /path/to/key)" \ # or EVAL_KEY_FILE=/path/to/key +// node backend/scripts/search-eval.mjs +// +// Reruns after the first are offline for query generation (cache hit) but still hit the +// live REST API for the actual searches. Set EVAL_SKIP_EXPLAIN=1 to skip the in-container +// diagnostic. All knobs are env vars (see CFG below). No secrets are written to disk. + +import { execFileSync } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const EVAL_DIR = resolve(__dirname, '..', '..', 'specs', 'search-overhaul', 'evals'); + +const PROMPT_VERSION = 'v1'; // bump to invalidate the paraphrase cache + +function readKeyFile() { + const f = process.env.EVAL_KEY_FILE; + if (f && existsSync(f)) return readFileSync(f, 'utf8'); + return ''; +} + +const CFG = { + baseUrl: process.env.EVAL_BASE_URL || 'http://127.0.0.1:8087', + loginUser: process.env.EVAL_USER || '', // required for a fresh login; never defaulted + loginPass: process.env.EVAL_PASS || '', // required for a fresh login; never defaulted + pgContainer: process.env.EVAL_PG_CONTAINER || 'mailflow-postgres', + pgUser: process.env.EVAL_PG_USER || 'mailflow', + pgDb: process.env.EVAL_PG_DB || 'mailflow', + backendContainer: process.env.EVAL_BACKEND_CONTAINER || 'mailflow-backend', + openaiBase: process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1', + openaiModel: process.env.OPENAI_MODEL || 'gpt-4o-mini', + openaiKey: (process.env.OPENAI_API_KEY || readKeyFile()).trim(), + seed: process.env.EVAL_SEED || 'mailflow-eval-2026-07-16', + nKeyword: Number(process.env.EVAL_N_KEYWORD || 15), + nParaphrase: Number(process.env.EVAL_N_PARAPHRASE || 25), + limit: 20, + spacingMs: Number(process.env.EVAL_REQUEST_SPACING_MS || 3300), // stay just under 20/min + llmSpacingMs: Number(process.env.EVAL_LLM_SPACING_MS || 3000), // pace flaky low-tier keys + generateOnly: process.env.EVAL_GENERATE_ONLY === '1', // populate the cache, then exit + skipExplain: process.env.EVAL_SKIP_EXPLAIN === '1', + cacheFile: resolve(EVAL_DIR, 'query-cache.json'), + outFile: process.env.EVAL_OUT || resolve(EVAL_DIR, `results-${new Date().toISOString().slice(0, 10)}.json`), +}; + +const MODES = ['lexical', 'vector', 'hybrid']; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// ── DB sampling (docker exec psql, read-only) ────────────────────────────────────────── +const FSEP = '\x1f'; +function psql(sql) { + const out = execFileSync( + 'docker', + ['exec', CFG.pgContainer, 'psql', '-U', CFG.pgUser, '-d', CFG.pgDb, '-tAF', FSEP, '-c', sql], + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + ); + return out.split('\n').filter((l) => l.length > 0).map((l) => l.split(FSEP)); +} + +function resolveScope() { + const rows = psql( + `SELECT u.id, (SELECT string_agg(id::text, ',') FROM email_accounts WHERE user_id=u.id AND enabled=true) + FROM users u WHERE u.username='${CFG.loginUser.replace(/'/g, "''")}'`, + ); + if (!rows.length) throw new Error(`no user ${CFG.loginUser}`); + const [userId, accts] = rows[0]; + return { userId, accountIds: (accts || '').split(',').filter(Boolean) }; +} + +// Sanitize subject/snippet in SQL so newlines/separators never break TSV row parsing. +const CLEAN = (col) => `regexp_replace(coalesce(${col},''), '\\s+', ' ', 'g')`; + +function sampleMessages({ salt, where, n }) { + const acctList = SCOPE.accountIds.map((a) => `'${a}'`).join(','); + return psql( + `SELECT id, ${CLEAN('subject')}, ${CLEAN('snippet')}, coalesce(from_name,'') + FROM messages m + WHERE m.is_deleted=false AND m.account_id IN (${acctList}) AND ${where} + ORDER BY md5(m.id::text || '${salt.replace(/'/g, "''")}') + LIMIT ${n}`, + ).map(([id, subject, snippet, fromName]) => ({ id, subject, snippet, fromName })); +} + +// ── KEYWORD query derivation ─────────────────────────────────────────────────────────── +const STOP = new Set([ + 'the', 'and', 'for', 'you', 'your', 'with', 'from', 'this', 'that', 'have', 'has', 'was', + 'are', 'will', 'not', 'new', 'can', 'all', 'our', 'out', 'get', 'now', 'about', 'been', + 'account', 'email', 'please', 'update', 'updated', 'notification', 're', 'fwd', 'fw', + 'order', 'invoice', 'payment', 'confirm', 'confirmation', 'reminder', 'receipt', 'alert', + 'security', 'verify', 'verification', 'here', 'more', 'just', 'been', 'they', 'them', + 'default', 'routing', 'transaction', 'needs', 'sent', 'may', 'copy', 'left', 'comment', +]); +function keywordTokens(subject) { + const toks = (subject.toLowerCase().match(/[a-z][a-z0-9'-]{3,}/g) || []) + .map((t) => t.replace(/^[-']+|[-']+$/g, '')) + .filter((t) => t.length >= 4 && !STOP.has(t)); + const uniq = [...new Set(toks)].sort((a, b) => b.length - a.length); + return uniq.slice(0, 2); +} + +function buildKeywordSet() { + const cands = sampleMessages({ + salt: `${CFG.seed}:kw`, + where: `length(coalesce(m.subject,'')) BETWEEN 10 AND 160`, + n: CFG.nKeyword * 4, + }); + const set = []; + for (const m of cands) { + if (set.length >= CFG.nKeyword) break; + const toks = keywordTokens(m.subject); + if (toks.length < 2) continue; + set.push({ id: m.id, set: 'keyword', query: toks.join(' '), subject: m.subject }); + } + return set; +} + +// ── PARAPHRASE query generation (LLM, cached) ────────────────────────────────────────── +async function openaiParaphrase(subject, snippet) { + const body = { + model: CFG.openaiModel, + temperature: 0.3, + max_tokens: 40, + messages: [ + { role: 'system', content: 'You write realistic email-search queries the way a busy person would type them months later from memory.' }, + { role: 'user', content: + `Email subject: ${subject}\nEmail preview: ${snippet}\n\n` + + 'Write ONE short natural-language search query (4-9 words) that a person might type to re-find THIS email later. ' + + 'Describe its topic or purpose in everyday words. Do NOT reuse the distinctive names, brands, product names, codes, or rare keywords from the subject — paraphrase them with common synonyms. ' + + 'Output only the query text, no quotes, no punctuation at the end.' }, + ], + }; + // Low-tier keys rate-limit bursts (429), sometimes even reporting it as a quota error; + // back off (3s,6s,12s,24s) and retry before giving up to the hand-written fallback. + const backoff = [3000, 6000, 12000, 24000]; + for (let attempt = 0; attempt <= backoff.length; attempt++) { + const res = await fetch(`${CFG.openaiBase}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${CFG.openaiKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (res.status === 429 && attempt < backoff.length) { await sleep(backoff[attempt]); continue; } + if (!res.ok) throw new Error(`openai ${res.status}: ${(await res.text()).slice(0, 120)}`); + const j = await res.json(); + return (j.choices?.[0]?.message?.content || '').trim().replace(/^["']|["']$/g, ''); + } + throw new Error('openai 429: retries exhausted'); +} + +// Hand-written fallbacks, only used if the LLM is unavailable AND nothing is cached. +function fallbackParaphrase(subject) { + const t = keywordTokens(subject); + return t.length ? `email about ${t.join(' and ')}` : 'that email i got recently'; +} + +async function buildParaphraseSet(cache) { + const cands = sampleMessages({ + salt: `${CFG.seed}:para`, + where: `length(coalesce(m.subject,'')) BETWEEN 12 AND 160 AND m.snippet IS NOT NULL AND length(m.snippet) >= 40`, + n: CFG.nParaphrase, + }); + const set = []; + let generated = 0; let usedFallback = 0; + for (const m of cands) { + const key = `${m.id}:${PROMPT_VERSION}`; + let query = cache[key]?.query; + if (!query) { + if (CFG.openaiKey) { + try { + query = await openaiParaphrase(m.subject, m.snippet); + generated++; + await sleep(CFG.llmSpacingMs); // pace LLM calls to avoid burst rate-limits on low-tier keys + } catch (err) { + console.warn(` paraphrase LLM failed for ${m.id.slice(0, 8)}: ${err.message}`); + } + } + if (!query) { query = fallbackParaphrase(m.subject); usedFallback++; } + cache[key] = { query, subject: m.subject, source: generated && query !== fallbackParaphrase(m.subject) ? 'llm' : 'fallback' }; + } + set.push({ id: m.id, set: 'paraphrase', query, subject: m.subject }); + } + // Tally the provenance of every query actually used (cache may hold llm/handwritten/fallback). + const sourceCounts = {}; + for (const m of cands) { + const s = cache[`${m.id}:${PROMPT_VERSION}`]?.source || 'unknown'; + sourceCounts[s] = (sourceCounts[s] || 0) + 1; + } + if (generated || usedFallback) console.log(` paraphrases: ${generated} generated, ${usedFallback} fallback, ${set.length - generated - usedFallback} cached`); + console.log(` paraphrase sources: ${JSON.stringify(sourceCounts)}`); + return { set, usedFallback, sourceCounts }; +} + +// ── Live REST search (session + 429-aware throttling) ────────────────────────────────── +let COOKIE = ''; +async function login() { + if (!CFG.loginUser) throw new Error('EVAL_USER is required for a fresh login (no default; local test creds)'); + if (!CFG.loginPass) throw new Error('EVAL_PASS is required for a fresh login (no default; local test creds)'); + const res = await fetch(`${CFG.baseUrl}/api/auth/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, + body: JSON.stringify({ username: CFG.loginUser, password: CFG.loginPass }), + }); + if (!res.ok) throw new Error(`login ${res.status}`); + const set = res.headers.getSetCookie?.() || []; + COOKIE = set.map((c) => c.split(';')[0]).join('; '); + if (!COOKIE) throw new Error('login returned no session cookie'); +} + +async function restSearch(query, mode) { + const url = `${CFG.baseUrl}/api/search/?q=${encodeURIComponent(query)}&mode=${mode}&limit=${CFG.limit}`; + for (let attempt = 0; attempt < 6; attempt++) { + const res = await fetch(url, { headers: { Cookie: COOKIE, 'X-Requested-With': 'XMLHttpRequest' } }); + if (res.status === 429) { + const retry = Number(res.headers.get('retry-after') || 5); + console.log(` 429 rate-limited, sleeping ${retry + 1}s`); + await sleep((retry + 1) * 1000); + continue; + } + if (!res.ok) throw new Error(`search ${res.status} for "${query}" (${mode})`); + return res.json(); + } + throw new Error(`search gave up after retries: "${query}" (${mode})`); +} + +// ── Metrics ──────────────────────────────────────────────────────────────────────────── +function rankOf(ids, targetId) { + const i = ids.indexOf(targetId); + return i < 0 ? null : i + 1; // 1-indexed +} +function jaccard(a, b) { + const sa = new Set(a); const sb = new Set(b); + if (!sa.size && !sb.size) return 1; + let inter = 0; for (const x of sa) if (sb.has(x)) inter++; + return inter / (sa.size + sb.size - inter); +} +function summarize(ranks) { + const n = ranks.length; + const rec = (k) => ranks.filter((r) => r != null && r <= k).length / n; + const mrr = ranks.reduce((s, r) => s + (r != null && r <= 20 ? 1 / r : 0), 0) / n; + const found = ranks.filter((r) => r != null).length; + return { n, recall_at_1: rec(1), recall_at_5: rec(5), recall_at_20: rec(20), mrr_at_20: mrr, found }; +} + +// ── In-container explain diagnostic (single node pass) ───────────────────────────────── +function explainDiagnostic(queries) { + const payload = JSON.stringify(queries.map((q) => ({ id: q.id, q: q.query, set: q.set }))); + const script = ` +import { search } from './src/services/search/searchService.js'; +import { parseQuery } from './src/services/search/queryParser.js'; +const userId = process.env.EVAL_USER_ID; +const queries = JSON.parse(process.env.EVAL_Q); +const out = []; +for (const item of queries) { + const parsed = parseQuery(item.q); + const row = { id: item.id, q: item.q, set: item.set }; + for (const mode of ['hybrid', 'vector']) { + try { + const r = await search({ userId, parsed, mode, limit: 20, explain: true }); + const hits = r.messages || []; + const ids = hits.map((h) => h.id); + const gi = ids.indexOf(item.id); + row[mode] = { + fellBack: !!r.fellBack, mode: r.mode, pool_saturated: !!r.pool_saturated, n: hits.length, + n_bm25: hits.filter((h) => h.score && h.score.bm25 != null).length, + n_vector: hits.filter((h) => h.score && h.score.vector != null).length, + n_both: hits.filter((h) => h.score && h.score.bm25 != null && h.score.vector != null).length, + n_subject_boosted: hits.filter((h) => h.score && h.score.subject_boosted).length, + gt_rank: gi < 0 ? null : gi + 1, + gt_score: gi < 0 ? null : hits[gi].score, + }; + } catch (e) { row[mode] = { error: String(e.message || e) }; } + } + out.push(row); +} +process.stdout.write(JSON.stringify(out)); +`; + const raw = execFileSync( + 'docker', + ['exec', '-e', `EVAL_Q=${payload}`, '-e', `EVAL_USER_ID=${SCOPE.userId}`, + CFG.backendContainer, 'node', '--input-type=module', '-e', script], + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }, + ); + return JSON.parse(raw); +} + +// ── Main ─────────────────────────────────────────────────────────────────────────────── +let SCOPE; +async function main() { + if (!existsSync(EVAL_DIR)) mkdirSync(EVAL_DIR, { recursive: true }); + SCOPE = resolveScope(); + console.log(`user=${SCOPE.userId.slice(0, 8)} accounts=${SCOPE.accountIds.length}`); + + const cache = existsSync(CFG.cacheFile) ? JSON.parse(readFileSync(CFG.cacheFile, 'utf8')) : {}; + + console.log('building keyword set...'); + const keywordSet = buildKeywordSet(); + console.log(`building paraphrase set (LLM=${CFG.openaiKey ? 'on' : 'off'})...`); + const { set: paraphraseSet, usedFallback, sourceCounts } = await buildParaphraseSet(cache); + // cacheFile holds paraphrases of real mail — gitignored, regenerated on demand; never committed. + writeFileSync(CFG.cacheFile, JSON.stringify(cache, null, 2)); + if (CFG.generateOnly) { + console.log(`generate-only: cache written (${paraphraseSet.length} paraphrases, ${usedFallback} still fallback). Exiting.`); + return; + } + + const queries = [...keywordSet, ...paraphraseSet]; + console.log(`total queries: ${queries.length} (${keywordSet.length} keyword, ${paraphraseSet.length} paraphrase)`); + + console.log('logging in...'); + await login(); + + // Run every query in every mode against the live REST API. + const perQuery = []; + let done = 0; + for (const q of queries) { + const rec = { id: q.id, set: q.set, query: q.query, subject: q.subject, byMode: {} }; + for (const mode of MODES) { + const r = await restSearch(q.query, mode); + const ids = (r.messages || []).map((m) => m.id); + rec.byMode[mode] = { ids, rank: rankOf(ids, q.id), fellBack: !!r.fellBack, total: r.total }; + await sleep(CFG.spacingMs); + } + perQuery.push(rec); + done++; + if (done % 5 === 0) console.log(` ${done}/${queries.length} queries searched`); + } + + // Explain diagnostics (in-container seam). + let explain = []; + if (!CFG.skipExplain) { + console.log('collecting hybrid/vector explain scores (in-container)...'); + try { explain = explainDiagnostic(queries); } + catch (e) { console.warn(` explain diagnostic failed: ${e.message}`); } + } + const explainById = Object.fromEntries(explain.map((e) => [`${e.set}:${e.id}`, e])); + + // Aggregate metrics. + const sets = ['keyword', 'paraphrase', 'overall']; + const metrics = {}; + for (const s of sets) { + metrics[s] = {}; + const rows = perQuery.filter((r) => s === 'overall' || r.set === s); + for (const mode of MODES) metrics[s][mode] = summarize(rows.map((r) => r.byMode[mode].rank)); + } + + // Cross-mode overlap (how identical are the result sets?). + const overlap = {}; + for (const s of ['keyword', 'paraphrase', 'overall']) { + const rows = perQuery.filter((r) => s === 'overall' || r.set === s); + const pairMean = (a, b) => rows.reduce((acc, r) => acc + jaccard(r.byMode[a].ids, r.byMode[b].ids), 0) / rows.length; + const identicalTop20 = (a, b) => rows.filter((r) => jaccard(r.byMode[a].ids, r.byMode[b].ids) === 1).length / rows.length; + overlap[s] = { + jaccard_hybrid_vector: pairMean('hybrid', 'vector'), + jaccard_hybrid_lexical: pairMean('hybrid', 'lexical'), + jaccard_vector_lexical: pairMean('vector', 'lexical'), + identical_top20_hybrid_vector: identicalTop20('hybrid', 'vector'), + identical_top20_hybrid_lexical: identicalTop20('hybrid', 'lexical'), + }; + } + + // Explain composition rollup (per set): how often did the BM25 leg actually contribute? + const explainRollup = {}; + for (const s of ['keyword', 'paraphrase']) { + const rows = explain.filter((e) => e.set === s && e.hybrid && !e.hybrid.error); + if (!rows.length) continue; + const avg = (f) => rows.reduce((a, e) => a + f(e), 0) / rows.length; + explainRollup[s] = { + n: rows.length, + queries_with_any_bm25_hit: rows.filter((e) => e.hybrid.n_bm25 > 0).length, + avg_hybrid_bm25_hits: avg((e) => e.hybrid.n_bm25), + avg_hybrid_vector_hits: avg((e) => e.hybrid.n_vector), + avg_hybrid_subject_boosted: avg((e) => e.hybrid.n_subject_boosted), + queries_pool_saturated: rows.filter((e) => e.hybrid.pool_saturated).length, + }; + } + + // ── Print human-readable tables ── + const pct = (x) => (x * 100).toFixed(1).padStart(5); + const num = (x) => x.toFixed(3); + console.log('\n================ RESULTS ================'); + for (const s of sets) { + console.log(`\n[${s.toUpperCase()}] (n=${metrics[s].lexical.n})`); + console.log(' mode R@1 R@5 R@20 MRR@20 found'); + for (const mode of MODES) { + const m = metrics[s][mode]; + console.log(` ${mode.padEnd(7)} ${pct(m.recall_at_1)}% ${pct(m.recall_at_5)}% ${pct(m.recall_at_20)}% ${num(m.mrr_at_20)} ${m.found}/${m.n}`); + } + } + console.log('\n[OVERLAP] mean Jaccard@20 of result sets'); + for (const s of ['keyword', 'paraphrase']) { + const o = overlap[s]; + console.log(` ${s.padEnd(11)} hybrid~vector=${num(o.jaccard_hybrid_vector)} hybrid~lexical=${num(o.jaccard_hybrid_lexical)} vector~lexical=${num(o.jaccard_vector_lexical)} (identical hybrid==vector top20: ${pct(o.identical_top20_hybrid_vector)}%)`); + } + console.log('\n[EXPLAIN] hybrid BM25-leg contribution'); + for (const s of ['keyword', 'paraphrase']) { + const e = explainRollup[s]; if (!e) continue; + console.log(` ${s.padEnd(11)} queries with >=1 BM25 hit: ${e.queries_with_any_bm25_hit}/${e.n} avg BM25 hits=${num(e.avg_hybrid_bm25_hits)} avg vector hits=${num(e.avg_hybrid_vector_hits)} avg subject-boosted=${num(e.avg_hybrid_subject_boosted)}`); + } + + // ── Write results JSON (NO secrets) ── + const results = { + generated_at: new Date().toISOString(), + config: { + baseUrl: CFG.baseUrl, seed: CFG.seed, limit: CFG.limit, + openaiModel: CFG.openaiKey ? CFG.openaiModel : null, + n_keyword: keywordSet.length, n_paraphrase: paraphraseSet.length, + paraphrase_fallback_used: usedFallback, + paraphrase_sources: sourceCounts, + }, + corpus: { messages: Number(psql('SELECT count(*) FROM messages')[0][0]), note: 'bodies mostly NULL; embeddings ~subject-only' }, + metrics, overlap, explainRollup, + perQuery: perQuery.map((r) => ({ + id: r.id, set: r.set, query: r.query, subject: r.subject, + rank: { lexical: r.byMode.lexical.rank, vector: r.byMode.vector.rank, hybrid: r.byMode.hybrid.rank }, + fellBack: { vector: r.byMode.vector.fellBack, hybrid: r.byMode.hybrid.fellBack }, + explain: explainById[`${r.set}:${r.id}`] || null, + })), + }; + writeFileSync(CFG.outFile, JSON.stringify(results, null, 2)); + console.log(`\nwrote ${CFG.outFile}`); + console.log(`wrote ${CFG.cacheFile}`); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/backend/scripts/vector-probe.js b/backend/scripts/vector-probe.js new file mode 100644 index 00000000..763bfd4d --- /dev/null +++ b/backend/scripts/vector-probe.js @@ -0,0 +1,38 @@ +// Dev probe: bring up the vector schema, insert N random D-dim vectors under a throwaway +// generation, then run an ANN query and print the nearest neighbors with scores. +// Usage: node scripts/vector-probe.js [N=100] [D=8] +// Requires DB_* env pointing at a pgvector-enabled Postgres. +import { randomUUID } from 'crypto'; +import { pool } from '../src/services/db.js'; +import { ensureVectorSchema, ensureVectorIndex, upsert, annSearch } from '../src/services/embeddings/vectorStore.js'; +import { createGeneration } from '../src/services/embeddings/generations.js'; +import { seedAccount, cleanupAccount } from '../src/services/embeddings/testSupport.js'; + +const N = Number(process.argv[2]) || 100; +const D = Number(process.argv[3]) || 8; + +function randVec(d) { return Array.from({ length: d }, () => Math.random() * 2 - 1); } + +const { vectorAvailable } = await ensureVectorSchema(); +if (!vectorAvailable) { console.error('vector unavailable — is this a pgvector image?'); process.exit(1); } +await ensureVectorIndex(D); + +const gen = await createGeneration('probe-model', D, `probe:${randomUUID()}`); +// Seed N messages (probe rows) so the ANN liveness EXISTS matches. Uses a throwaway account. +const { accountId: acctId, userId } = await seedAccount(pool, 'probe'); +const chunks = []; +for (let i = 0; i < N; i++) { + const m = await pool.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1,$2,'INBOX',$3) RETURNING id`, + [acctId, 900000 + i, `probe ${i}`]); + chunks.push({ messageId: m.rows[0].id, chunkIndex: 0, vector: randVec(D), sourceCharLen: 8, chunkCharStart: 0, chunkCharEnd: 8, truncated: false }); +} +await upsert(gen, chunks); + +const q = randVec(D); +console.log(`query = [${q.map((x) => x.toFixed(3)).join(', ')}]`); +const hits = await annSearch(gen, q, 5); +for (const h of hits) console.log(` rank ${h.rank} msg ${h.messageId} score ${h.score.toFixed(4)}`); + +await cleanupAccount(pool, userId); +await pool.end(); +process.exit(0); diff --git a/backend/src/index.js b/backend/src/index.js index 8ba433a4..fbf24a36 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -11,13 +11,14 @@ import { redisClient } from './services/redis.js'; import sendRoutes from './routes/send.js'; import draftRoutes from './routes/draft.js'; -import oauthRoutes from './routes/oauth.js'; +import oauthRoutes, { refreshMicrosoftToken } from './routes/oauth.js'; import integrationsRoutes, { loadIntegrationConfigs } from './routes/integrations.js'; import authRoutes from './routes/auth.js'; import accountRoutes from './routes/accounts.js'; import mailRoutes from './routes/mail.js'; import searchRoutes from './routes/search.js'; import adminRoutes from './routes/admin.js'; +import indexingRoutes from './routes/indexing.js'; import totpRoutes from './routes/totp.js'; import oidcApiRouter, { oidcBrowserRouter } from './routes/oidc.js'; import rulesRoutes from './routes/rules.js'; @@ -25,19 +26,38 @@ import blockListRoutes from './routes/blockList.js'; import contactsRoutes from './routes/contacts.js'; import todoistRoutes from './routes/todoist.js'; import aiRoutes from './routes/ai.js'; +import aiEmbeddingsRoutes from './routes/aiEmbeddings.js'; import categoriesRoutes from './routes/categories.js'; import gtdRoutes from './routes/gtd.js'; import senderFaviconsRoutes from './routes/senderFavicons.js'; import carddavRouter from './routes/carddav.js'; import carddavAccountRouter from './routes/carddavAccount.js'; +import { + createMcpRuntimeDependencies, + entityTooLargeResponse, + mcpBodyLimit, + mountMcp, +} from './mcp/server.js'; +import apiTokensRoutes from './routes/apiTokens.js'; +import mcpDeletionsRoutes from './routes/mcpDeletions.js'; +import mcpAccountStagesRoutes from './routes/mcpAccountStages.js'; +import { createComposeSessionsRouter } from './routes/composeSessions.js'; import { startCardavScheduler } from './services/carddavSync.js'; -import { encryptExistingCredentials, query } from './services/db.js'; -import { runMigrations } from './services/migrations.js'; +import { scheduleFtsBackfill } from './services/search/ftsBackfill.js'; +import { encryptExistingCredentials, query, withTransaction } from './services/db.js'; +import { runMigrations, warnOnCollationMismatch } from './services/migrations.js'; +import { ensureVectorSchema } from './services/embeddings/vectorStore.js'; +import { startEmbeddingScheduler } from './services/embeddings/scheduler.js'; import { parseVCard } from './utils/vcard.js'; import { reloadAuthSettings } from './services/authLimiter.js'; import { setupWebSocket } from './services/websocket.js'; import { ImapManager } from './services/imapManager.js'; import { getUpdateStatus } from './services/updateCheck.js'; +import * as sendService from './services/sendService.js'; +import * as outboxService from './services/outboxService.js'; +import * as draftService from './services/draftService.js'; +import * as composeSessionService from './services/composeSessionService.js'; +import * as composeSessionLifecycle from './services/composeSessionLifecycle.js'; const packageMeta = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf-8')); let buildMeta = {}; @@ -113,15 +133,20 @@ app.use((req, res, next) => { // 25 MB attachment limit → ~34 MB base64 on the wire; add headroom for the rest of the payload. app.use('/api/mail/send', express.json({ limit: '35mb' })); app.use('/api/mail/draft', express.json({ limit: '35mb' })); +app.use( + '/api/compose-sessions/:id/attachments', + express.raw({ type: 'application/octet-stream', limit: '25mb' }), +); +app.use('/api/compose-sessions', express.json({ limit: '35mb' })); +app.use('/mcp', mcpBodyLimit()); // A pet-import body carries a base64 spritesheet (~33% larger than the 5 MB sheet cap // enforced after decode in gtdPet.importPet), so it needs more than the global 1 MB. app.use('/api/gtd/pet/import', express.json({ limit: '8mb' })); app.use(express.json({ limit: '1mb' })); // Return a clean JSON error when the body parser rejects an oversized payload. app.use((err, req, res, next) => { - if (err.type === 'entity.too.large') { - return res.status(413).json({ error: 'Request too large. Total attachment size must not exceed 25 MB.' }); - } + const response = entityTooLargeResponse(err, req.path); + if (response) return res.status(response.status).json(response.body); next(err); }); app.use(sessionMiddleware); @@ -168,6 +193,7 @@ app.use('/api/mail', sendRoutes); app.use('/api/mail', draftRoutes); app.use('/api/search', searchRoutes); app.use('/api/admin', adminRoutes); +app.use('/api/admin/indexing', indexingRoutes); app.use('/api/totp', totpRoutes); app.use('/api/rules', rulesRoutes); app.use('/api/block-list', blockListRoutes); @@ -175,6 +201,15 @@ app.use('/api/contacts', contactsRoutes); app.use('/api/todoist', todoistRoutes); app.use('/api/carddav', carddavAccountRouter); app.use('/api', aiRoutes); +app.use('/api', aiEmbeddingsRoutes); +app.use('/api/tokens', apiTokensRoutes); +app.use('/api/mcp-deletions', mcpDeletionsRoutes); +app.use('/api/mcp-account-stages', mcpAccountStagesRoutes); +app.use('/api/compose-sessions', createComposeSessionsRouter({ + query, + withTransaction, + broadcast: (event, userId) => imapManager.broadcast(event, userId), +})); app.use('/api', categoriesRoutes); // Mounted at the /api/gtd subtree (not bare /api) so gtd.js's router-level // requireAuth cannot intercept the unauthenticated /api/health and /api/version @@ -187,6 +222,24 @@ app.use('/carddav', carddavRouter); // RFC 6764 well-known redirect — handle all methods so PROPFIND probes also redirect app.all('/.well-known/carddav', (req, res) => res.redirect(308, '/carddav/')); +// MCP Streamable-HTTP endpoint. Bearer-authenticated (mcp/auth.js), intentionally +// outside the /api CSRF gate and session middleware — auth is a token, not a cookie. +mountMcp(app, { + ...createMcpRuntimeDependencies({ + imapManager, + refreshMicrosoftToken, + redisClient, + sendService, + outboxService, + draftService, + composeSessionService, + composeSessionLifecycle, + query, + withTransaction, + broadcast: (event, userId) => imapManager.broadcast(event, userId), + }), +}); + app.get('/api/health', (req, res) => res.json({ status: 'ok' })); app.get('/api/version', (_req, res) => res.json({ version: APP_VERSION, sha: process.env.BUILD_SHA || 'dev' })); // Server-side update check (#261). Cached in updateCheck.js so repeated hits never @@ -212,6 +265,11 @@ setupWebSocket(wss, sessionMiddleware, imapManager); // Run pending schema migrations then start await runMigrations(); +// Loud, best-effort drift check: a Postgres image swap (e.g. postgres:16-alpine → +// pgvector/pgvector:pg16) changes the libc collation and silently corrupts text-index +// ordering until a REINDEX. Logs the remedy; never blocks the boot. +await warnOnCollationMismatch(); + // One-time backfill: populate photo_data from existing vcard column for contacts // that were synced before CardDAV PUT started persisting photo_data. async function backfillContactPhotos() { @@ -240,12 +298,26 @@ await encryptExistingCredentials(); // Load OAuth integration configs from DB into process.env await loadIntegrationConfigs(); +// Best-effort vector schema bring-up. On stock postgres:16-alpine this logs +// "Vector disabled" and semantic search stays off; lexical is unaffected. +await ensureVectorSchema(); + +// Periodic embedding nudge — drives any building/active generation toward coverage. +startEmbeddingScheduler(); + // Start background snooze watcher — polls every 60 seconds to restore snoozed messages imapManager.startSnoozeWatcher(); +outboxService.startOutboxWorker( + { imapManager, refreshMicrosoftToken, redisClient, query }, + { tickMs: 5000 }, +); // Schedule periodic CardDAV contact sync for any connected accounts. startCardavScheduler(); +// Postgres-only drainer: populate search_fts for pre-existing rows (no IMAP). +scheduleFtsBackfill(); + // Re-connect all enabled IMAP accounts on startup with bounded concurrency so a // large user base doesn't hammer IMAP servers and the DB connection pool at once. try { diff --git a/backend/src/mcp/accountAdapter.js b/backend/src/mcp/accountAdapter.js new file mode 100644 index 00000000..870fa4bd --- /dev/null +++ b/backend/src/mcp/accountAdapter.js @@ -0,0 +1,493 @@ +// Account/alias/compose/outbox SQL seam for MCP write tools. Callers pass account +// ids or a user id already resolved from the bearer token; every lookup keeps +// that boundary in SQL. +import { query } from '../services/db.js'; +import { safeAccount, SAFE_FIELDS } from '../services/accountFields.js'; +import { sanitizeSignature } from '../services/emailSanitizer.js'; +import { + DEFAULT_GTD_FOLDERS, + findGtdFolderCollisions, + invalidateGtdConfigCache, + sanitizeGtdFoldersDetailed, +} from '../services/gtdConfig.js'; +import { invalidateOwnerAddressesCache } from '../services/gtdTransitions.js'; +import { applyInboxRules, toRuleMessage } from '../services/inboxRules.js'; +import { DETAIL_COLUMNS } from './engineAdapter.js'; + +const COMPOSE_SOURCE_COLUMNS = DETAIL_COLUMNS + + ', m.uid, m.reply_to, m.in_reply_to, m.thread_references'; + +export async function getAccountRow(accountId, accountIds) { + if (!accountIds?.includes(accountId)) return null; + const { rows } = await query( + 'SELECT * FROM email_accounts WHERE id = $1 AND id = ANY($2)', + [accountId, accountIds], + ); + return rows[0] || null; +} + +export async function getAccountByEmail(email, accountIds) { + if (!accountIds?.length) return { error: `account_not_found: ${email}` }; + const { rows } = await query( + 'SELECT * FROM email_accounts WHERE email_address = $1 AND id = ANY($2)', + [email, accountIds], + ); + return rows[0] || { error: `account_not_found: ${email}` }; +} + +export async function listAliases(accountId) { + const { rows } = await query( + 'SELECT * FROM account_aliases WHERE account_id = $1 ORDER BY created_at', + [accountId], + ); + return rows; +} + +export async function resolveAlias(accountId, aliasEmail) { + // Keep the same predicate as mail/identity.js intentionally: this adapter lets + // MCP tools preflight an email selector as row-or-null, while identity.js must + // still hard-fail aliasId/aliasEmail selectors for existing REST service callers. + const { rows } = await query( + 'SELECT * FROM account_aliases WHERE account_id = $1 AND LOWER(email) = LOWER($2) LIMIT 1', + [accountId, aliasEmail], + ); + return rows[0] || null; +} + +export async function getComposeSource(messageId, accountIds) { + if (!accountIds?.length) return null; + const { rows } = await query( + `SELECT ${COMPOSE_SOURCE_COLUMNS} + FROM messages m + WHERE m.id = $1 AND m.account_id = ANY($2)`, + [messageId, accountIds], + ); + return rows[0] || null; +} + +export async function getOutboxRowByMessageId(messageId, userId) { + const { rows } = await query( + `SELECT * + FROM outbox_messages + WHERE message_id = $1 AND user_id = $2 AND status = 'pending' + LIMIT 1`, + [messageId, userId], + ); + return rows[0] || null; +} + +export async function deleteMessageRow(accountId, uid, folder) { + const result = await query( + 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', + [accountId, uid, folder], + ); + return result.rowCount; +} + +export async function listDraftRows(accountId, { limit, offset, folder } = {}) { + const params = [accountId]; + const where = ['account_id = $1', 'is_deleted = false']; + if (folder) { + params.push(folder); + where.push(`folder = $${params.length}`); + } + params.push(limit, offset); + const limitParam = `$${params.length - 1}`; + const offsetParam = `$${params.length}`; + const { rows } = await query( + `SELECT * FROM messages + WHERE ${where.join(' AND ')} + ORDER BY date DESC NULLS LAST + LIMIT ${limitParam} OFFSET ${offsetParam}`, + params, + ); + return rows; +} + +export async function getDraftRow(accountId, folder, uid) { + const { rows } = await query( + `SELECT * FROM messages + WHERE account_id = $1 AND folder = $2 AND uid = $3 AND is_deleted = false + LIMIT 1`, + [accountId, folder, uid], + ); + return rows[0] || null; +} + +export async function getUserPreferences(userId) { + const { rows } = await query( + 'SELECT preferences FROM users WHERE id = $1', + [userId], + ); + return rows[0]?.preferences || {}; +} + +export async function listAccountsSafe(accountIds) { + if (!accountIds?.length) return []; + const { rows } = await query( + `SELECT ${SAFE_FIELDS.join(', ')} + FROM email_accounts + WHERE id = ANY($1) + ORDER BY sort_order, created_at`, + [accountIds], + ); + + const aliasesByAccount = new Map(); + if (rows.length) { + const aliases = await query( + `SELECT id, account_id, name, email, reply_to, signature, created_at + FROM account_aliases WHERE account_id = ANY($1) ORDER BY created_at`, + [rows.map((account) => account.id)], + ); + for (const alias of aliases.rows) { + if (!aliasesByAccount.has(alias.account_id)) aliasesByAccount.set(alias.account_id, []); + aliasesByAccount.get(alias.account_id).push({ + ...alias, + signature: alias.signature ? sanitizeSignature(alias.signature) : alias.signature, + }); + } + } + + return rows.map((row) => ({ + ...safeAccount(row), + aliases: aliasesByAccount.get(row.id) || [], + })); +} + +const MCP_ACCOUNT_UPDATE_FIELDS = [ + 'name', + 'sender_name', + 'color', + 'sort_order', + 'folder_mappings', + 'signature', + 'categorization_enabled', + 'gtd_enabled', + 'gtd_folders', + 'enabled', +]; + +export async function updateAccountSettings({ accountId, accountIds, updates }) { + if (!accountIds?.includes(accountId)) return { error: 'Account not found', status: 404 }; + const check = await query( + 'SELECT id, gtd_folders FROM email_accounts WHERE id = $1 AND id = ANY($2)', + [accountId, accountIds], + ); + if (!check.rows.length) return { error: 'Account not found', status: 404 }; + + let gtdFoldersValue; + let gtdRejected = []; + let gtdFoldersChanged = false; + if ('gtd_folders' in updates) { + const { folders, rejected, reserved } = sanitizeGtdFoldersDetailed(updates.gtd_folders); + if (reserved.length) { + return { + error: 'A GTD state cannot map to a reserved system folder', + reserved, + status: 400, + }; + } + const collisions = findGtdFolderCollisions({ + ...DEFAULT_GTD_FOLDERS, + ...folders, + }); + if (collisions.length) { + return { + error: 'Two GTD states cannot map to the same folder', + collisions, + status: 400, + }; + } + gtdFoldersValue = folders; + gtdRejected = rejected; + const before = sanitizeGtdFoldersDetailed(check.rows[0].gtd_folders).folders; + gtdFoldersChanged = JSON.stringify(before) !== JSON.stringify(folders); + } + + const sets = []; + const values = []; + for (const key of MCP_ACCOUNT_UPDATE_FIELDS) { + if (!(key in updates)) continue; + sets.push(`${key} = $${values.length + 1}`); + const value = key === 'signature' + ? sanitizeSignature(updates[key]) || null + : key === 'gtd_enabled' + ? !!updates[key] + : key === 'gtd_folders' + ? gtdFoldersValue + : updates[key]; + values.push(value); + } + if (!sets.length) return { error: 'No valid fields to update', status: 400 }; + + const idParam = values.length + 1; + const scopeParam = values.length + 2; + values.push(accountId, accountIds); + const result = await query( + `UPDATE email_accounts SET ${sets.join(', ')} + WHERE id = $${idParam} AND id = ANY($${scopeParam}) RETURNING *`, + values, + ); + if (!result.rows.length) return { error: 'Account not found', status: 404 }; + + const updated = result.rows[0]; + const account = safeAccount(updated); + if ('gtd_folders' in updates) account.gtd_folders_rejected = gtdRejected; + if ('gtd_enabled' in updates || 'gtd_folders' in updates) { + invalidateGtdConfigCache(accountId); + } + if ('enabled' in updates || 'gtd_enabled' in updates || 'gtd_folders' in updates) { + const { reconcileConnectionState } = await import('../services/accountService.js'); + reconcileConnectionState({ + id: accountId, + updates, + before: { gtdFoldersChanged }, + updated, + }); + } + return { account }; +} + +async function accountOwnedByUser(accountId, accountIds, userId) { + if (!accountIds?.includes(accountId)) return false; + const { rows } = await query( + 'SELECT id FROM email_accounts WHERE id = $1 AND user_id = $2 AND id = ANY($3)', + [accountId, userId, accountIds], + ); + return rows.length > 0; +} + +export async function createAlias({ + accountId, + accountIds, + userId, + fields, +}) { + if (!(await accountOwnedByUser(accountId, accountIds, userId))) return null; + const result = await query( + `INSERT INTO account_aliases (account_id, name, email, reply_to, signature) + VALUES ($1, $2, $3, $4, $5) RETURNING *`, + [ + accountId, + fields.name, + fields.email, + fields.reply_to || null, + sanitizeSignature(fields.signature) || null, + ], + ); + invalidateOwnerAddressesCache(accountId); + return result.rows[0]; +} + +async function ownedAlias(accountId, accountIds, userId, aliasId) { + if (!accountIds?.includes(accountId)) return null; + const { rows } = await query( + `SELECT a.id, a.account_id FROM account_aliases a + JOIN email_accounts e ON a.account_id = e.id + WHERE a.id = $1 AND e.user_id = $2 AND e.id = $3 AND e.id = ANY($4)`, + [aliasId, userId, accountId, accountIds], + ); + return rows[0] || null; +} + +export async function updateAlias({ + accountId, + accountIds, + userId, + aliasId, + fields, +}) { + const owned = await ownedAlias(accountId, accountIds, userId, aliasId); + if (!owned) return null; + const result = await query( + `UPDATE account_aliases + SET name = $1, email = $2, reply_to = $3, signature = $4 + WHERE id = $5 RETURNING *`, + [ + fields.name, + fields.email, + fields.reply_to || null, + sanitizeSignature(fields.signature) || null, + aliasId, + ], + ); + invalidateOwnerAddressesCache(accountId); + return result.rows[0] || null; +} + +export async function deleteAlias({ + accountId, + accountIds, + userId, + aliasId, +}) { + const owned = await ownedAlias(accountId, accountIds, userId, aliasId); + if (!owned) return false; + await query('DELETE FROM account_aliases WHERE id = $1', [aliasId]); + invalidateOwnerAddressesCache(accountId); + return true; +} + +export async function listRules({ userId, accountId }) { + const params = [userId]; + const accountFilter = accountId + ? ' AND (account_id IS NULL OR account_id = $2)' + : ''; + if (accountId) params.push(accountId); + const { rows } = await query( + `SELECT * FROM inbox_rules WHERE user_id = $1${accountFilter} + ORDER BY priority, created_at`, + params, + ); + return rows; +} + +async function moveDestinationError(accountId, actions) { + const moveAction = actions.find( + (action) => action.type === 'move' && action.value?.trim(), + ); + if (!moveAction || !accountId) return null; + const { rows } = await query( + `SELECT COUNT(*) AS total, COUNT(*) FILTER (WHERE path = $2) AS match + FROM folders WHERE account_id = $1`, + [accountId, moveAction.value.trim()], + ); + const { total, match } = rows[0]; + if (parseInt(total, 10) > 0 && parseInt(match, 10) === 0) { + return 'Move destination folder not found for this account'; + } + return null; +} + +export async function createRule({ + userId, + accountId, + name, + conditionLogic, + conditions, + actions, + enabled, + stopProcessing, +}) { + const folderError = await moveDestinationError(accountId, actions); + if (folderError) return { error: folderError, status: 400 }; + const countResult = await query( + 'SELECT COUNT(*) AS cnt FROM inbox_rules WHERE user_id = $1', + [userId], + ); + const priority = parseInt(countResult.rows[0].cnt, 10); + const result = await query( + `INSERT INTO inbox_rules + (user_id, account_id, name, enabled, stop_processing, priority, condition_logic, conditions, actions) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + RETURNING *`, + [ + userId, + accountId || null, + name || '', + enabled !== false, + !!stopProcessing, + priority, + conditionLogic === 'OR' ? 'OR' : 'AND', + JSON.stringify(conditions), + JSON.stringify(actions), + ], + ); + return result.rows[0]; +} + +export async function updateRule({ + userId, + ruleId, + accountId, + name, + conditionLogic, + conditions, + actions, + enabled, + stopProcessing, +}) { + const folderError = await moveDestinationError(accountId, actions); + if (folderError) return { error: folderError, status: 400 }; + const result = await query( + `UPDATE inbox_rules + SET name = $1, account_id = $2, enabled = $3, stop_processing = $4, + condition_logic = $5, conditions = $6, actions = $7, updated_at = NOW() + WHERE id = $8 AND user_id = $9 + RETURNING *`, + [ + name || '', + accountId || null, + enabled !== false, + !!stopProcessing, + conditionLogic === 'OR' ? 'OR' : 'AND', + JSON.stringify(conditions), + JSON.stringify(actions), + ruleId, + userId, + ], + ); + return result.rows[0] || null; +} + +export async function deleteRule({ userId, ruleId }) { + const result = await query( + 'DELETE FROM inbox_rules WHERE id = $1 AND user_id = $2 RETURNING id', + [ruleId, userId], + ); + return result.rows.length > 0; +} + +export async function runRules({ userId, accountIds, imapManager }) { + let processed = 0; + let matched = 0; + for (const accountId of accountIds || []) { + try { + const rulesCheck = await query( + `SELECT COUNT(*) AS cnt FROM inbox_rules + WHERE user_id = $1 AND enabled = true + AND (account_id IS NULL OR account_id = $2)`, + [userId, accountId], + ); + if (parseInt(rulesCheck.rows[0].cnt, 10) === 0) continue; + + const accountResult = await query( + 'SELECT * FROM email_accounts WHERE id = $1 AND id = ANY($2)', + [accountId, accountIds], + ); + const account = accountResult.rows[0]; + if (!account) continue; + + const batchSize = 500; + let lastId = null; + while (true) { + const messageResult = await query( + `SELECT id, uid, folder, from_email, from_name, to_addresses, + subject, has_attachments, is_read + FROM messages + WHERE account_id = $1 AND lower(folder) = 'inbox' + ${lastId ? 'AND id > $3' : ''} + ORDER BY id + LIMIT $2`, + lastId + ? [accountId, batchSize, lastId] + : [accountId, batchSize], + ); + if (!messageResult.rows.length) break; + lastId = messageResult.rows.at(-1).id; + const messages = messageResult.rows.map(toRuleMessage); + const { remaining } = await applyInboxRules( + messages, + account, + imapManager, + ); + processed += messages.length; + matched += messages.length - remaining.length; + if (messageResult.rows.length < batchSize) break; + } + } catch (error) { + console.error(`MCP run_rules error for account ${accountId}:`, error.message); + } + } + return { processed, matched }; +} diff --git a/backend/src/mcp/accountAdapter.phase4.test.js b/backend/src/mcp/accountAdapter.phase4.test.js new file mode 100644 index 00000000..360d421e --- /dev/null +++ b/backend/src/mcp/accountAdapter.phase4.test.js @@ -0,0 +1,338 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { SAFE_FIELDS } from '../services/accountFields.js'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('../services/accountService.js', () => ({ + reconcileConnectionState: vi.fn(), +})); +vi.mock('../services/gtdConfig.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + invalidateGtdConfigCache: vi.fn(), + }; +}); +vi.mock('../services/gtdTransitions.js', () => ({ + invalidateOwnerAddressesCache: vi.fn(), +})); +vi.mock('../services/inboxRules.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + applyInboxRules: vi.fn(), + toRuleMessage: vi.fn(actual.toRuleMessage), + }; +}); + +import { query } from '../services/db.js'; +import { reconcileConnectionState } from '../services/accountService.js'; +import { invalidateGtdConfigCache } from '../services/gtdConfig.js'; +import { invalidateOwnerAddressesCache } from '../services/gtdTransitions.js'; +import { applyInboxRules } from '../services/inboxRules.js'; +import * as accountAdapter from './accountAdapter.js'; + +const ACCOUNT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const SECOND_ACCOUNT_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; +const accountIds = [ACCOUNT_ID, SECOND_ACCOUNT_ID]; + +function exported(name) { + expect(accountAdapter[name], `${name} must be exported`).toBeTypeOf('function'); + return accountAdapter[name]; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('listAccountsSafe', () => { + it('passes an exact SAFE_FIELDS SQL column list and attaches aliases in one query', async () => { + const account = Object.fromEntries(SAFE_FIELDS.map((field) => [field, null])); + account.id = ACCOUNT_ID; + account.email_address = 'owner@example.com'; + query + .mockResolvedValueOnce({ rows: [account] }) + .mockResolvedValueOnce({ + rows: [{ + id: 'alias-1', + account_id: ACCOUNT_ID, + name: 'Alias', + email: 'alias@example.com', + signature: 'safe', + }], + }); + + const result = await exported('listAccountsSafe')([ACCOUNT_ID]); + + expect(result).toEqual([expect.objectContaining({ + id: ACCOUNT_ID, + email_address: 'owner@example.com', + aliases: [expect.objectContaining({ + id: 'alias-1', + signature: 'safe', + })], + })]); + const accountSql = query.mock.calls[0][0]; + const columns = accountSql + .slice(accountSql.indexOf('SELECT') + 6, accountSql.indexOf('FROM email_accounts')) + .split(',') + .map((column) => column.trim()) + .filter(Boolean); + expect(columns).toEqual(SAFE_FIELDS); + expect(accountSql).not.toMatch(/\bauth_pass\b|\boauth_access_token\b|\boauth_refresh_token\b/); + expect(query.mock.calls[1][0]).toMatch( + /SELECT id, account_id, name, email, reply_to, signature, created_at[\s\S]*account_id = ANY\(\$1\)/, + ); + expect(query.mock.calls[1][1]).toEqual([[ACCOUNT_ID]]); + }); + + it('does not query for an empty account scope', async () => { + await expect(exported('listAccountsSafe')([])).resolves.toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('updateAccountSettings', () => { + it('rejects reserved and colliding GTD folder mappings before UPDATE', async () => { + query.mockResolvedValueOnce({ rows: [{ id: ACCOUNT_ID, gtd_folders: {} }] }); + const reserved = await exported('updateAccountSettings')({ + accountId: ACCOUNT_ID, + accountIds, + updates: { gtd_folders: { todo: 'INBOX' } }, + }); + expect(reserved).toEqual({ + error: 'A GTD state cannot map to a reserved system folder', + reserved: ['todo'], + status: 400, + }); + expect(query).toHaveBeenCalledTimes(1); + + query.mockReset(); + query.mockResolvedValueOnce({ rows: [{ id: ACCOUNT_ID, gtd_folders: {} }] }); + const collision = await exported('updateAccountSettings')({ + accountId: ACCOUNT_ID, + accountIds, + updates: { gtd_folders: { todo: 'Work', watch: 'Work' } }, + }); + expect(collision.error).toBe('Two GTD states cannot map to the same folder'); + expect(collision.collisions).toEqual([{ folder: 'Work', states: ['todo', 'watch'] }]); + expect(query).toHaveBeenCalledTimes(1); + }); + + it('returns a safe row, reports rejected folders, invalidates GTD, and reconciles once', async () => { + const updated = Object.fromEntries(SAFE_FIELDS.map((field) => [field, null])); + updated.id = ACCOUNT_ID; + updated.protocol = 'imap'; + updated.enabled = true; + updated.gtd_enabled = true; + query + .mockResolvedValueOnce({ rows: [{ id: ACCOUNT_ID, gtd_folders: {} }] }) + .mockResolvedValueOnce({ rows: [updated] }); + + const result = await exported('updateAccountSettings')({ + accountId: ACCOUNT_ID, + accountIds, + updates: { + name: 'Updated', + signature: 'safe', + gtd_enabled: true, + gtd_folders: { todo: 'Work', watch: '../bad' }, + }, + }); + + expect(result.account).toEqual(expect.objectContaining({ + id: ACCOUNT_ID, + signature: null, + gtd_folders_rejected: ['watch'], + })); + const [sql, values] = query.mock.calls[1]; + expect(sql).toMatch(/UPDATE email_accounts SET/); + expect(sql).toMatch(/WHERE id = \$\d+ AND id = ANY\(\$\d+\) RETURNING \*/); + expect(values).toContain('safe'); + expect(invalidateGtdConfigCache).toHaveBeenCalledTimes(1); + expect(invalidateGtdConfigCache).toHaveBeenCalledWith(ACCOUNT_ID); + expect(reconcileConnectionState).toHaveBeenCalledTimes(1); + expect(reconcileConnectionState).toHaveBeenCalledWith(expect.objectContaining({ + id: ACCOUNT_ID, + updates: expect.objectContaining({ gtd_enabled: true }), + before: { gtdFoldersChanged: true }, + updated, + })); + }); + + it('returns Account not found without updating when the scoped row is absent', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(exported('updateAccountSettings')({ + accountId: ACCOUNT_ID, + accountIds, + updates: { name: 'Nope' }, + })).resolves.toEqual({ error: 'Account not found', status: 404 }); + expect(query).toHaveBeenCalledTimes(1); + expect(reconcileConnectionState).not.toHaveBeenCalled(); + }); +}); + +describe('alias mutations', () => { + const fields = { + name: 'Alias', + email: 'alias@example.com', + reply_to: 'reply@example.com', + signature: 'Sig', + }; + + it.each([ + ['createAlias', { fields }], + ['updateAlias', { aliasId: 'alias-1', fields }], + ['deleteAlias', { aliasId: 'alias-1' }], + ])('%s rejects ownership misses before mutation', async (name, extra) => { + query.mockResolvedValueOnce({ rows: [] }); + + const result = await exported(name)({ + accountId: ACCOUNT_ID, + accountIds, + userId: 'wrong-user', + ...extra, + }); + + expect(result === null || result === false).toBe(true); + expect(query).toHaveBeenCalledTimes(1); + expect(invalidateOwnerAddressesCache).not.toHaveBeenCalled(); + }); + + it.each([ + ['createAlias', { + extra: { fields }, + responses: [{ rows: [{ id: ACCOUNT_ID }] }, { rows: [{ id: 'alias-1' }] }], + expected: { id: 'alias-1' }, + }], + ['updateAlias', { + extra: { aliasId: 'alias-1', fields }, + responses: [ + { rows: [{ id: 'alias-1', account_id: ACCOUNT_ID }] }, + { rows: [{ id: 'alias-1' }] }, + ], + expected: { id: 'alias-1' }, + }], + ['deleteAlias', { + extra: { aliasId: 'alias-1' }, + responses: [ + { rows: [{ id: 'alias-1', account_id: ACCOUNT_ID }] }, + { rows: [], rowCount: 1 }, + ], + expected: true, + }], + ])('%s invalidates owner addresses exactly once after success', async (name, spec) => { + for (const response of spec.responses) query.mockResolvedValueOnce(response); + + await expect(exported(name)({ + accountId: ACCOUNT_ID, + accountIds, + userId: 'user-1', + ...spec.extra, + })).resolves.toEqual(spec.expected); + expect(invalidateOwnerAddressesCache).toHaveBeenCalledTimes(1); + expect(invalidateOwnerAddressesCache).toHaveBeenCalledWith(ACCOUNT_ID); + }); +}); + +describe('rule persistence', () => { + it('lists global and account-specific rules for an optional account filter', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'rule-1' }] }); + + await expect(exported('listRules')({ + userId: 'user-1', + accountId: ACCOUNT_ID, + })).resolves.toEqual([{ id: 'rule-1' }]); + expect(query).toHaveBeenCalledWith( + expect.stringMatching(/user_id = \$1 AND \(account_id IS NULL OR account_id = \$2\).*ORDER BY priority, created_at/s), + ['user-1', ACCOUNT_ID], + ); + }); + + it('creates a user-owned rule with REST defaults and priority', async () => { + query + .mockResolvedValueOnce({ rows: [{ cnt: '4' }] }) + .mockResolvedValueOnce({ rows: [{ id: 'rule-1' }] }); + + await expect(exported('createRule')({ + userId: 'user-1', + accountId: ACCOUNT_ID, + name: 'Rule', + conditionLogic: 'OR', + conditions: [{ field: 'from', value: 'example.com' }], + actions: [{ type: 'archive' }], + enabled: true, + stopProcessing: false, + })).resolves.toEqual({ id: 'rule-1' }); + expect(query.mock.calls[1][0]).toMatch(/INSERT INTO inbox_rules/); + expect(query.mock.calls[1][1]).toEqual([ + 'user-1', + ACCOUNT_ID, + 'Rule', + true, + false, + 4, + 'OR', + JSON.stringify([{ field: 'from', value: 'example.com' }]), + JSON.stringify([{ type: 'archive' }]), + ]); + }); + + it('updates and deletes only rules owned by the scoped user', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await expect(exported('updateRule')({ + userId: 'wrong-user', + ruleId: 'rule-1', + accountId: null, + name: 'Rule', + conditionLogic: 'AND', + conditions: [], + actions: [], + })).resolves.toBeNull(); + expect(query.mock.calls[0][0]).toMatch(/WHERE id = \$8 AND user_id = \$9/); + + query.mockReset(); + query.mockResolvedValueOnce({ rows: [] }); + await expect(exported('deleteRule')({ + userId: 'wrong-user', + ruleId: 'rule-1', + })).resolves.toBe(false); + expect(query).toHaveBeenCalledWith( + 'DELETE FROM inbox_rules WHERE id = $1 AND user_id = $2 RETURNING id', + ['rule-1', 'wrong-user'], + ); + }); +}); + +describe('runRules', () => { + it('skips accounts with no enabled rules and batches inbox messages for the rest', async () => { + const imapManager = { marker: 'injected' }; + const account = { id: SECOND_ACCOUNT_ID, email_address: 'second@example.com' }; + query + .mockResolvedValueOnce({ rows: [{ cnt: '0' }] }) + .mockResolvedValueOnce({ rows: [{ cnt: '1' }] }) + .mockResolvedValueOnce({ rows: [account] }) + .mockResolvedValueOnce({ + rows: [ + { id: 'm1', uid: 1, folder: 'INBOX', subject: 'One' }, + { id: 'm2', uid: 2, folder: 'INBOX', subject: 'Two' }, + ], + }); + applyInboxRules.mockResolvedValue({ remaining: [{ id: 'm2' }] }); + + const result = await exported('runRules')({ + userId: 'user-1', + accountIds, + imapManager, + }); + + expect(result).toEqual({ processed: 2, matched: 1 }); + expect(applyInboxRules).toHaveBeenCalledTimes(1); + expect(applyInboxRules.mock.calls[0][1]).toEqual(account); + expect(applyInboxRules.mock.calls[0][2]).toBe(imapManager); + const messageSql = query.mock.calls[3][0]; + expect(messageSql).toMatch(/lower\(folder\) = 'inbox'/); + expect(messageSql).toMatch(/ORDER BY id[\s\S]*LIMIT \$2/); + expect(query.mock.calls[3][1]).toEqual([SECOND_ACCOUNT_ID, 500]); + }); +}); diff --git a/backend/src/mcp/accountAdapter.test.js b/backend/src/mcp/accountAdapter.test.js new file mode 100644 index 00000000..59cd9da8 --- /dev/null +++ b/backend/src/mcp/accountAdapter.test.js @@ -0,0 +1,257 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); + +import { query } from '../services/db.js'; +import { + deleteMessageRow, + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + getOutboxRowByMessageId, + getUserPreferences, + listAliases, + listDraftRows, + resolveAlias, +} from './accountAdapter.js'; + +describe('outbox row lookups', () => { + beforeEach(() => query.mockReset()); + + it('finds a pending outbox row by message id within the caller user scope', async () => { + const row = { + id: 'outbox-1', + user_id: 'user-1', + message_id: '', + status: 'pending', + }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect( + getOutboxRowByMessageId('', 'user-1'), + ).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/message_id = \$1/); + expect(sql).toMatch(/user_id = \$2/); + expect(sql).toMatch(/status = 'pending'/); + expect(params).toEqual(['', 'user-1']); + }); + + it('returns null when no pending row matches that message and user', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect( + getOutboxRowByMessageId('', 'user-1'), + ).resolves.toBeNull(); + }); +}); + +describe('deleteMessageRow', () => { + beforeEach(() => query.mockReset()); + + it('uses the same account/uid/folder delete tuple as draftService', async () => { + query.mockResolvedValueOnce({ rowCount: 1, rows: [] }); + + await expect( + deleteMessageRow('account-1', 42, 'Sent'), + ).resolves.toBe(1); + expect(query).toHaveBeenCalledWith( + 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', + ['account-1', 42, 'Sent'], + ); + }); +}); + +describe('getAccountRow', () => { + beforeEach(() => query.mockReset()); + + it('returns a full account row only when its id is in the caller account scope', async () => { + const row = { + id: 'acc-1', + email_address: 'owner@example.com', + smtp_host: 'smtp.example.com', + auth_pass: 'encrypted-secret', + }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(getAccountRow('acc-1', ['acc-1', 'acc-2'])).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/SELECT \* FROM email_accounts/); + expect(sql).toMatch(/id = \$1 AND id = ANY\(\$2\)/); + expect(params).toEqual(['acc-1', ['acc-1', 'acc-2']]); + }); + + it('returns null for an out-of-scope account without querying', async () => { + await expect(getAccountRow('acc-foreign', ['acc-1'])).resolves.toBeNull(); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('getAccountByEmail', () => { + beforeEach(() => query.mockReset()); + + it('returns the account row matched by email within the caller account scope', async () => { + const row = { id: 'acc-2', email_address: 'owner@example.com', auth_pass: 'secret' }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(getAccountByEmail('owner@example.com', ['acc-1', 'acc-2'])).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/email_address = \$1/); + expect(sql).toMatch(/id = ANY\(\$2\)/); + expect(params).toEqual(['owner@example.com', ['acc-1', 'acc-2']]); + }); + + it('returns account_not_found on a scoped lookup miss', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(getAccountByEmail('missing@example.com', ['acc-1'])).resolves.toEqual({ + error: 'account_not_found: missing@example.com', + }); + }); + + it('returns account_not_found for an empty account scope without querying', async () => { + await expect(getAccountByEmail('owner@example.com', [])).resolves.toEqual({ + error: 'account_not_found: owner@example.com', + }); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('alias lookups', () => { + beforeEach(() => query.mockReset()); + + it('lists aliases only for the supplied scoped account id', async () => { + const rows = [{ id: 'alias-1', account_id: 'acc-1', email: 'alias@example.com' }]; + query.mockResolvedValueOnce({ rows }); + + await expect(listAliases('acc-1')).resolves.toBe(rows); + expect(query.mock.calls[0][0]).toMatch(/account_id = \$1/); + expect(query.mock.calls[0][1]).toEqual(['acc-1']); + }); + + it('resolves an alias email only within the supplied scoped account id', async () => { + const alias = { id: 'alias-1', account_id: 'acc-1', email: 'Alias@Example.com' }; + query.mockResolvedValueOnce({ rows: [alias] }); + + await expect(resolveAlias('acc-1', 'alias@example.com')).resolves.toBe(alias); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = \$1/); + expect(sql).toMatch(/LOWER\(email\) = LOWER\(\$2\)/); + expect(params).toEqual(['acc-1', 'alias@example.com']); + }); + + it('returns null when the alias email is not configured on that account', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(resolveAlias('acc-1', 'missing@example.com')).resolves.toBeNull(); + }); +}); + +describe('getComposeSource', () => { + beforeEach(() => query.mockReset()); + + it('returns a compose source selected through the caller account scope', async () => { + const row = { + id: 'message-1', + account_id: 'acc-1', + uid: 42, + folder: 'INBOX', + message_id: '', + reply_to: [{ email: 'reply@example.com' }], + in_reply_to: '', + thread_references: ' ', + body_text: 'Hello', + body_html: '

Hello

', + }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(getComposeSource('message-1', ['acc-1'])).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/m\.account_id = ANY\(\$2\)/); + for (const column of ['uid', 'reply_to', 'in_reply_to', 'thread_references', 'body_text', 'body_html']) { + expect(sql).toContain(`m.${column}`); + } + expect(params).toEqual(['message-1', ['acc-1']]); + }); + + it('returns null when the message is outside the caller account scope', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(getComposeSource('message-foreign', ['acc-1'])).resolves.toBeNull(); + }); + + it('returns null for an empty account scope without querying', async () => { + await expect(getComposeSource('message-1', [])).resolves.toBeNull(); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('draft row lookups', () => { + beforeEach(() => query.mockReset()); + + it('lists live draft rows only for the supplied scoped account and folder', async () => { + const rows = [{ account_id: 'acc-1', folder: 'Drafts', uid: 7 }]; + query.mockResolvedValueOnce({ rows }); + + await expect(listDraftRows('acc-1', { + limit: 20, + offset: 5, + folder: 'Drafts', + })).resolves.toBe(rows); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = \$1/); + expect(sql).toMatch(/folder = \$2/); + expect(sql).toMatch(/is_deleted = false/); + expect(sql).toMatch(/LIMIT \$3 OFFSET \$4/); + expect(params).toEqual(['acc-1', 'Drafts', 20, 5]); + }); + + it('lists live draft rows for a scoped account without widening through a folder predicate', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await listDraftRows('acc-1', { limit: 10, offset: 0 }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = \$1/); + expect(sql).not.toMatch(/folder = \$2/); + expect(params).toEqual(['acc-1', 10, 0]); + }); + + it('gets a draft row by scoped account, folder, and uid', async () => { + const row = { account_id: 'acc-1', folder: 'Drafts', uid: 7 }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(getDraftRow('acc-1', 'Drafts', 7)).resolves.toBe(row); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = \$1 AND folder = \$2 AND uid = \$3/); + expect(sql).toMatch(/is_deleted = false/); + expect(params).toEqual(['acc-1', 'Drafts', 7]); + }); + + it('returns null when a draft lookup misses its scoped account/folder/uid tuple', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(getDraftRow('acc-1', 'Drafts', 999)).resolves.toBeNull(); + }); +}); + +describe('getUserPreferences', () => { + beforeEach(() => query.mockReset()); + + it('returns preferences only for the supplied user id', async () => { + const preferences = { plaintextEmail: true, undoSendSeconds: 30 }; + query.mockResolvedValueOnce({ rows: [{ preferences }] }); + + await expect(getUserPreferences('user-1')).resolves.toBe(preferences); + expect(query.mock.calls[0]).toEqual([ + 'SELECT preferences FROM users WHERE id = $1', + ['user-1'], + ]); + }); + + it('returns an empty object when the scoped user row is absent', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(getUserPreferences('user-missing')).resolves.toEqual({}); + }); +}); diff --git a/backend/src/mcp/accountTools.js b/backend/src/mcp/accountTools.js new file mode 100644 index 00000000..13a4a988 --- /dev/null +++ b/backend/src/mcp/accountTools.js @@ -0,0 +1,501 @@ +import { + createAlias, + createRule, + deleteAlias, + deleteRule, + getAccountRow, + listAccountsSafe, + listRules, + runRules, + updateAccountSettings, + updateAlias, + updateRule, +} from './accountAdapter.js'; +import { resolveAccountScope } from './engineAdapter.js'; +import { errorResult, jsonResult } from './result.js'; +import { + hasHeaderInjectionChars, +} from '../services/emailSanitizer.js'; +import { + normalizeActions, + validateConditions, +} from '../routes/rules.js'; + +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); +const CREATE_ANNOTATIONS = annotations(); +const IDEMPOTENT_WRITE_ANNOTATIONS = annotations({ idempotentHint: true }); +const DESTRUCTIVE_ANNOTATIONS = annotations({ destructiveHint: true }); +const DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS = annotations({ + destructiveHint: true, + idempotentHint: true, +}); + +export const listAccountsDef = { + name: 'list_accounts', + description: 'List the connected email accounts and their aliases. Never returns credentials.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { type: 'object', properties: {} }, +}; + +export const addAccountDef = { + name: 'add_account', + description: + 'Stage a new email account for setup. Does NOT accept passwords or OAuth tokens — the user must complete authentication in the Mailflow settings UI. Returns a stage_id.', + annotations: CREATE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['name', 'email_address'], + properties: { + name: { type: 'string' }, + email_address: { type: 'string' }, + sender_name: { type: 'string' }, + color: { type: 'string' }, + protocol: { type: 'string', enum: ['imap'] }, + imap_host: { type: 'string' }, + imap_port: { type: 'integer' }, + smtp_host: { type: 'string' }, + smtp_port: { type: 'integer' }, + smtp_tls: { type: 'string', enum: ['STARTTLS', 'SSL', 'none'] }, + auth_user: { type: 'string' }, + signature: { type: 'string' }, + }, + }, +}; + +export const updateAccountSettingsDef = { + name: 'update_account_settings', + description: + 'Update non-secret account settings. Credentials, hosts, ports, and TLS settings must be changed in the Mailflow settings UI.', + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account'], + properties: { + account: { type: 'string', description: 'Account email address (use list_accounts)' }, + name: { type: 'string' }, + sender_name: { type: 'string' }, + color: { type: 'string' }, + sort_order: { type: 'integer' }, + folder_mappings: { type: 'object' }, + signature: { type: 'string' }, + categorization_enabled: { type: 'boolean' }, + gtd_enabled: { type: 'boolean' }, + gtd_folders: { type: 'object' }, + enabled: { type: 'boolean' }, + }, + }, +}; + +export const testAccountConnectionDef = { + name: 'test_account_connection', + description: + 'Test IMAP and SMTP connectivity for an existing account using its stored credentials. Does not send mail.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account'], + properties: { + account: { + type: 'string', + description: 'Account email address (use list_accounts)', + }, + }, + }, +}; + +const aliasFields = { + name: { type: 'string' }, + email: { type: 'string' }, + reply_to: { type: 'string' }, + signature: { type: 'string' }, +}; + +export const createAliasDef = { + name: 'create_alias', + description: 'Create a send-as alias on a scoped account.', + annotations: CREATE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'name', 'email'], + properties: { + account: { type: 'string', description: 'Account email address (use list_accounts)' }, + ...aliasFields, + }, + }, +}; + +export const updateAliasDef = { + name: 'update_alias', + description: 'Update a send-as alias owned by a scoped account.', + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'alias_id', 'name', 'email'], + properties: { + account: { type: 'string', description: 'Account email address (use list_accounts)' }, + alias_id: { type: 'string' }, + ...aliasFields, + }, + }, +}; + +export const deleteAliasDef = { + name: 'delete_alias', + description: 'Delete a send-as alias owned by a scoped account.', + annotations: DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'alias_id'], + properties: { + account: { type: 'string', description: 'Account email address (use list_accounts)' }, + alias_id: { type: 'string' }, + }, + }, +}; + +export const listRulesDef = { + name: 'list_rules', + description: 'List inbox rules owned by the token user, optionally including only global and one account’s rules.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Optional account email address' }, + }, + }, +}; + +const ruleFields = { + name: { type: 'string' }, + account_id: { + type: 'string', + description: 'Optional account email address; the field name matches the REST rule shape', + }, + condition_logic: { type: 'string', enum: ['AND', 'OR'] }, + conditions: { type: 'array', items: { type: 'object' } }, + actions: { type: 'array', items: { type: 'object' } }, + enabled: { type: 'boolean' }, + stop_processing: { type: 'boolean' }, +}; + +export const createRuleDef = { + name: 'create_rule', + description: 'Create an inbox rule for all accounts or one scoped account.', + annotations: CREATE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['name', 'conditions', 'actions'], + properties: ruleFields, + }, +}; + +export const updateRuleDef = { + name: 'update_rule', + description: 'Replace the settings of a user-owned inbox rule.', + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['rule_id', 'name', 'conditions', 'actions'], + properties: { + rule_id: { type: 'string' }, + ...ruleFields, + }, + }, +}; + +export const deleteRuleDef = { + name: 'delete_rule', + description: 'Delete a user-owned inbox rule.', + annotations: DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['rule_id'], + properties: { + rule_id: { type: 'string' }, + }, + }, +}; + +export const runRulesDef = { + name: 'run_rules', + description: + 'Run enabled inbox rules against current INBOX messages for all scoped accounts or one account. Rule actions can move, archive, or delete messages.', + annotations: DESTRUCTIVE_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Optional account email address' }, + }, + }, +}; + +async function resolvedAccountId(account, scope) { + const resolved = await resolveAccountScope(account, scope.accountIds); + if (resolved.error) return resolved; + const accountId = resolved.accountIds?.[0]; + if (!accountId) return { error: `account not found: ${account}` }; + return { accountId }; +} + +export async function handleListAccounts(_args, scope) { + return jsonResult({ accounts: await listAccountsSafe(scope.accountIds) }); +} + +const SECRET_ACCOUNT_FIELDS = [ + 'auth_pass', + 'oauth_access_token', + 'oauth_refresh_token', +]; +const SECRET_REJECTION = + 'Passwords and OAuth tokens cannot be sent over MCP; finish authentication in the Mailflow settings UI'; + +export async function handleAddAccount(args, scope) { + if (SECRET_ACCOUNT_FIELDS.some((field) => Object.prototype.hasOwnProperty.call(args, field))) { + return errorResult(SECRET_REJECTION); + } + try { + const { stageAccount } = await import('../services/accountService.js'); + const result = await stageAccount({ userId: scope.userId, payload: args }); + if (result?.error) return errorResult(result.error); + return jsonResult({ + stage_id: result.id, + next_step: + 'Open Mailflow Settings > Accounts to finish setup, or POST /api/mcp-account-stages/:id/execute (session-authed) with credentials', + }); + } catch (error) { + return errorResult(error.message); + } +} + +const ACCOUNT_UPDATE_FIELDS = new Set([ + 'name', + 'sender_name', + 'color', + 'sort_order', + 'folder_mappings', + 'signature', + 'categorization_enabled', + 'gtd_enabled', + 'gtd_folders', + 'enabled', +]); +const UI_ONLY_ACCOUNT_FIELDS = new Set([ + 'auth_user', + 'auth_pass', + 'imap_host', + 'imap_port', + 'imap_tls', + 'imap_skip_tls_verify', + 'smtp_host', + 'smtp_port', + 'smtp_tls', +]); + +export async function handleUpdateAccountSettings(args, scope) { + for (const field of Object.keys(args)) { + if (UI_ONLY_ACCOUNT_FIELDS.has(field)) { + return errorResult( + `${field} cannot be changed over MCP; use the Mailflow settings UI`, + ); + } + if (field !== 'account' && !ACCOUNT_UPDATE_FIELDS.has(field)) { + return errorResult(`${field} is not an account setting supported over MCP`); + } + } + if ('name' in args && hasHeaderInjectionChars(args.name)) { + return errorResult('Name cannot contain control characters'); + } + if ( + 'sender_name' in args + && args.sender_name + && hasHeaderInjectionChars(args.sender_name) + ) { + return errorResult('Sender name cannot contain control characters'); + } + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const updates = Object.fromEntries( + Object.entries(args).filter(([field]) => ACCOUNT_UPDATE_FIELDS.has(field)), + ); + const result = await updateAccountSettings({ + accountId: resolved.accountId, + accountIds: scope.accountIds, + updates, + }); + if (result.error) return errorResult(result.error); + return jsonResult(result.account); +} + +export async function handleTestAccountConnection(args, scope) { + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const account = await getAccountRow(resolved.accountId, scope.accountIds); + if (!account) return errorResult(`account not found: ${args.account}`); + const { testConnection } = await import('../services/connectionTest.js'); + return jsonResult({ + account: args.account, + ...await testConnection(account), + }); +} + +function aliasFieldsFrom(args) { + return { + name: args.name, + email: args.email, + reply_to: args.reply_to, + signature: args.signature, + }; +} + +function aliasValidation(args) { + if (!args.name || !args.email) return 'Name and email required'; + if ( + hasHeaderInjectionChars(args.name) + || hasHeaderInjectionChars(args.email) + || hasHeaderInjectionChars(args.reply_to) + ) { + return 'Fields cannot contain control characters'; + } + return null; +} + +export async function handleCreateAlias(args, scope) { + const validation = aliasValidation(args); + if (validation) return errorResult(validation); + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const alias = await createAlias({ + accountId: resolved.accountId, + accountIds: scope.accountIds, + userId: scope.userId, + fields: aliasFieldsFrom(args), + }); + if (!alias) return errorResult('Account not found'); + return jsonResult(alias); +} + +export async function handleUpdateAlias(args, scope) { + const validation = aliasValidation(args); + if (validation) return errorResult(validation); + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const alias = await updateAlias({ + accountId: resolved.accountId, + accountIds: scope.accountIds, + userId: scope.userId, + aliasId: args.alias_id, + fields: aliasFieldsFrom(args), + }); + if (!alias) return errorResult('Alias not found'); + return jsonResult(alias); +} + +export async function handleDeleteAlias(args, scope) { + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + const deleted = await deleteAlias({ + accountId: resolved.accountId, + accountIds: scope.accountIds, + userId: scope.userId, + aliasId: args.alias_id, + }); + if (!deleted) return errorResult('Alias not found'); + return jsonResult({ ok: true }); +} + +export async function handleListRules(args, scope) { + let accountId; + if (args.account) { + const resolved = await resolvedAccountId(args.account, scope); + if (resolved.error) return errorResult(resolved.error); + accountId = resolved.accountId; + } + return jsonResult({ + rules: await listRules({ userId: scope.userId, accountId }), + }); +} + +async function normalizedRuleInput(args, scope) { + if (!Array.isArray(args.conditions) || !Array.isArray(args.actions)) { + return { error: 'conditions and actions must be arrays' }; + } + const conditionError = validateConditions(args.conditions); + if (conditionError) return { error: conditionError }; + let accountId = null; + if (args.account_id) { + const resolved = await resolvedAccountId(args.account_id, scope); + if (resolved.error) return resolved; + accountId = resolved.accountId; + } + const actions = normalizeActions(args.actions) + .filter((action) => accountId || action.type !== 'move'); + return { + accountId, + name: args.name, + conditionLogic: args.condition_logic, + conditions: args.conditions, + actions, + enabled: args.enabled, + stopProcessing: args.stop_processing, + }; +} + +export async function handleCreateRule(args, scope) { + const input = await normalizedRuleInput(args, scope); + if (input.error) return errorResult(input.error); + const rule = await createRule({ userId: scope.userId, ...input }); + if (rule?.error) return errorResult(rule.error); + return jsonResult(rule); +} + +export async function handleUpdateRule(args, scope) { + const input = await normalizedRuleInput(args, scope); + if (input.error) return errorResult(input.error); + const rule = await updateRule({ + userId: scope.userId, + ruleId: args.rule_id, + ...input, + }); + if (rule?.error) return errorResult(rule.error); + if (!rule) return errorResult('Rule not found'); + return jsonResult(rule); +} + +export async function handleDeleteRule(args, scope) { + const deleted = await deleteRule({ + userId: scope.userId, + ruleId: args.rule_id, + }); + if (!deleted) return errorResult('Rule not found'); + return jsonResult({ ok: true }); +} + +export async function handleRunRules(args, scope, deps = {}) { + let accountIds = scope.accountIds; + if (args.account) { + const resolved = await resolveAccountScope(args.account, scope.accountIds); + if (resolved.error) return errorResult(resolved.error); + accountIds = resolved.accountIds; + } + return jsonResult(await runRules({ + userId: scope.userId, + accountIds, + imapManager: deps.imapManager, + })); +} diff --git a/backend/src/mcp/accountTools.test.js b/backend/src/mcp/accountTools.test.js new file mode 100644 index 00000000..cc825eef --- /dev/null +++ b/backend/src/mcp/accountTools.test.js @@ -0,0 +1,457 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; +import { SAFE_FIELDS } from '../services/accountFields.js'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('./accountAdapter.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getAccountRow: vi.fn(), + listAccountsSafe: vi.fn(), + updateAccountSettings: vi.fn(), + createAlias: vi.fn(), + updateAlias: vi.fn(), + deleteAlias: vi.fn(), + listRules: vi.fn(), + createRule: vi.fn(), + updateRule: vi.fn(), + deleteRule: vi.fn(), + runRules: vi.fn(), + }; +}); +vi.mock('./engineAdapter.js', async (orig) => { + const actual = await orig(); + return { ...actual, resolveAccountScope: vi.fn() }; +}); +vi.mock('../services/accountService.js', () => ({ + stageAccount: vi.fn(), + reconcileConnectionState: vi.fn(), +})); +vi.mock('../services/connectionTest.js', () => ({ testConnection: vi.fn() })); +vi.mock('../services/gtdConfig.js', () => ({ + DEFAULT_GTD_FOLDERS: { + todo: 'Todo', + watch: 'Watch', + delegated: 'Delegated', + someday: 'Someday', + reference: 'Reference', + }, + GTD_STATES: ['todo', 'watch', 'delegated', 'someday', 'reference'], + sanitizeGtdFoldersDetailed: vi.fn((input) => ({ + folders: input || {}, + rejected: [], + reserved: [], + })), + findGtdFolderCollisions: vi.fn(() => []), + invalidateGtdConfigCache: vi.fn(), + getGtdFolderSet: vi.fn(async () => new Set()), +})); +vi.mock('../services/gtdTransitions.js', () => ({ + invalidateOwnerAddressesCache: vi.fn(), +})); +vi.mock('../services/inboxRules.js', () => ({ + applyInboxRules: vi.fn(), + isDangerousRegex: vi.fn(() => false), + matchingRules: vi.fn(() => []), + toRuleMessage: vi.fn((message) => message), +})); +vi.mock('../routes/rules.js', () => ({ + validateConditions: vi.fn(), + normalizeActions: vi.fn(), +})); + +const db = await import('../services/db.js'); +const accountAdapter = await import('./accountAdapter.js'); +const realAccountAdapter = await vi.importActual('./accountAdapter.js'); +const engineAdapter = await import('./engineAdapter.js'); +const accountService = await import('../services/accountService.js'); +const connectionTest = await import('../services/connectionTest.js'); +const rulesRoute = await import('../routes/rules.js'); +const accountTools = await import('./accountTools.js').catch(() => ({})); +const registeredTools = await import('./tools.js'); + +const ACCOUNT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const OTHER_ACCOUNT_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; +const ACCOUNT_EMAIL = 'owner@example.com'; +const scope = { + userId: 'user-1', + accountIds: [ACCOUNT_ID, OTHER_ACCOUNT_ID], + scopes: ['read', 'write', 'settings'], +}; +const deps = { imapManager: { marker: 'injected-imap-manager' } }; + +function handler(name) { + expect(accountTools[name], `${name} must be exported`).toBeTypeOf('function'); + return accountTools[name]; +} + +function jsonOf(result) { + return JSON.parse(result.content[0].text); +} + +beforeEach(() => { + vi.clearAllMocks(); + engineAdapter.resolveAccountScope.mockImplementation(async (account, accountIds) => ( + account + ? { accountIds: [ACCOUNT_ID] } + : { accountIds } + )); + rulesRoute.validateConditions.mockReturnValue(null); + rulesRoute.normalizeActions.mockImplementation((actions) => actions); +}); + +describe('account/settings tool definitions and registration', () => { + it('registers all twelve tools with their required scopes and annotation hints', () => { + const expected = { + list_accounts: ['read', true, false, true], + add_account: ['settings', false, false, false], + update_account_settings: ['settings', false, false, true], + test_account_connection: ['settings', true, false, true], + create_alias: ['settings', false, false, false], + update_alias: ['settings', false, false, true], + delete_alias: ['settings', false, true, true], + list_rules: ['settings', true, false, true], + create_rule: ['settings', false, false, false], + update_rule: ['settings', false, false, true], + delete_rule: ['settings', false, true, true], + run_rules: [['settings', 'write'], false, true, false], + }; + const defs = new Map(registeredTools.TOOL_DEFS.map((def) => [def.name, def])); + + for (const [name, [requiredScope, readOnlyHint, destructiveHint, idempotentHint]] of Object.entries(expected)) { + expect(defs.get(name)?.annotations).toEqual({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); + expect(registeredTools.TOOL_SCOPES[name]).toEqual(requiredScope); + expect(registeredTools.HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('documents and schemas the no-secrets staged account flow', () => { + const def = registeredTools.TOOL_DEFS.find(({ name }) => name === 'add_account'); + expect(def?.description).toMatch(/does not accept passwords or OAuth tokens/i); + expect(def?.description).toMatch(/stage_id/i); + expect(def?.inputSchema.required).toEqual(['name', 'email_address']); + expect(def?.inputSchema.properties).not.toHaveProperty('auth_pass'); + expect(def?.inputSchema.properties).not.toHaveProperty('oauth_access_token'); + expect(def?.inputSchema.properties).not.toHaveProperty('oauth_refresh_token'); + }); +}); + +describe('list_accounts', () => { + it('returns safe accounts with aliases through the adapter seam', async () => { + const accounts = [{ + id: ACCOUNT_ID, + email_address: ACCOUNT_EMAIL, + aliases: [{ id: 'alias-1', email: 'alias@example.com' }], + }]; + accountAdapter.listAccountsSafe.mockResolvedValue(accounts); + + const result = await handler('handleListAccounts')({}, scope, deps); + + expect(jsonOf(result)).toEqual({ accounts }); + expect(accountAdapter.listAccountsSafe).toHaveBeenCalledWith(scope.accountIds); + }); + + it('passes exactly SAFE_FIELDS to the account query and never selects credentials', async () => { + const account = Object.fromEntries(SAFE_FIELDS.map((field) => [field, null])); + account.id = ACCOUNT_ID; + db.query + .mockResolvedValueOnce({ rows: [account] }) + .mockResolvedValueOnce({ rows: [] }); + + await realAccountAdapter.listAccountsSafe([ACCOUNT_ID]); + + const accountSql = db.query.mock.calls[0][0]; + const columns = accountSql + .slice(accountSql.indexOf('SELECT') + 6, accountSql.indexOf('FROM email_accounts')) + .split(',') + .map((column) => column.trim()) + .filter(Boolean); + expect(columns).toEqual(SAFE_FIELDS); + expect(accountSql).not.toMatch(/\bauth_pass\b|\boauth_access_token\b|\boauth_refresh_token\b/); + expect(db.query.mock.calls[1][0]).toMatch(/account_aliases WHERE account_id = ANY\(\$1\)/); + }); +}); + +describe('add_account', () => { + it.each(['auth_pass', 'oauth_access_token', 'oauth_refresh_token'])( + 'rejects %s even when its value is empty before staging', + async (secretField) => { + const result = await handler('handleAddAccount')({ + name: 'Mail', + email_address: ACCOUNT_EMAIL, + [secretField]: '', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'Passwords and OAuth tokens cannot be sent over MCP; finish authentication in the Mailflow settings UI', + ); + expect(accountService.stageAccount).not.toHaveBeenCalled(); + }, + ); + + it('returns the stage id and the exact human-authentication next step', async () => { + accountService.stageAccount.mockResolvedValue({ id: 'stage-1' }); + + const result = await handler('handleAddAccount')({ + name: 'Mail', + email_address: ACCOUNT_EMAIL, + imap_host: 'imap.example.com', + }, scope, deps); + + expect(jsonOf(result)).toEqual({ + stage_id: 'stage-1', + next_step: 'Open Mailflow Settings > Accounts to finish setup, or POST /api/mcp-account-stages/:id/execute (session-authed) with credentials', + }); + expect(accountService.stageAccount).toHaveBeenCalledWith({ + userId: scope.userId, + payload: { + name: 'Mail', + email_address: ACCOUNT_EMAIL, + imap_host: 'imap.example.com', + }, + }); + }); + + it('surfaces host/port validation errors from stageAccount', async () => { + accountService.stageAccount.mockResolvedValue({ + error: 'IMAP: Port 25 is not allowed. Allowed: 143, 993', + status: 400, + }); + + const result = await handler('handleAddAccount')({ + name: 'Mail', + email_address: ACCOUNT_EMAIL, + imap_port: 25, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('IMAP: Port 25 is not allowed. Allowed: 143, 993'); + }); +}); + +describe('update_account_settings', () => { + it.each([ + 'auth_user', + 'auth_pass', + 'imap_host', + 'imap_port', + 'imap_tls', + 'imap_skip_tls_verify', + 'smtp_host', + 'smtp_port', + 'smtp_tls', + ])('hard-errors on excluded field %s instead of silently dropping it', async (field) => { + const result = await handler('handleUpdateAccountSettings')({ + account: ACCOUNT_EMAIL, + [field]: field.endsWith('port') ? 993 : 'value', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + `${field} cannot be changed over MCP; use the Mailflow settings UI`, + ); + expect(engineAdapter.resolveAccountScope).not.toHaveBeenCalled(); + expect(accountAdapter.updateAccountSettings).not.toHaveBeenCalled(); + }); + + it('resolves the account and returns the updated safe row', async () => { + const account = { id: ACCOUNT_ID, email_address: ACCOUNT_EMAIL, enabled: false }; + accountAdapter.updateAccountSettings.mockResolvedValue({ account }); + + const result = await handler('handleUpdateAccountSettings')({ + account: ACCOUNT_EMAIL, + enabled: false, + signature: 'Bye', + }, scope, deps); + + expect(jsonOf(result)).toEqual(account); + expect(accountAdapter.updateAccountSettings).toHaveBeenCalledWith({ + accountId: ACCOUNT_ID, + accountIds: scope.accountIds, + updates: { enabled: false, signature: 'Bye' }, + }); + }); +}); + +describe('test_account_connection', () => { + it('loads the scoped full account row and delegates both probes', async () => { + const fullRow = { + id: ACCOUNT_ID, + email_address: ACCOUNT_EMAIL, + auth_pass: 'encrypted-secret', + }; + accountAdapter.getAccountRow.mockResolvedValue(fullRow); + connectionTest.testConnection.mockResolvedValue({ + imap: { ok: true }, + smtp: { ok: false, error: 'SMTP authentication failed' }, + }); + + const result = await handler('handleTestAccountConnection')({ + account: ACCOUNT_EMAIL, + }, scope, deps); + + expect(jsonOf(result)).toEqual({ + account: ACCOUNT_EMAIL, + imap: { ok: true }, + smtp: { ok: false, error: 'SMTP authentication failed' }, + }); + expect(accountAdapter.getAccountRow).toHaveBeenCalledWith(ACCOUNT_ID, scope.accountIds); + expect(connectionTest.testConnection).toHaveBeenCalledWith(fullRow); + }); + + it('returns the account-scope error before touching the full-row adapter', async () => { + engineAdapter.resolveAccountScope.mockResolvedValue({ + error: 'account not found: foreign@example.com', + }); + + const result = await handler('handleTestAccountConnection')({ + account: 'foreign@example.com', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: foreign@example.com'); + expect(accountAdapter.getAccountRow).not.toHaveBeenCalled(); + expect(connectionTest.testConnection).not.toHaveBeenCalled(); + }); +}); + +describe('alias CRUD', () => { + it.each([ + ['handleCreateAlias', 'createAlias', { account: 'foreign@example.com', name: 'Alias', email: 'alias@example.com' }], + ['handleUpdateAlias', 'updateAlias', { account: 'foreign@example.com', alias_id: 'alias-1', name: 'Alias', email: 'alias@example.com' }], + ['handleDeleteAlias', 'deleteAlias', { account: 'foreign@example.com', alias_id: 'alias-1' }], + ])('%s rejects an out-of-scope account before the mutation adapter', async (handlerName, adapterName, args) => { + engineAdapter.resolveAccountScope.mockResolvedValue({ + error: 'account not found: foreign@example.com', + }); + + const result = await handler(handlerName)(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: foreign@example.com'); + expect(accountAdapter[adapterName]).not.toHaveBeenCalled(); + }); + +}); + +describe('rules tools', () => { + it('list_rules resolves an optional account and returns user-owned rules', async () => { + const rules = [{ id: 'rule-1', account_id: ACCOUNT_ID }]; + accountAdapter.listRules.mockResolvedValue(rules); + + const result = await handler('handleListRules')({ account: ACCOUNT_EMAIL }, scope, deps); + + expect(jsonOf(result)).toEqual({ rules }); + expect(accountAdapter.listRules).toHaveBeenCalledWith({ + userId: scope.userId, + accountId: ACCOUNT_ID, + }); + }); + + it.each([ + ['handleCreateRule', 'createRule'], + ['handleUpdateRule', 'updateRule'], + ])('%s calls the imported condition validator and action normalizer', async (handlerName, adapterName) => { + const conditions = [{ field: 'from', operator: 'contains', value: 'example.com' }]; + const actions = [{ type: 'archive' }, { type: 'move', value: 'Archive ' }]; + const normalized = [{ type: 'archive' }]; + rulesRoute.normalizeActions.mockReturnValue(normalized); + accountAdapter[adapterName].mockResolvedValue({ id: 'rule-1' }); + const args = { + name: 'Archive example mail', + account_id: ACCOUNT_EMAIL, + conditions, + actions, + condition_logic: 'OR', + enabled: true, + stop_processing: true, + }; + if (handlerName === 'handleUpdateRule') args.rule_id = 'rule-1'; + + const result = await handler(handlerName)(args, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(rulesRoute.validateConditions).toHaveBeenCalledWith(conditions); + expect(rulesRoute.normalizeActions).toHaveBeenCalledWith(actions); + expect(accountAdapter[adapterName]).toHaveBeenCalledWith(expect.objectContaining({ + userId: scope.userId, + accountId: ACCOUNT_ID, + conditions, + actions: normalized, + conditionLogic: 'OR', + enabled: true, + stopProcessing: true, + })); + }); + + it('surfaces validation errors before normalizing or writing', async () => { + rulesRoute.validateConditions.mockReturnValue('Condition value cannot be empty'); + + const result = await handler('handleCreateRule')({ + name: 'Bad', + conditions: [{ field: 'from', value: '' }], + actions: [], + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('Condition value cannot be empty'); + expect(rulesRoute.normalizeActions).not.toHaveBeenCalled(); + expect(accountAdapter.createRule).not.toHaveBeenCalled(); + }); + + it('delete_rule returns an error for a missing user-owned rule', async () => { + accountAdapter.deleteRule.mockResolvedValue(false); + + const result = await handler('handleDeleteRule')({ rule_id: 'missing' }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('Rule not found'); + }); +}); + +describe('run_rules', () => { + it('uses every scoped account when account is omitted and threads deps.imapManager', async () => { + accountAdapter.runRules.mockResolvedValue({ processed: 7, matched: 3 }); + + const result = await handler('handleRunRules')({}, scope, deps); + + expect(jsonOf(result)).toEqual({ processed: 7, matched: 3 }); + expect(accountAdapter.runRules).toHaveBeenCalledWith({ + userId: scope.userId, + accountIds: scope.accountIds, + imapManager: deps.imapManager, + }); + }); + + it('returns an account scope error without invoking the run loop', async () => { + engineAdapter.resolveAccountScope.mockResolvedValue({ + error: 'account not found: foreign@example.com', + }); + + const result = await handler('handleRunRules')({ + account: 'foreign@example.com', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: foreign@example.com'); + expect(accountAdapter.runRules).not.toHaveBeenCalled(); + }); + +}); + +describe('mock-drift guard: mocked seams exist on their real modules', () => { + it.each([ + ['accountAdapter', () => accountAdapter, './accountAdapter.js'], + ['engineAdapter', () => engineAdapter, './engineAdapter.js'], + ])('%s mock surface matches the real module', async (_name, getMock, path) => { + const real = await vi.importActual(path); + expect(mockSurfaceDrift(getMock(), real)).toEqual([]); + }); +}); diff --git a/backend/src/mcp/auth.js b/backend/src/mcp/auth.js new file mode 100644 index 00000000..b2e3964e --- /dev/null +++ b/backend/src/mcp/auth.js @@ -0,0 +1,62 @@ +import crypto from 'crypto'; +import { query } from '../services/db.js'; + +export const ALL_SCOPES = ['read', 'write', 'send', 'settings']; + +export function expandScopes(scopes) { + const expanded = new Set(Array.isArray(scopes) && scopes.length ? scopes : ['read']); + if (expanded.has('write') || expanded.has('send') || expanded.has('settings')) { + expanded.add('read'); + } + return [...expanded]; +} + +export function hasScope(scope, required) { + if (!required) return true; + const requiredScopes = Array.isArray(required) ? required : [required]; + const grantedScopes = scope.scopes || []; + return requiredScopes.every((requiredScope) => grantedScopes.includes(requiredScope)); +} + +// Plaintext tokens are shown once at mint time; we persist only the hash. +export function generateToken() { + return 'mcp_' + crypto.randomBytes(32).toString('base64url'); +} + +export function hashToken(plaintext) { + return crypto.createHash('sha256').update(plaintext, 'utf8').digest('hex'); +} + +// The scope object pins multi-user isolation: every MCP tool call is bounded to +// exactly this user's enabled accounts. msgvault is single-archive; Mailflow is not. +export async function resolveScope(userId, scopes) { + const { rows } = await query( + 'SELECT id FROM email_accounts WHERE user_id = $1 AND enabled = true', + [userId], + ); + return { userId, accountIds: rows.map((r) => r.id), scopes: expandScopes(scopes) }; +} + +export async function mcpBearerAuth(req, res, next) { + try { + const header = req.get('Authorization') || ''; + const m = /^Bearer\s+(.+)$/i.exec(header); + if (!m) return res.status(401).json({ error: 'invalid_token' }); + + const { rows } = await query( + 'SELECT id, user_id, scopes FROM api_tokens WHERE token_hash = $1', + [hashToken(m[1].trim())], + ); + if (!rows.length) return res.status(401).json({ error: 'invalid_token' }); + + // Best-effort recency stamp; never block the request on it. + await query('UPDATE api_tokens SET last_used_at = NOW() WHERE id = $1', [rows[0].id]) + .catch(() => {}); + + req.mcpTokenId = rows[0].id; // rate-limit key: per token, not per IP + req.mcpScope = await resolveScope(rows[0].user_id, rows[0].scopes); + next(); + } catch (err) { + next(err); + } +} diff --git a/backend/src/mcp/auth.test.js b/backend/src/mcp/auth.test.js new file mode 100644 index 00000000..691b3070 --- /dev/null +++ b/backend/src/mcp/auth.test.js @@ -0,0 +1,133 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +import { query } from '../services/db.js'; +import { + ALL_SCOPES, + expandScopes, + generateToken, + hasScope, + hashToken, + resolveScope, + mcpBearerAuth, +} from './auth.js'; + +function mockRes() { + return { statusCode: 200, body: null, status(c){this.statusCode=c;return this;}, json(b){this.body=b;return this;} }; +} + +describe('token helpers', () => { + it('exports the complete set of supported scopes', () => { + expect(ALL_SCOPES).toEqual(['read', 'write', 'send', 'settings']); + }); + + it('adds implied read access to elevated scopes', () => { + expect(expandScopes(['write'])).toEqual(['write', 'read']); + expect(expandScopes(['send'])).toEqual(['send', 'read']); + expect(expandScopes(['settings'])).toEqual(['settings', 'read']); + }); + + it('defaults empty or non-array scope values to read', () => { + expect(expandScopes([])).toEqual(['read']); + expect(expandScopes()).toEqual(['read']); + expect(expandScopes('send')).toEqual(['read']); + }); + + it('checks a single required scope', () => { + const scope = { scopes: ['read', 'write'] }; + expect(hasScope(scope, 'write')).toBe(true); + expect(hasScope(scope, 'send')).toBe(false); + expect(hasScope(scope)).toBe(true); + }); + + it('requires every scope when given an array', () => { + const scope = { scopes: ['read', 'write', 'settings'] }; + expect(hasScope(scope, ['settings', 'write'])).toBe(true); + expect(hasScope(scope, ['settings', 'send'])).toBe(false); + }); + + it('generateToken is prefixed and unique', () => { + const a = generateToken(); const b = generateToken(); + expect(a).toMatch(/^mcp_[A-Za-z0-9_-]{20,}$/); + expect(a).not.toEqual(b); + }); + it('hashToken is deterministic hex SHA-256 and hides the plaintext', () => { + const h = hashToken('mcp_secret'); + expect(h).toMatch(/^[0-9a-f]{64}$/); + expect(h).toEqual(hashToken('mcp_secret')); + expect(h).not.toContain('secret'); + }); +}); + +describe('resolveScope', () => { + it("returns only the token owner's enabled account ids and expanded scopes", async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-1' }, { id: 'acc-2' }] }); + const scope = await resolveScope('user-1', ['send']); + expect(scope).toEqual({ + userId: 'user-1', + accountIds: ['acc-1', 'acc-2'], + scopes: ['send', 'read'], + }); + expect(query).toHaveBeenCalledWith( + expect.stringMatching(/FROM email_accounts WHERE user_id = \$1 AND enabled = true/), + ['user-1'], + ); + }); +}); + +describe('mcpBearerAuth', () => { + beforeEach(() => query.mockReset()); + + it('rejects a request with no Authorization header', async () => { + const req = { get: () => undefined }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(res.statusCode).toBe(401); + expect(res.body).toEqual({ error: 'invalid_token' }); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects a malformed (non-Bearer) header', async () => { + const req = { get: () => 'Basic abc' }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(res.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects an unknown / revoked token', async () => { + query.mockResolvedValueOnce({ rows: [] }); // no api_tokens row + const req = { get: () => 'Bearer mcp_revoked' }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(res.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('accepts a valid token and attaches the user-scoped account ids', async () => { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok-1', user_id: 'user-1', scopes: ['write'] }] }) // token lookup + .mockResolvedValueOnce({ rows: [] }) // last_used_at UPDATE + .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }); // resolveScope + const req = { get: (h) => (h === 'Authorization' ? 'Bearer mcp_good' : undefined) }; + const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(next).toHaveBeenCalledOnce(); + expect(req.mcpScope).toEqual({ + userId: 'user-1', + accountIds: ['acc-1'], + scopes: ['write', 'read'], + }); + // token lookup must be by HASH, never the plaintext + expect(query.mock.calls[0][0]).toMatch(/SELECT id, user_id, scopes FROM api_tokens/); + expect(query.mock.calls[0][1]).toEqual([hashToken('mcp_good')]); + }); + + it('isolates users: the scope carries only the resolved owner', async () => { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok-2', user_id: 'user-2' }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [{ id: 'acc-9' }] }); + const req = { get: () => 'Bearer mcp_u2' }; const res = mockRes(); const next = vi.fn(); + await mcpBearerAuth(req, res, next); + expect(req.mcpScope.userId).toBe('user-2'); + expect(req.mcpScope.accountIds).toEqual(['acc-9']); + }); +}); diff --git a/backend/src/mcp/bodyMatch.js b/backend/src/mcp/bodyMatch.js new file mode 100644 index 00000000..a73b4a0b --- /dev/null +++ b/backend/src/mcp/bodyMatch.js @@ -0,0 +1,78 @@ +// All offsets are UTF-8 BYTES (msgvault wire contract). We operate on Buffers, +// never JS string .length (UTF-16 code units). Case-insensitive matching mirrors +// msgvault: find in the lowercased buffer, slice the original at the same offsets +// (ASCII-faithful; identical behavior to strings.ToLower + strings.Index in Go). +import { SNIPPET_BYTES, isRuneStart, lineNumberAt } from '../utils/textExcerpt.js'; + +export function contextWindow(bodyLen, pos, termLen, contextChars) { + let start = pos - Math.floor((contextChars - termLen) / 2); + let end = start + contextChars; + if (start < 0) { + start = 0; + end = Math.min(bodyLen, contextChars); + } else if (end > bodyLen) { + end = bodyLen; + start = Math.max(0, end - contextChars); + } + return [start, end]; +} + +export function bodyByteSliceRange(buf, start, end) { + if (start < 0) start = 0; + if (end > buf.length) end = buf.length; + if (start >= buf.length) return { text: '', adjStart: buf.length, adjEnd: buf.length }; + let adjStart = start; + let adjEnd = end <= start ? Math.min(buf.length, start + 1) : end; + while (adjStart < adjEnd && !isRuneStart(buf[adjStart])) adjStart++; + while (adjEnd > adjStart && adjEnd < buf.length && !isRuneStart(buf[adjEnd])) adjEnd--; + return { text: buf.toString('utf8', adjStart, adjEnd), adjStart, adjEnd }; +} + +function bodyByteSlice(buf, start, end) { + return bodyByteSliceRange(buf, start, end).text; +} + +export function findTermMatches(body, term) { + if (!body || !term) return []; + const buf = Buffer.from(body, 'utf8'); + const lower = Buffer.from(body.toLowerCase(), 'utf8'); + const t = Buffer.from(term.toLowerCase(), 'utf8'); + const termLen = t.length; + const matches = []; + let from = 0; + for (;;) { + const idx = lower.indexOf(t, from); + if (idx < 0) break; + from = idx + 1; + const [start, end] = contextWindow(buf.length, idx, termLen, SNIPPET_BYTES); + matches.push({ char_offset: idx, snippet: bodyByteSlice(buf, start, end), line: lineNumberAt(buf, idx) }); + } + return matches; +} + +export function extractContextChar(body, terms, contextChars) { + if (!body || !terms || !terms.length || contextChars <= 0) return null; + const buf = Buffer.from(body, 'utf8'); + const lower = Buffer.from(body.toLowerCase(), 'utf8'); + const spans = []; + for (const term of terms) { + if (!term || term.length < 2) continue; + const t = Buffer.from(term.toLowerCase(), 'utf8'); + let from = 0; + for (;;) { + const idx = lower.indexOf(t, from); + if (idx < 0) break; + from = idx + 1; + spans.push(contextWindow(buf.length, idx, t.length, contextChars)); + } + } + if (!spans.length) return null; + spans.sort((a, b) => (a[0] === b[0] ? a[1] - b[1] : a[0] - b[0])); + const merged = [spans[0].slice()]; + for (const s of spans.slice(1)) { + const last = merged[merged.length - 1]; + if (s[0] <= last[1]) last[1] = Math.max(last[1], s[1]); + else merged.push(s.slice()); + } + return merged.map(([s, e]) => bodyByteSlice(buf, s, e)); +} diff --git a/backend/src/mcp/bodyMatch.test.js b/backend/src/mcp/bodyMatch.test.js new file mode 100644 index 00000000..b494a8b4 --- /dev/null +++ b/backend/src/mcp/bodyMatch.test.js @@ -0,0 +1,46 @@ +import { describe, it, expect } from 'vitest'; +import { contextWindow, bodyByteSliceRange, extractContextChar, findTermMatches } from './bodyMatch.js'; + +describe('contextWindow', () => { + it('centers within bounds', () => { + expect(contextWindow(1000, 500, 10, 300)).toEqual([355, 655]); + }); + it('clamps at the start', () => { + expect(contextWindow(1000, 5, 10, 300)).toEqual([0, 300]); + }); + it('clamps at the end', () => { + expect(contextWindow(300, 290, 4, 300)).toEqual([0, 300]); + }); +}); + +describe('byte-offset fidelity on multibyte bodies', () => { + // "café — " then the term. 'é' is 2 bytes (0xC3 0xA9), '—' is 3 bytes. + const body = 'café — meeting notes'; + it('findTermMatches reports the BYTE offset of the term, not the code-unit index', () => { + const m = findTermMatches(body, 'meeting'); + // "café — " = c a f é(2) space —(3) space = 1+1+1+2+1+3+1 = 10 bytes + expect(m).toHaveLength(1); + expect(m[0].char_offset).toBe(10); + expect(m[0].line).toBe(1); + expect(m[0].snippet).toContain('meeting'); + }); + it('bodyByteSliceRange never splits a rune', () => { + const buf = Buffer.from(body, 'utf8'); + // Ask for a window that would cut the 'é' (bytes 3-4) in half at byte 4. + const { text } = bodyByteSliceRange(buf, 0, 4); + expect(Buffer.from(text, 'utf8').every((b) => b !== undefined)).toBe(true); + expect(text).toBe('caf'); // é dropped rather than split + }); +}); + +describe('extractContextChar', () => { + it('merges overlapping windows into snippet strings', () => { + const body = 'alpha beta alpha'; + const snippets = extractContextChar(body, ['alpha'], 300); + expect(snippets).toHaveLength(1); // both hits merge into one window + expect(snippets[0]).toContain('alpha'); + }); + it('ignores terms shorter than 2 chars', () => { + expect(extractContextChar('a a a', ['a'], 300)).toBeNull(); + }); +}); diff --git a/backend/src/mcp/composeSessionTools.js b/backend/src/mcp/composeSessionTools.js new file mode 100644 index 00000000..0e6439f9 --- /dev/null +++ b/backend/src/mcp/composeSessionTools.js @@ -0,0 +1,634 @@ +import { createHash } from 'node:crypto'; +import * as defaultAccountAdapter from './accountAdapter.js'; +import { errorResult, jsonResult } from './result.js'; +import { writeError } from './writeResult.js'; +import { resolveFromIdentity as defaultResolveFromIdentity } from '../services/mail/identity.js'; +import { buildReferences as defaultBuildReferences } from '../services/replyService.js'; + +const annotations = (readOnlyHint, destructiveHint, idempotentHint) => ({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, +}); + +const slotSchema = { type: 'integer', minimum: 1, maximum: 9 }; +const expectedRevisionSchema = { type: 'integer', minimum: 1 }; + +const editableProperties = { + to: { type: 'array', items: { type: 'string' } }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string' }, + body_html: { type: 'string' }, + alias: { type: 'string' }, + priority: { type: 'string', enum: ['high', 'normal', 'low'] }, +}; + +const sessionMutationProperties = { + slot: slotSchema, + expected_revision: expectedRevisionSchema, +}; + +export function sessionRef(args) { + const slot = Number(args.slot); + if (!Number.isInteger(slot) || slot < 1 || slot > 9) { + throw Object.assign(new Error('slot must be an integer from 1 to 9'), { + code: 'invalid_slot', status: 400, expose: true, + }); + } + return { slot }; +} + +export const listComposeSessionsDef = { + name: 'list_compose_sessions', + description: 'List the current user\'s live compose sessions and available slots.', + inputSchema: { + type: 'object', + properties: {}, + }, + annotations: annotations(true, false, true), +}; + +export const getComposeSessionDef = { + name: 'get_compose_session', + description: 'Get a complete live compose session by its user-facing slot.', + inputSchema: { + type: 'object', + required: ['slot'], + properties: { slot: slotSchema }, + }, + annotations: annotations(true, false, true), +}; + +export const createComposeSessionDef = { + name: 'create_compose_session', + description: 'Create a live compose session in a requested or lowest available slot.', + inputSchema: { + type: 'object', + properties: { + slot: slotSchema, + account: { type: 'string' }, + ...editableProperties, + reply_to_message_id: { type: 'string' }, + }, + }, + annotations: annotations(false, false, false), +}; + +export const updateComposeSessionDef = { + name: 'update_compose_session', + description: 'Update explicitly provided fields on a live compose session.', + inputSchema: { + type: 'object', + required: ['slot', 'expected_revision'], + properties: { + ...sessionMutationProperties, + ...editableProperties, + reply_to_message_id: { type: 'string' }, + }, + }, + annotations: annotations(false, false, true), +}; + +export const minimizeComposeSessionDef = { + name: 'minimize_compose_session', + description: 'Minimize a live compose session.', + inputSchema: { + type: 'object', + required: ['slot', 'expected_revision'], + properties: sessionMutationProperties, + }, + annotations: annotations(false, false, true), +}; + +export const restoreComposeSessionDef = { + name: 'restore_compose_session', + description: 'Restore a minimized live compose session.', + inputSchema: { + type: 'object', + required: ['slot', 'expected_revision'], + properties: sessionMutationProperties, + }, + annotations: annotations(false, false, true), +}; + +export const addComposeAttachmentDef = { + name: 'add_compose_attachment', + description: 'Add a base64-encoded attachment to a live compose session.', + inputSchema: { + type: 'object', + required: ['slot', 'expected_revision', 'filename', 'content'], + properties: { + ...sessionMutationProperties, + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }, + annotations: annotations(false, false, true), +}; + +export const removeComposeAttachmentDef = { + name: 'remove_compose_attachment', + description: 'Remove an attachment from a live compose session.', + inputSchema: { + type: 'object', + required: ['slot', 'expected_revision', 'attachment_id'], + properties: { + ...sessionMutationProperties, + attachment_id: { type: 'string' }, + }, + }, + annotations: annotations(false, true, true), +}; + +export const closeComposeSessionDef = { + name: 'close_compose_session', + description: 'Safely close a compose session, saving meaningful content as an IMAP draft.', + inputSchema: { + type: 'object', + required: ['slot', 'expected_revision'], + properties: { + ...sessionMutationProperties, + ...editableProperties, + reply_to_message_id: { type: 'string' }, + }, + }, + annotations: annotations(false, false, true), +}; + +export const discardComposeSessionDef = { + name: 'discard_compose_session', + description: 'Permanently discard a live compose session and free its slot.', + inputSchema: { + type: 'object', + required: ['slot', 'expected_revision'], + properties: sessionMutationProperties, + }, + annotations: annotations(false, true, true), +}; + +export const sendComposeSessionDef = { + name: 'send_compose_session', + description: 'Send a live compose session and free its slot after enqueue or delivery succeeds.', + inputSchema: { + type: 'object', + required: ['slot', 'expected_revision'], + properties: { + ...sessionMutationProperties, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, + }, + }, + annotations: annotations(false, true, false), +}; + +const EDITABLE_FIELDS = ['to', 'cc', 'bcc', 'subject', 'priority']; +const ALL_SLOTS = Array.from({ length: 9 }, (_value, index) => index + 1); +const MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024; + +function hasOwn(value, key) { + return Object.prototype.hasOwnProperty.call(value, key); +} + +function unsupported() { + return writeError('unsupported', 'compose session tools require composeSessionService'); +} + +function unsupportedLifecycle() { + return writeError('unsupported', 'compose session tools require composeSessionLifecycle'); +} + +function clientId(scope) { + return `mcp:${scope.tokenId ?? scope.mcpTokenId ?? scope.userId}`; +} + +function sessionResult(session) { + return jsonResult({ + session_id: session.id, + slot: session.slot, + revision: session.revision, + state: session.presentationState, + session, + }); +} + +function presentationResult(session) { + return jsonResult({ + session_id: session.id, + slot: session.slot, + revision: session.revision, + state: session.presentationState, + }); +} + +function attachmentMetadata(attachment) { + return { + id: attachment.id, + filename: attachment.filename, + contentType: attachment.contentType, + byteCount: attachment.byteCount, + createdAt: attachment.createdAt, + }; +} + +function attachmentInputError(message) { + throw Object.assign(new Error(message), { + code: 'invalid_arguments', status: 400, expose: true, + }); +} + +function decodeAttachmentContent(content) { + if (typeof content !== 'string') { + attachmentInputError('content must be canonical base64'); + } + if (content === '') { + attachmentInputError('attachment content must not be empty'); + } + + const match = /^([A-Za-z0-9+/]+)(={0,2})$/.exec(content); + if (!match) attachmentInputError('content must be canonical base64'); + const [, unpadded, padding] = match; + const remainder = unpadded.length % 4; + const validPadding = (remainder === 0 && padding.length === 0) + || (remainder === 2 && (padding.length === 0 || padding.length === 2)) + || (remainder === 3 && (padding.length === 0 || padding.length === 1)); + if (!validPadding) attachmentInputError('content must be canonical base64'); + + const decodedLength = Math.floor((unpadded.length * 3) / 4); + if (decodedLength > MAX_ATTACHMENT_BYTES) { + throw Object.assign(new Error('attachment content must not exceed 25 MiB'), { + code: 'attachment_too_large', status: 413, expose: true, + }); + } + + const decoded = Buffer.from(content, 'base64'); + const normalized = decoded.toString('base64').replace(/=+$/, ''); + if (normalized !== unpadded) attachmentInputError('content must be canonical base64'); + if (decoded.length === 0) attachmentInputError('attachment content must not be empty'); + return decoded; +} + +function resultForError(error) { + if (error?.expose === true && error.code === 'compose_conflict') { + return errorResult(JSON.stringify({ + error: 'compose_conflict', + message: error.message, + current_revision: error.details?.currentRevision, + conflicting_fields: error.details?.conflictingFields, + remote_values: error.details?.remoteValues, + })); + } + if (error?.expose === true && error.code) return writeError(error.code, error.message); + throw error; +} + +async function asToolResult(callback) { + try { + return await callback(); + } catch (error) { + return resultForError(error); + } +} + +function serviceFor(deps, method) { + const service = deps?.composeSessionService; + return typeof service?.[method] === 'function' ? service : null; +} + +function lifecycleFor(deps, method) { + const lifecycle = deps?.composeSessionLifecycle; + return typeof lifecycle?.[method] === 'function' ? lifecycle : null; +} + +function referenceList(value) { + if (Array.isArray(value)) return value; + if (typeof value !== 'string' || !value.trim()) return []; + return value.match(/<[^>]+>/g) || [value.trim()]; +} + +async function threadingChanges(args, scope, deps) { + if (!hasOwn(args, 'reply_to_message_id')) return {}; + if (args.reply_to_message_id === '') { + return { inReplyTo: null, references: [] }; + } + const accountAdapter = deps.accountAdapter || defaultAccountAdapter; + const source = await accountAdapter.getComposeSource( + args.reply_to_message_id, + scope.accountIds, + ); + if (!source) { + throw Object.assign(new Error(args.reply_to_message_id), { + code: 'message_not_found', + expose: true, + }); + } + const buildReferences = deps.buildReferences || defaultBuildReferences; + const threading = buildReferences(source); + return { + inReplyTo: threading.inReplyTo, + references: referenceList(threading.references), + }; +} + +async function editableChanges(args, scope, deps) { + const changes = {}; + for (const field of EDITABLE_FIELDS) { + if (hasOwn(args, field)) changes[field] = args[field]; + } + if (hasOwn(args, 'body_html')) { + changes.body = args.body_html; + changes.bodyIsHtml = true; + } else if (hasOwn(args, 'body')) { + changes.body = args.body; + changes.bodyIsHtml = false; + } + return { ...changes, ...await threadingChanges(args, scope, deps) }; +} + +async function aliasChanges(args, scope, deps, ref) { + if (!hasOwn(args, 'alias')) return {}; + if (args.alias === '') return { aliasId: null }; + const service = serviceFor(deps, 'getComposeSession'); + if (!service) return null; + const current = await service.getComposeSession({ + userId: scope.userId, + ...ref, + }, deps); + const accountAdapter = deps.accountAdapter || defaultAccountAdapter; + const account = await accountAdapter.getAccountRow(current.accountId, scope.accountIds); + if (!account) { + throw Object.assign(new Error(current.accountId), { + code: 'account_not_found', expose: true, + }); + } + const resolveFromIdentity = deps.resolveFromIdentity || defaultResolveFromIdentity; + const identity = await resolveFromIdentity(account, { aliasEmail: args.alias }, deps); + return { aliasId: identity.aliasId }; +} + +function invalidUndoSeconds() { + throw Object.assign( + new Error('undo_send_seconds must be an integer from 0 to 120'), + { code: 'invalid_compose_undo_seconds', status: 400, expose: true }, + ); +} + +function deterministicSendKey(args, scope, ref) { + const context = JSON.stringify({ + requestId: scope.requestId ?? scope.mcpRequestId ?? null, + tokenId: scope.tokenId ?? scope.mcpTokenId ?? null, + userId: scope.userId, + slot: ref.slot, + expectedRevision: args.expected_revision, + }); + return `mcp-compose:${createHash('sha256').update(context, 'utf8').digest('hex')}`; +} + +export async function handleListComposeSessions(_args, scope, deps = {}) { + const service = serviceFor(deps, 'listComposeSessions'); + if (!service) return unsupported(); + return asToolResult(async () => { + const sessions = await service.listComposeSessions({ userId: scope.userId }, deps); + const occupiedSlots = [...new Set(sessions.map(item => item.slot))].sort((a, b) => a - b); + const occupied = new Set(occupiedSlots); + return jsonResult({ + sessions, + occupied_slots: occupiedSlots, + available_slots: ALL_SLOTS.filter(slot => !occupied.has(slot)), + }); + }); +} + +export async function handleGetComposeSession(args, scope, deps = {}) { + const service = serviceFor(deps, 'getComposeSession'); + if (!service) return unsupported(); + return asToolResult(async () => { + const session = await service.getComposeSession({ + userId: scope.userId, + ...sessionRef(args), + }, deps); + return sessionResult(session); + }); +} + +export async function handleCreateComposeSession(args, scope, deps = {}) { + const service = serviceFor(deps, 'createComposeSession'); + if (!service) return unsupported(); + return asToolResult(async () => { + const requestedSlot = hasOwn(args, 'slot') ? sessionRef(args).slot : undefined; + let account; + if (hasOwn(args, 'account')) { + const accountAdapter = deps.accountAdapter || defaultAccountAdapter; + account = await accountAdapter.getAccountByEmail(args.account, scope.accountIds); + if (account?.error) return errorResult(account.error); + } + let aliasId; + if (hasOwn(args, 'alias')) { + if (args.alias === '') { + aliasId = null; + } else { + if (!account) { + return writeError('invalid_arguments', 'alias requires account'); + } + const resolveFromIdentity = deps.resolveFromIdentity || defaultResolveFromIdentity; + const identity = await resolveFromIdentity(account, { aliasEmail: args.alias }, deps); + aliasId = identity.aliasId; + } + } + const changes = await editableChanges(args, scope, deps); + if (account) changes.accountId = account.id; + if (hasOwn(args, 'alias')) changes.aliasId = aliasId; + const input = { + userId: scope.userId, + changes, + clientId: clientId(scope), + }; + if (requestedSlot !== undefined) input.requestedSlot = requestedSlot; + const session = await service.createComposeSession(input, deps); + return sessionResult(session); + }); +} + +export async function handleUpdateComposeSession(args, scope, deps = {}) { + const service = serviceFor(deps, 'patchComposeSession'); + if (!service) return unsupported(); + return asToolResult(async () => { + const ref = sessionRef(args); + const changes = await editableChanges(args, scope, deps); + if (hasOwn(args, 'alias')) { + if (args.alias === '') { + changes.aliasId = null; + } else { + if (typeof service.getComposeSession !== 'function') return unsupported(); + const current = await service.getComposeSession({ + userId: scope.userId, + ...ref, + }, deps); + const accountAdapter = deps.accountAdapter || defaultAccountAdapter; + const account = await accountAdapter.getAccountRow(current.accountId, scope.accountIds); + if (!account) return writeError('account_not_found', current.accountId); + const resolveFromIdentity = deps.resolveFromIdentity || defaultResolveFromIdentity; + const identity = await resolveFromIdentity(account, { aliasEmail: args.alias }, deps); + changes.aliasId = identity.aliasId; + } + } + const session = await service.patchComposeSession({ + userId: scope.userId, + ...ref, + expectedRevision: args.expected_revision, + changes, + clientId: clientId(scope), + }, deps); + return sessionResult(session); + }); +} + +async function handlePresentation(args, scope, deps, state) { + const service = serviceFor(deps, 'setComposePresentation'); + if (!service) return unsupported(); + return asToolResult(async () => { + const session = await service.setComposePresentation({ + userId: scope.userId, + ...sessionRef(args), + expectedRevision: args.expected_revision, + state, + clientId: clientId(scope), + }, deps); + return presentationResult(session); + }); +} + +export async function handleMinimizeComposeSession(args, scope, deps = {}) { + return handlePresentation(args, scope, deps, 'minimized'); +} + +export async function handleRestoreComposeSession(args, scope, deps = {}) { + return handlePresentation(args, scope, deps, 'expanded'); +} + +export async function handleAddComposeAttachment(args, scope, deps = {}) { + const service = serviceFor(deps, 'addComposeAttachment'); + if (!service) return unsupported(); + return asToolResult(async () => { + const ref = sessionRef(args); + const content = decodeAttachmentContent(args.content); + const result = await service.addComposeAttachment({ + userId: scope.userId, + ...ref, + expectedRevision: args.expected_revision, + filename: args.filename, + content, + contentType: args.content_type, + clientId: clientId(scope), + }, deps); + return jsonResult({ + session_id: result.sessionId, + slot: result.slot, + revision: result.revision, + attachment: attachmentMetadata(result.attachment), + }); + }); +} + +export async function handleRemoveComposeAttachment(args, scope, deps = {}) { + const service = serviceFor(deps, 'removeComposeAttachment'); + if (!service) return unsupported(); + return asToolResult(async () => { + const result = await service.removeComposeAttachment({ + userId: scope.userId, + ...sessionRef(args), + expectedRevision: args.expected_revision, + attachmentId: args.attachment_id, + clientId: clientId(scope), + }, deps); + return jsonResult({ + session_id: result.sessionId, + slot: result.slot, + revision: result.revision, + removed_attachment_id: result.removedAttachmentId, + }); + }); +} + +export async function handleCloseComposeSession(args, scope, deps = {}) { + const lifecycle = lifecycleFor(deps, 'closeComposeSession'); + if (!lifecycle) return unsupportedLifecycle(); + return asToolResult(async () => { + const ref = sessionRef(args); + const changes = await editableChanges(args, scope, deps); + const alias = await aliasChanges(args, scope, deps, ref); + if (alias === null) return unsupported(); + Object.assign(changes, alias); + const result = await lifecycle.closeComposeSession({ + userId: scope.userId, + ...ref, + expectedRevision: args.expected_revision, + changes, + }, deps); + return jsonResult({ + closed: result.closed, + freed_slot: result.slot, + draft: result.draft ? { + account: result.draft.account, + draft_uid: result.draft.uid, + folder: result.draft.folder, + message_id: result.draft.messageId, + } : null, + }); + }); +} + +export async function handleDiscardComposeSession(args, scope, deps = {}) { + const lifecycle = lifecycleFor(deps, 'discardComposeSession'); + if (!lifecycle) return unsupportedLifecycle(); + return asToolResult(async () => { + const result = await lifecycle.discardComposeSession({ + userId: scope.userId, + ...sessionRef(args), + expectedRevision: args.expected_revision, + }, deps); + return jsonResult({ discarded: result.discarded, freed_slot: result.slot }); + }); +} + +export async function handleSendComposeSession(args, scope, deps = {}) { + const lifecycle = lifecycleFor(deps, 'sendComposeSession'); + if (!lifecycle) return unsupportedLifecycle(); + return asToolResult(async () => { + const ref = sessionRef(args); + if (hasOwn(args, 'undo_send_seconds') && ( + !Number.isInteger(args.undo_send_seconds) + || args.undo_send_seconds < 0 + || args.undo_send_seconds > 120 + )) invalidUndoSeconds(); + const input = { + userId: scope.userId, + ...ref, + expectedRevision: args.expected_revision, + idempotencyKey: hasOwn(args, 'idempotency_key') + ? args.idempotency_key + : deterministicSendKey(args, scope, ref), + }; + if (hasOwn(args, 'undo_send_seconds')) input.undoSendSeconds = args.undo_send_seconds; + const result = await lifecycle.sendComposeSession(input, deps); + if (result.queued === true) { + return jsonResult({ + queued: true, + freed_slot: ref.slot, + outbox_id: result.outboxId, + send_at: result.sendAt, + undo_seconds: result.undoSeconds, + }); + } + return jsonResult({ + sent: true, + freed_slot: ref.slot, + message_id: result.messageId, + sent_copy_saved: result.sentCopySaved, + receipt: result.receipt, + }); + }); +} diff --git a/backend/src/mcp/composeSessionTools.test.js b/backend/src/mcp/composeSessionTools.test.js new file mode 100644 index 00000000..fdb0fe0b --- /dev/null +++ b/backend/src/mcp/composeSessionTools.test.js @@ -0,0 +1,1524 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { + addComposeAttachmentDef, + closeComposeSessionDef, + createComposeSessionDef, + discardComposeSessionDef, + getComposeSessionDef, + handleCloseComposeSession, + handleCreateComposeSession, + handleDiscardComposeSession, + handleGetComposeSession, + handleAddComposeAttachment, + handleListComposeSessions, + handleMinimizeComposeSession, + handleRemoveComposeAttachment, + handleRestoreComposeSession, + handleSendComposeSession, + handleUpdateComposeSession, + listComposeSessionsDef, + minimizeComposeSessionDef, + removeComposeAttachmentDef, + restoreComposeSessionDef, + sendComposeSessionDef, + updateComposeSessionDef, +} from './composeSessionTools.js'; + +const definitions = [ + listComposeSessionsDef, + getComposeSessionDef, + createComposeSessionDef, + updateComposeSessionDef, + minimizeComposeSessionDef, + restoreComposeSessionDef, + addComposeAttachmentDef, + removeComposeAttachmentDef, + closeComposeSessionDef, + discardComposeSessionDef, + sendComposeSessionDef, +]; + +const slotSchema = { type: 'integer', minimum: 1, maximum: 9 }; +const revisionSchema = { type: 'integer', minimum: 1 }; +const replyToMessageIdSchema = { type: 'string' }; +const editableProperties = { + to: { type: 'array', items: { type: 'string' } }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string' }, + body_html: { type: 'string' }, + alias: { type: 'string' }, + priority: { type: 'string', enum: ['high', 'normal', 'low'] }, +}; + +describe('compose-session tool definitions', () => { + it('publishes the exact definition inventory in approved order', () => { + expect(definitions.map(definition => definition.name)).toEqual([ + 'list_compose_sessions', + 'get_compose_session', + 'create_compose_session', + 'update_compose_session', + 'minimize_compose_session', + 'restore_compose_session', + 'add_compose_attachment', + 'remove_compose_attachment', + 'close_compose_session', + 'discard_compose_session', + 'send_compose_session', + ]); + }); + + it('uses scope-neutral annotations with destructive operations identified', () => { + expect(definitions.map(definition => [definition.name, definition.annotations])).toEqual([ + ['list_compose_sessions', annotations(true, false, true)], + ['get_compose_session', annotations(true, false, true)], + ['create_compose_session', annotations(false, false, false)], + ['update_compose_session', annotations(false, false, true)], + ['minimize_compose_session', annotations(false, false, true)], + ['restore_compose_session', annotations(false, false, true)], + ['add_compose_attachment', annotations(false, false, true)], + ['remove_compose_attachment', annotations(false, true, true)], + ['close_compose_session', annotations(false, false, true)], + ['discard_compose_session', annotations(false, true, true)], + ['send_compose_session', annotations(false, true, false)], + ]); + }); + + it('defines the exact list, get, create, and update input schemas', () => { + expect(listComposeSessionsDef.inputSchema).toEqual({ + type: 'object', + properties: {}, + }); + expect(getComposeSessionDef.inputSchema).toEqual({ + type: 'object', + required: ['slot'], + properties: { slot: slotSchema }, + }); + expect(createComposeSessionDef.inputSchema).toEqual({ + type: 'object', + properties: { + slot: slotSchema, + account: { type: 'string' }, + ...editableProperties, + reply_to_message_id: replyToMessageIdSchema, + }, + }); + expect(updateComposeSessionDef.inputSchema).toEqual({ + type: 'object', + required: ['slot', 'expected_revision'], + properties: { + slot: slotSchema, + expected_revision: revisionSchema, + ...editableProperties, + reply_to_message_id: replyToMessageIdSchema, + }, + }); + }); + + it('defines revision-guarded presentation schemas', () => { + const expected = { + type: 'object', + required: ['slot', 'expected_revision'], + properties: { + slot: slotSchema, + expected_revision: revisionSchema, + }, + }; + expect(minimizeComposeSessionDef.inputSchema).toEqual(expected); + expect(restoreComposeSessionDef.inputSchema).toEqual(expected); + }); + + it('defines the exact attachment schemas', () => { + expect(addComposeAttachmentDef.inputSchema).toEqual({ + type: 'object', + required: ['slot', 'expected_revision', 'filename', 'content'], + properties: { + slot: slotSchema, + expected_revision: revisionSchema, + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }); + expect(removeComposeAttachmentDef.inputSchema).toEqual({ + type: 'object', + required: ['slot', 'expected_revision', 'attachment_id'], + properties: { + slot: slotSchema, + expected_revision: revisionSchema, + attachment_id: { type: 'string' }, + }, + }); + }); + + it('defines the exact close, discard, and send schemas', () => { + expect(closeComposeSessionDef.inputSchema).toEqual({ + type: 'object', + required: ['slot', 'expected_revision'], + properties: { + slot: slotSchema, + expected_revision: revisionSchema, + ...editableProperties, + reply_to_message_id: replyToMessageIdSchema, + }, + }); + expect(discardComposeSessionDef.inputSchema).toEqual({ + type: 'object', + required: ['slot', 'expected_revision'], + properties: { + slot: slotSchema, + expected_revision: revisionSchema, + }, + }); + expect(sendComposeSessionDef.inputSchema).toEqual({ + type: 'object', + required: ['slot', 'expected_revision'], + properties: { + slot: slotSchema, + expected_revision: revisionSchema, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, + }, + }); + }); + + it('keeps every definition plain JSON-serializable data', () => { + expect(JSON.parse(JSON.stringify(definitions))).toEqual(definitions); + }); +}); + +function annotations(readOnlyHint, destructiveHint, idempotentHint) { + return { readOnlyHint, destructiveHint, idempotentHint, openWorldHint: false }; +} + +const scope = { + userId: 'user-1', + accountIds: ['account-1'], + scopes: ['read', 'write'], +}; + +const account = { + id: 'account-1', + email_address: 'sender@example.com', + sender_name: 'Sender', +}; + +const identity = { + fromName: 'Team', + fromEmail: 'team@example.com', + fromReplyTo: null, + signature: null, + aliasId: 'alias-1', +}; + +const session = { + id: '11111111-1111-4111-8111-111111111111', + slot: 2, + revision: 7, + presentationState: 'expanded', + subject: 'Subject', +}; + +function payload(result) { + return JSON.parse(result.content[0].text); +} + +function dependencies(overrides = {}) { + const composeSessionService = { + listComposeSessions: vi.fn().mockResolvedValue([]), + getComposeSession: vi.fn().mockResolvedValue(session), + createComposeSession: vi.fn().mockResolvedValue(session), + patchComposeSession: vi.fn().mockResolvedValue(session), + setComposePresentation: vi.fn().mockResolvedValue(session), + addComposeAttachment: vi.fn().mockResolvedValue({ + sessionId: session.id, + slot: session.slot, + revision: session.revision + 1, + attachment: { + id: '22222222-2222-4222-8222-222222222222', + filename: 'note.txt', + contentType: 'text/plain', + byteCount: 5, + createdAt: '2026-08-01T00:00:00.000Z', + content: Buffer.from('synthetic attachment bytes'), + }, + }), + removeComposeAttachment: vi.fn().mockResolvedValue({ + sessionId: session.id, + slot: session.slot, + revision: session.revision + 1, + removedAttachmentId: '22222222-2222-4222-8222-222222222222', + }), + }; + const accountAdapter = { + getAccountByEmail: vi.fn().mockResolvedValue(account), + getAccountRow: vi.fn().mockResolvedValue(account), + getComposeSource: vi.fn().mockResolvedValue({ + id: 'message-1', + message_id: '', + thread_references: '', + }), + }; + const composeSessionLifecycle = { + closeComposeSession: vi.fn().mockResolvedValue({ + closed: true, + slot: session.slot, + draft: null, + }), + discardComposeSession: vi.fn().mockResolvedValue({ + discarded: true, + slot: session.slot, + }), + sendComposeSession: vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: { + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: '', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Synthetic subject', + attachments: [], + messageId: '', + sentCopySaved: true, + folder: 'Sent', + }, + }), + }; + return { + composeSessionService, + composeSessionLifecycle, + accountAdapter, + resolveFromIdentity: vi.fn().mockResolvedValue(identity), + buildReferences: vi.fn().mockReturnValue({ + inReplyTo: '', + references: ' ', + }), + draftService: { + saveDraft: vi.fn(), + deleteDraft: vi.fn(), + }, + sendService: { sendOrEnqueue: vi.fn() }, + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('compose-session read handlers', () => { + it('lists summary sessions with occupied and available slots', async () => { + const deps = dependencies(); + const summaries = [ + { ...session, slot: 2 }, + { ...session, id: '77777777-7777-4777-8777-777777777777', slot: 7 }, + ]; + deps.composeSessionService.listComposeSessions.mockResolvedValue(summaries); + + const result = payload(await handleListComposeSessions({}, scope, deps)); + + expect(deps.composeSessionService.listComposeSessions).toHaveBeenCalledWith( + { userId: 'user-1' }, + deps, + ); + expect(result).toEqual({ + sessions: summaries, + occupied_slots: [2, 7], + available_slots: [1, 3, 4, 5, 6, 8, 9], + }); + }); + + it('gets a complete session by slot and returns the stable envelope', async () => { + const deps = dependencies(); + + const result = payload(await handleGetComposeSession({ slot: 2 }, scope, deps)); + + expect(deps.composeSessionService.getComposeSession).toHaveBeenCalledWith( + { userId: 'user-1', slot: 2 }, + deps, + ); + expect(result).toEqual({ + session_id: session.id, + slot: 2, + revision: 7, + state: 'expanded', + session, + }); + }); + + it.each([0, 10, 2.5])('rejects invalid read slot %s before the service call', async (slot) => { + const deps = dependencies(); + + const result = await handleGetComposeSession({ slot }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe( + 'invalid_slot: slot must be an integer from 1 to 9', + ); + expect(deps.composeSessionService.getComposeSession).not.toHaveBeenCalled(); + }); + + it('returns a stable unsupported read error when the service dependency is missing', async () => { + const result = await handleListComposeSessions({}, scope, {}); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe( + 'unsupported: compose session tools require composeSessionService', + ); + }); +}); + +describe('compose-session create handlers', () => { + it('creates in the lowest free slot with no invented changes', async () => { + const deps = dependencies(); + + const result = payload(await handleCreateComposeSession({}, scope, deps)); + + expect(deps.composeSessionService.createComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + changes: {}, + clientId: 'mcp:user-1', + }, deps); + expect(result).toEqual({ + session_id: session.id, + slot: 2, + revision: 7, + state: 'expanded', + session, + }); + }); + + it('creates in a requested slot and resolves account, alias, HTML body, and reply source', async () => { + const deps = dependencies(); + + await handleCreateComposeSession({ + slot: 4, + account: 'sender@example.com', + alias: 'team@example.com', + to: ['recipient@example.com'], + cc: [], + bcc: [], + subject: 'Threaded subject', + body: 'plain fallback', + body_html: '

HTML body

', + priority: 'high', + reply_to_message_id: 'message-1', + }, { ...scope, tokenId: 'token-1' }, deps); + + expect(deps.accountAdapter.getAccountByEmail).toHaveBeenCalledWith( + 'sender@example.com', + ['account-1'], + ); + expect(deps.resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasEmail: 'team@example.com' }, + deps, + ); + expect(deps.accountAdapter.getComposeSource).toHaveBeenCalledWith( + 'message-1', + ['account-1'], + ); + expect(deps.buildReferences).toHaveBeenCalledWith(expect.objectContaining({ + id: 'message-1', + })); + expect(deps.composeSessionService.createComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + requestedSlot: 4, + changes: { + accountId: 'account-1', + aliasId: 'alias-1', + to: ['recipient@example.com'], + cc: [], + bcc: [], + subject: 'Threaded subject', + body: '

HTML body

', + bodyIsHtml: true, + priority: 'high', + inReplyTo: '', + references: ['', ''], + }, + clientId: 'mcp:token-1', + }, deps); + }); + + it('maps an explicitly empty plain body as a clear without inventing other fields', async () => { + const deps = dependencies(); + + await handleCreateComposeSession({ body: '' }, scope, deps); + + expect(deps.composeSessionService.createComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + changes: { body: '', bodyIsHtml: false }, + clientId: 'mcp:user-1', + }, deps); + }); + + it('rejects an unknown scoped account without calling identity or session services', async () => { + const deps = dependencies(); + deps.accountAdapter.getAccountByEmail.mockResolvedValue({ + error: 'account_not_found: missing@example.com', + }); + + const result = await handleCreateComposeSession({ + account: 'missing@example.com', + alias: 'team@example.com', + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe('account_not_found: missing@example.com'); + expect(deps.resolveFromIdentity).not.toHaveBeenCalled(); + expect(deps.composeSessionService.createComposeSession).not.toHaveBeenCalled(); + }); + + it('rejects an out-of-scope reply source before creating', async () => { + const deps = dependencies(); + deps.accountAdapter.getComposeSource.mockResolvedValue(null); + + const result = await handleCreateComposeSession({ + reply_to_message_id: 'missing-message', + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe('message_not_found: missing-message'); + expect(deps.composeSessionService.createComposeSession).not.toHaveBeenCalled(); + }); + + it('hard-fails an unknown alias through the shared identity path', async () => { + const deps = dependencies(); + deps.resolveFromIdentity.mockRejectedValue(Object.assign( + new Error('Alias not found'), + { code: 'alias_not_found', expose: true }, + )); + + const result = await handleCreateComposeSession({ + account: 'sender@example.com', + alias: 'unknown@example.com', + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe('alias_not_found: Alias not found'); + expect(deps.composeSessionService.createComposeSession).not.toHaveBeenCalled(); + }); +}); + +describe('compose-session update handlers', () => { + it('updates only explicitly present fields, including empty list and plain-body clears', async () => { + const deps = dependencies(); + + await handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + subject: '', + to: [], + cc: [], + bcc: [], + body: '', + priority: 'low', + }, scope, deps); + + expect(deps.composeSessionService.patchComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 2, + expectedRevision: 7, + changes: { + to: [], + cc: [], + bcc: [], + subject: '', + body: '', + bodyIsHtml: false, + priority: 'low', + }, + clientId: 'mcp:user-1', + }, deps); + }); + + it('maps HTML and reply updates without preserving an omitted plain body', async () => { + const deps = dependencies(); + + await handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + body_html: '', + reply_to_message_id: 'message-1', + }, scope, deps); + + expect(deps.composeSessionService.patchComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 2, + expectedRevision: 7, + changes: { + body: '', + bodyIsHtml: true, + inReplyTo: '', + references: ['', ''], + }, + clientId: 'mcp:user-1', + }, deps); + }); + + it('resolves an alias against the current scoped session account', async () => { + const deps = dependencies(); + deps.composeSessionService.getComposeSession.mockResolvedValue({ + ...session, + accountId: 'account-1', + }); + + await handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + alias: 'team@example.com', + }, scope, deps); + + expect(deps.composeSessionService.getComposeSession).toHaveBeenCalledWith( + { userId: 'user-1', slot: 2 }, + deps, + ); + expect(deps.accountAdapter.getAccountRow).toHaveBeenCalledWith( + 'account-1', + ['account-1'], + ); + expect(deps.resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasEmail: 'team@example.com' }, + deps, + ); + expect(deps.composeSessionService.patchComposeSession).toHaveBeenCalledWith( + expect.objectContaining({ changes: { aliasId: 'alias-1' } }), + deps, + ); + }); + + it('clears an alias explicitly without requiring an account lookup', async () => { + const deps = dependencies(); + + await handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + alias: '', + }, scope, deps); + + expect(deps.composeSessionService.getComposeSession).not.toHaveBeenCalled(); + expect(deps.accountAdapter.getAccountRow).not.toHaveBeenCalled(); + expect(deps.resolveFromIdentity).not.toHaveBeenCalled(); + expect(deps.composeSessionService.patchComposeSession).toHaveBeenCalledWith( + expect.objectContaining({ changes: { aliasId: null } }), + deps, + ); + }); + + it('clears reply threading explicitly without resolving a source', async () => { + const deps = dependencies(); + + await handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + reply_to_message_id: '', + }, scope, deps); + + expect(deps.accountAdapter.getComposeSource).not.toHaveBeenCalled(); + expect(deps.buildReferences).not.toHaveBeenCalled(); + expect(deps.composeSessionService.patchComposeSession).toHaveBeenCalledWith( + expect.objectContaining({ + changes: { inReplyTo: null, references: [] }, + }), + deps, + ); + }); + + it('returns ordinary exposed update errors through writeError', async () => { + const deps = dependencies(); + deps.composeSessionService.patchComposeSession.mockRejectedValue(Object.assign( + new Error('Compose session not found'), + { code: 'compose_session_not_found', expose: true }, + )); + + const result = await handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + subject: 'Updated', + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe( + 'compose_session_not_found: Compose session not found', + ); + }); + + it('returns compose conflicts as structured snake-case JSON', async () => { + const deps = dependencies(); + deps.composeSessionService.patchComposeSession.mockRejectedValue(Object.assign( + new Error('Compose session changed in the requested fields'), + { + code: 'compose_conflict', + expose: true, + details: { + currentRevision: 8, + conflictingFields: ['subject'], + remoteValues: { subject: 'Remote subject' }, + }, + }, + )); + + const result = await handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + subject: 'Local subject', + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(payload(result)).toEqual({ + error: 'compose_conflict', + message: 'Compose session changed in the requested fields', + current_revision: 8, + conflicting_fields: ['subject'], + remote_values: { subject: 'Remote subject' }, + }); + }); + + it('rethrows an unexposed coded compose conflict for the MCP internal-error boundary', async () => { + const deps = dependencies(); + deps.composeSessionService.patchComposeSession.mockRejectedValue(Object.assign( + new Error('internal conflict detail'), + { + code: 'compose_conflict', + expose: false, + details: { + currentRevision: 8, + conflictingFields: ['subject'], + remoteValues: { subject: 'Remote subject' }, + }, + }, + )); + + await expect(handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + subject: 'Local subject', + }, scope, deps)).rejects.toThrow('internal conflict detail'); + }); + + it('rethrows an unexposed ordinary coded failure for the MCP internal-error boundary', async () => { + const deps = dependencies(); + deps.composeSessionService.patchComposeSession.mockRejectedValue(Object.assign( + new Error('duplicate key value exposes internals'), + { code: '23505' }, + )); + + await expect(handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + subject: 'Updated', + }, scope, deps)).rejects.toThrow('duplicate key value exposes internals'); + }); + + it('rethrows unknown update failures for the MCP internal-error boundary', async () => { + const deps = dependencies(); + deps.composeSessionService.patchComposeSession.mockRejectedValue( + new Error('database unavailable'), + ); + + await expect(handleUpdateComposeSession({ + slot: 2, + expected_revision: 7, + subject: 'Updated', + }, scope, deps)).rejects.toThrow('database unavailable'); + }); +}); + +describe('compose-session presentation handlers', () => { + it('minimizes and restores with exact revision guards and a stable token client id', async () => { + const deps = dependencies(); + deps.composeSessionService.setComposePresentation + .mockResolvedValueOnce({ ...session, revision: 8, presentationState: 'minimized' }) + .mockResolvedValueOnce({ ...session, revision: 9, presentationState: 'expanded' }); + const tokenScope = { ...scope, tokenId: 'token-1' }; + + const minimized = payload(await handleMinimizeComposeSession({ + slot: 2, + expected_revision: 7, + }, tokenScope, deps)); + const restored = payload(await handleRestoreComposeSession({ + slot: 2, + expected_revision: 8, + }, tokenScope, deps)); + + expect(deps.composeSessionService.setComposePresentation.mock.calls).toEqual([ + [{ + userId: 'user-1', + slot: 2, + expectedRevision: 7, + state: 'minimized', + clientId: 'mcp:token-1', + }, deps], + [{ + userId: 'user-1', + slot: 2, + expectedRevision: 8, + state: 'expanded', + clientId: 'mcp:token-1', + }, deps], + ]); + expect(minimized).toEqual({ + session_id: session.id, + slot: 2, + revision: 8, + state: 'minimized', + }); + expect(restored).toEqual({ + session_id: session.id, + slot: 2, + revision: 9, + state: 'expanded', + }); + }); +}); + +describe('compose-session attachment handlers', () => { + const attachmentId = '22222222-2222-4222-8222-222222222222'; + + it('keeps attachment removal explicitly destructive', () => { + expect(removeComposeAttachmentDef.annotations).toEqual( + annotations(false, true, true), + ); + }); + + it('decodes canonical base64 into a Buffer and forwards exact attachment scope', async () => { + const deps = dependencies(); + const result = payload(await handleAddComposeAttachment({ + slot: 2, + expected_revision: 7, + filename: 'note.txt', + content: Buffer.from('hello').toString('base64'), + content_type: 'text/plain', + }, { ...scope, tokenId: 'token-1' }, deps)); + + expect(deps.composeSessionService.addComposeAttachment).toHaveBeenCalledTimes(1); + const [input, serviceDeps] = deps.composeSessionService.addComposeAttachment.mock.calls[0]; + expect(serviceDeps).toBe(deps); + expect(input).toEqual({ + userId: 'user-1', + slot: 2, + expectedRevision: 7, + filename: 'note.txt', + content: expect.any(Buffer), + contentType: 'text/plain', + clientId: 'mcp:token-1', + }); + expect(Buffer.isBuffer(input.content)).toBe(true); + expect(input.content.equals(Buffer.from('hello'))).toBe(true); + expect(result).toEqual({ + session_id: session.id, + slot: 2, + revision: 8, + attachment: { + id: attachmentId, + filename: 'note.txt', + contentType: 'text/plain', + byteCount: 5, + createdAt: '2026-08-01T00:00:00.000Z', + }, + }); + expect(JSON.stringify(result)).not.toContain('hello'); + }); + + it('accepts a canonical unpadded encoding and lets the service apply its content-type default', async () => { + const deps = dependencies(); + deps.composeSessionService.addComposeAttachment.mockResolvedValue({ + sessionId: session.id, + slot: 2, + revision: 8, + attachment: { + id: attachmentId, + filename: 'one-byte.bin', + contentType: 'application/octet-stream', + byteCount: 1, + createdAt: '2026-08-01T00:00:00.000Z', + }, + }); + + const result = payload(await handleAddComposeAttachment({ + slot: 2, + expected_revision: 7, + filename: 'one-byte.bin', + content: 'YQ', + }, scope, deps)); + + expect(deps.composeSessionService.addComposeAttachment).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 2, + expectedRevision: 7, + filename: 'one-byte.bin', + content: Buffer.from('a'), + contentType: undefined, + clientId: 'mcp:user-1', + }, deps); + expect(result.attachment.contentType).toBe('application/octet-stream'); + }); + + it.each([ + ['invalid alphabet', 'YWJj*'], + ['URL-safe alphabet', '-_8='], + ['embedded whitespace', 'Y WJj'], + ['missing required padding characters', 'YQ='], + ['excess padding', 'YQ==='], + ['padding on an unpadded quantum', 'YWJj='], + ['misplaced padding', '=YQ='], + ['impossible encoded length', 'A'], + ['non-canonical pad bits', 'YR=='], + ])('rejects malformed attachment base64 with %s before service', async (_case, content) => { + const deps = dependencies(); + + const result = await handleAddComposeAttachment({ + slot: 2, + expected_revision: 7, + filename: 'bad.bin', + content, + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe( + 'invalid_arguments: content must be canonical base64', + ); + expect(deps.composeSessionService.addComposeAttachment).not.toHaveBeenCalled(); + }); + + it('rejects an empty decoded attachment before service', async () => { + const deps = dependencies(); + + const result = await handleAddComposeAttachment({ + slot: 2, + expected_revision: 7, + filename: 'empty.bin', + content: '', + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe( + 'invalid_arguments: attachment content must not be empty', + ); + expect(deps.composeSessionService.addComposeAttachment).not.toHaveBeenCalled(); + }); + + it('allows exactly 25 MiB and rejects one decoded byte more before service', async () => { + const deps = dependencies(); + const maxBytes = 25 * 1024 * 1024; + const exactContent = Buffer.alloc(maxBytes).toString('base64'); + + await handleAddComposeAttachment({ + slot: 2, + expected_revision: 7, + filename: 'exact.bin', + content: exactContent, + }, scope, deps); + + expect(deps.composeSessionService.addComposeAttachment).toHaveBeenCalledTimes(1); + const exactInput = deps.composeSessionService.addComposeAttachment.mock.calls[0][0]; + expect(Buffer.isBuffer(exactInput.content)).toBe(true); + expect(exactInput.content.length).toBe(maxBytes); + + deps.composeSessionService.addComposeAttachment.mockClear(); + const tooLargeContent = Buffer.alloc(maxBytes + 1).toString('base64'); + const result = await handleAddComposeAttachment({ + slot: 2, + expected_revision: 7, + filename: 'too-large.bin', + content: tooLargeContent, + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe( + 'attachment_too_large: attachment content must not exceed 25 MiB', + ); + expect(deps.composeSessionService.addComposeAttachment).not.toHaveBeenCalled(); + }); + + it('removes only the requested attachment from the user-owned slot and revision', async () => { + const deps = dependencies(); + deps.composeSessionService.removeComposeAttachment.mockResolvedValue({ + sessionId: session.id, + slot: 7, + revision: 12, + removedAttachmentId: attachmentId, + }); + + const result = payload(await handleRemoveComposeAttachment({ + slot: 7, + expected_revision: 11, + attachment_id: attachmentId, + }, { ...scope, mcpTokenId: 'token-2' }, deps)); + + expect(deps.composeSessionService.removeComposeAttachment).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 7, + expectedRevision: 11, + attachmentId, + clientId: 'mcp:token-2', + }, deps); + expect(result).toEqual({ + session_id: session.id, + slot: 7, + revision: 12, + removed_attachment_id: attachmentId, + }); + }); + + it.each([ + ['add', handleAddComposeAttachment, 'addComposeAttachment', { + slot: 2, + expected_revision: 7, + filename: 'note.txt', + content: 'aGVsbG8=', + }], + ['remove', handleRemoveComposeAttachment, 'removeComposeAttachment', { + slot: 2, + expected_revision: 7, + attachment_id: attachmentId, + }], + ])('maps attachment %s revision conflicts through the shared result shape', async ( + _operation, + handler, + method, + args, + ) => { + const deps = dependencies(); + deps.composeSessionService[method].mockRejectedValue(Object.assign( + new Error('Compose session changed in the requested fields'), + { + code: 'compose_conflict', + expose: true, + details: { + currentRevision: 8, + conflictingFields: ['attachments'], + remoteValues: { attachments: [attachmentId] }, + }, + }, + )); + + const result = await handler(args, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(payload(result)).toEqual({ + error: 'compose_conflict', + message: 'Compose session changed in the requested fields', + current_revision: 8, + conflicting_fields: ['attachments'], + remote_values: { attachments: [attachmentId] }, + }); + }); + + it('maps exposed attachment errors and rethrows unexposed failures', async () => { + const deps = dependencies(); + deps.composeSessionService.addComposeAttachment.mockRejectedValueOnce(Object.assign( + new Error('Attachment filename must be a non-empty string'), + { code: 'invalid_attachment_filename', expose: true }, + )); + + const exposed = await handleAddComposeAttachment({ + slot: 2, + expected_revision: 7, + filename: '', + content: 'YQ==', + }, scope, deps); + + expect(exposed).toMatchObject({ isError: true }); + expect(exposed.content[0].text).toBe( + 'invalid_attachment_filename: Attachment filename must be a non-empty string', + ); + + deps.composeSessionService.removeComposeAttachment.mockRejectedValueOnce( + Object.assign(new Error('database unavailable'), { code: 'XX000' }), + ); + await expect(handleRemoveComposeAttachment({ + slot: 2, + expected_revision: 7, + attachment_id: attachmentId, + }, scope, deps)).rejects.toThrow('database unavailable'); + }); + + it('validates attachment slots and reports missing injected methods before mutation', async () => { + const deps = dependencies(); + + const invalid = await handleRemoveComposeAttachment({ + slot: 10, + expected_revision: 7, + attachment_id: attachmentId, + }, scope, deps); + + expect(invalid).toMatchObject({ isError: true }); + expect(invalid.content[0].text).toBe( + 'invalid_slot: slot must be an integer from 1 to 9', + ); + expect(deps.composeSessionService.removeComposeAttachment).not.toHaveBeenCalled(); + + const unsupportedResult = await handleAddComposeAttachment({ + slot: 2, + expected_revision: 7, + filename: 'note.txt', + content: 'YQ==', + }, scope, { composeSessionService: {} }); + expect(unsupportedResult).toMatchObject({ isError: true }); + expect(unsupportedResult.content[0].text).toBe( + 'unsupported: compose session tools require composeSessionService', + ); + }); +}); + +describe('compose-session terminal handlers', () => { + const immediateReceipt = { + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: '', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Synthetic terminal subject', + attachments: [{ filename: 'synthetic.txt', size: 9 }], + messageId: '', + sentCopySaved: true, + folder: 'Sent', + sharedReceiptMarker: { preserve: true }, + }; + + it('closes through one atomic lifecycle call with every explicit clear and HTML body mode', async () => { + const deps = dependencies(); + deps.composeSessionLifecycle.closeComposeSession.mockResolvedValue({ + closed: true, + slot: 4, + draft: { + accountId: 'account-1', + account: 'sender@example.com', + uid: 91, + folder: 'Drafts', + messageId: '', + }, + }); + + const result = payload(await handleCloseComposeSession({ + slot: 4, + expected_revision: 13, + to: [], + cc: [], + bcc: [], + subject: '', + body: 'ignored because explicit HTML wins', + body_html: '', + alias: '', + priority: 'normal', + reply_to_message_id: '', + }, scope, deps)); + + expect(deps.composeSessionLifecycle.closeComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 4, + expectedRevision: 13, + changes: { + to: [], + cc: [], + bcc: [], + subject: '', + body: '', + bodyIsHtml: true, + aliasId: null, + priority: 'normal', + inReplyTo: null, + references: [], + }, + }, deps); + expect(deps.composeSessionService.patchComposeSession).not.toHaveBeenCalled(); + expect(deps.draftService.saveDraft).not.toHaveBeenCalled(); + expect(result).toEqual({ + closed: true, + freed_slot: 4, + draft: { + account: 'sender@example.com', + draft_uid: 91, + folder: 'Drafts', + message_id: '', + }, + }); + }); + + it.each([ + ['plain empty body', { body: '' }, { body: '', bodyIsHtml: false }], + ['HTML empty body', { body_html: '' }, { body: '', bodyIsHtml: true }], + ['no body field', { subject: 'Only this field' }, { subject: 'Only this field' }], + ])('preserves close final-patch presence for %s without inventing dirty fields', async ( + _case, + finalFields, + expectedChanges, + ) => { + const deps = dependencies(); + + await handleCloseComposeSession({ + slot: 2, + expected_revision: 7, + ...finalFields, + }, scope, deps); + + expect(deps.composeSessionLifecycle.closeComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 2, + expectedRevision: 7, + changes: expectedChanges, + }, deps); + expect(deps.composeSessionService.patchComposeSession).not.toHaveBeenCalled(); + }); + + it('resolves an explicit reply target and alias against the current owned account before atomic close', async () => { + const deps = dependencies(); + deps.composeSessionService.getComposeSession.mockResolvedValue({ + ...session, + accountId: 'account-1', + }); + + await handleCloseComposeSession({ + slot: 2, + expected_revision: 7, + alias: 'team@example.com', + reply_to_message_id: 'message-1', + }, scope, deps); + + expect(deps.composeSessionService.getComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 2, + }, deps); + expect(deps.accountAdapter.getAccountRow).toHaveBeenCalledWith( + 'account-1', + ['account-1'], + ); + expect(deps.resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasEmail: 'team@example.com' }, + deps, + ); + expect(deps.accountAdapter.getComposeSource).toHaveBeenCalledWith( + 'message-1', + ['account-1'], + ); + expect(deps.composeSessionLifecycle.closeComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 2, + expectedRevision: 7, + changes: { + aliasId: 'alias-1', + inReplyTo: '', + references: ['', ''], + }, + }, deps); + expect(deps.composeSessionService.patchComposeSession).not.toHaveBeenCalled(); + }); + + it('returns draft null for an empty close without post-terminal account work', async () => { + const deps = dependencies(); + deps.composeSessionLifecycle.closeComposeSession.mockResolvedValue({ + closed: true, + slot: 9, + draft: null, + }); + + const result = payload(await handleCloseComposeSession({ + slot: 9, + expected_revision: 3, + }, scope, deps)); + + expect(result).toEqual({ closed: true, freed_slot: 9, draft: null }); + expect(deps.accountAdapter.getAccountRow).not.toHaveBeenCalled(); + }); + + it('discards with the exact revision and exposes only the freed-slot envelope', async () => { + const deps = dependencies(); + deps.composeSessionLifecycle.discardComposeSession.mockResolvedValue({ + discarded: true, + slot: 7, + }); + + const result = payload(await handleDiscardComposeSession({ + slot: 7, + expected_revision: 11, + }, scope, deps)); + + expect(deps.composeSessionLifecycle.discardComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 7, + expectedRevision: 11, + }, deps); + expect(result).toEqual({ discarded: true, freed_slot: 7 }); + }); + + it.each([ + ['close', handleCloseComposeSession, 'closeComposeSession', { + slot: 2, + expected_revision: 7, + }], + ['discard', handleDiscardComposeSession, 'discardComposeSession', { + slot: 2, + expected_revision: 7, + }], + ['send', handleSendComposeSession, 'sendComposeSession', { + slot: 2, + expected_revision: 7, + }], + ])('leaves source-draft preservation and destructive work lifecycle-owned for %s', async ( + _operation, + handler, + method, + args, + ) => { + const deps = dependencies(); + + await handler(args, scope, deps); + + expect(deps.composeSessionLifecycle[method]).toHaveBeenCalledTimes(1); + expect(deps.draftService.saveDraft).not.toHaveBeenCalled(); + expect(deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(deps.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('returns an immediate send identity while preserving the shared receipt verbatim', async () => { + const deps = dependencies(); + deps.composeSessionLifecycle.sendComposeSession.mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: immediateReceipt, + }); + + const result = payload(await handleSendComposeSession({ + slot: 6, + expected_revision: 21, + undo_send_seconds: 0, + idempotency_key: 'caller-stable-key', + }, scope, deps)); + + expect(deps.composeSessionLifecycle.sendComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 6, + expectedRevision: 21, + undoSendSeconds: 0, + idempotencyKey: 'caller-stable-key', + }, deps); + expect(result).toEqual({ + sent: true, + freed_slot: 6, + message_id: '', + sent_copy_saved: true, + receipt: immediateReceipt, + }); + expect(result.receipt).toEqual(immediateReceipt); + }); + + it('returns queued outbox identity unchanged and forwards the upper undo boundary', async () => { + const deps = dependencies(); + deps.composeSessionLifecycle.sendComposeSession.mockResolvedValue({ + queued: true, + outboxId: '33333333-3333-4333-8333-333333333333', + sendAt: '2026-08-01T00:02:00.000Z', + undoSeconds: 120, + }); + + const result = payload(await handleSendComposeSession({ + slot: 8, + expected_revision: 34, + undo_send_seconds: 120, + idempotency_key: 'queued-stable-key', + }, scope, deps)); + + expect(deps.composeSessionLifecycle.sendComposeSession).toHaveBeenCalledWith({ + userId: 'user-1', + slot: 8, + expectedRevision: 34, + undoSendSeconds: 120, + idempotencyKey: 'queued-stable-key', + }, deps); + expect(result).toEqual({ + queued: true, + freed_slot: 8, + outbox_id: '33333333-3333-4333-8333-333333333333', + send_at: '2026-08-01T00:02:00.000Z', + undo_seconds: 120, + }); + }); + + it('derives a deterministic bounded request key from stable MCP context when omitted', async () => { + const deps = dependencies(); + const requestScope = { + ...scope, + requestId: 'request-1', + tokenId: 'token-1', + }; + const args = { slot: 3, expected_revision: 9 }; + + await handleSendComposeSession(args, requestScope, deps); + await handleSendComposeSession(args, requestScope, deps); + await handleSendComposeSession({ ...args, expected_revision: 10 }, requestScope, deps); + + const keys = deps.composeSessionLifecycle.sendComposeSession.mock.calls + .map(([input]) => input.idempotencyKey); + expect(keys[0]).toBe(keys[1]); + expect(keys[0]).not.toBe(keys[2]); + expect(keys[0]).toMatch(/^mcp-compose:[a-f0-9]{64}$/); + expect(keys.every(key => key.length <= 128)).toBe(true); + expect(JSON.stringify(keys)).not.toContain('request-1'); + expect(JSON.stringify(keys)).not.toContain('token-1'); + expect(JSON.stringify(keys)).not.toContain('user-1'); + }); + + it.each([-1, 121, 1.5, '30', null, true])( + 'rejects invalid direct-call undo seconds %j before lifecycle', + async (undoSendSeconds) => { + const deps = dependencies(); + + const result = await handleSendComposeSession({ + slot: 2, + expected_revision: 7, + undo_send_seconds: undoSendSeconds, + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe( + 'invalid_compose_undo_seconds: undo_send_seconds must be an integer from 0 to 120', + ); + expect(deps.composeSessionLifecycle.sendComposeSession).not.toHaveBeenCalled(); + }, + ); + + it('omits undo seconds but still supplies deterministic idempotency to lifecycle', async () => { + const deps = dependencies(); + + await handleSendComposeSession({ + slot: 2, + expected_revision: 7, + }, scope, deps); + + expect(deps.composeSessionLifecycle.sendComposeSession.mock.calls[0][0]) + .not.toHaveProperty('undoSendSeconds'); + expect(deps.composeSessionLifecycle.sendComposeSession.mock.calls[0][0].idempotencyKey) + .toMatch(/^mcp-compose:[a-f0-9]{64}$/); + }); + + it.each([ + ['close', handleCloseComposeSession, 'closeComposeSession', { + slot: 2, + expected_revision: 7, + subject: 'Changed', + }], + ['discard', handleDiscardComposeSession, 'discardComposeSession', { + slot: 2, + expected_revision: 7, + }], + ['send', handleSendComposeSession, 'sendComposeSession', { + slot: 2, + expected_revision: 7, + }], + ])('maps terminal %s conflicts through the shared structured shape', async ( + _operation, + handler, + method, + args, + ) => { + const deps = dependencies(); + deps.composeSessionLifecycle[method].mockRejectedValue(Object.assign( + new Error('Compose session changed in the requested fields'), + { + code: 'compose_conflict', + expose: true, + details: { + currentRevision: 8, + conflictingFields: ['subject'], + remoteValues: { subject: 'Remote synthetic subject' }, + }, + }, + )); + + const result = await handler(args, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(payload(result)).toEqual({ + error: 'compose_conflict', + message: 'Compose session changed in the requested fields', + current_revision: 8, + conflicting_fields: ['subject'], + remote_values: { subject: 'Remote synthetic subject' }, + }); + }); + + it.each([ + ['close', handleCloseComposeSession, 'closeComposeSession', { + slot: 2, + expected_revision: 7, + }], + ['discard', handleDiscardComposeSession, 'discardComposeSession', { + slot: 2, + expected_revision: 7, + }], + ['send', handleSendComposeSession, 'sendComposeSession', { + slot: 2, + expected_revision: 7, + }], + ])('maps exposed terminal %s errors and rethrows unexposed failures', async ( + _operation, + handler, + method, + args, + ) => { + const deps = dependencies(); + deps.composeSessionLifecycle[method] + .mockRejectedValueOnce(Object.assign(new Error('Synthetic exposed failure'), { + code: 'compose_operation_in_progress', + expose: true, + })) + .mockRejectedValueOnce(Object.assign(new Error('private database diagnostics'), { + code: 'XX000', + })); + + const exposed = await handler(args, scope, deps); + expect(exposed).toMatchObject({ isError: true }); + expect(exposed.content[0].text).toBe( + 'compose_operation_in_progress: Synthetic exposed failure', + ); + await expect(handler(args, scope, deps)).rejects.toThrow( + 'private database diagnostics', + ); + }); + + it.each([ + ['close', handleCloseComposeSession, 'closeComposeSession'], + ['discard', handleDiscardComposeSession, 'discardComposeSession'], + ['send', handleSendComposeSession, 'sendComposeSession'], + ])('reports a missing injected lifecycle method for %s without other terminal work', async ( + _operation, + handler, + method, + ) => { + const deps = dependencies(); + delete deps.composeSessionLifecycle[method]; + + const result = await handler({ slot: 2, expected_revision: 7 }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe( + 'unsupported: compose session tools require composeSessionLifecycle', + ); + expect(deps.composeSessionService.patchComposeSession).not.toHaveBeenCalled(); + expect(deps.draftService.saveDraft).not.toHaveBeenCalled(); + expect(deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(deps.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it.each([0, 10, 2.5])('validates terminal slot %s before lifecycle', async (slot) => { + const deps = dependencies(); + + const result = await handleDiscardComposeSession({ + slot, + expected_revision: 7, + }, scope, deps); + + expect(result).toMatchObject({ isError: true }); + expect(result.content[0].text).toBe( + 'invalid_slot: slot must be an integer from 1 to 9', + ); + expect(deps.composeSessionLifecycle.discardComposeSession).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/mcp/composeTools.js b/backend/src/mcp/composeTools.js new file mode 100644 index 00000000..4ef212ba --- /dev/null +++ b/backend/src/mcp/composeTools.js @@ -0,0 +1,258 @@ +import { + getAccountRow, + getComposeSource, + getUserPreferences, + listAliases, +} from './accountAdapter.js'; +import { jsonResult } from './result.js'; +import { buildWriteReceipt, writeError } from './writeResult.js'; +import { normalizeUndoWindow } from '../services/outboxService.js'; +import * as replyService from '../services/replyService.js'; + +const annotations = (readOnlyHint, destructiveHint, idempotentHint) => ({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, +}); + +const replyProperties = { + message_id: { + type: 'string', + description: 'Id of the message being replied to (from search/list/get_message)', + }, + body: { type: 'string' }, + body_html: { type: 'string' }, + to: { + type: 'array', + items: { type: 'string' }, + description: 'REPLACES the computed To. Displaced recipients move to Cc rather than being dropped. Mutually exclusive with to_add.', + }, + cc: { + type: 'array', + items: { type: 'string' }, + description: 'REPLACES the computed Cc.', + }, + bcc: { type: 'array', items: { type: 'string' } }, + to_add: { + type: 'array', + items: { type: 'string' }, + description: 'Appends to the computed To.', + }, + cc_add: { type: 'array', items: { type: 'string' } }, + bcc_add: { type: 'array', items: { type: 'string' } }, + remove: { + type: 'array', + items: { type: 'string' }, + description: 'Email addresses to drop from To/Cc/Bcc after computation.', + }, + no_quote: { + type: 'boolean', + description: 'Omit the quoted original.', + }, + include_inline_images: { + type: 'boolean', + description: 'Re-fetch cid: images from the original so the quote renders (default: replaced with [inline image: name]).', + }, + alias: { type: 'string' }, + attachments: {}, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, +}; + +export const replyEmailDef = { + name: 'reply_email', + description: 'Reply to the sender of a message.', + inputSchema: { + type: 'object', + required: ['message_id', 'body'], + properties: replyProperties, + }, + annotations: annotations(false, true, false), +}; + +export const replyAllEmailDef = { + name: 'reply_all_email', + description: 'Reply to the sender and all original To/Cc recipients. Bcc recipients of the original are not recoverable (they are not stored). Your own addresses and aliases are excluded from the recipients automatically.', + inputSchema: { + type: 'object', + required: ['message_id', 'body'], + properties: replyProperties, + }, + annotations: annotations(false, true, false), +}; + +export const forwardEmailDef = { + name: 'forward_email', + description: "Forward a message. Carries the original's attachments by default via forwardedAttachments: [{messageId, part}] — re-fetched from IMAP inside sendService, never round-tripped through the MCP wire.", + inputSchema: { + type: 'object', + required: ['message_id', 'to'], + properties: { + message_id: { type: 'string' }, + to: { type: 'array', items: { type: 'string' } }, + note: { type: 'string' }, + skip_attachments: { type: 'boolean' }, + alias: { type: 'string' }, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, + }, + }, + annotations: annotations(false, true, false), +}; + +function errorFrom(err) { + if (err?.code) return writeError(err.code, err.message); + return writeError('invalid_arguments', err?.message || 'compose operation failed'); +} + +async function composeContext(messageId, scope) { + const message = await getComposeSource(messageId, scope.accountIds); + if (!message) { + return { error: writeError('message_not_found', messageId) }; + } + const account = await getAccountRow(message.account_id, scope.accountIds); + if (!account) { + return { error: writeError('account_not_found', message.account_id) }; + } + const aliases = await listAliases(account.id); + return { message, account, aliases }; +} + +function recipientsComputed(message, account, aliases) { + return { + reply_target: replyService.pickReplyTarget(message).email || '', + excluded_self: [...replyService.selfAddressSet(account, aliases)].sort(), + }; +} + +function receiptForResult(result, compose, forwarded) { + const receipt = { + ...result.receipt, + ...(compose.inReplyTo !== undefined ? { inReplyTo: compose.inReplyTo } : {}), + ...(compose.references !== undefined ? { references: compose.references } : {}), + }; + if (forwarded) { + receipt.attachments = (receipt.attachments || []).map(attachment => ({ + ...attachment, + source: 'forwarded', + })); + } + return receipt; +} + +function sendResult(result, compose, resultFields = {}, forwarded = false) { + if (result.queued) { + return jsonResult(buildWriteReceipt({ + subject: compose.subject, + inReplyTo: compose.inReplyTo, + references: compose.references, + }, { + queued: true, + outboxId: result.outboxId, + sendAt: result.sendAt, + undoSeconds: result.undoSeconds, + ...resultFields, + note: 'Cancel with unsend_email before send_at.', + })); + } + return jsonResult(buildWriteReceipt( + receiptForResult(result, compose, forwarded), + { sent: true, ...resultFields }, + )); +} + +async function sendCompose(compose, args, scope, deps, resultFields, forwarded = false) { + const preferences = await getUserPreferences(scope.userId); + const result = await deps.sendService.sendOrEnqueue({ + ...compose, + undoSeconds: normalizeUndoWindow( + args.undo_send_seconds, + preferences?.undoSendSeconds, + ), + idempotencyKey: args.idempotency_key, + }, deps); + return sendResult(result, compose, resultFields, forwarded); +} + +function replyInput(args, context, replyAll) { + const bodyIsHtml = args.body_html !== undefined; + return { + message: context.message, + account: context.account, + aliases: context.aliases, + replyAll, + body: bodyIsHtml ? args.body_html : (args.body || ''), + bodyIsHtml, + to: args.to, + cc: args.cc, + bcc: args.bcc, + toAdd: args.to_add, + ccAdd: args.cc_add, + bccAdd: args.bcc_add, + remove: args.remove, + noQuote: args.no_quote, + includeInlineImages: args.include_inline_images, + alias: args.alias, + }; +} + +async function handleReply(args, scope, deps, replyAll) { + if (!deps?.sendService) { + return writeError('unsupported', 'compose tools require sendService'); + } + try { + const context = await composeContext(args.message_id, scope); + if (context.error) return context.error; + if ( + args.include_inline_images && + !args.no_quote && + !deps.imapManager?.fetchAttachment + ) { + return writeError('unsupported', 'include_inline_images requires imapManager'); + } + const compose = await replyService.buildReply( + replyInput(args, context, replyAll), + deps, + ); + return sendCompose(compose, args, scope, deps, { + recipientsComputed: recipientsComputed( + context.message, + context.account, + context.aliases, + ), + }); + } catch (err) { + return errorFrom(err); + } +} + +export async function handleReplyEmail(args, scope, deps = {}) { + return handleReply(args, scope, deps, false); +} + +export async function handleReplyAllEmail(args, scope, deps = {}) { + return handleReply(args, scope, deps, true); +} + +export async function handleForwardEmail(args, scope, deps = {}) { + if (!deps?.sendService) { + return writeError('unsupported', 'compose tools require sendService'); + } + try { + const context = await composeContext(args.message_id, scope); + if (context.error) return context.error; + const compose = await replyService.buildForward({ + message: context.message, + account: context.account, + aliases: context.aliases, + to: args.to, + note: args.note, + skipAttachments: args.skip_attachments, + alias: args.alias, + }, deps); + return sendCompose(compose, args, scope, deps, {}, true); + } catch (err) { + return errorFrom(err); + } +} diff --git a/backend/src/mcp/composeTools.test.js b/backend/src/mcp/composeTools.test.js new file mode 100644 index 00000000..d8fb181a --- /dev/null +++ b/backend/src/mcp/composeTools.test.js @@ -0,0 +1,559 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./accountAdapter.js', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + getAccountRow: vi.fn(), + getComposeSource: vi.fn(), + getUserPreferences: vi.fn(), + listAliases: vi.fn(), + }; +}); + +import { + getAccountRow, + getComposeSource, + getUserPreferences, + listAliases, +} from './accountAdapter.js'; +import * as composeTools from './composeTools.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; + +const replyProperties = { + message_id: { + type: 'string', + description: 'Id of the message being replied to (from search/list/get_message)', + }, + body: { type: 'string' }, + body_html: { type: 'string' }, + to: { + type: 'array', + items: { type: 'string' }, + description: 'REPLACES the computed To. Displaced recipients move to Cc rather than being dropped. Mutually exclusive with to_add.', + }, + cc: { + type: 'array', + items: { type: 'string' }, + description: 'REPLACES the computed Cc.', + }, + bcc: { type: 'array', items: { type: 'string' } }, + to_add: { + type: 'array', + items: { type: 'string' }, + description: 'Appends to the computed To.', + }, + cc_add: { type: 'array', items: { type: 'string' } }, + bcc_add: { type: 'array', items: { type: 'string' } }, + remove: { + type: 'array', + items: { type: 'string' }, + description: 'Email addresses to drop from To/Cc/Bcc after computation.', + }, + no_quote: { + type: 'boolean', + description: 'Omit the quoted original.', + }, + include_inline_images: { + type: 'boolean', + description: 'Re-fetch cid: images from the original so the quote renders (default: replaced with [inline image: name]).', + }, + alias: { type: 'string' }, + attachments: {}, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, +}; + +const sendAnnotations = { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, +}; + +const scope = { + userId: 'user-1', + accountIds: ['account-1'], + scopes: ['send'], +}; +const account = { + id: 'account-1', + user_id: 'user-1', + email_address: 'me@example.com', + sender_name: 'Me', +}; +const aliases = [{ + id: 'alias-1', + email: 'team@example.com', + reply_to: 'alias-replies@example.com', +}]; +const message = { + id: '11111111-1111-4111-8111-111111111111', + account_id: 'account-1', + uid: 42, + folder: 'INBOX', + subject: 'Topic', + from_name: 'Sender', + from_email: 'sender@example.com', + reply_to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + to_addresses: [ + { name: 'Me', email: 'me@example.com' }, + { name: 'Team', email: 'team@example.com' }, + { name: 'Other', email: 'other@example.com' }, + ], + cc_addresses: [ + { name: 'Colleague', email: 'colleague@example.com' }, + { name: 'Alias Reply', email: 'alias-replies@example.com' }, + ], + body_text: 'Original text', + body_html: '

Original HTML

', + attachments: [{ + part: '2', + filename: 'deck.pdf', + type: 'application/pdf', + size: 2_144_000, + }], + message_id: '', + thread_references: '', +}; +const replyReceipt = { + from: { name: 'Team', email: 'team@example.com' }, + to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + cc: [], + bcc: [], + subject: 'Re: Topic', + attachments: [], + messageId: '', + sentCopySaved: true, + folder: 'Sent', +}; + +function deps(overrides = {}) { + return { + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: replyReceipt, + }), + }, + ...overrides, + }; +} + +function payload(result) { + return JSON.parse(result.content[0].text); +} + +beforeEach(() => { + vi.clearAllMocks(); + getComposeSource.mockResolvedValue(message); + getAccountRow.mockResolvedValue(account); + listAliases.mockResolvedValue(aliases); + getUserPreferences.mockResolvedValue({}); +}); + +describe('compose tool definitions and registration', () => { + it('publishes the Phase 1 schemas, descriptions, annotations, scopes, and handlers', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + + expect(defs.get('reply_email')).toEqual({ + name: 'reply_email', + description: 'Reply to the sender of a message.', + inputSchema: { + type: 'object', + required: ['message_id', 'body'], + properties: replyProperties, + }, + annotations: sendAnnotations, + }); + expect(defs.get('reply_all_email')).toEqual({ + name: 'reply_all_email', + description: 'Reply to the sender and all original To/Cc recipients. Bcc recipients of the original are not recoverable (they are not stored). Your own addresses and aliases are excluded from the recipients automatically.', + inputSchema: { + type: 'object', + required: ['message_id', 'body'], + properties: replyProperties, + }, + annotations: sendAnnotations, + }); + expect(defs.get('forward_email')).toEqual({ + name: 'forward_email', + description: "Forward a message. Carries the original's attachments by default via forwardedAttachments: [{messageId, part}] — re-fetched from IMAP inside sendService, never round-tripped through the MCP wire.", + inputSchema: { + type: 'object', + required: ['message_id', 'to'], + properties: { + message_id: { type: 'string' }, + to: { type: 'array', items: { type: 'string' } }, + note: { type: 'string' }, + skip_attachments: { type: 'boolean' }, + alias: { type: 'string' }, + undo_send_seconds: { type: 'integer', minimum: 0, maximum: 120 }, + idempotency_key: { type: 'string' }, + }, + }, + annotations: sendAnnotations, + }); + + for (const [name, handler] of [ + ['reply_email', composeTools.handleReplyEmail], + ['reply_all_email', composeTools.handleReplyAllEmail], + ['forward_email', composeTools.handleForwardEmail], + ]) { + expect(TOOL_SCOPES[name]).toBe('send'); + expect(HANDLERS[name]).toBe(handler); + } + }); +}); + +describe('reply_email', () => { + it('returns message_not_found without disclosing an out-of-scope message', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(null); + + const result = await composeTools.handleReplyEmail({ + message_id: 'foreign-message', + body: 'Reply', + }, scope, service); + + expect(getComposeSource).toHaveBeenCalledWith('foreign-message', ['account-1']); + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message_not_found: foreign-message'); + expect(getAccountRow).not.toHaveBeenCalled(); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('returns account_not_found if the compose source account disappears', async () => { + const service = deps(); + getAccountRow.mockResolvedValue(null); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + }, scope, service); + + expect(getAccountRow).toHaveBeenCalledWith('account-1', ['account-1']); + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account_not_found: account-1'); + expect(listAliases).not.toHaveBeenCalled(); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('preserves alias_not_found from replyService', async () => { + const service = deps(); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + alias: 'missing@example.com', + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('alias_not_found: Alias not found'); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it.each([ + [{ + to: Array.from({ length: 101 }, (_, i) => `person-${i}@example.com`), + }, 'too_many_recipients: Too many recipients (max 100)'], + [{ + to: ['replacement@example.com'], + to_add: ['additional@example.com'], + }, 'invalid_arguments: to and toAdd are mutually exclusive'], + ])('preserves recipient computation errors for %j', async (recipientArgs, expected) => { + const service = deps(); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + ...recipientArgs, + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(expected); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('sends immediately and returns threading plus recipients_computed', async () => { + const service = deps(); + + const wireResult = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Thanks', + no_quote: true, + undo_send_seconds: 0, + idempotency_key: 'reply-1', + }, scope, service); + + expect(wireResult.isError).not.toBe(true); + expect(listAliases).toHaveBeenCalledWith('account-1'); + expect(getUserPreferences).toHaveBeenCalledWith('user-1'); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith({ + account, + aliasId: 'alias-1', + userId: 'user-1', + to: ['Reply Desk '], + cc: [], + bcc: [], + subject: 'Re: Topic', + body: 'Thanks', + bodyIsHtml: false, + quotedBody: '', + quotedBodyHtml: null, + inReplyTo: '', + references: ' ', + undoSeconds: 0, + idempotencyKey: 'reply-1', + }, service); + expect(payload(wireResult)).toEqual({ + sent: true, + recipients_computed: { + reply_target: 'reply@example.com', + excluded_self: [ + 'alias-replies@example.com', + 'me@example.com', + 'team@example.com', + ], + }, + message_id: '', + in_reply_to: '', + references: ' ', + from: { name: 'Team', email: 'team@example.com' }, + to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + cc: [], + bcc: [], + subject: 'Re: Topic', + attachments: [], + sent_copy_saved: true, + folder: 'Sent', + }); + }); + + it('uses HTML body selection and passes additive recipient adjustments to replyService', async () => { + const service = deps(); + + await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Plain fallback', + body_html: '

HTML reply

', + to_add: ['additional@example.com'], + cc_add: ['added-copy@example.com'], + bcc_add: ['added-blind@example.com'], + remove: ['reply@example.com'], + no_quote: true, + }, scope, service); + + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + body: '

HTML reply

', + bodyIsHtml: true, + to: ['additional@example.com'], + cc: ['added-copy@example.com'], + bcc: ['added-blind@example.com'], + }), + service, + ); + }); + + it('maps a missing inline-image fetch dependency to unsupported', async () => { + const service = deps(); + getComposeSource.mockResolvedValue({ + ...message, + body_html: '

Original

', + attachments: [{ + part: '2', + cid: 'image-1', + filename: 'inline.png', + type: 'image/png', + }], + }); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + include_inline_images: true, + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'unsupported: include_inline_images requires imapManager', + ); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('does not require imapManager when no_quote disables inline-image fetching', async () => { + const service = deps(); + + const result = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Reply', + no_quote: true, + include_inline_images: true, + }, scope, service); + + expect(result.isError).not.toBe(true); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledOnce(); + }); + + it('returns the queued write shape with threading and recipient computation', async () => { + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + queued: true, + outboxId: 'outbox-reply-1', + sendAt: new Date('2026-07-28T10:00:30.000Z'), + undoSeconds: 30, + }), + }, + }); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 30 }); + + const wireResult = await composeTools.handleReplyEmail({ + message_id: message.id, + body: 'Queued reply', + }, scope, service); + + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + undoSeconds: 30, + idempotencyKey: undefined, + }), + service, + ); + expect(payload(wireResult)).toEqual({ + queued: true, + outbox_id: 'outbox-reply-1', + send_at: '2026-07-28T10:00:30Z', + undo_seconds: 30, + recipients_computed: { + reply_target: 'reply@example.com', + excluded_self: [ + 'alias-replies@example.com', + 'me@example.com', + 'team@example.com', + ], + }, + in_reply_to: '', + references: ' ', + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Re: Topic', + attachments: [], + note: 'Cancel with unsend_email before send_at.', + }); + }); +}); + +describe('reply_all_email', () => { + it('excludes account and alias identities while preserving non-self recipients', async () => { + const service = deps(); + + const result = await composeTools.handleReplyAllEmail({ + message_id: message.id, + body: 'Reply all', + no_quote: true, + }, scope, service); + + const [input] = service.sendService.sendOrEnqueue.mock.calls[0]; + expect(input.to).toEqual(['Reply Desk ']); + expect(input.cc).toEqual([ + 'Other ', + 'Colleague ', + ]); + expect(payload(result).recipients_computed).toEqual({ + reply_target: 'reply@example.com', + excluded_self: [ + 'alias-replies@example.com', + 'me@example.com', + 'team@example.com', + ], + }); + }); +}); + +describe('forward_email', () => { + it('passes forward options and tags delivered attachments as forwarded', async () => { + const forwardReceipt = { + ...replyReceipt, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + subject: 'Fwd: Topic', + attachments: [{ filename: 'deck.pdf', size: 2_144_000 }], + messageId: '', + }; + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: forwardReceipt, + }), + }, + }); + + const wireResult = await composeTools.handleForwardEmail({ + message_id: message.id, + to: ['Recipient '], + note: 'Please review.', + alias: 'team@example.com', + undo_send_seconds: 0, + idempotency_key: 'forward-1', + }, scope, service); + + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + account, + aliasId: 'alias-1', + userId: 'user-1', + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Fwd: Topic', + body: 'Please review.', + bodyIsHtml: false, + forwardedAttachments: [{ + messageId: message.id, + part: '2', + }], + undoSeconds: 0, + idempotencyKey: 'forward-1', + }), + service, + ); + expect(payload(wireResult).attachments).toEqual([{ + filename: 'deck.pdf', + size: 2_144_000, + source: 'forwarded', + }]); + }); +}); + +describe('missing dependencies', () => { + it.each([ + ['reply_email', composeTools.handleReplyEmail, { + message_id: message.id, + body: 'Reply', + }], + ['reply_all_email', composeTools.handleReplyAllEmail, { + message_id: message.id, + body: 'Reply all', + }], + ['forward_email', composeTools.handleForwardEmail, { + message_id: message.id, + to: ['recipient@example.com'], + }], + ])('%s degrades to unsupported when sendService is missing', async (_name, handler, args) => { + const result = await handler(args, scope, {}); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'unsupported: compose tools require sendService', + ); + expect(getComposeSource).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/mcp/draftTools.js b/backend/src/mcp/draftTools.js new file mode 100644 index 00000000..5e742c81 --- /dev/null +++ b/backend/src/mcp/draftTools.js @@ -0,0 +1,432 @@ +import { + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + listDraftRows, +} from './accountAdapter.js'; +import { newPaginatedResponse, toRFC3339 } from './envelope.js'; +import { errorResult, jsonResult } from './result.js'; +import { buildWriteReceipt, writeError } from './writeResult.js'; +import { mapRecipientList } from '../services/mail/addresses.js'; +import { resolveFromIdentity } from '../services/mail/identity.js'; +import { buildReferences } from '../services/replyService.js'; + +const annotations = (readOnlyHint, destructiveHint, idempotentHint) => ({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, +}); + +const attachmentSchema = { + type: 'array', + items: { + type: 'object', + required: ['filename', 'content'], + properties: { + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }, +}; + +const draftComposeProperties = { + account: { + type: 'string', + description: 'Account email address (see get_stats for available accounts)', + }, + to: { + type: 'array', + items: { type: 'string' }, + description: "Recipients as 'Name ' or bare 'email'", + }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string', description: 'Plain-text body' }, + body_html: { + type: 'string', + description: 'HTML body. When set, takes precedence over body for the HTML part.', + }, + reply_to_message_id: { + type: 'string', + description: 'Message id to thread this draft under; sets In-Reply-To and References.', + }, + alias: { + type: 'string', + description: 'Send-as alias email. Must be a configured alias on the account — errors if unknown, never silently falls back.', + }, + attachments: attachmentSchema, +}; + +export const createDraftDef = { + name: 'create_draft', + description: "Create a draft email in the account's Drafts folder. Does NOT send. Returns the draft's uid and folder for later update_draft/send_draft/delete_draft calls.", + inputSchema: { + type: 'object', + required: ['account'], + properties: draftComposeProperties, + }, + annotations: annotations(false, false, false), +}; + +export const updateDraftDef = { + name: 'update_draft', + description: 'Update an existing draft. IMAP has no update-in-place for messages: this appends a new draft and deletes the old one, so the draft_uid CHANGES. Use the returned draft_uid for subsequent calls.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + ...draftComposeProperties, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + }, + }, + annotations: annotations(false, true, false), +}; + +export const listDraftsDef = { + name: 'list_drafts', + description: 'List drafts, newest first. Paginate with offset/limit (default limit 20, max 100).', + inputSchema: { + type: 'object', + properties: { + account: { + type: 'string', + description: 'Filter by account email address (use get_stats to list available accounts)', + }, + limit: { type: 'integer', default: 20, maximum: 100 }, + offset: { type: 'integer', default: 0, minimum: 0 }, + }, + }, + annotations: annotations(true, false, true), +}; + +export const getDraftDef = { + name: 'get_draft', + description: 'Get a draft including body text, body HTML, threading headers, and attachment metadata.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + account: { + type: 'string', + description: 'Account email address (see get_stats for available accounts)', + }, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + }, + }, + annotations: annotations(true, false, true), +}; + +export const deleteDraftDef = { + name: 'delete_draft', + description: 'PERMANENTLY deletes the draft from IMAP — it does not go to Trash and cannot be recovered.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + account: { + type: 'string', + description: 'Account email address (see get_stats for available accounts)', + }, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + }, + }, + annotations: annotations(false, true, true), +}; + +function unsupported(deps) { + if (deps?.draftService) return null; + return writeError('unsupported', 'draft tools require draftService'); +} + +function draftFolder(account, requestedFolder) { + return requestedFolder || account?.folder_mappings?.drafts || 'Drafts'; +} + +async function accountForEmail(email, scope) { + const account = await getAccountByEmail(email, scope.accountIds); + return account?.error ? { error: errorResult(account.error) } : { account }; +} + +function addressString(address) { + if (typeof address === 'string') return address; + if (!address?.email) return ''; + return address.name ? `${address.name} <${address.email}>` : address.email; +} + +function addressStrings(value) { + return (Array.isArray(value) ? value : []).map(addressString).filter(Boolean); +} + +function attachmentInput(attachment) { + return { + filename: attachment.filename, + content: attachment.content, + contentType: attachment.content_type ?? attachment.contentType, + }; +} + +function attachmentSize(attachment) { + if (Number.isFinite(Number(attachment?.size))) return Number(attachment.size); + if (typeof attachment?.content !== 'string') return 0; + try { + return Buffer.from(attachment.content, 'base64').length; + } catch { + return 0; + } +} + +function receiptForDraft(identity, compose, saved) { + return buildWriteReceipt({ + messageId: saved.messageId, + from: { + name: identity?.fromName || '', + email: identity?.fromEmail || '', + }, + to: mapRecipientList(compose.to), + cc: mapRecipientList(compose.cc), + bcc: mapRecipientList(compose.bcc), + subject: compose.subject || '', + attachments: (compose.attachments || []).map(attachment => ({ + filename: attachment.filename, + size: attachmentSize(attachment), + })), + }); +} + +function draftResult(saved, identity, compose) { + return jsonResult({ + draft_uid: saved.uid, + folder: saved.folder, + message_id: saved.messageId, + receipt: receiptForDraft(identity, compose, saved), + }); +} + +function errorFrom(exception) { + if (exception?.code) return writeError(exception.code, exception.message); + if (/drafts folder/i.test(exception?.message || '')) { + return writeError('no_drafts_folder', exception.message); + } + return writeError('invalid_arguments', exception?.message || 'draft operation failed'); +} + +async function threadingFor(messageId, scope) { + if (!messageId) return {}; + const source = await getComposeSource(messageId, scope.accountIds); + if (!source) return { error: writeError('message_not_found', messageId) }; + return buildReferences(source); +} + +export async function handleCreateDraft(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await accountForEmail(args.account, scope); + if (resolved.error) return resolved.error; + const identity = await resolveFromIdentity( + resolved.account, + { aliasEmail: args.alias }, + deps, + ); + const threading = await threadingFor(args.reply_to_message_id, scope); + if (threading.error) return threading.error; + const compose = { + userId: scope.userId, + account: resolved.account, + aliasEmail: args.alias, + to: args.to || [], + cc: args.cc || [], + bcc: args.bcc || [], + subject: args.subject || '', + body: args.body_html !== undefined ? args.body_html : (args.body || ''), + bodyIsHtml: args.body_html !== undefined, + attachments: (args.attachments || []).map(attachmentInput), + inReplyTo: threading.inReplyTo, + references: threading.references, + }; + const saved = await deps.draftService.saveDraft(compose, deps); + return draftResult(saved, identity, compose); + } catch (err) { + return errorFrom(err); + } +} + +function hasOwn(object, key) { + return Object.prototype.hasOwnProperty.call(object, key); +} + +function mergedCompose(args, existing, account) { + const explicitBody = hasOwn(args, 'body'); + const explicitHtml = hasOwn(args, 'body_html'); + const useHtml = explicitHtml || (!explicitBody && Boolean(existing.body_html)); + const body = explicitHtml + ? args.body_html + : explicitBody + ? args.body + : useHtml + ? existing.body_html + : (existing.body_text || ''); + return { + userId: undefined, + account, + aliasEmail: hasOwn(args, 'alias') + ? args.alias + : (existing.from_email && existing.from_email !== account.email_address + ? existing.from_email + : undefined), + to: hasOwn(args, 'to') ? args.to : addressStrings(existing.to_addresses), + cc: hasOwn(args, 'cc') ? args.cc : addressStrings(existing.cc_addresses), + bcc: hasOwn(args, 'bcc') ? args.bcc : addressStrings(existing.bcc_addresses), + subject: hasOwn(args, 'subject') ? args.subject : (existing.subject || ''), + body, + bodyIsHtml: useHtml, + attachments: hasOwn(args, 'attachments') + ? (args.attachments || []).map(attachmentInput) + : (existing.attachments || []).map(attachmentInput), + inReplyTo: existing.in_reply_to || undefined, + references: existing.thread_references || undefined, + }; +} + +export async function handleUpdateDraft(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await accountForEmail(args.account, scope); + if (resolved.error) return resolved.error; + const folder = draftFolder(resolved.account, args.folder); + const existing = await getDraftRow(resolved.account.id, folder, args.draft_uid); + if (!existing) return writeError('draft_not_found', args.draft_uid); + + const compose = mergedCompose(args, existing, resolved.account); + compose.userId = scope.userId; + if (args.reply_to_message_id) { + const threading = await threadingFor(args.reply_to_message_id, scope); + if (threading.error) return threading.error; + compose.inReplyTo = threading.inReplyTo; + compose.references = threading.references; + } + const identity = await resolveFromIdentity( + resolved.account, + { aliasEmail: compose.aliasEmail }, + deps, + ); + compose.existingUid = existing.uid; + compose.existingFolder = existing.folder; + const saved = await deps.draftService.saveDraft(compose, deps); + return draftResult(saved, identity, compose); + } catch (err) { + return errorFrom(err); + } +} + +function draftSummary(row) { + return { + draft_uid: row.uid, + folder: row.folder, + subject: row.subject || '', + to: row.to_addresses || [], + cc: row.cc_addresses || [], + snippet: row.snippet || '', + date: toRFC3339(row.date), + has_attachments: Boolean( + row.has_attachments || (Array.isArray(row.attachments) && row.attachments.length), + ), + }; +} + +async function listAccounts(args, scope) { + if (args.account) { + const resolved = await accountForEmail(args.account, scope); + return resolved.error ? resolved : { accounts: [resolved.account] }; + } + const accounts = []; + for (const accountId of scope.accountIds || []) { + const account = await getAccountRow(accountId, scope.accountIds); + if (account) accounts.push(account); + } + return { accounts }; +} + +export async function handleListDrafts(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await listAccounts(args, scope); + if (resolved.error) return resolved.error; + const rows = []; + for (const account of resolved.accounts) { + const accountRows = await listDraftRows(account.id, { + limit: Number.MAX_SAFE_INTEGER, + offset: 0, + folder: draftFolder(account), + }); + rows.push(...accountRows); + } + rows.sort((a, b) => String(b.date || '').localeCompare(String(a.date || ''))); + const limit = Math.min(Math.max(Number.parseInt(args.limit, 10) || 20, 1), 100); + const offset = Math.max(Number.parseInt(args.offset, 10) || 0, 0); + const page = rows.slice(offset, offset + limit).map(draftSummary); + return jsonResult(newPaginatedResponse(page, rows.length, offset)); + } catch (err) { + return errorFrom(err); + } +} + +function fullDraft(row) { + return { + draft_uid: row.uid, + folder: row.folder, + subject: row.subject || '', + to: row.to_addresses || [], + cc: row.cc_addresses || [], + bcc: row.bcc_addresses || [], + body_text: row.body_text || '', + body_html: row.body_html || '', + in_reply_to: row.in_reply_to || null, + references: row.thread_references || null, + attachments: row.attachments || [], + }; +} + +export async function handleGetDraft(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await accountForEmail(args.account, scope); + if (resolved.error) return resolved.error; + const folder = draftFolder(resolved.account, args.folder); + const row = await getDraftRow(resolved.account.id, folder, args.draft_uid); + if (!row) return writeError('draft_not_found', args.draft_uid); + return jsonResult(fullDraft(row)); + } catch (err) { + return errorFrom(err); + } +} + +export async function handleDeleteDraft(args, scope, deps = {}) { + const missing = unsupported(deps); + if (missing) return missing; + try { + const resolved = await accountForEmail(args.account, scope); + if (resolved.error) return resolved.error; + const folder = draftFolder(resolved.account, args.folder); + const row = await getDraftRow(resolved.account.id, folder, args.draft_uid); + if (!row) return writeError('draft_not_found', args.draft_uid); + await deps.draftService.deleteDraft({ + account: resolved.account, + uid: row.uid, + folder: row.folder, + }, deps); + return jsonResult({ deleted: true, draft_uid: row.uid, folder: row.folder }); + } catch (err) { + return errorFrom(err); + } +} diff --git a/backend/src/mcp/draftTools.test.js b/backend/src/mcp/draftTools.test.js new file mode 100644 index 00000000..200f88c7 --- /dev/null +++ b/backend/src/mcp/draftTools.test.js @@ -0,0 +1,467 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./accountAdapter.js', () => ({ + getAccountByEmail: vi.fn(), + getAccountRow: vi.fn(), + getComposeSource: vi.fn(), + getDraftRow: vi.fn(), + listDraftRows: vi.fn(), +})); +vi.mock('../services/mail/identity.js', () => ({ + resolveFromIdentity: vi.fn(), +})); +vi.mock('../services/replyService.js', () => ({ + buildReferences: vi.fn(), +})); + +import { + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + listDraftRows, +} from './accountAdapter.js'; +import { + createDraftDef, + deleteDraftDef, + getDraftDef, + handleCreateDraft, + handleDeleteDraft, + handleGetDraft, + handleListDrafts, + handleUpdateDraft, + listDraftsDef, + updateDraftDef, +} from './draftTools.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; +import { resolveFromIdentity } from '../services/mail/identity.js'; +import { buildReferences } from '../services/replyService.js'; + +const scope = { + userId: 'user-1', + accountIds: ['account-1'], + scopes: ['read', 'write'], +}; +const account = { + id: 'account-1', + email_address: 'sender@example.com', + sender_name: 'Sender', + folder_mappings: { drafts: 'Drafts' }, +}; +const identity = { + fromName: 'Sender', + fromEmail: 'sender@example.com', + fromReplyTo: null, + signature: null, + aliasId: null, +}; + +function payload(result) { + return JSON.parse(result.content[0].text); +} + +function deps(overrides = {}) { + return { + draftService: { + saveDraft: vi.fn().mockResolvedValue({ + uid: 42, + folder: 'Drafts', + messageId: '', + }), + deleteDraft: vi.fn().mockResolvedValue({ ok: true }), + }, + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + getAccountByEmail.mockResolvedValue(account); + getAccountRow.mockResolvedValue(account); + resolveFromIdentity.mockResolvedValue(identity); + buildReferences.mockReturnValue({ + inReplyTo: '', + references: ' ', + }); +}); + +describe('draft tool definitions and registration', () => { + it('publishes the plan schemas, descriptions, annotations, scopes, and handlers', () => { + expect(createDraftDef).toEqual({ + name: 'create_draft', + description: "Create a draft email in the account's Drafts folder. Does NOT send. Returns the draft's uid and folder for later update_draft/send_draft/delete_draft calls.", + inputSchema: { + type: 'object', + required: ['account'], + properties: { + account: { type: 'string', description: 'Account email address (see get_stats for available accounts)' }, + to: { type: 'array', items: { type: 'string' }, description: "Recipients as 'Name ' or bare 'email'" }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string', description: 'Plain-text body' }, + body_html: { type: 'string', description: 'HTML body. When set, takes precedence over body for the HTML part.' }, + reply_to_message_id: { type: 'string', description: 'Message id to thread this draft under; sets In-Reply-To and References.' }, + alias: { type: 'string', description: 'Send-as alias email. Must be a configured alias on the account — errors if unknown, never silently falls back.' }, + attachments: { + type: 'array', + items: { + type: 'object', + required: ['filename', 'content'], + properties: { + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }, + }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: false, + idempotentHint: false, + openWorldHint: false, + }, + }); + expect(updateDraftDef.description).toContain( + 'IMAP has no update-in-place for messages: this appends a new draft and deletes the old one, so the draft_uid CHANGES. Use the returned draft_uid for subsequent calls.', + ); + expect(updateDraftDef.inputSchema.required).toEqual(['account', 'draft_uid']); + expect(updateDraftDef.inputSchema.properties).toEqual(expect.objectContaining({ + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + })); + expect(listDraftsDef.inputSchema.properties.limit).toEqual({ + type: 'integer', + default: 20, + maximum: 100, + }); + expect(getDraftDef.inputSchema.required).toEqual(['account', 'draft_uid']); + expect(deleteDraftDef.description).toContain( + 'PERMANENTLY deletes the draft from IMAP — it does not go to Trash and cannot be recovered.', + ); + + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const [name, requiredScope, handler] of [ + ['create_draft', 'write', handleCreateDraft], + ['update_draft', 'write', handleUpdateDraft], + ['list_drafts', 'read', handleListDrafts], + ['get_draft', 'read', handleGetDraft], + ['delete_draft', 'write', handleDeleteDraft], + ]) { + expect(defs.get(name)).toBeTruthy(); + expect(TOOL_SCOPES[name]).toBe(requiredScope); + expect(HANDLERS[name]).toBe(handler); + } + }); +}); + +describe('create_draft', () => { + it('resolves the scoped account and alias, threads the draft, saves it, and returns a write receipt', async () => { + const service = deps(); + getComposeSource.mockResolvedValue({ + id: 'message-1', + message_id: '', + thread_references: '', + }); + resolveFromIdentity.mockResolvedValue({ + ...identity, + fromName: 'Team', + fromEmail: 'team@example.com', + aliasId: 'alias-1', + }); + + const result = payload(await handleCreateDraft({ + account: 'sender@example.com', + to: ['Recipient '], + cc: ['copy@example.com'], + subject: 'Subject', + body: 'plain fallback', + body_html: '

Hello

', + reply_to_message_id: 'message-1', + alias: 'team@example.com', + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + content_type: 'text/plain', + }], + }, scope, service)); + + expect(getAccountByEmail).toHaveBeenCalledWith('sender@example.com', ['account-1']); + expect(resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasEmail: 'team@example.com' }, + service, + ); + expect(getComposeSource).toHaveBeenCalledWith('message-1', ['account-1']); + expect(buildReferences).toHaveBeenCalledWith(expect.objectContaining({ id: 'message-1' })); + expect(service.draftService.saveDraft).toHaveBeenCalledWith({ + userId: 'user-1', + account, + aliasEmail: 'team@example.com', + to: ['Recipient '], + cc: ['copy@example.com'], + bcc: [], + subject: 'Subject', + body: '

Hello

', + bodyIsHtml: true, + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + contentType: 'text/plain', + }], + inReplyTo: '', + references: ' ', + }, service); + expect(result).toEqual({ + draft_uid: 42, + folder: 'Drafts', + message_id: '', + receipt: { + message_id: '', + from: { name: 'Team', email: 'team@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [{ name: '', email: 'copy@example.com' }], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + }, + }); + }); + + it('does not reveal an out-of-scope account and never calls the service', async () => { + const service = deps(); + getAccountByEmail.mockResolvedValue({ error: 'account_not_found: foreign@example.com' }); + + const result = await handleCreateDraft( + { account: 'foreign@example.com' }, + scope, + service, + ); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account_not_found: foreign@example.com'); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); + + it('hard-fails an unknown alias and preserves its stable error code', async () => { + const service = deps(); + resolveFromIdentity.mockRejectedValue( + Object.assign(new Error('Alias not found'), { code: 'alias_not_found' }), + ); + + const result = await handleCreateDraft({ + account: 'sender@example.com', + alias: 'unknown@example.com', + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('alias_not_found: Alias not found'); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); + + it('returns message_not_found for an out-of-scope reply source', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(null); + + const result = await handleCreateDraft({ + account: 'sender@example.com', + reply_to_message_id: 'foreign-message', + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message_not_found: foreign-message'); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); +}); + +describe('update_draft', () => { + it('carries over every unsupplied compose field and returns the replacement uid', async () => { + const service = deps(); + getDraftRow.mockResolvedValue({ + uid: 9, + folder: 'Drafts', + to_addresses: [{ name: 'Original', email: 'original@example.com' }], + cc_addresses: [{ name: '', email: 'copy@example.com' }], + bcc_addresses: [{ name: '', email: 'blind@example.com' }], + subject: 'Original subject', + body_text: 'Original body', + body_html: null, + in_reply_to: '', + thread_references: ' ', + attachments: [], + }); + + const result = payload(await handleUpdateDraft({ + account: 'sender@example.com', + draft_uid: 9, + body: 'Edited body only', + }, scope, service)); + + expect(getDraftRow).toHaveBeenCalledWith('account-1', 'Drafts', 9); + expect(service.draftService.saveDraft).toHaveBeenCalledWith(expect.objectContaining({ + account, + to: ['Original '], + cc: ['copy@example.com'], + bcc: ['blind@example.com'], + subject: 'Original subject', + body: 'Edited body only', + bodyIsHtml: false, + inReplyTo: '', + references: ' ', + existingUid: 9, + existingFolder: 'Drafts', + }), service); + expect(result.draft_uid).toBe(42); + expect(result.draft_uid).not.toBe(9); + expect(result.receipt.to).toEqual([{ name: 'Original', email: 'original@example.com' }]); + }); + + it('returns draft_not_found without saving when the existing row is absent', async () => { + const service = deps(); + getDraftRow.mockResolvedValue(null); + + const result = await handleUpdateDraft({ + account: 'sender@example.com', + draft_uid: 404, + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('draft_not_found: 404'); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); +}); + +describe('list_drafts and get_draft', () => { + it('lists shaped draft summaries with the shared pagination envelope', async () => { + listDraftRows.mockResolvedValue([ + { + uid: 7, + folder: 'Drafts', + subject: 'Draft subject', + to_addresses: [{ name: '', email: 'to@example.com' }], + cc_addresses: [], + snippet: 'Draft snippet', + date: '2026-07-28T10:00:00.000Z', + has_attachments: true, + }, + ]); + + const result = payload(await handleListDrafts({ + account: 'sender@example.com', + limit: 20, + offset: 0, + }, scope, deps())); + + expect(listDraftRows).toHaveBeenCalledWith('account-1', { + limit: Number.MAX_SAFE_INTEGER, + offset: 0, + folder: 'Drafts', + }); + expect(result).toEqual({ + data: [{ + draft_uid: 7, + folder: 'Drafts', + subject: 'Draft subject', + to: [{ name: '', email: 'to@example.com' }], + cc: [], + snippet: 'Draft snippet', + date: '2026-07-28T10:00:00Z', + has_attachments: true, + }], + total: 1, + returned: 1, + offset: 0, + has_more: false, + }); + }); + + it('uses every scoped account when list_drafts omits account', async () => { + const multiScope = { ...scope, accountIds: ['account-1', 'account-2'] }; + getAccountRow + .mockResolvedValueOnce(account) + .mockResolvedValueOnce({ + ...account, + id: 'account-2', + email_address: 'other@example.com', + }); + listDraftRows + .mockResolvedValueOnce([{ uid: 1, folder: 'Drafts', date: '2026-01-01T00:00:00Z' }]) + .mockResolvedValueOnce([{ uid: 2, folder: 'Drafts', date: '2026-02-01T00:00:00Z' }]); + + const result = payload(await handleListDrafts({}, multiScope, deps())); + + expect(getAccountRow).toHaveBeenCalledTimes(2); + expect(result.data.map(row => row.draft_uid)).toEqual([2, 1]); + expect(result.total).toBe(2); + }); + + it('returns a full shaped draft without leaking storage-only fields', async () => { + getDraftRow.mockResolvedValue({ + uid: 7, + folder: 'Drafts', + subject: 'Draft subject', + to_addresses: [{ name: '', email: 'to@example.com' }], + cc_addresses: [], + bcc_addresses: [], + body_text: 'Hello', + body_html: '

Hello

', + in_reply_to: '', + thread_references: ' ', + attachments: [{ filename: 'note.txt', size: 5, part: '2' }], + account_id: 'account-1', + auth_pass: 'must-not-leak', + }); + + const result = payload(await handleGetDraft({ + account: 'sender@example.com', + draft_uid: 7, + }, scope, deps())); + + expect(result).toEqual({ + draft_uid: 7, + folder: 'Drafts', + subject: 'Draft subject', + to: [{ name: '', email: 'to@example.com' }], + cc: [], + bcc: [], + body_text: 'Hello', + body_html: '

Hello

', + in_reply_to: '', + references: ' ', + attachments: [{ filename: 'note.txt', size: 5, part: '2' }], + }); + }); +}); + +describe('delete_draft and missing dependencies', () => { + it('permanently deletes a scoped draft and returns its identifier', async () => { + const service = deps(); + getDraftRow.mockResolvedValue({ uid: 7, folder: 'Drafts' }); + + const result = payload(await handleDeleteDraft({ + account: 'sender@example.com', + draft_uid: 7, + }, scope, service)); + + expect(service.draftService.deleteDraft).toHaveBeenCalledWith({ + account, + uid: 7, + folder: 'Drafts', + }, service); + expect(result).toEqual({ deleted: true, draft_uid: 7, folder: 'Drafts' }); + }); + + it.each([ + ['create', handleCreateDraft, { account: 'sender@example.com' }], + ['update', handleUpdateDraft, { account: 'sender@example.com', draft_uid: 1 }], + ['list', handleListDrafts, { account: 'sender@example.com' }], + ['get', handleGetDraft, { account: 'sender@example.com', draft_uid: 1 }], + ['delete', handleDeleteDraft, { account: 'sender@example.com', draft_uid: 1 }], + ])('%s degrades to unsupported when draftService is absent', async (_name, handler, args) => { + const result = await handler(args, scope, {}); + expect(result.isError).toBe(true); + expect(result.content[0].text).toMatch(/^unsupported: /); + }); +}); diff --git a/backend/src/mcp/engineAdapter.js b/backend/src/mcp/engineAdapter.js new file mode 100644 index 00000000..c4c56fce --- /dev/null +++ b/backend/src/mcp/engineAdapter.js @@ -0,0 +1,411 @@ +// The query.Engine port: all message/aggregate/deletion SQL lives here, scoped by +// accountIds, so MCP handlers never touch db.js (the no-SQL-in-handlers invariant). +// Row → msgvault-shape mappers pin the wire-casing split: MessageSummary/Detail carry +// snake_case json keys, but Address/AttachmentInfo/AccountInfo/AggregateRow/TotalStats +// have NO Go json tags → Go-default CAPITALIZED keys. D6: ids are UUID strings, +// conversation_id = thread_id. +import { query, withTransaction } from '../services/db.js'; +import { buildOperatorClauses, freeTextTermClause, hasSearchableToken } from '../services/search/lexicalRepo.js'; + +export const SUMMARY_COLUMNS = ` + m.id, m.account_id, m.message_id, m.thread_id, m.subject, m.snippet, + m.from_email, m.from_name, m.to_addresses, m.cc_addresses, m.date, + m.has_attachments, m.attachments, m.flags, m.folder`; + +export const DETAIL_COLUMNS = SUMMARY_COLUMNS + `, m.body_text, m.body_html`; + +export function mapAddrs(jsonb) { + return (jsonb || []).map((a) => ({ Email: a.email || '', Name: a.name || '' })); +} + +// msgvault labels ≈ Gmail labels; Mailflow's closest analogue is the folder plus IMAP flags. +export function mapLabels(row) { + const flags = Array.isArray(row.flags) ? row.flags : []; + return [row.folder, ...flags].filter(Boolean); +} + +function toISO(d) { + return d instanceof Date ? d.toISOString() : (d || ''); +} + +export function rowToMessageSummary(row) { + const atts = Array.isArray(row.attachments) ? row.attachments : []; + const s = { + id: row.id, + source_id: row.account_id, + source_message_id: row.message_id || '', + conversation_id: row.thread_id || '', + source_conversation_id: row.thread_id || '', + subject: row.subject || '', + snippet: row.snippet || '', + from_email: row.from_email || '', + from_name: row.from_name || '', + sent_at: toISO(row.date), + size_estimate: 0, // divergence: Mailflow stores no byte size + has_attachments: !!row.has_attachments, + attachment_count: atts.length, + labels: mapLabels(row), + message_type: 'email', + }; + const to = mapAddrs(row.to_addresses); + const cc = mapAddrs(row.cc_addresses); + if (to.length) s.to = to; // omitempty parity + if (cc.length) s.cc = cc; + return s; +} + +function mapAttachments(atts) { + return (Array.isArray(atts) ? atts : []).map((a, i) => ({ + ID: i, Filename: a.filename || '', MimeType: a.type || '', Size: a.size || 0, + ContentHash: '', URL: '', StoragePath: '', // content tools are non-goals; synthetic index id + })); +} + +export function rowToMessageDetail(row) { + const base = rowToMessageSummary(row); + return { + ...base, + from: row.from_email ? [{ Email: row.from_email, Name: row.from_name || '' }] : [], + to: mapAddrs(row.to_addresses), + cc: mapAddrs(row.cc_addresses), + bcc: [], + body_text: row.body_text || '', + body_html: row.body_html || '', + attachments: mapAttachments(row.attachments), + }; +} + +// Raw detail row (not yet mapped) for one message, scoped. Handlers map/page it. +export async function getMessage(id, accountIds) { + if (!accountIds || !accountIds.length) return null; + const { rows } = await query( + `SELECT ${DETAIL_COLUMNS} FROM messages m WHERE m.id = $1 AND m.account_id = ANY($2)`, + [id, accountIds], + ); + return rows[0] || null; +} + +export async function getMessageSummariesByIDs(ids, accountIds) { + if (!ids || !ids.length) return []; + const { rows } = await query( + `SELECT ${SUMMARY_COLUMNS} FROM messages m WHERE m.id = ANY($1) AND m.account_id = ANY($2)`, + [ids, accountIds], + ); + const byId = new Map(rows.map((r) => [r.id, rowToMessageSummary(r)])); + return ids.map((id) => byId.get(id)).filter(Boolean); // preserve input order +} + +export async function getMessageBodiesByIDs(ids, accountIds) { + const out = new Map(); + if (!ids || !ids.length) return out; + const { rows } = await query( + `SELECT m.id, m.body_text, m.body_html FROM messages m WHERE m.id = ANY($1) AND m.account_id = ANY($2)`, + [ids, accountIds], + ); + for (const r of rows) out.set(r.id, { body_text: r.body_text, body_html: r.body_html }); + return out; +} + +export async function listAccounts(accountIds) { + if (!accountIds || !accountIds.length) return []; + const { rows } = await query( + `SELECT id, protocol, email_address, name FROM email_accounts WHERE id = ANY($1) ORDER BY sort_order`, + [accountIds], + ); + return rows.map((a) => ({ ID: a.id, SourceType: a.protocol || 'imap', Identifier: a.email_address, DisplayName: a.name || '' })); +} + +// Resolve an optional `account` email to a single scoped id (msgvault getAccountID). +// Returns { accountIds } narrowed to the match, or { error } when unknown (msgvault +// errors rather than silently widening). Empty account = no narrowing (all of the +// token's accounts). Shared by the search AND message/aggregate/deletion handlers so +// the `account` argument narrows scope everywhere it is advertised. +export async function resolveAccountScope(account, accountIds) { + if (!account) return { accountIds }; + const accounts = await listAccounts(accountIds); + const match = accounts.find((a) => a.Identifier === account); + if (!match) return { error: `account not found: ${account}` }; + return { accountIds: [match.ID] }; +} + +// Cheap owner-scope membership check for a single message id (find_similar seed). +export async function messageInScope(id, accountIds) { + if (!accountIds || !accountIds.length) return false; + const { rows } = await query('SELECT 1 FROM messages WHERE id = $1 AND account_id = ANY($2) LIMIT 1', [id, accountIds]); + return rows.length > 0; +} + +// Port of msgvault getDateArg (internal/mcp/handlers.go): structured after/before +// tool args are strict YYYY-MM-DD (the format every tool schema advertises). An +// unparseable value throws msgvault's exact error message instead of surfacing a +// raw Postgres timestamptz cast failure; a valid one binds as midnight UTC — the +// same instant lexicalRepo's buildOperatorClauses gives query-string dates — so +// structured args and before:/after: operators sit on one clock. Callers apply +// `before` EXCLUSIVELY (<): msgvault uses < and lexicalRepo already does; the +// list/aggregate/domain paths here had drifted to <=. +function parseDateArg(key, value) { + if (!value) return null; + const d = /^\d{4}-\d{2}-\d{2}$/.test(value) ? new Date(`${value}T00:00:00.000Z`) : null; + // Round-trip guard: Date.parse ROLLS an out-of-range day over ("2025-02-31" + // → Mar 2) instead of rejecting it; msgvault's time.Parse rejects. + if (!d || isNaN(d) || !d.toISOString().startsWith(value)) { + throw new Error(`invalid ${key} date "${value}": expected YYYY-MM-DD`); + } + return d.toISOString(); +} + +// Shared by every structured-arg path below: validate both bounds up front, +// then push the >= / < clauses through the caller's bind. +function pushDateClauses(where, bind, after, before) { + const afterISO = parseDateArg('after', after); + const beforeISO = parseDateArg('before', before); + if (afterISO) where.push(`m.date >= ${bind(afterISO)}`); + if (beforeISO) where.push(`m.date < ${bind(beforeISO)}`); +} + +// Newest-first message list, scoped and filtered. Returns MessageSummary[] (mapped +// here so the handler only pages). `from` filters from_email when it looks like an +// address, else from_name (msgvault list_messages semantics). +export async function listMessages({ accountIds, from, to, label, hasAttachment, after, before, conversationId, limit, offset }) { + if (!accountIds || !accountIds.length) return []; + const args = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + + if (from) { + where.push(from.includes('@') + ? `m.from_email ILIKE ${bind('%' + from + '%')}` + : `m.from_name ILIKE ${bind('%' + from + '%')}`); + } + if (to) where.push(`m.to_addresses::text ILIKE ${bind('%' + to + '%')}`); + if (label) where.push(`(m.folder = ${bind(label)} OR m.flags @> ${bind(JSON.stringify([label]))}::jsonb)`); + if (hasAttachment) where.push('m.has_attachments = true'); + pushDateClauses(where, bind, after, before); + if (conversationId) where.push(`m.thread_id = ${bind(conversationId)}`); + + const sql = `SELECT ${SUMMARY_COLUMNS} FROM messages m WHERE ${where.join(' AND ')} + ORDER BY m.date DESC NULLS LAST LIMIT ${bind(limit)} OFFSET ${bind(offset)}`; + const { rows } = await query(sql, args); + return rows.map(rowToMessageSummary); +} + +const MAX_STAGE_DELETION = 100000; // msgvault maxStageDeletionResults + +// A free-text term contributes a staging predicate only when it is positive, +// ≥2 chars, and carries a searchable token — the same hygiene the lexical and +// semantic paths apply. Negated terms are deliberately NOT enforced on the +// staging path (unlike lexical search's FTS exclusion), so a negation-only +// query yields zero predicates; countEnforceableQueryPredicates below is what +// lets the handler refuse that rather than stage every live message. +function usablePositiveTerm(t) { + return !t.negate && t.value.length >= 2 && hasSearchableToken(t.value); +} + +// Count the enforceable row predicates a parsed query would contribute to a +// staging WHERE: supported structured filters (buildOperatorClauses; in:/ +// unsupported operators contribute nothing) plus usable positive free-text +// terms. Mirrors resolveStageDeletionIds' predicate construction exactly (same +// builder, same term hygiene) so the two can never disagree about whether a +// query is enforceable. Zero here means the WHERE would carry only account + +// liveness — i.e. the query would soft-delete the whole (capped) mailbox — so +// the stage_deletion handler refuses it. This is the final safety net, reached +// only for queries that carry no unsupported operator yet still yield no +// predicate (e.g. all stopwords or punctuation): the handler already rejects any +// unsupported operator up front via unsupportedSearchOperatorMessage (msgvault +// parity, internal/mcp/handlers.go — so a mixed query like `invoice +// label:promotions` is refused, not silently widened to every "invoice" match). +export function countEnforceableQueryPredicates(parsed) { + if (!parsed) return 0; + let n = buildOperatorClauses(parsed.filters, () => '$0').length; + for (const t of parsed.terms || []) if (usablePositiveTerm(t)) n++; + return n; +} + +// Resolve candidate message ids for staging, scoped to accountIds. Query path builds +// its free-text predicates from lexicalRepo's freeTextTermClause — the EXACT builder +// the search path uses (ranked search_fts match + un-backfilled fallback + stopword +// vacuity) — so the search preview and the staged set can never diverge: `the invoice` +// stages the invoice matches the preview showed, not the zero rows the legacy +// ILIKE/plainto fork produced once "the" normalized to an empty tsquery. Structured +// operators share buildOperatorClauses the same way; the structured-filter path +// builds from the discrete filter args. +async function resolveStageDeletionIds({ accountIds, parsed, from, domain, label, hasAttachment, after, before }) { + const params = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { params.push(v); return `$${params.length}`; }; + + if (parsed) { + for (const cond of buildOperatorClauses(parsed.filters, bind)) where.push(cond); + for (const t of parsed.terms || []) { + if (!usablePositiveTerm(t)) continue; + where.push(freeTextTermClause(t.value, false, bind)); + } + } else { + if (from) { const p = bind('%' + from + '%'); where.push(`(m.from_email ILIKE ${p} OR m.from_name ILIKE ${p})`); } + if (domain) where.push(`m.from_email ILIKE ${bind('%@' + domain)}`); + if (label) where.push(`(m.folder = ${bind(label)} OR m.flags @> ${bind(JSON.stringify([label]))}::jsonb)`); + if (hasAttachment) where.push('m.has_attachments = true'); + pushDateClauses(where, bind, after, before); + } + + const sql = `SELECT m.id FROM messages m WHERE ${where.join(' AND ')} LIMIT ${bind(MAX_STAGE_DELETION)}`; + const { rows } = await query(sql, params); + return rows.map((r) => r.id); +} + +// Record a STAGED batch + its members scoped to the token user. NEVER flips +// is_deleted — execution is a separate, session-authed step. +export async function stageDeletion(opts) { + const ids = await resolveStageDeletionIds(opts); + if (!ids.length) return { batchId: null, messageCount: 0 }; + return withTransaction(async (client) => { + const { rows } = await client.query( + "INSERT INTO mcp_deletion_batches (user_id, description, status, message_count) VALUES ($1, $2, 'staged', $3) RETURNING id", + [opts.userId, opts.description || '', ids.length], + ); + const batchId = rows[0].id; + await client.query( + 'INSERT INTO mcp_deletion_batch_messages (batch_id, message_id) SELECT $1, unnest($2::uuid[])', + [batchId, ids], + ); + return { batchId, messageCount: ids.length }; + }); +} + +// Session-authed soft-delete of a STAGED batch, scoped to the owner's accounts. +// Returns the updated row count, or null when the batch is absent/not owned/not staged. +export async function executeDeletionBatch(batchId, userId) { + return withTransaction(async (client) => { + const { rows } = await client.query( + "SELECT id FROM mcp_deletion_batches WHERE id = $1 AND user_id = $2 AND status = 'staged' FOR UPDATE", + [batchId, userId], + ); + if (!rows.length) return null; + const upd = await client.query( + `UPDATE messages SET is_deleted = true + WHERE id IN (SELECT message_id FROM mcp_deletion_batch_messages WHERE batch_id = $1) + AND account_id IN (SELECT id FROM email_accounts WHERE user_id = $2)`, + [batchId, userId], + ); + await client.query( + "UPDATE mcp_deletion_batches SET status = 'executed', executed_at = NOW() WHERE id = $1", + [batchId], + ); + return upd.rowCount; + }); +} + +// Discard a staged batch (owner-scoped). Cascades member rows; never touches messages. +export async function unstageDeletionBatch(batchId, userId) { + const { rowCount } = await query( + 'DELETE FROM mcp_deletion_batches WHERE id = $1 AND user_id = $2', + [batchId, userId], + ); + return rowCount > 0; +} + +// Archive overview scoped to accountIds. TotalStats has NO Go json tags → CAPITALIZED +// keys. TotalSize is a body-byte proxy and AttachmentSize is 0 (Mailflow stores no +// byte sizes — documented divergence). +export async function getTotalStats(accountIds) { + const empty = { + MessageCount: 0, ActiveMessageCount: 0, SourceDeletedMessageCount: 0, + TotalSize: 0, AttachmentCount: 0, AttachmentSize: 0, LabelCount: 0, + AccountCount: (accountIds && accountIds.length) || 0, + }; + if (!accountIds || !accountIds.length) return empty; + const { rows } = await query( + `SELECT + COUNT(*)::bigint AS message_count, + COUNT(*) FILTER (WHERE is_deleted = false)::bigint AS active_count, + COUNT(*) FILTER (WHERE is_deleted = true)::bigint AS deleted_count, + COALESCE(SUM(octet_length(COALESCE(body_text, ''))), 0)::bigint AS total_size, + COALESCE(SUM(jsonb_array_length(COALESCE(attachments, '[]'::jsonb))), 0)::bigint AS attachment_count, + COUNT(DISTINCT folder)::bigint AS label_count + FROM messages WHERE account_id = ANY($1)`, + [accountIds], + ); + const r = rows[0] || {}; + return { + MessageCount: Number(r.message_count || 0), + ActiveMessageCount: Number(r.active_count || 0), + SourceDeletedMessageCount: Number(r.deleted_count || 0), + TotalSize: Number(r.total_size || 0), + AttachmentCount: Number(r.attachment_count || 0), + AttachmentSize: 0, + LabelCount: Number(r.label_count || 0), + AccountCount: accountIds.length, + }; +} + +// group_by → grouping SQL. `recipient` needs a lateral unnest of to_addresses; +// the rest are scalar expressions over messages. `time` buckets by calendar year. +const AGG_KEY_EXPR = { + sender: 'm.from_email', + domain: "split_part(m.from_email, '@', 2)", + label: 'm.folder', + time: "to_char(m.date, 'YYYY')", +}; + +// Grouped statistics (top senders/recipients/domains/labels, or volume by year). +// Returns AggregateRow[] — capitalized keys (no Go json tags). AttachmentSize is 0 +// (Mailflow stores no per-attachment byte totals — documented divergence). +export async function aggregate(groupBy, { accountIds, after, before, limit }) { + if (!accountIds || !accountIds.length) return []; + const args = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + pushDateClauses(where, bind, after, before); + + const measures = ` + COUNT(*)::bigint AS count, + COALESCE(SUM(octet_length(COALESCE(m.body_text, ''))), 0)::bigint AS total_size, + COALESCE(SUM(jsonb_array_length(COALESCE(m.attachments, '[]'::jsonb))), 0)::bigint AS attachment_count, + COUNT(*) OVER ()::bigint AS total_unique`; + + let sql; + if (groupBy === 'recipient') { + sql = `SELECT (ra->>'email') AS key, ${measures} + FROM messages m, LATERAL jsonb_array_elements(COALESCE(m.to_addresses, '[]'::jsonb)) AS ra + WHERE ${where.join(' AND ')} + GROUP BY key ORDER BY count DESC LIMIT ${bind(limit)}`; + } else { + const expr = AGG_KEY_EXPR[groupBy]; + sql = `SELECT ${expr} AS key, ${measures} + FROM messages m + WHERE ${where.join(' AND ')} + GROUP BY key ORDER BY count DESC LIMIT ${bind(limit)}`; + } + const { rows } = await query(sql, args); + return rows.map((r) => ({ + Key: r.key == null ? '' : String(r.key), + Count: Number(r.count || 0), + TotalSize: Number(r.total_size || 0), + AttachmentSize: 0, + AttachmentCount: Number(r.attachment_count || 0), + TotalUnique: Number(r.total_unique || 0), + })); +} + +// Messages where any participant (from/to/cc) belongs to one of the domains. +// Returns MessageSummary[], newest-first, scoped to accountIds. +export async function searchByDomains(domains, after, before, limit, offset, accountIds) { + if (!accountIds || !accountIds.length || !domains.length) return []; + const args = [accountIds]; + const where = ['m.account_id = ANY($1)', 'm.is_deleted = false']; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + + const domainConds = domains.map((d) => { + const from = bind('%@' + d); + const to = bind('%' + d + '%'); + const cc = bind('%' + d + '%'); + return `(m.from_email ILIKE ${from} OR m.to_addresses::text ILIKE ${to} OR m.cc_addresses::text ILIKE ${cc})`; + }); + where.push('(' + domainConds.join(' OR ') + ')'); + pushDateClauses(where, bind, after, before); + + const sql = `SELECT ${SUMMARY_COLUMNS} FROM messages m WHERE ${where.join(' AND ')} + ORDER BY m.date DESC NULLS LAST LIMIT ${bind(limit)} OFFSET ${bind(offset)}`; + const { rows } = await query(sql, args); + return rows.map(rowToMessageSummary); +} diff --git a/backend/src/mcp/engineAdapter.test.js b/backend/src/mcp/engineAdapter.test.js new file mode 100644 index 00000000..5f352705 --- /dev/null +++ b/backend/src/mcp/engineAdapter.test.js @@ -0,0 +1,309 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn() })); +import { query, withTransaction } from '../services/db.js'; +import { rowToMessageSummary, rowToMessageDetail, mapAddrs, getMessageSummariesByIDs, listAccounts, listMessages, getTotalStats, aggregate, searchByDomains, stageDeletion, resolveAccountScope, messageInScope, countEnforceableQueryPredicates } from './engineAdapter.js'; +import { parseQuery } from '../services/search/queryParser.js'; +import { freeTextTermClause, searchLexical } from '../services/search/lexicalRepo.js'; + +const row = { + id: '11111111-1111-1111-1111-111111111111', account_id: 'acc-1', message_id: '', + thread_id: 'tid-9', subject: 'Hi', snippet: 's', from_email: 'a@b.com', from_name: 'A', + to_addresses: [{ name: 'C', email: 'c@d.com' }], cc_addresses: [], + date: '2024-01-01T00:00:00.000Z', has_attachments: true, + attachments: [{ part: '2', filename: 'f.pdf', type: 'application/pdf', size: 10 }], + flags: ['\\Seen'], folder: 'INBOX', body_text: 'hello', body_html: '

hello

', +}; + +describe('mapAddrs', () => { + it('emits capitalized Email/Name keys (msgvault no-json-tag quirk)', () => { + expect(mapAddrs([{ name: 'C', email: 'c@d.com' }])).toEqual([{ Email: 'c@d.com', Name: 'C' }]); + expect(mapAddrs(null)).toEqual([]); + }); +}); + +describe('rowToMessageSummary', () => { + it('maps thread_id to conversation_id and keeps the uuid id as a string', () => { + const s = rowToMessageSummary(row); + expect(s.id).toBe('11111111-1111-1111-1111-111111111111'); + expect(s.conversation_id).toBe('tid-9'); + expect(s.source_conversation_id).toBe('tid-9'); + expect(s.source_message_id).toBe(''); + expect(s.source_id).toBe('acc-1'); + expect(s.to).toEqual([{ Email: 'c@d.com', Name: 'C' }]); + expect(s.attachment_count).toBe(1); + expect(s.size_estimate).toBe(0); + expect(s.labels).toContain('INBOX'); + expect(s.labels).toContain('\\Seen'); + expect(s.message_type).toBe('email'); + }); + + it('omits empty to/cc (omitempty parity)', () => { + const s = rowToMessageSummary({ ...row, to_addresses: [], cc_addresses: [] }); + expect(s).not.toHaveProperty('to'); + expect(s).not.toHaveProperty('cc'); + }); +}); + +describe('rowToMessageDetail', () => { + it('adds body + capitalized AttachmentInfo', () => { + const d = rowToMessageDetail(row); + expect(d.body_text).toBe('hello'); + expect(d.body_html).toBe('

hello

'); + expect(d.from).toEqual([{ Email: 'a@b.com', Name: 'A' }]); + expect(d.attachments[0]).toEqual({ ID: 0, Filename: 'f.pdf', MimeType: 'application/pdf', Size: 10, ContentHash: '', URL: '', StoragePath: '' }); + }); +}); + +describe('getMessageSummariesByIDs', () => { + beforeEach(() => query.mockReset()); + it('scopes to accountIds and preserves input order', async () => { + query.mockResolvedValueOnce({ rows: [{ ...row, id: 'b' }, { ...row, id: 'a' }] }); + const out = await getMessageSummariesByIDs(['a', 'b'], ['acc-1']); + expect(out.map((m) => m.id)).toEqual(['a', 'b']); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/account_id = ANY/); + expect(params[1]).toContain('acc-1'); // accountIds is the array bound to ANY($2) + }); + it('returns [] for empty ids without hitting the DB', async () => { + expect(await getMessageSummariesByIDs([], ['acc-1'])).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('listAccounts', () => { + beforeEach(() => query.mockReset()); + it('maps to capitalized AccountInfo scoped to accountIds', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-1', protocol: 'imap', email_address: 'a@b.com', name: 'Work' }] }); + const out = await listAccounts(['acc-1']); + expect(out).toEqual([{ ID: 'acc-1', SourceType: 'imap', Identifier: 'a@b.com', DisplayName: 'Work' }]); + expect(query.mock.calls[0][1]).toEqual([['acc-1']]); + }); + it('returns [] for empty scope without querying', async () => { + expect(await listAccounts([])).toEqual([]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('resolveAccountScope', () => { + beforeEach(() => query.mockReset()); + it('no account → full scope, no lookup', async () => { + expect(await resolveAccountScope('', ['acc-1', 'acc-2'])).toEqual({ accountIds: ['acc-1', 'acc-2'] }); + expect(query).not.toHaveBeenCalled(); + }); + it('known account email → narrowed to its id', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-2', protocol: 'imap', email_address: 'c@d.com', name: 'W' }] }); + expect(await resolveAccountScope('c@d.com', ['acc-2'])).toEqual({ accountIds: ['acc-2'] }); + }); + it('unknown account email → error (msgvault getAccountID parity)', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'acc-1', protocol: 'imap', email_address: 'a@b.com', name: 'W' }] }); + expect(await resolveAccountScope('nope@x.com', ['acc-1'])).toEqual({ error: 'account not found: nope@x.com' }); + }); +}); + +describe('messageInScope', () => { + beforeEach(() => query.mockReset()); + it('true when the id belongs to one of the accounts, scoped', async () => { + query.mockResolvedValueOnce({ rows: [{ '?column?': 1 }] }); + expect(await messageInScope('m1', ['acc-1'])).toBe(true); + expect(query.mock.calls[0][1]).toEqual(['m1', ['acc-1']]); + }); + it('false for a foreign/absent id', async () => { + query.mockResolvedValueOnce({ rows: [] }); + expect(await messageInScope('m1', ['acc-1'])).toBe(false); + }); + it('false for empty scope without querying', async () => { + expect(await messageInScope('m1', [])).toBe(false); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('getTotalStats', () => { + beforeEach(() => query.mockReset()); + it('returns capitalized TotalStats keys scoped to accountIds', async () => { + query.mockResolvedValueOnce({ rows: [{ message_count: '10', active_count: '9', deleted_count: '1', total_size: '5000', attachment_count: '3', label_count: '2' }] }); + const s = await getTotalStats(['acc-1', 'acc-2']); + expect(s).toEqual({ + MessageCount: 10, ActiveMessageCount: 9, SourceDeletedMessageCount: 1, + TotalSize: 5000, AttachmentCount: 3, AttachmentSize: 0, LabelCount: 2, AccountCount: 2, + }); + expect(query.mock.calls[0][1]).toEqual([['acc-1', 'acc-2']]); + }); + it('returns a zeroed struct for empty scope without querying', async () => { + const s = await getTotalStats([]); + expect(s.MessageCount).toBe(0); + expect(s.AccountCount).toBe(0); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('aggregate', () => { + beforeEach(() => query.mockReset()); + it('groups sender by from_email and maps capitalized AggregateRow keys, scoped', async () => { + query.mockResolvedValueOnce({ rows: [{ key: 'a@b.com', count: '3', total_size: '10', attachment_count: '1', total_unique: '5' }] }); + const out = await aggregate('sender', { accountIds: ['acc-1'], limit: 50 }); + expect(out).toEqual([{ Key: 'a@b.com', Count: 3, TotalSize: 10, AttachmentSize: 0, AttachmentCount: 1, TotalUnique: 5 }]); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/m\.from_email AS key/); + expect(sql).toMatch(/account_id = ANY\(\$1\)/); + expect(params[0]).toEqual(['acc-1']); + }); + + it("time buckets by calendar year (to_char YYYY)", async () => { + query.mockResolvedValueOnce({ rows: [{ key: '2024', count: '9', total_size: '0', attachment_count: '0', total_unique: '2' }] }); + const out = await aggregate('time', { accountIds: ['acc-1'], limit: 50 }); + expect(out[0].Key).toBe('2024'); + expect(query.mock.calls[0][0]).toMatch(/to_char\(m\.date, 'YYYY'\) AS key/); + }); + + it('recipient unnests to_addresses via a lateral join', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await aggregate('recipient', { accountIds: ['acc-1'], limit: 50 }); + expect(query.mock.calls[0][0]).toMatch(/LATERAL jsonb_array_elements/); + }); +}); + +describe('searchByDomains', () => { + beforeEach(() => query.mockReset()); + it('matches any participant at a domain, scoped, and maps capitalized summaries', async () => { + query.mockResolvedValueOnce({ rows: [{ ...row, id: 'm1' }] }); + const out = await searchByDomains(['gobright.com'], null, null, 100, 0, ['acc-1']); + expect(out[0].id).toBe('m1'); + expect(out[0].to).toEqual([{ Email: 'c@d.com', Name: 'C' }]); // capitalized address keys + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/from_email ILIKE/); + expect(sql).toMatch(/to_addresses::text ILIKE/); + expect(sql).toMatch(/cc_addresses::text ILIKE/); + expect(params[0]).toEqual(['acc-1']); + expect(params).toContain('%@gobright.com'); + }); +}); + +describe('stageDeletion', () => { + beforeEach(() => { query.mockReset(); withTransaction.mockReset(); }); + + it('resolves ids scoped to accountIds, records a STAGED batch + members, never soft-deletes', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'm1' }, { id: 'm2' }] }); // id resolution + const clientCalls = []; + const client = { query: vi.fn(async (text) => { + clientCalls.push(text); + if (/INSERT INTO mcp_deletion_batches/.test(text)) return { rows: [{ id: 'batch-1' }] }; + return { rows: [] }; + }) }; + withTransaction.mockImplementation(async (fn) => fn(client)); + const out = await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], domain: 'linkedin.com', description: 'filter' }); + expect(out).toEqual({ batchId: 'batch-1', messageCount: 2 }); + expect(query.mock.calls[0][1][0]).toEqual(['acc-1']); // id resolution scoped + expect(clientCalls.some((t) => /INSERT INTO mcp_deletion_batches/.test(t) && /'staged'/.test(t))).toBe(true); + expect(clientCalls.some((t) => /INSERT INTO mcp_deletion_batch_messages/.test(t))).toBe(true); + expect(clientCalls.some((t) => /UPDATE messages SET is_deleted/.test(t))).toBe(false); // never hard/soft deletes here + }); + + it('returns a zero count without opening a transaction when nothing matches', async () => { + query.mockResolvedValueOnce({ rows: [] }); + const out = await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], from: 'x' }); + expect(out).toEqual({ batchId: null, messageCount: 0 }); + expect(withTransaction).not.toHaveBeenCalled(); + }); +}); + +describe('structured after/before args (Wave D Fix 5)', () => { + beforeEach(() => { query.mockReset(); withTransaction.mockReset(); }); + + it('rejects a malformed after/before with msgvault getDateArg wording, before any SQL runs', async () => { + await expect(listMessages({ accountIds: ['a'], after: 'notadate', limit: 50, offset: 0 })) + .rejects.toThrow('invalid after date "notadate": expected YYYY-MM-DD'); + // Strict YYYY-MM-DD (msgvault time.Parse("2006-01-02")) — no loose forms. + await expect(aggregate('sender', { accountIds: ['a'], before: '2025-1-2', limit: 10 })) + .rejects.toThrow('invalid before date "2025-1-2": expected YYYY-MM-DD'); + await expect(searchByDomains(['x.com'], null, '2025-02-31', 10, 0, ['a'])) + .rejects.toThrow('invalid before date "2025-02-31": expected YYYY-MM-DD'); + await expect(stageDeletion({ userId: 'u', accountIds: ['a'], before: 'garbage' })) + .rejects.toThrow('invalid before date "garbage": expected YYYY-MM-DD'); + expect(query).not.toHaveBeenCalled(); + }); + + it('binds valid dates as midnight-UTC ISO with an EXCLUSIVE before (<), matching lexicalRepo', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await listMessages({ accountIds: ['a'], after: '2025-01-02', before: '2025-02-03', limit: 50, offset: 0 }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.date >= $2'); + expect(sql).toContain('m.date < $3'); + expect(sql).not.toContain('m.date <='); + expect(params[1]).toBe('2025-01-02T00:00:00.000Z'); + expect(params[2]).toBe('2025-02-03T00:00:00.000Z'); + }); +}); + +describe('stage_deletion / search predicate parity (Wave D Fix 3)', () => { + beforeEach(() => { query.mockReset(); withTransaction.mockReset(); }); + + // Rebuild the term clauses through lexicalRepo's shared owner with the same + // ordinals both paths allocate ($1 = accountIds, terms from $2). + function expectedClauses(terms) { + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { clauses: terms.map((t) => freeTextTermClause(t, false, bind)), params }; + } + + it('builds the staging WHERE from the ranked/search_fts builder the search path uses, stopword guard included', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], parsed: parseQuery('the invoice') }); + const [sql, params] = query.mock.calls[0]; + const expected = expectedClauses(['the', 'invoice']); + for (const clause of expected.clauses) expect(sql).toContain(clause); + expect(params.slice(1, 5)).toEqual(expected.params); + // Ranked construction, not the legacy ILIKE/plainto fork the staging path + // used to build (preview and staged set must agree): search_fts-first match + // with the stopword vacuity guard, so `the invoice` stages the invoice set + // instead of zero rows ("the" normalizes to an empty tsquery). + expect(sql).toContain('m.search_fts @@'); + expect(sql).toContain('numnode('); + }); + + it('emits byte-identical term predicates to a lexical search of the same query', async () => { + const searchCalls = []; + const client = async (text, params) => { searchCalls.push({ text, params }); return { rows: [] }; }; + await searchLexical(client, { + parsed: parseQuery('the invoice'), accountIds: ['acc-1'], + folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + query.mockResolvedValueOnce({ rows: [] }); + await stageDeletion({ userId: 'u1', accountIds: ['acc-1'], parsed: parseQuery('the invoice') }); + const stagingSql = query.mock.calls[0][0]; + const searchSql = searchCalls[0].text; + const expected = expectedClauses(['the', 'invoice']); + for (const clause of expected.clauses) { + expect(searchSql).toContain(clause); // same ordinals in both WHEREs… + expect(stagingSql).toContain(clause); // …so the clause text is identical + } + // And the bind values line up pairwise (like + fts per term). + expect(searchCalls[0].params.slice(1, 5)).toEqual(query.mock.calls[0][1].slice(1, 5)); + }); +}); + +describe('countEnforceableQueryPredicates', () => { + const n = (q) => countEnforceableQueryPredicates(parseQuery(q)); + + it('counts supported structured filters and usable positive free-text terms', () => { + expect(n('from:amazon invoice')).toBe(2); // from: + invoice + expect(n('is:unread')).toBe(1); // one structured predicate + expect(n('invoice urgent')).toBe(2); // two positive terms + }); + + it('returns 0 when every token is discarded (the stage-everything hazard)', () => { + expect(n('label:promotions')).toBe(0); // unsupported operator only + expect(n('-newsletter')).toBe(0); // negation-only (not enforced on staging) + expect(n('a')).toBe(0); // sub-2-char + expect(n('!!!')).toBe(0); // punctuation-only + expect(n('in:inbox')).toBe(0); // folder scope is not a row predicate + }); + + it('mirrors resolveStageDeletionIds: a real term survives alongside an unsupported operator', () => { + expect(n('invoice label:promotions')).toBe(1); + }); + + it('treats a null parsed query (structured-filter path) as zero', () => { + expect(countEnforceableQueryPredicates(null)).toBe(0); + }); +}); diff --git a/backend/src/mcp/envelope.js b/backend/src/mcp/envelope.js new file mode 100644 index 00000000..d143b7fc --- /dev/null +++ b/backend/src/mcp/envelope.js @@ -0,0 +1,30 @@ +export const TOTAL_COUNT_UNKNOWN = -1; + +// msgvault wire timestamps are Go-marshaled RFC3339 with no sub-second part +// (SQLite second-precision sent_at; vector/stats.go formatTime uses +// time.RFC3339 explicitly). engineAdapter's toISO emits Date.toISOString() +// milliseconds — an internal convention REST shares — so the MCP layer +// re-formats at emission instead of touching engineAdapter (Wave D's file). +// TODO(seam): fold into engineAdapter.toISO if it ever becomes wire-final. +export function toRFC3339(value) { + if (typeof value !== 'string' || !value) return value; + return value.replace(/\.\d+(?=(?:Z|[+-]\d\d:\d\d)$)/, ''); +} + +// Re-shape one MessageSummary for the wire: today only sent_at needs +// re-formatting. Non-mutating; tolerates partial summaries from tests. +export function wireSummary(summary) { + if (!summary || typeof summary !== 'object') return summary; + if (typeof summary.sent_at !== 'string' || !summary.sent_at) return summary; + return { ...summary, sent_at: toRFC3339(summary.sent_at) }; +} + +export function newPaginatedResponse(data, total, offset) { + const d = data || []; + return { data: d, total, returned: d.length, offset, has_more: offset + d.length < total }; +} + +export function newPaginatedResponseNoTotal(data, offset, hasMore) { + const d = data || []; + return { data: d, total: TOTAL_COUNT_UNKNOWN, returned: d.length, offset, has_more: hasMore }; +} diff --git a/backend/src/mcp/envelope.test.js b/backend/src/mcp/envelope.test.js new file mode 100644 index 00000000..c4bdb9c4 --- /dev/null +++ b/backend/src/mcp/envelope.test.js @@ -0,0 +1,48 @@ +import { describe, it, expect } from 'vitest'; +import { newPaginatedResponse, newPaginatedResponseNoTotal, TOTAL_COUNT_UNKNOWN, toRFC3339, wireSummary } from './envelope.js'; + +describe('newPaginatedResponse', () => { + it('computes has_more from offset+returned vs total', () => { + expect(newPaginatedResponse([{ id: 'a' }, { id: 'b' }], 10, 0)).toEqual({ + data: [{ id: 'a' }, { id: 'b' }], total: 10, returned: 2, offset: 0, has_more: true, + }); + expect(newPaginatedResponse([{ id: 'a' }], 1, 0).has_more).toBe(false); + }); + it('coerces null data to an empty array', () => { + expect(newPaginatedResponse(null, 0, 0).data).toEqual([]); + }); +}); + +describe('newPaginatedResponseNoTotal', () => { + it('always reports total -1 and echoes has_more', () => { + const r = newPaginatedResponseNoTotal([{ id: 'a' }], 20, true); + expect(r).toEqual({ data: [{ id: 'a' }], total: TOTAL_COUNT_UNKNOWN, returned: 1, offset: 20, has_more: true }); + expect(TOTAL_COUNT_UNKNOWN).toBe(-1); + }); +}); + +describe('toRFC3339', () => { + it('strips fractional seconds (msgvault wire timestamps are Go RFC3339, no millis)', () => { + expect(toRFC3339('2024-01-01T00:00:00.000Z')).toBe('2024-01-01T00:00:00Z'); + expect(toRFC3339('2024-01-01T00:00:00.123Z')).toBe('2024-01-01T00:00:00Z'); + expect(toRFC3339('2024-01-01T00:00:00.123456+02:00')).toBe('2024-01-01T00:00:00+02:00'); + }); + it('passes through already-clean, empty, and non-string values', () => { + expect(toRFC3339('2024-01-01T00:00:00Z')).toBe('2024-01-01T00:00:00Z'); + expect(toRFC3339('')).toBe(''); + expect(toRFC3339(undefined)).toBe(undefined); + }); +}); + +describe('wireSummary', () => { + it('reformats sent_at to RFC3339 without mutating the input', () => { + const s = { id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }; + expect(wireSummary(s)).toEqual({ id: 'm1', sent_at: '2024-01-01T00:00:00Z' }); + expect(s.sent_at).toBe('2024-01-01T00:00:00.000Z'); + }); + it('leaves summaries without a sent_at string untouched', () => { + const s = { id: 'm1' }; + expect(wireSummary(s)).toBe(s); + expect(wireSummary(null)).toBe(null); + }); +}); diff --git a/backend/src/mcp/goldenParity.test.js b/backend/src/mcp/goldenParity.test.js new file mode 100644 index 00000000..2e20086b --- /dev/null +++ b/backend/src/mcp/goldenParity.test.js @@ -0,0 +1,968 @@ +// Golden-fixture parity: msgvault's wire shapes (internal/query/models.go, +// internal/mcp/handlers.go, internal/vector/stats.go) transcribed as key+type +// shapes and diffed field-for-field against Mailflow's output. Drives REAL code +// end-to-end with only I/O mocked, so the diff pins the true wire shape. +// +// Divergences the diff intentionally accepts: D6 UUID-string ids (token 'uuid' +// matches any string), ≤1 semantic excerpt, and the capitalized-key split +// (Address/AttachmentInfo/AccountInfo/AggregateRow/TotalStats have no Go json +// tags → Go-default caps; MessageSummary/getMessageResponse are snake_case). +// +// Shapes live inline here (not separate fixtures/*.json files) — a deliberate +// consolidation; the diffing is identical and the reference sits beside the test. +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { readFileSync } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn() })); +vi.mock('../services/search/queryParser.js', () => ({ parseQuery: vi.fn(() => ({ filters: [], terms: [{ value: 'hello', negate: false }], unsupported: [], errors: [] })) })); +vi.mock('../services/search/searchService.js', () => ({ search: vi.fn() })); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchFromChunk: vi.fn(), matchesInMessage: vi.fn() })); +vi.mock('../services/embeddings/generations.js', () => ({ activeGeneration: vi.fn(), buildingGeneration: vi.fn(), chunkCount: vi.fn() })); +vi.mock('../services/embeddings/hybrid.js', () => ({ resolveActiveGenerationFromConfig: vi.fn() })); +vi.mock('../services/embeddings/vectorStore.js', () => ({ loadVector: vi.fn(), annSearch: vi.fn() })); +vi.mock('../services/embeddings/config.js', () => ({ generationFingerprint: vi.fn(() => 'fp'), resolveEmbedConfig: vi.fn(async () => ({ enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 })) })); +vi.mock('../services/mailbox/move.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + bulkMoveToFolder: vi.fn(), + resolveMovedIds: vi.fn(), + }; +}); +vi.mock('../services/mailbox/archive.js', async (orig) => { + const actual = await orig(); + return { ...actual, bulkArchive: vi.fn() }; +}); +vi.mock('../services/mailbox/trash.js', async (orig) => { + const actual = await orig(); + return { ...actual, bulkTrash: vi.fn() }; +}); +vi.mock('../services/mailbox/snooze.js', async (orig) => { + const actual = await orig(); + return { ...actual, snoozeConversation: vi.fn() }; +}); +vi.mock('../services/gtd/actions.js', async (orig) => { + const actual = await orig(); + return { ...actual, gtdDone: vi.fn() }; +}); + +import { query, withTransaction } from '../services/db.js'; +import { search } from '../services/search/searchService.js'; +import * as generations from '../services/embeddings/generations.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; +import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { ALL_SCOPES } from './auth.js'; +import { rowToMessageSummary, rowToMessageDetail } from './engineAdapter.js'; +import { handleSearchMetadata, handleSearchMessageBodies, handleSemanticSearchMessages } from './searchTools.js'; +import { + handleGetMessage, handleListMessages, handleGetStats, handleAggregate, + handleFindSimilarMessages, handleSearchInMessage, handleStageDeletion, handleSearchByDomains, +} from './messageTools.js'; +import { + handleCreateDraft, handleDeleteDraft, handleUpdateDraft, +} from './draftTools.js'; +import { + handleListOutbox, handleRecallEmail, handleSendDraft, handleSendEmail, + handleUnsendEmail, +} from './sendTools.js'; +import { + handleForwardEmail, handleReplyAllEmail, handleReplyEmail, +} from './composeTools.js'; +import { + handleArchiveMessages, + handleGtdDone, + handleMoveMessages, + handleSnoozeMessage, + handleTrashMessages, +} from './mailboxTools.js'; +import { + bulkMoveToFolder, + resolveMovedIds, +} from '../services/mailbox/move.js'; +import { bulkArchive } from '../services/mailbox/archive.js'; +import { bulkTrash } from '../services/mailbox/trash.js'; +import { snoozeConversation } from '../services/mailbox/snooze.js'; +import { gtdDone } from '../services/gtd/actions.js'; +import { HANDLERS } from './tools.js'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; + +// ---- transcribed reference shapes --------------------------------------------- +const Address = { Email: 'string', Name: 'string' }; +const AttachmentInfo = { ID: 'int', Filename: 'string', MimeType: 'string', Size: 'int', ContentHash: 'string', URL: 'string', StoragePath: 'string' }; +const Generation = { id: 'int', model: 'string', dimension: 'int', fingerprint: 'string', state: 'string' }; +const MessageSummary = { + id: 'uuid', 'source_id?': 'uuid', source_message_id: 'string', conversation_id: 'uuid', + source_conversation_id: 'string', subject: 'string', snippet: 'string', + from_email: 'string', from_name: 'string', 'to?': [Address], 'cc?': [Address], + sent_at: 'string', size_estimate: 'int', has_attachments: 'bool', + attachment_count: 'int', labels: ['string'], message_type: 'string', +}; +const Match = { snippet: 'string', 'char_offset?': 'int', 'line?': 'int', 'score?': 'number' }; +// hybridScoreBreakdown (handlers.go:563-572): every field omitempty — rrf only +// fuses in mode=hybrid, subject_boosted only when true. +const HybridScore = { 'rrf?': 'number', 'bm25?': 'number', 'vector?': 'number', 'subject_boosted?': 'bool' }; +// searchMessageItem (handlers.go:331-342): MessageSummary + matches/ +// matches_truncated/score, all three Go-omitempty. +const SearchMessageItem = { ...MessageSummary, 'matches?': [Match], 'matches_truncated?': 'bool', 'score?': HybridScore }; +// searchMessageBodiesResponse (handlers.go:586-595): paginated (no-total) + +// mode/pool_saturated/generation — shared by keyword and vector/hybrid modes. +const SearchBodiesEnvelope = { + data: [SearchMessageItem], total: 'int', returned: 'int', offset: 'int', has_more: 'bool', + mode: 'string', pool_saturated: 'bool', generation: Generation, +}; +const WriteAddress = { name: 'string', email: 'string' }; +const WriteAttachment = { filename: 'string', size: 'int', 'source?': 'string' }; +const WriteReceipt = { + from: WriteAddress, + to: [WriteAddress], + cc: [WriteAddress], + bcc: [WriteAddress], + subject: 'string', + attachments: [WriteAttachment], +}; +const ImmediateSend = { + sent: 'bool', + message_id: 'string', + ...WriteReceipt, + sent_copy_saved: 'bool', + folder: 'string', +}; +const QueuedSend = { + queued: 'bool', + outbox_id: 'string', + send_at: 'string', + undo_seconds: 'int', + from: {}, + to: [WriteAddress], + cc: [WriteAddress], + bcc: [WriteAddress], + subject: 'string', + attachments: [WriteAttachment], + note: 'string', +}; +const RecipientsComputed = { + reply_target: 'string', + excluded_self: ['string'], +}; +const SHAPES = { + message_summary: MessageSummary, + message_detail: { ...MessageSummary, from: [Address], to: [Address], cc: [Address], bcc: [Address], body_text: 'string', body_html: 'string', attachments: [AttachmentInfo] }, + search_metadata: { data: [MessageSummary], total: 'int', returned: 'int', offset: 'int', has_more: 'bool' }, + list_messages: { data: [MessageSummary], total: 'int', returned: 'int', offset: 'int', has_more: 'bool' }, + get_message: { + id: 'uuid', source_message_id: 'string', conversation_id: 'uuid', source_conversation_id: 'string', + subject: 'string', 'message_type?': 'string', snippet: 'string', sent_at: 'string', + size_estimate: 'int', has_attachments: 'bool', from: [Address], to: [Address], cc: [Address], bcc: [Address], + body_text: 'string', body_html: 'string', 'body_format?': 'string', body_length: 'int', + body_returned: 'int', offset: 'int', has_more: 'bool', labels: ['string'], attachments: [AttachmentInfo], + }, + get_stats: { + stats: { MessageCount: 'int', ActiveMessageCount: 'int', SourceDeletedMessageCount: 'int', TotalSize: 'int', AttachmentCount: 'int', AttachmentSize: 'int', LabelCount: 'int', AccountCount: 'int' }, + accounts: [{ ID: 'uuid', SourceType: 'string', Identifier: 'string', DisplayName: 'string' }], + 'vector_search?': { enabled: 'bool', active_generation: { id: 'int', model: 'string', dimension: 'int', fingerprint: 'string', state: 'string', 'activated_at?': 'string', message_count: 'int' }, 'building_generation?': {}, missing_embeddings_total: 'int' }, + }, + aggregate: [{ Key: 'string', Count: 'int', TotalSize: 'int', AttachmentSize: 'int', AttachmentCount: 'int', TotalUnique: 'int' }], + find_similar: { seed_message_id: 'uuid', returned: 'int', generation: Generation, messages: [MessageSummary] }, + search_in_message: { data: [Match], total: 'int', returned: 'int', offset: 'int', has_more: 'bool' }, + stage_deletion: { batch_id: 'uuid', message_count: 'int', status: 'string', next_step: 'string' }, + search_message_bodies: SearchBodiesEnvelope, + semantic_search_messages: SearchBodiesEnvelope, + search_by_domains: [MessageSummary], // raw array, no envelope (handlers.go:1984-1989) + ping: { pong: 'bool' }, // Mailflow-specific health tool — no msgvault counterpart (documented divergence) + write_receipt: WriteReceipt, + create_or_update_draft: { + draft_uid: 'int', + folder: 'string', + message_id: 'string', + receipt: { message_id: 'string', ...WriteReceipt }, + }, + delete_draft: { deleted: 'bool', draft_uid: 'int', folder: 'string' }, + immediate_send: ImmediateSend, + queued_send: QueuedSend, + reply_send: { + sent: 'bool', + recipients_computed: RecipientsComputed, + message_id: 'string', + in_reply_to: 'string', + references: 'string', + ...WriteReceipt, + sent_copy_saved: 'bool', + folder: 'string', + }, + forward_send: { + ...ImmediateSend, + attachments: [{ filename: 'string', size: 'int', source: 'string' }], + }, + unsend_email: { + cancelled: 'bool', + outbox_id: 'string', + subject: 'string', + to: ['string'], + }, + list_outbox: { + data: [{ + id: 'string', + subject: 'string', + to_preview: ['string'], + send_at: 'string', + }], + total: 'int', + returned: 'int', + offset: 'int', + has_more: 'bool', + }, + recall_cancelled_before_send: { + recalled: 'string', + outbox_id: 'string', + subject: 'string', + to: ['string'], + }, + recall_not_possible: { + recalled: 'string', + note: 'string', + sent_copy_deleted: 'bool', + followup_draft: { draft_uid: 'int', folder: 'string' }, + }, +}; + +// ---- diffKeys: [] on parity, else a list of divergence paths ------------------- +function typeOk(val, token) { + const t = token.replace(/\?$/, ''); + if (t.includes('uuid')) return typeof val === 'string'; // D6 id divergence + if (t === 'string') return typeof val === 'string'; + if (t === 'int' || t === 'number' || t === 'float') return typeof val === 'number'; + if (t === 'bool') return typeof val === 'boolean'; + return true; +} +function isOptional(shapeKey, shapeVal) { + return shapeKey.endsWith('?') || (typeof shapeVal === 'string' && shapeVal.endsWith('?')); +} +function diffKeys(actual, shape, path = '$') { + const out = []; + if (typeof shape === 'string') { + if (!typeOk(actual, shape)) out.push(`${path}: type mismatch (want ${shape}, got ${typeof actual})`); + return out; + } + if (Array.isArray(shape)) { + if (!Array.isArray(actual)) { out.push(`${path}: want array, got ${typeof actual}`); return out; } + actual.forEach((a, i) => out.push(...diffKeys(a, shape[0], `${path}[${i}]`))); + return out; + } + if (actual === null || typeof actual !== 'object' || Array.isArray(actual)) { out.push(`${path}: want object, got ${actual === null ? 'null' : typeof actual}`); return out; } + const realKeys = new Set(); + for (const sk of Object.keys(shape)) { + const k = sk.endsWith('?') ? sk.slice(0, -1) : sk; + realKeys.add(k); + if (!(k in actual)) { if (!isOptional(sk, shape[sk])) out.push(`${path}.${k}: missing`); continue; } + out.push(...diffKeys(actual[k], shape[sk], `${path}.${k}`)); + } + for (const k of Object.keys(actual)) if (!realKeys.has(k)) out.push(`${path}.${k}: extra (not in msgvault shape)`); + return out; +} + +const realRow = { + id: '11111111-1111-1111-1111-111111111111', account_id: 'acc-1', message_id: '', thread_id: 'tid-9', + subject: 'Hi', snippet: 's', from_email: 'a@b.com', from_name: 'A', + to_addresses: [{ name: 'C', email: 'c@d.com' }], cc_addresses: [{ name: 'E', email: 'e@f.com' }], + date: new Date('2024-01-01T00:00:00Z'), has_attachments: true, + attachments: [{ part: '2', filename: 'f.pdf', type: 'application/pdf', size: 10 }], + flags: ['\\Seen'], folder: 'INBOX', body_text: 'hello world', body_html: '

hello world

', +}; +const writeAccount = { + id: 'acc-1', + user_id: 'u', + email_address: 'sender@example.com', + sender_name: 'Sender', + signature: null, + folder_mappings: { drafts: 'Drafts' }, +}; +const composeSource = { + id: '22222222-2222-4222-8222-222222222222', + account_id: 'acc-1', + uid: 42, + folder: 'Sent', + message_id: '', + thread_references: '', + subject: 'Topic', + from_name: 'Original Sender', + from_email: 'original@example.com', + reply_to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + to_addresses: [{ name: 'Sender', email: 'sender@example.com' }], + cc_addresses: [{ name: 'Colleague', email: 'colleague@example.com' }], + body_text: 'Original body', + body_html: '

Original body

', + attachments: [{ part: '2', filename: 'deck.pdf', size: 2_144_000 }], + date: new Date('2026-07-28T09:00:00Z'), +}; +const immediateReceipt = { + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + messageId: '', + sentCopySaved: true, + folder: 'Sent', +}; +const jsonOf = (r) => JSON.parse(r.content[0].text); +const scope = { userId: 'u', accountIds: ['acc-1'], scopes: ALL_SCOPES }; + +function writeDeps(overrides = {}) { + return { + draftService: { + saveDraft: vi.fn().mockResolvedValue({ + uid: 42, + folder: 'Drafts', + messageId: '', + }), + deleteDraft: vi.fn().mockResolvedValue({ ok: true }), + }, + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: immediateReceipt, + }), + }, + outboxService: { + normalizeUndoWindow: vi.fn((requested, preference) => requested ?? preference ?? 0), + cancel: vi.fn().mockResolvedValue({ cancelled: true }), + listPending: vi.fn().mockResolvedValue([]), + }, + imapManager: { + permanentDeleteMessage: vi.fn().mockResolvedValue(undefined), + }, + ...overrides, + }; +} + +function mockQueryRows(...rowSets) { + for (const rows of rowSets) { + query.mockResolvedValueOnce({ rows, rowCount: rows.length }); + } +} + +beforeEach(() => { + query.mockReset(); withTransaction.mockReset(); search.mockReset(); matchFromChunk.mockReset(); + generations.activeGeneration.mockReset(); generations.buildingGeneration.mockReset(); generations.chunkCount.mockReset(); + resolveActiveGenerationFromConfig.mockReset(); loadVector.mockReset(); annSearch.mockReset(); + bulkMoveToFolder.mockReset(); resolveMovedIds.mockReset(); bulkArchive.mockReset(); + bulkTrash.mockReset(); snoozeConversation.mockReset(); gtdDone.mockReset(); +}); + +// Every function this suite vi.mock()s must actually exist on its real module — +// a renamed/never-implemented seam (e.g. generations.chunkCount) otherwise passes +// here while throwing live. Catches the missing/renamed-export drift class only, +// not value-shape drift. +describe('mock-drift guard: mocked seams exist on their real modules', () => { + // hybrid.js is intentionally omitted: importActual runs its module body, which + // references vectorStore.fusedSearch — not in this suite's vectorStore mock. + it.each([ + ['generations', () => generations, '../services/embeddings/generations.js'], + ['vectorStore', () => ({ loadVector, annSearch }), '../services/embeddings/vectorStore.js'], + ['searchService', () => ({ search }), '../services/search/searchService.js'], + ])('%s mock surface matches the real module', async (_name, getMock, path) => { + const real = await vi.importActual(path); + expect(mockSurfaceDrift(getMock(), real)).toEqual([]); + }); +}); + +describe('golden parity: structural mappers', () => { + it('rowToMessageSummary matches MessageSummary (snake_case + capitalized Address, id divergence excepted)', () => { + // Round-trip through JSON so absent optional keys behave like the wire. + const s = JSON.parse(JSON.stringify(rowToMessageSummary(realRow))); + expect(diffKeys(s, SHAPES.message_summary)).toEqual([]); + }); + it('rowToMessageDetail matches MessageDetail (capitalized AttachmentInfo)', () => { + const d = JSON.parse(JSON.stringify(rowToMessageDetail(realRow))); + expect(diffKeys(d, SHAPES.message_detail)).toEqual([]); + }); +}); + +describe('golden parity: message tool envelopes', () => { + // These unchanged get/list cases are the read-wire regression guard for the + // write-handler `(args, scope, deps)` stack. + it('get_message → getMessageResponse', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); + const b = jsonOf(await handleGetMessage({ id: realRow.id }, scope)); + expect(diffKeys(b, SHAPES.get_message)).toEqual([]); + }); + + it('list_messages → paginated MessageSummary envelope', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); // listMessages + const b = jsonOf(await handleListMessages({}, scope)); + expect(diffKeys(b, SHAPES.list_messages)).toEqual([]); + }); + + it('aggregate → capitalized AggregateRow array', async () => { + query.mockResolvedValueOnce({ rows: [{ key: 'a@b.com', count: '3', total_size: '10', attachment_count: '1', total_unique: '5' }] }); + const b = jsonOf(await handleAggregate({ group_by: 'sender' }, scope)); + expect(diffKeys(b, SHAPES.aggregate)).toEqual([]); + }); + + it('get_stats → {stats, accounts, vector_search} with capitalized structs', async () => { + query + .mockResolvedValueOnce({ rows: [{ message_count: '10', active_count: '9', deleted_count: '1', total_size: '5000', attachment_count: '3', label_count: '2' }] }) // getTotalStats + .mockResolvedValueOnce({ rows: [{ id: 'acc-1', protocol: 'imap', email_address: 'a@b.com', name: 'Work' }] }) // listAccounts + .mockResolvedValueOnce({ rows: [{ n: '4' }] }); // collectStats missingCount + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 1536, fingerprint: 'fp', state: 'active', activatedAt: 1704067200 }); // epoch seconds → RFC3339 wire + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(1000); + const b = jsonOf(await handleGetStats({}, scope)); + expect(diffKeys(b, SHAPES.get_stats)).toEqual([]); + // RFC3339 without sub-second digits (vector/stats.go:146-153 formatTime). + expect(b.vector_search.active_generation.activated_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('find_similar_messages → seed/returned/generation/messages', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 }, + generation: { id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([{ messageId: realRow.id, score: 0.9, rank: 1 }]); + query + .mockResolvedValueOnce({ rows: [{ '?column?': 1 }] }) // messageInScope(seed) + .mockResolvedValueOnce({ rows: [realRow] }); // getMessageSummariesByIDs + const b = jsonOf(await handleFindSimilarMessages({ message_id: realRow.id }, scope)); + expect(diffKeys(b, SHAPES.find_similar)).toEqual([]); + }); + + it('search_in_message (keyword) → Match envelope with byte char_offset', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); // getMessage + const b = jsonOf(await handleSearchInMessage({ id: realRow.id, query: 'hello' }, scope)); + expect(diffKeys(b, SHAPES.search_in_message)).toEqual([]); + expect(b.data[0].char_offset).toBe(Buffer.from(realRow.body_text, 'utf8').indexOf(Buffer.from('hello', 'utf8'))); + }); + + it('stage_deletion → {batch_id, message_count, status, next_step}', async () => { + query.mockResolvedValueOnce({ rows: [{ id: realRow.id }] }); // id resolution + withTransaction.mockImplementation(async (fn) => fn({ query: vi.fn(async (text) => (/INSERT INTO mcp_deletion_batches/.test(text) ? { rows: [{ id: 'batch-1' }] } : { rows: [] })) })); + const b = jsonOf(await handleStageDeletion({ domain: 'linkedin.com' }, scope)); + expect(diffKeys(b, SHAPES.stage_deletion)).toEqual([]); + expect(b.status).toBe('pending'); // msgvault manifest.StatusPending literal (manifest.go:25) + }); +}); + +describe('golden parity: search tool envelopes', () => { + it('search_metadata → paginated MessageSummary envelope (real hydration)', async () => { + search.mockResolvedValue({ messages: [{ id: realRow.id }], total: 1, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + query.mockResolvedValueOnce({ rows: [realRow] }); // getMessageSummariesByIDs hydration + const b = jsonOf(await handleSearchMetadata({ query: 'x' }, scope)); + expect(diffKeys(b, SHAPES.search_metadata)).toEqual([]); + // Go time.Time wire format: RFC3339 with no sub-second digits. + expect(b.data[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('search_message_bodies → searchMessageItem envelope; matches/matches_truncated omitted when empty/false (handlers.go:339-341)', async () => { + search.mockResolvedValue({ + messages: [ + { id: realRow.id, body_text: 'hello world, hello again' }, // 1 merged excerpt + { id: 'no-hit-1111-1111-1111-111111111111', body_text: 'nothing relevant' }, // 0 excerpts + ], + mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false }, + }); + query.mockResolvedValueOnce({ rows: [realRow, { ...realRow, id: 'no-hit-1111-1111-1111-111111111111' }] }); // hydration + const b = jsonOf(await handleSearchMessageBodies({ query: 'hello' }, scope)); + expect(diffKeys(b, SHAPES.search_message_bodies)).toEqual([]); + expect(b.mode).toBe('keyword'); + expect(b.total).toBe(-1); // body search never counts + expect(b.generation).toEqual({ id: 0, model: '', dimension: 0, fingerprint: '', state: '' }); + // diffKeys accepts optional keys whether present or absent, so the + // omitempty contract is asserted explicitly on both sides: + expect(b.data[0]).toHaveProperty('matches'); + expect(b.data[0]).not.toHaveProperty('matches_truncated'); // false → omitted + expect(b.data[1]).not.toHaveProperty('matches'); // empty → omitted + expect(b.data[1]).not.toHaveProperty('matches_truncated'); + }); + + it('semantic_search_messages → mode/pool_saturated/generation envelope; explain score omitempty (handlers.go:563-572)', async () => { + search.mockResolvedValue({ + messages: [{ + message_id: realRow.id, id: realRow.id, + best_chunk: { chunk_index: 0, char_start: 0, char_end: 10, score: 0.9 }, + score: { rrf: 0.03, bm25: 1.2, vector: 0.9, subject_boosted: false }, + }], + mode: 'vector', page: { offset: 0, limit: 20, hasMore: false }, + pool_saturated: true, + generation: { id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + matchFromChunk.mockResolvedValue({ char_offset: 5, snippet: 'hello', line: 1, score: 0.9 }); + query.mockResolvedValueOnce({ rows: [realRow] }); // hydration + const b = jsonOf(await handleSemanticSearchMessages({ query: 'hello', mode: 'vector', explain: true }, scope)); + expect(diffKeys(b, SHAPES.semantic_search_messages)).toEqual([]); + expect(b.mode).toBe('vector'); + expect(b.pool_saturated).toBe(true); + expect(b.total).toBe(-1); + expect(b.generation).toEqual({ id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }); + // omitempty asserted explicitly: no rrf in mode=vector (nothing to fuse), + // no subject_boosted when false. + expect(b.data[0].score).toEqual({ bm25: 1.2, vector: 0.9 }); + expect(b.data[0].matches).toHaveLength(1); // ≤1 excerpt — documented divergence from msgvault's ≤5 + }); + + it('search_by_domains → raw MessageSummary array, no envelope (handlers.go:1951-1989)', async () => { + query.mockResolvedValueOnce({ rows: [realRow] }); + const b = jsonOf(await handleSearchByDomains({ domains: 'b.com' }, scope)); + expect(diffKeys(b, SHAPES.search_by_domains)).toEqual([]); + expect(b[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('ping → {pong:true} (Mailflow-specific health tool)', async () => { + const b = jsonOf(await HANDLERS.ping({}, scope)); + expect(diffKeys(b, SHAPES.ping)).toEqual([]); + expect(b.pong).toBe(true); + }); +}); + +describe('golden parity: write tool envelopes', () => { + it('create_draft → draft identifier plus nested write receipt', async () => { + const deps = writeDeps(); + mockQueryRows([writeAccount]); + + const b = jsonOf(await handleCreateDraft({ + account: 'sender@example.com', + to: ['Recipient '], + subject: 'Subject', + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + content_type: 'text/plain', + }], + }, scope, deps)); + + expect(diffKeys(b, SHAPES.create_or_update_draft)).toEqual([]); + }); + + it('update_draft → replacement identifier plus nested write receipt', async () => { + const deps = writeDeps(); + deps.draftService.saveDraft.mockResolvedValue({ + uid: 43, + folder: 'Drafts', + messageId: '', + }); + mockQueryRows( + [writeAccount], + [{ + uid: 42, + folder: 'Drafts', + from_email: 'sender@example.com', + to_addresses: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc_addresses: [], + bcc_addresses: [], + subject: 'Subject', + body_text: 'Original body', + body_html: null, + attachments: [], + }], + ); + + const b = jsonOf(await handleUpdateDraft({ + account: 'sender@example.com', + draft_uid: 42, + body: 'Updated body', + }, scope, deps)); + + expect(diffKeys(b, SHAPES.create_or_update_draft)).toEqual([]); + expect(b.draft_uid).toBe(43); + }); + + it('delete_draft → permanent-deletion identifier envelope', async () => { + const deps = writeDeps(); + mockQueryRows( + [writeAccount], + [{ uid: 42, folder: 'Drafts' }], + ); + + const b = jsonOf(await handleDeleteDraft({ + account: 'sender@example.com', + draft_uid: 42, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.delete_draft)).toEqual([]); + }); + + it('send_email immediate → full write receipt', async () => { + const deps = writeDeps(); + mockQueryRows([writeAccount], [{ preferences: { undoSendSeconds: 0 } }]); + + const b = jsonOf(await handleSendEmail({ + account: 'sender@example.com', + to: ['Recipient '], + subject: 'Subject', + undo_send_seconds: 0, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.immediate_send)).toEqual([]); + }); + + it('send_email queued → placeholder receipt with undo metadata', async () => { + const deps = writeDeps(); + deps.outboxService.normalizeUndoWindow.mockReturnValue(30); + deps.sendService.sendOrEnqueue.mockResolvedValue({ + queued: true, + outboxId: 'outbox-1', + sendAt: new Date('2026-07-28T10:00:30.000Z'), + undoSeconds: 30, + }); + mockQueryRows([writeAccount], [{ preferences: { undoSendSeconds: 30 } }]); + + const b = jsonOf(await handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + subject: 'Queued subject', + }, scope, deps)); + + expect(diffKeys(b, SHAPES.queued_send)).toEqual([]); + expect(b.from).toEqual({}); + expect(b.to).toEqual([]); + }); + + it('send_draft immediate → same full write receipt as send_email', async () => { + const deps = writeDeps(); + mockQueryRows( + [writeAccount], + [{ + uid: 42, + folder: 'Drafts', + from_email: 'sender@example.com', + to_addresses: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc_addresses: [], + bcc_addresses: [], + subject: 'Subject', + body_text: 'Draft body', + body_html: '', + }], + [{ preferences: { undoSendSeconds: 0 } }], + ); + + const b = jsonOf(await handleSendDraft({ + account: 'sender@example.com', + draft_uid: 42, + undo_send_seconds: 0, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.immediate_send)).toEqual([]); + }); + + it.each([ + ['reply_email', handleReplyEmail], + ['reply_all_email', handleReplyAllEmail], + ])('%s → threading plus recipients_computed receipt', async (_name, handler) => { + const deps = writeDeps(); + deps.sendService.sendOrEnqueue.mockResolvedValue({ + ok: true, + receipt: { + ...immediateReceipt, + to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + subject: 'Re: Topic', + attachments: [], + messageId: '', + }, + }); + mockQueryRows( + [composeSource], + [writeAccount], + [], + [{ preferences: { undoSendSeconds: 0 } }], + ); + + const b = jsonOf(await handler({ + message_id: composeSource.id, + body: 'Thanks', + no_quote: true, + undo_send_seconds: 0, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.reply_send)).toEqual([]); + expect(b.recipients_computed.reply_target).toBe('reply@example.com'); + }); + + it('forward_email → immediate receipt with forwarded attachment source', async () => { + const deps = writeDeps(); + deps.sendService.sendOrEnqueue.mockResolvedValue({ + ok: true, + receipt: { + ...immediateReceipt, + subject: 'Fwd: Topic', + attachments: [{ filename: 'deck.pdf', size: 2_144_000 }], + messageId: '', + }, + }); + mockQueryRows( + [composeSource], + [writeAccount], + [], + [{ preferences: { undoSendSeconds: 0 } }], + ); + + const b = jsonOf(await handleForwardEmail({ + message_id: composeSource.id, + to: ['Recipient '], + undo_send_seconds: 0, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.forward_send)).toEqual([]); + expect(b.attachments[0].source).toBe('forwarded'); + }); + + it('unsend_email → cancelled outbox receipt', async () => { + const deps = writeDeps(); + deps.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: new Date('2026-07-28T10:00:30.000Z'), + }]); + + const b = jsonOf(await handleUnsendEmail({ + outbox_id: 'outbox-1', + }, scope, deps)); + + expect(diffKeys(b, SHAPES.unsend_email)).toEqual([]); + }); + + it('list_outbox → no-total pagination envelope', async () => { + const deps = writeDeps(); + deps.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: new Date('2026-07-28T10:00:30.000Z'), + }]); + + const b = jsonOf(await handleListOutbox({}, scope, deps)); + + expect(diffKeys(b, SHAPES.list_outbox)).toEqual([]); + expect(b.total).toBe(-1); + }); + + it('recall_email pending → cancelled_before_send envelope', async () => { + const deps = writeDeps(); + deps.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + }]); + + const b = jsonOf(await handleRecallEmail({ + outbox_id: 'outbox-1', + }, scope, deps)); + + expect(diffKeys(b, SHAPES.recall_cancelled_before_send)).toEqual([]); + expect(b.recalled).toBe('cancelled_before_send'); + }); + + it('recall_email delivered → not_possible envelope with follow-up draft', async () => { + const deps = writeDeps(); + deps.draftService.saveDraft.mockResolvedValue({ + uid: 57, + folder: 'Drafts', + messageId: '', + }); + mockQueryRows( + [], + [composeSource], + [writeAccount], + ); + query.mockResolvedValueOnce({ rows: [], rowCount: 1 }); + + const b = jsonOf(await handleRecallEmail({ + message_id: composeSource.id, + }, scope, deps)); + + expect(diffKeys(b, SHAPES.recall_not_possible)).toEqual([]); + expect(b.recalled).toBe('not_possible'); + expect(deps.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); +}); + +describe('golden parity: mailbox tool envelopes', () => { + const id = '33333333-3333-4333-8333-333333333333'; + const secondId = '44444444-4444-4444-8444-444444444444'; + const newId = '55555555-5555-4555-8555-555555555555'; + const deps = { imapManager: {} }; + + it('pins move_messages receipt keys', async () => { + bulkMoveToFolder.mockResolvedValue({ + ok: true, + movedDetails: [{ id, accountId: 'acc-1', uid: 110 }], + failed: [], + skippedAccounts: [], + }); + resolveMovedIds.mockResolvedValue([{ id: newId, uid: 110 }]); + + expect(jsonOf(await handleMoveMessages({ + message_ids: [id], + folder: 'Projects', + }, scope, deps))).toEqual({ + ok: true, + moved: [{ id, new_id: newId, uid: 110, folder: 'Projects' }], + failed: [], + skipped_accounts: [], + resync_pending: false, + note: 'message ids change on move; use new_id for follow-up calls', + }); + }); + + it('pins archive_messages receipt keys including destination_untracked', async () => { + bulkArchive.mockResolvedValue({ + ok: true, + archivedDetails: [ + { + id, + accountId: 'acc-1', + folder: 'Archive', + uid: 110, + destinationUntracked: false, + }, + { + id: secondId, + accountId: 'acc-1', + folder: '[Gmail]/All Mail', + uid: 111, + destinationUntracked: true, + }, + ], + failed: [], + noArchiveFolder: [], + }); + resolveMovedIds.mockResolvedValue([{ id: newId, uid: 110 }]); + + expect(jsonOf(await handleArchiveMessages({ + message_ids: [id, secondId], + }, scope, deps))).toEqual({ + ok: true, + archived: [ + { + id, + new_id: newId, + uid: 110, + folder: 'Archive', + destination_untracked: false, + }, + { + id: secondId, + new_id: null, + uid: 111, + folder: '[Gmail]/All Mail', + destination_untracked: true, + }, + ], + failed: [], + no_archive_folder: [], + resync_pending: false, + note: 'message ids change on archive; use new_id for follow-up calls', + }); + }); + + it('pins trash_messages receipt keys including the refusal partition', async () => { + bulkTrash.mockResolvedValue({ + ok: true, + trashedDetails: [{ + id, + accountId: 'acc-1', + folder: 'Trash', + uid: 110, + }], + failed: [], + refused: [{ + id: secondId, + folder: 'Trash', + reason: 'already_in_trash_permanent_delete_required', + }], + }); + resolveMovedIds.mockResolvedValue([{ id: newId, uid: 110 }]); + + expect(jsonOf(await handleTrashMessages({ + message_ids: [id, secondId], + }, scope, deps))).toEqual({ + ok: true, + trashed: [{ id, new_id: newId, folder: 'Trash' }], + failed: [], + refused: [{ + id: secondId, + folder: 'Trash', + reason: 'already_in_trash_permanent_delete_required', + }], + resync_pending: false, + next_step: 'use stage_deletion for permanent removal', + }); + }); + + it('pins snooze_message receipt keys', async () => { + snoozeConversation.mockResolvedValue({ + ok: true, + movedCount: 2, + movedIds: [id, secondId], + folder: 'Snoozed', + }); + + expect(jsonOf(await handleSnoozeMessage({ + message_id: id, + until: new Date(Date.now() + 60_000).toISOString(), + }, scope, deps))).toEqual({ + ok: true, + moved_count: 2, + sibling_ids: [secondId], + folder: 'Snoozed', + }); + }); + + it('pins gtd_done partial-success receipt keys', async () => { + gtdDone.mockResolvedValue({ + ok: true, + removed: ['Watch'], + archived: false, + noArchiveFolder: false, + archiveFailed: true, + }); + + const result = await handleGtdDone({ + message_id: id, + states: ['watch'], + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual({ + ok: true, + removed: ['Watch'], + archived: false, + no_archive_folder: false, + archive_failed: true, + }); + }); +}); + +describe('SQL is confined to MCP adapter seams', () => { + const here = dirname(fileURLToPath(import.meta.url)); + const read = (n) => readFileSync(join(here, n), 'utf8'); + for (const file of ['engineAdapter.js', 'accountAdapter.js']) { + it(`${file} is an explicitly allowed SQL seam`, () => { + expect(read(file)).toMatch(/from '\.\.\/services\/db\.js'/); + }); + } + for (const file of [ + 'searchTools.js', + 'messageTools.js', + 'mailboxTools.js', + 'triageTools.js', + 'composeTools.js', + 'draftTools.js', + 'sendTools.js', + 'writeResult.js', + 'accountTools.js', + ]) { + it(`${file} contains no raw SQL or db.js import`, () => { + const src = read(file); + expect(src).not.toMatch(/\bSELECT\b|\bINSERT\b|\bUPDATE\b|\bDELETE FROM\b/); + expect(src).not.toMatch(/from '\.\.\/services\/db\.js'/); + expect(src).not.toMatch(/\bpool\b/); + }); + } +}); diff --git a/backend/src/mcp/mailboxTools.js b/backend/src/mcp/mailboxTools.js new file mode 100644 index 00000000..abb6600d --- /dev/null +++ b/backend/src/mcp/mailboxTools.js @@ -0,0 +1,748 @@ +import { getAccountRow } from './accountAdapter.js'; +import { errorResult, jsonResult } from './result.js'; +import { + countMessagesIn, + createFolder, + deleteFolder, + listFolders, + renameFolder, +} from '../services/mailbox/folders.js'; +import { + bulkMoveToFolder, + resolveMovedIds, +} from '../services/mailbox/move.js'; +import { bulkArchive } from '../services/mailbox/archive.js'; +import { bulkTrash } from '../services/mailbox/trash.js'; +import { bulkSetRead, setStarred } from '../services/mailbox/flags.js'; +import { runInBatches } from '../services/mailbox/batch.js'; +import { markNotSpam, markSpam } from '../services/mailbox/spamLabel.js'; +import { + snoozeConversation, + unsnoozeConversation, +} from '../services/mailbox/snooze.js'; +import { setCategory } from '../services/mailbox/category.js'; +import { + gtdClassify, + gtdDone, + gtdUnclassify, +} from '../services/gtd/actions.js'; +import { GTD_STATES } from '../services/gtdConfig.js'; +import { + areValidUUIDs, + isValidFolderName, + UUID_RE, +} from '../utils/validation.js'; + +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); +const CREATE_ANNOTATIONS = annotations(); +const IDEMPOTENT_WRITE_ANNOTATIONS = annotations({ idempotentHint: true }); +const DESTRUCTIVE_WRITE_ANNOTATIONS = annotations({ destructiveHint: true }); +const DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS = annotations({ + destructiveHint: true, + idempotentHint: true, +}); + +export function messageIdsArg(args) { + const ids = args.message_ids; + if (!Array.isArray(ids) || ids.length === 0) { + return { error: 'message_ids must contain at least one id' }; + } + if (ids.length > 500) return { error: 'Too many ids — maximum 500 per request' }; + if (!areValidUUIDs(ids)) return { error: 'Invalid message id format' }; + return { value: ids }; +} + +export function messageIdArg(args) { + const id = args.message_id; + if (!id || typeof id !== 'string') return { error: 'message_id parameter is required' }; + if (!UUID_RE.test(id)) return { error: 'Invalid message id format' }; + return { value: id }; +} + +export const listFoldersDef = { + name: 'list_folders', + description: 'List folders and live message counts for scoped accounts. Pass account to narrow the result to one account.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string' }, + }, + }, +}; + +export const createFolderDef = { + name: 'create_folder', + description: 'Create a folder in one scoped account. The name must be an explicit valid folder component.', + annotations: CREATE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'name'], + properties: { + account: { type: 'string' }, + name: { type: 'string' }, + parent_path: { type: 'string' }, + }, + }, +}; + +export const renameFolderDef = { + name: 'rename_folder', + description: 'Rename the final component of an existing folder path. Both the account and source path must be explicit.', + annotations: DESTRUCTIVE_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'path', 'new_name'], + properties: { + account: { type: 'string' }, + path: { type: 'string' }, + new_name: { type: 'string' }, + }, + }, +}; + +export const deleteFolderDef = { + name: 'delete_folder', + description: 'Delete a folder and its tracked messages. The live message count must exactly match expected_message_count.', + annotations: DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['account', 'path', 'expected_message_count'], + properties: { + account: { type: 'string' }, + path: { type: 'string' }, + expected_message_count: { type: 'integer', minimum: 0 }, + }, + }, +}; + +const messageIdsSchema = { + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, +}; + +export const moveMessagesDef = { + name: 'move_messages', + description: 'Move explicit messages to a destination folder and return their replacement ids. Message ids change on move, and non-UIDPLUS servers may report resync_pending.', + annotations: DESTRUCTIVE_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids', 'folder'], + properties: { + message_ids: messageIdsSchema, + folder: { type: 'string' }, + }, + }, +}; + +export const archiveMessagesDef = { + name: 'archive_messages', + description: 'Archive explicit messages and return their replacement ids. Gmail All Mail destinations are reported as destination_untracked.', + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids'], + properties: { + message_ids: messageIdsSchema, + }, + }, +}; + +export const trashMessagesDef = { + name: 'trash_messages', + description: 'Move explicit messages to Trash and return their replacement ids. Messages requiring permanent deletion are refused.', + annotations: DESTRUCTIVE_IDEMPOTENT_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids'], + properties: { + message_ids: messageIdsSchema, + }, + }, +}; + +function messageFlagDef(name, description) { + return { + name, + description, + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids'], + properties: { + message_ids: messageIdsSchema, + }, + }, + }; +} + +export const markReadDef = messageFlagDef( + 'mark_read', + 'Mark explicit messages as read. The updated receipt excludes messages already read.', +); +export const markUnreadDef = messageFlagDef( + 'mark_unread', + 'Mark explicit messages as unread. The updated receipt excludes messages already unread.', +); +export const starMessageDef = messageFlagDef( + 'star_message', + 'Star explicit messages. The updated receipt excludes messages already starred.', +); +export const unstarMessageDef = messageFlagDef( + 'unstar_message', + 'Unstar explicit messages. The updated receipt excludes messages already unstarred.', +); + +function singleMessageDef(name, description, extraProperties = {}, extraRequired = []) { + return { + name, + description, + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_id', ...extraRequired], + properties: { + message_id: { type: 'string' }, + ...extraProperties, + }, + }, + }; +} + +export const markSpamDef = singleMessageDef( + 'mark_spam', + 'Mark one explicit message as spam and move it to the configured spam folder.', +); +export const markNotSpamDef = singleMessageDef( + 'mark_not_spam', + 'Mark one explicit spam-folder message as not spam and move it to Inbox.', +); +export const snoozeMessageDef = singleMessageDef( + 'snooze_message', + 'Snooze one message and its reply-chain siblings to the Snoozed folder for up to 30 days.', + { until: { type: 'string', format: 'date-time' } }, + ['until'], +); +export const unsnoozeMessageDef = singleMessageDef( + 'unsnooze_message', + 'Restore one snoozed message and its reply-chain siblings to their original folder.', + { mark_unread: { type: 'boolean', default: false } }, +); +const CATEGORIES = [ + 'primary', + 'newsletter', + 'promotion', + 'automated', + 'social', +]; +export const setCategoryDef = singleMessageDef( + 'set_category', + 'Set the category of one explicit message.', + { category: { type: 'string', enum: CATEGORIES } }, + ['category'], +); +export const gtdClassifyDef = singleMessageDef( + 'gtd_classify', + 'Apply or remove one GTD state label from an explicit message.', + { + state: { type: 'string', enum: GTD_STATES }, + remove: { type: 'boolean', default: false }, + }, + ['state'], +); +export const gtdDoneDef = singleMessageDef( + 'gtd_done', + 'Remove GTD state labels and archive the Inbox copy of one explicit message.', + { + states: { + type: 'array', + items: { type: 'string', enum: GTD_STATES }, + minItems: 1, + }, + }, +); + +function folderWire(row) { + return { + path: row.path, + name: row.name, + delimiter: row.delimiter, + special_use: row.special_use, + total_count: Number(row.total_count || 0), + unread_count: Number(row.unread_count || 0), + message_count: Number(row.total_count || 0), + }; +} + +async function scopedAccountIds(accountId, scope) { + if (!accountId) return scope.accountIds; + const account = await getAccountRow(accountId, scope.accountIds); + return account ? [account.id] : null; +} + +async function requireAccount(accountId, scope) { + if (!accountId || typeof accountId !== 'string') { + return { error: 'account parameter is required' }; + } + const account = await getAccountRow(accountId, scope.accountIds); + if (!account) return { error: `account not found: ${accountId}` }; + return { account }; +} + +function serviceError(result) { + return errorResult(result.error || 'Mailbox operation failed'); +} + +export async function handleListFolders(args, scope, deps) { + const accountIds = await scopedAccountIds(args.account, scope); + if (!accountIds) return errorResult(`account not found: ${args.account}`); + + const results = await Promise.all(accountIds.map(accountId => listFolders(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId, + }))); + const failed = results.find(result => !result.ok); + if (failed) return errorResult(failed.error); + return jsonResult({ folders: results.flatMap(result => result.folders.map(folderWire)) }); +} + +export async function handleCreateFolder(args, scope, deps) { + if (!isValidFolderName(args.name?.trim())) return errorResult('Invalid folder name'); + if (args.parent_path !== undefined && !isValidFolderName(args.parent_path)) { + return errorResult('Invalid parent folder path'); + } + const scoped = await requireAccount(args.account, scope); + if (scoped.error) return errorResult(scoped.error); + const result = await createFolder(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: scoped.account.id, + name: args.name.trim(), + parentPath: args.parent_path, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, path: result.path }); +} + +export async function handleRenameFolder(args, scope, deps) { + if (!isValidFolderName(args.path)) return errorResult('Invalid folder path'); + if (!isValidFolderName(args.new_name?.trim())) return errorResult('Invalid folder name'); + const scoped = await requireAccount(args.account, scope); + if (scoped.error) return errorResult(scoped.error); + const result = await renameFolder(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: scoped.account.id, + oldPath: args.path, + newName: args.new_name.trim(), + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, old_path: args.path, new_path: result.newPath }); +} + +export async function handleDeleteFolder(args, scope, deps) { + if (!isValidFolderName(args.path)) return errorResult('Invalid folder path'); + if (!Number.isInteger(args.expected_message_count) || args.expected_message_count < 0) { + return errorResult('expected_message_count must be a non-negative integer'); + } + const scoped = await requireAccount(args.account, scope); + if (scoped.error) return errorResult(scoped.error); + + const listed = await listFolders(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: scoped.account.id, + }); + if (!listed.ok) return serviceError(listed); + if (!listed.folders.some(folder => folder.path === args.path)) { + return errorResult(`folder not found: ${args.path}`); + } + + const count = await countMessagesIn(scoped.account.id, args.path); + if (args.expected_message_count !== count) { + return errorResult( + `folder "${args.path}" holds ${count} messages, not ${args.expected_message_count}; ` + + 're-check with list_folders and pass the current count to confirm', + ); + } + + const result = await deleteFolder(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: scoped.account.id, + path: args.path, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, deleted: args.path, message_count: count }); +} + +export async function handleMoveMessages(args, scope, deps) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + if (!isValidFolderName(args.folder)) return errorResult('Invalid destination folder'); + + const result = await bulkMoveToFolder(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: ids.value, + folder: args.folder, + }); + if (!result.ok) return serviceError(result); + + const movedDetails = result.movedDetails || []; + const failed = result.failed || []; + const skippedAccounts = result.skippedAccounts || []; + if (movedDetails.length === 0 && failed.length === 0 && skippedAccounts.length > 0) { + const accountIds = skippedAccounts.map(item => item.account_id).join(', '); + return errorResult( + `destination folder not found for account ${accountIds}: ${args.folder}`, + ); + } + + const resolvedByAccount = new Map(); + for (const detail of movedDetails) { + if (detail.uid == null) continue; + if (!resolvedByAccount.has(detail.accountId)) resolvedByAccount.set(detail.accountId, []); + resolvedByAccount.get(detail.accountId).push(detail.uid); + } + const newIds = new Map(); + await Promise.all([...resolvedByAccount].map(async ([accountId, uids]) => { + const rows = await resolveMovedIds(accountId, args.folder, uids); + for (const row of rows) newIds.set(`${accountId}:${String(row.uid)}`, row.id); + })); + + const moved = movedDetails.map(detail => ({ + id: detail.id, + new_id: detail.uid == null + ? null + : (newIds.get(`${detail.accountId}:${String(detail.uid)}`) || null), + uid: detail.uid, + folder: args.folder, + })); + const resyncPending = movedDetails.some((detail, index) => ( + detail.uid == null || moved[index].new_id == null + )); + return jsonResult({ + ok: true, + moved, + failed, + skipped_accounts: skippedAccounts, + resync_pending: resyncPending, + note: 'message ids change on move; use new_id for follow-up calls', + }); +} + +async function resolveDestinationIds(details) { + const grouped = new Map(); + for (const detail of details) { + if (detail.uid == null || detail.destinationUntracked) continue; + const key = `${detail.accountId}:${detail.folder}`; + if (!grouped.has(key)) { + grouped.set(key, { + accountId: detail.accountId, + folder: detail.folder, + uids: [], + }); + } + grouped.get(key).uids.push(detail.uid); + } + + const resolved = new Map(); + await Promise.all([...grouped.values()].map(async ({ accountId, folder, uids }) => { + const rows = await resolveMovedIds(accountId, folder, uids); + for (const row of rows) { + resolved.set(`${accountId}:${folder}:${String(row.uid)}`, row.id); + } + })); + return resolved; +} + +export async function handleArchiveMessages(args, scope, deps) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + + const result = await bulkArchive(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: ids.value, + }); + if (!result.ok) return serviceError(result); + + const details = result.archivedDetails || []; + const resolved = await resolveDestinationIds(details); + const archived = details.map(detail => ({ + id: detail.id, + new_id: detail.destinationUntracked || detail.uid == null + ? null + : (resolved.get( + `${detail.accountId}:${detail.folder}:${String(detail.uid)}`, + ) || null), + uid: detail.uid, + folder: detail.folder, + destination_untracked: Boolean(detail.destinationUntracked), + })); + const resyncPending = details.some((detail, index) => ( + !detail.destinationUntracked + && (detail.uid == null || archived[index].new_id == null) + )); + + return jsonResult({ + ok: true, + archived, + failed: result.failed || [], + no_archive_folder: result.noArchiveFolder || [], + resync_pending: resyncPending, + note: 'message ids change on archive; use new_id for follow-up calls', + }); +} + +export async function handleTrashMessages(args, scope, deps) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + + const result = await bulkTrash(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: ids.value, + allowPermanent: false, + }); + if (!result.ok) return serviceError(result); + + const details = result.trashedDetails || []; + const resolved = await resolveDestinationIds(details); + const trashed = details.map(detail => ({ + id: detail.id, + new_id: detail.uid == null + ? null + : (resolved.get( + `${detail.accountId}:${detail.folder}:${String(detail.uid)}`, + ) || null), + folder: detail.folder, + })); + const resyncPending = details.some((detail, index) => ( + detail.uid == null || trashed[index].new_id == null + )); + + return jsonResult({ + ok: true, + trashed, + failed: result.failed || [], + refused: result.refused || [], + resync_pending: resyncPending, + next_step: 'use stage_deletion for permanent removal', + }); +} + +function createFlagHandler({ kind, value }) { + return async function handleFlag(args, scope, deps) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + + if (kind === 'read') { + const result = await bulkSetRead(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: ids.value, + read: value, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, updated: result.updated || [] }); + } + + const results = await runInBatches(ids.value, 3, id => setStarred( + deps.imapManager, + { + userId: scope.userId, + accountIds: scope.accountIds, + id, + starred: value, + }, + )); + const updated = []; + results.forEach((result, index) => { + if ( + result.status === 'fulfilled' + && result.value.ok + && result.value.updated + ) { + updated.push(ids.value[index]); + } + }); + return jsonResult({ ok: true, updated }); + }; +} + +export const handleMarkRead = createFlagHandler({ kind: 'read', value: true }); +export const handleMarkUnread = createFlagHandler({ kind: 'read', value: false }); +export const handleStarMessage = createFlagHandler({ kind: 'starred', value: true }); +export const handleUnstarMessage = createFlagHandler({ kind: 'starred', value: false }); + +function createSpamHandler(service) { + return async function handleSpamLabel(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + + const result = await service(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ + ok: true, + folder: result.body.folder, + new_uid: result.body.newUid ?? null, + already_in_folder: Boolean(result.body.alreadyInFolder), + }); + }; +} + +export const handleMarkSpam = createSpamHandler(markSpam); +export const handleMarkNotSpam = createSpamHandler(markNotSpam); + +export async function handleSnoozeMessage(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + if (!args.until) return errorResult('until is required'); + + const until = new Date(args.until); + if (Number.isNaN(until.getTime())) { + return errorResult('until must be a valid ISO date'); + } + const now = Date.now(); + if (until.getTime() <= now) return errorResult('until must be in the future'); + if (until.getTime() > now + 30 * 86_400_000) { + return errorResult('until must be within 30 days'); + } + + const result = await snoozeConversation(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + until, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ + ok: true, + moved_count: result.movedCount, + sibling_ids: (result.movedIds || []).filter(movedId => movedId !== id.value), + folder: result.folder, + }); +} + +export async function handleUnsnoozeMessage(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + if ( + args.mark_unread !== undefined + && typeof args.mark_unread !== 'boolean' + ) { + return errorResult('mark_unread must be a boolean'); + } + + const result = await unsnoozeConversation(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + markUnread: args.mark_unread ?? false, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ + ok: true, + restored: result.restored, + folder: result.folder, + }); +} + +export async function handleSetCategory(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + if (!CATEGORIES.includes(args.category)) return errorResult('Invalid category'); + + const result = await setCategory(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + category: args.category, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ ok: true, category: result.category }); +} + +export async function handleGtdClassify(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + if (!GTD_STATES.includes(args.state)) { + return errorResult(`Unknown GTD state: ${args.state}`); + } + if (args.remove !== undefined && typeof args.remove !== 'boolean') { + return errorResult('remove must be a boolean'); + } + + const remove = args.remove ?? false; + const service = remove ? gtdUnclassify : gtdClassify; + const result = await service(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + messageId: id.value, + state: args.state, + }); + if (!result.ok) return serviceError(result); + + const receipt = { + ok: true, + state: args.state, + folder: result.folder ?? null, + }; + if (remove) receipt.removed = Boolean(result.removed); + return jsonResult(receipt); +} + +export async function handleGtdDone(args, scope, deps) { + const id = messageIdArg(args); + if (id.error) return errorResult(id.error); + + let states = 'all'; + if (args.states !== undefined) { + if (!Array.isArray(args.states) || args.states.length === 0) { + return errorResult('states must be a non-empty array'); + } + const unknown = args.states.find(state => !GTD_STATES.includes(state)); + if (unknown) return errorResult(`Unknown GTD state: ${unknown}`); + states = args.states; + } + + const result = await gtdDone(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: id.value, + states, + }); + if (!result.ok) return serviceError(result); + return jsonResult({ + ok: true, + removed: result.removed, + archived: result.archived, + no_archive_folder: result.noArchiveFolder, + archive_failed: result.archiveFailed, + }); +} diff --git a/backend/src/mcp/mailboxTools.test.js b/backend/src/mcp/mailboxTools.test.js new file mode 100644 index 00000000..6d823d54 --- /dev/null +++ b/backend/src/mcp/mailboxTools.test.js @@ -0,0 +1,1146 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./accountAdapter.js', async (orig) => { + const actual = await orig(); + return { ...actual, getAccountRow: vi.fn() }; +}); +vi.mock('../services/mailbox/folders.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + listFolders: vi.fn(), + createFolder: vi.fn(), + renameFolder: vi.fn(), + deleteFolder: vi.fn(), + countMessagesIn: vi.fn(), + }; +}); +vi.mock('../services/mailbox/move.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + bulkMoveToFolder: vi.fn(), + resolveMovedIds: vi.fn(), + }; +}); +vi.mock('../services/mailbox/archive.js', async (orig) => { + const actual = await orig(); + return { ...actual, bulkArchive: vi.fn() }; +}); +vi.mock('../services/mailbox/trash.js', async (orig) => { + const actual = await orig(); + return { ...actual, bulkTrash: vi.fn() }; +}); +vi.mock('../services/mailbox/flags.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + bulkSetRead: vi.fn(), + setStarred: vi.fn(), + }; +}); +vi.mock('../services/mailbox/batch.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + runInBatches: vi.fn(actual.runInBatches), + }; +}); +vi.mock('../services/mailbox/spamLabel.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + markSpam: vi.fn(), + markNotSpam: vi.fn(), + }; +}); +vi.mock('../services/mailbox/snooze.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + snoozeConversation: vi.fn(), + unsnoozeConversation: vi.fn(), + }; +}); +vi.mock('../services/mailbox/category.js', async (orig) => { + const actual = await orig(); + return { ...actual, setCategory: vi.fn() }; +}); +vi.mock('../services/gtd/actions.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + gtdClassify: vi.fn(), + gtdUnclassify: vi.fn(), + gtdDone: vi.fn(), + }; +}); + +import { getAccountRow } from './accountAdapter.js'; +import { + countMessagesIn, + createFolder, + deleteFolder, + listFolders, + renameFolder, +} from '../services/mailbox/folders.js'; +import { + bulkMoveToFolder, + resolveMovedIds, +} from '../services/mailbox/move.js'; +import { bulkArchive } from '../services/mailbox/archive.js'; +import { bulkTrash } from '../services/mailbox/trash.js'; +import { bulkSetRead, setStarred } from '../services/mailbox/flags.js'; +import { runInBatches } from '../services/mailbox/batch.js'; +import { markNotSpam, markSpam } from '../services/mailbox/spamLabel.js'; +import { + snoozeConversation, + unsnoozeConversation, +} from '../services/mailbox/snooze.js'; +import { setCategory } from '../services/mailbox/category.js'; +import { + gtdClassify, + gtdDone, + gtdUnclassify, +} from '../services/gtd/actions.js'; +import * as mailboxTools from './mailboxTools.js'; +import { handleListFolders } from './mailboxTools.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; + +const ACCOUNT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const scope = { userId: 'user-1', accountIds: [ACCOUNT_ID] }; +const deps = { imapManager: { marker: 'imap' } }; + +beforeEach(() => { + vi.clearAllMocks(); + getAccountRow.mockResolvedValue({ id: ACCOUNT_ID }); +}); + +describe('mailbox tool definitions and registration', () => { + it('registers list_folders as a read tool', () => { + expect(TOOL_DEFS.map(def => def.name)).toContain('list_folders'); + expect(TOOL_SCOPES.list_folders).toBe('read'); + expect(HANDLERS.list_folders).toBeTypeOf('function'); + }); + + it('registers all folder mutations with write scope', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const [name, required] of [ + ['create_folder', ['account', 'name']], + ['rename_folder', ['account', 'path', 'new_name']], + ['delete_folder', ['account', 'path', 'expected_message_count']], + ]) { + expect(defs.get(name)?.inputSchema.required).toEqual(required); + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('registers move_messages with explicit bulk ids and destination folder', () => { + const def = TOOL_DEFS.find(entry => entry.name === 'move_messages'); + expect(def.inputSchema.required).toEqual(['message_ids', 'folder']); + expect(def.inputSchema.properties.message_ids).toEqual({ + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, + }); + expect(def.inputSchema.properties).not.toHaveProperty('label'); + expect(TOOL_SCOPES.move_messages).toBe('write'); + expect(HANDLERS.move_messages).toBeTypeOf('function'); + }); + + it('registers archive_messages and trash_messages without a delete_messages alias', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const name of ['archive_messages', 'trash_messages']) { + expect(defs.get(name)?.inputSchema.required).toEqual(['message_ids']); + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + expect(defs.has('delete_messages')).toBe(false); + expect(HANDLERS).not.toHaveProperty('delete_messages'); + }); + + it('registers all four message flag tools with explicit bulk ids and write scope', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const name of [ + 'mark_read', + 'mark_unread', + 'star_message', + 'unstar_message', + ]) { + expect(defs.get(name)?.inputSchema.required).toEqual(['message_ids']); + expect(defs.get(name)?.inputSchema.properties.message_ids).toEqual({ + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, + }); + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('registers mark_spam and mark_not_spam with one explicit message id', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const name of ['mark_spam', 'mark_not_spam']) { + expect(defs.get(name)?.inputSchema.required).toEqual(['message_id']); + expect(defs.get(name)?.inputSchema.properties.message_id).toEqual({ + type: 'string', + }); + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('registers snooze_message and unsnooze_message with their explicit arguments', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + expect(defs.get('snooze_message')?.inputSchema.required).toEqual([ + 'message_id', + 'until', + ]); + expect(defs.get('unsnooze_message')?.inputSchema.required).toEqual([ + 'message_id', + ]); + expect( + defs.get('unsnooze_message')?.inputSchema.properties.mark_unread, + ).toEqual({ type: 'boolean', default: false }); + for (const name of ['snooze_message', 'unsnooze_message']) { + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); + + it('registers set_category, gtd_classify, and gtd_done with write scope', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + expect(defs.get('set_category')?.inputSchema.required).toEqual([ + 'message_id', + 'category', + ]); + expect(defs.get('gtd_classify')?.inputSchema.required).toEqual([ + 'message_id', + 'state', + ]); + expect(defs.get('gtd_done')?.inputSchema.required).toEqual(['message_id']); + for (const name of ['set_category', 'gtd_classify', 'gtd_done']) { + expect(TOOL_SCOPES[name]).toBe('write'); + expect(HANDLERS[name]).toBeTypeOf('function'); + } + }); +}); + +describe('mailbox tool metadata completeness', () => { + const expected = { + list_folders: [true, false, true, false], + create_folder: [false, false, false, false], + rename_folder: [false, true, false, false], + delete_folder: [false, true, true, false], + move_messages: [false, true, false, false], + archive_messages: [false, false, true, false], + trash_messages: [false, true, true, false], + mark_read: [false, false, true, false], + mark_unread: [false, false, true, false], + star_message: [false, false, true, false], + unstar_message: [false, false, true, false], + mark_spam: [false, false, true, false], + mark_not_spam: [false, false, true, false], + snooze_message: [false, false, true, false], + unsnooze_message: [false, false, true, false], + set_category: [false, false, true, false], + gtd_classify: [false, false, true, false], + gtd_done: [false, false, true, false], + }; + + it('pins all 18 mailbox annotation rows from the protocol table', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + expect(Object.keys(expected)).toHaveLength(18); + for (const [name, [ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint, + ]] of Object.entries(expected)) { + expect(defs.get(name)?.annotations).toEqual({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint, + }); + } + }); + + it('gives every mailbox write definition a scope and four boolean hints', () => { + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const name of Object.keys(expected).filter(name => name !== 'list_folders')) { + expect(TOOL_SCOPES[name]).toBe('write'); + const annotations = defs.get(name)?.annotations; + expect(Object.keys(annotations || {}).sort()).toEqual([ + 'destructiveHint', + 'idempotentHint', + 'openWorldHint', + 'readOnlyHint', + ]); + expect(Object.values(annotations).every(value => typeof value === 'boolean')).toBe(true); + } + }); +}); + +describe('shared mailbox validators', () => { + it('enforces explicit 1-500 UUID message_ids arrays', () => { + expect(mailboxTools.messageIdsArg({})).toEqual({ + error: 'message_ids must contain at least one id', + }); + expect(mailboxTools.messageIdsArg({ message_ids: [] })).toEqual({ + error: 'message_ids must contain at least one id', + }); + expect(mailboxTools.messageIdsArg({ + message_ids: Array.from({ length: 501 }, () => ACCOUNT_ID), + })).toEqual({ error: 'Too many ids — maximum 500 per request' }); + expect(mailboxTools.messageIdsArg({ message_ids: ['not-a-uuid'] })).toEqual({ + error: 'Invalid message id format', + }); + expect(mailboxTools.messageIdsArg({ message_ids: [ACCOUNT_ID] })).toEqual({ + value: [ACCOUNT_ID], + }); + }); + + it('requires a single UUID message_id', () => { + expect(mailboxTools.messageIdArg({})).toEqual({ + error: 'message_id parameter is required', + }); + expect(mailboxTools.messageIdArg({ message_id: 'not-a-uuid' })).toEqual({ + error: 'Invalid message id format', + }); + expect(mailboxTools.messageIdArg({ message_id: ACCOUNT_ID })).toEqual({ + value: ACCOUNT_ID, + }); + }); +}); + +describe('list_folders', () => { + it('scope-checks the account and projects the curated folder shape', async () => { + listFolders.mockResolvedValue({ + ok: true, + folders: [{ + account_id: ACCOUNT_ID, + path: 'INBOX', + name: 'Inbox', + delimiter: '/', + special_use: '\\Inbox', + total_count: 9, + unread_count: 2, + ignored_column: 'not-on-wire', + }], + }); + + const result = await handleListFolders({ account: ACCOUNT_ID }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + folders: [{ + path: 'INBOX', + name: 'Inbox', + delimiter: '/', + special_use: '\\Inbox', + total_count: 9, + unread_count: 2, + message_count: 9, + }], + }); + expect(getAccountRow).toHaveBeenCalledWith(ACCOUNT_ID, scope.accountIds); + expect(listFolders).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: ACCOUNT_ID, + }); + }); + + it('rejects an out-of-scope account before calling the folder service', async () => { + getAccountRow.mockResolvedValue(null); + + const result = await handleListFolders({ account: 'outside' }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: outside'); + expect(listFolders).not.toHaveBeenCalled(); + }); +}); + +describe('folder mutations', () => { + it('creates a scoped folder and returns the resolved path', async () => { + createFolder.mockResolvedValue({ ok: true, path: 'Projects/Client' }); + + const result = await mailboxTools.handleCreateFolder({ + account: ACCOUNT_ID, + name: 'Client', + parent_path: 'Projects', + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + path: 'Projects/Client', + }); + expect(createFolder).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: ACCOUNT_ID, + name: 'Client', + parentPath: 'Projects', + }); + }); + + it('rejects an invalid folder name before calling the service', async () => { + const result = await mailboxTools.handleCreateFolder({ + account: ACCOUNT_ID, + name: 'bad\u0000name', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('Invalid folder name'); + expect(createFolder).not.toHaveBeenCalled(); + }); + + it('renames the final folder component and names both paths in the receipt', async () => { + renameFolder.mockResolvedValue({ ok: true, newPath: 'Projects/New' }); + + const result = await mailboxTools.handleRenameFolder({ + account: ACCOUNT_ID, + path: 'Projects/Old', + new_name: 'New', + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + old_path: 'Projects/Old', + new_path: 'Projects/New', + }); + expect(renameFolder).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: ACCOUNT_ID, + oldPath: 'Projects/Old', + newName: 'New', + }); + }); + + it('refuses delete_folder when the live count differs from confirmation', async () => { + listFolders.mockResolvedValue({ + ok: true, + folders: [{ path: 'Projects', total_count: 7 }], + }); + countMessagesIn.mockResolvedValue(7); + + const result = await mailboxTools.handleDeleteFolder({ + account: ACCOUNT_ID, + path: 'Projects', + expected_message_count: 3, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'folder "Projects" holds 7 messages, not 3; re-check with list_folders and pass the current count to confirm', + ); + expect(getAccountRow).toHaveBeenCalledWith(ACCOUNT_ID, scope.accountIds); + expect(countMessagesIn).toHaveBeenCalledWith(ACCOUNT_ID, 'Projects'); + expect(deleteFolder).not.toHaveBeenCalled(); + }); + + it('checks folder existence before reading its live count', async () => { + listFolders.mockResolvedValue({ ok: true, folders: [] }); + + const result = await mailboxTools.handleDeleteFolder({ + account: ACCOUNT_ID, + path: 'Missing', + expected_message_count: 0, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('folder not found: Missing'); + expect(countMessagesIn).not.toHaveBeenCalled(); + expect(deleteFolder).not.toHaveBeenCalled(); + }); + + it('deletes only after the scope, existence, and count guards pass', async () => { + listFolders.mockResolvedValue({ + ok: true, + folders: [{ path: 'Projects', total_count: 2 }], + }); + countMessagesIn.mockResolvedValue(2); + deleteFolder.mockResolvedValue({ ok: true }); + + const result = await mailboxTools.handleDeleteFolder({ + account: ACCOUNT_ID, + path: 'Projects', + expected_message_count: 2, + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + deleted: 'Projects', + message_count: 2, + }); + expect(deleteFolder).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + accountId: ACCOUNT_ID, + path: 'Projects', + }); + }); +}); + +describe('move_messages', () => { + const SECOND_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; + const NEW_ID = 'cccccccc-cccc-4ccc-8ccc-cccccccccccc'; + + it('rejects malformed ids before calling the move service', async () => { + const result = await mailboxTools.handleMoveMessages({ + message_ids: ['not-a-uuid'], + folder: 'Projects', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('Invalid message id format'); + expect(bulkMoveToFolder).not.toHaveBeenCalled(); + }); + + it('resolves UIDPLUS destination ids and reports non-UIDPLUS resyncs', async () => { + bulkMoveToFolder.mockResolvedValue({ + ok: true, + moved: [ACCOUNT_ID, SECOND_ID], + movedDetails: [ + { id: ACCOUNT_ID, accountId: ACCOUNT_ID, uid: 110 }, + { id: SECOND_ID, accountId: ACCOUNT_ID, uid: null }, + ], + failed: [], + skippedAccounts: [], + }); + resolveMovedIds.mockResolvedValue([{ id: NEW_ID, uid: 110 }]); + + const result = await mailboxTools.handleMoveMessages({ + message_ids: [ACCOUNT_ID, SECOND_ID], + folder: 'Projects', + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + moved: [ + { id: ACCOUNT_ID, new_id: NEW_ID, uid: 110, folder: 'Projects' }, + { id: SECOND_ID, new_id: null, uid: null, folder: 'Projects' }, + ], + failed: [], + skipped_accounts: [], + resync_pending: true, + note: 'message ids change on move; use new_id for follow-up calls', + }); + expect(bulkMoveToFolder).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: [ACCOUNT_ID, SECOND_ID], + folder: 'Projects', + }); + expect(resolveMovedIds).toHaveBeenCalledWith(ACCOUNT_ID, 'Projects', [110]); + }); + + it('turns a fully skipped destination into an error', async () => { + bulkMoveToFolder.mockResolvedValue({ + ok: true, + moved: [], + movedDetails: [], + failed: [], + skippedAccounts: [{ + account_id: ACCOUNT_ID, + reason: 'folder_not_found', + }], + }); + + const result = await mailboxTools.handleMoveMessages({ + message_ids: [ACCOUNT_ID], + folder: 'Missing', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + `destination folder not found for account ${ACCOUNT_ID}: Missing`, + ); + expect(resolveMovedIds).not.toHaveBeenCalled(); + }); + + it('keeps failed and skipped partitions on a partial success receipt', async () => { + bulkMoveToFolder.mockResolvedValue({ + ok: true, + moved: [ACCOUNT_ID], + movedDetails: [{ id: ACCOUNT_ID, accountId: ACCOUNT_ID, uid: 110 }], + failed: [{ id: SECOND_ID, reason: 'IMAP move failed' }], + skippedAccounts: [{ account_id: 'other-account', reason: 'folder_not_found' }], + }); + resolveMovedIds.mockResolvedValue([{ id: NEW_ID, uid: 110 }]); + + const result = await mailboxTools.handleMoveMessages({ + message_ids: [ACCOUNT_ID, SECOND_ID], + folder: 'Projects', + }, scope, deps); + + const receipt = JSON.parse(result.content[0].text); + expect(result.isError).toBeUndefined(); + expect(receipt.failed).toEqual([{ id: SECOND_ID, reason: 'IMAP move failed' }]); + expect(receipt.skipped_accounts).toEqual([ + { account_id: 'other-account', reason: 'folder_not_found' }, + ]); + }); +}); + +describe('archive_messages', () => { + const NEW_ID = 'cccccccc-cccc-4ccc-8ccc-cccccccccccc'; + const ALL_MAIL_ID = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd'; + + it.each([ + [{}, 'message_ids must contain at least one id'], + [{ message_ids: [] }, 'message_ids must contain at least one id'], + [{ + message_ids: Array.from({ length: 501 }, () => ACCOUNT_ID), + }, 'Too many ids — maximum 500 per request'], + [{ message_ids: ['not-a-uuid'] }, 'Invalid message id format'], + ])('rejects invalid explicit ids before calling the archive service', async (args, error) => { + const result = await mailboxTools.handleArchiveMessages(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(bulkArchive).not.toHaveBeenCalled(); + }); + + it('returns durable ids and marks Gmail All Mail destinations as untracked', async () => { + bulkArchive.mockResolvedValue({ + ok: true, + archived: [ACCOUNT_ID, ALL_MAIL_ID], + archivedDetails: [ + { + id: ACCOUNT_ID, + accountId: ACCOUNT_ID, + folder: 'Archive', + uid: 110, + destinationUntracked: false, + }, + { + id: ALL_MAIL_ID, + accountId: ACCOUNT_ID, + folder: '[Gmail]/All Mail', + uid: 111, + destinationUntracked: true, + }, + ], + failed: [], + noArchiveFolder: ['account-without-archive'], + }); + resolveMovedIds.mockResolvedValue([{ id: NEW_ID, uid: 110 }]); + + const result = await mailboxTools.handleArchiveMessages({ + message_ids: [ACCOUNT_ID, ALL_MAIL_ID], + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + archived: [ + { + id: ACCOUNT_ID, + new_id: NEW_ID, + uid: 110, + folder: 'Archive', + destination_untracked: false, + }, + { + id: ALL_MAIL_ID, + new_id: null, + uid: 111, + folder: '[Gmail]/All Mail', + destination_untracked: true, + }, + ], + failed: [], + no_archive_folder: ['account-without-archive'], + resync_pending: false, + note: 'message ids change on archive; use new_id for follow-up calls', + }); + expect(bulkArchive).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: [ACCOUNT_ID, ALL_MAIL_ID], + }); + expect(resolveMovedIds).toHaveBeenCalledWith(ACCOUNT_ID, 'Archive', [110]); + }); +}); + +describe('trash_messages', () => { + const NEW_ID = 'cccccccc-cccc-4ccc-8ccc-cccccccccccc'; + + it.each([ + [{}, 'message_ids must contain at least one id'], + [{ message_ids: [] }, 'message_ids must contain at least one id'], + [{ + message_ids: Array.from({ length: 501 }, () => ACCOUNT_ID), + }, 'Too many ids — maximum 500 per request'], + [{ message_ids: ['not-a-uuid'] }, 'Invalid message id format'], + ])('rejects invalid explicit ids before calling the trash service', async (args, error) => { + const result = await mailboxTools.handleTrashMessages(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(bulkTrash).not.toHaveBeenCalled(); + }); + + it('never enables permanent deletion and preserves the refusal partition', async () => { + bulkTrash.mockResolvedValue({ + ok: true, + deleted: [ACCOUNT_ID], + trashedDetails: [{ + id: ACCOUNT_ID, + accountId: ACCOUNT_ID, + folder: 'Trash', + uid: 110, + }], + failed: [], + refused: [{ + id: 'dddddddd-dddd-4ddd-8ddd-dddddddddddd', + folder: 'Trash', + reason: 'already_in_trash_permanent_delete_required', + }], + }); + resolveMovedIds.mockResolvedValue([{ id: NEW_ID, uid: 110 }]); + + const result = await mailboxTools.handleTrashMessages({ + message_ids: [ + ACCOUNT_ID, + 'dddddddd-dddd-4ddd-8ddd-dddddddddddd', + ], + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + trashed: [{ + id: ACCOUNT_ID, + new_id: NEW_ID, + folder: 'Trash', + }], + failed: [], + refused: [{ + id: 'dddddddd-dddd-4ddd-8ddd-dddddddddddd', + folder: 'Trash', + reason: 'already_in_trash_permanent_delete_required', + }], + resync_pending: false, + next_step: 'use stage_deletion for permanent removal', + }); + expect(bulkTrash).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: [ + ACCOUNT_ID, + 'dddddddd-dddd-4ddd-8ddd-dddddddddddd', + ], + allowPermanent: false, + }); + }); +}); + +describe('mark_read and mark_unread', () => { + it.each([ + ['handleMarkRead', true], + ['handleMarkUnread', false], + ])('returns only ids changed by %s', async (handlerName, read) => { + bulkSetRead.mockResolvedValue({ ok: true, updated: [ACCOUNT_ID] }); + + const result = await mailboxTools[handlerName]({ + message_ids: [ACCOUNT_ID], + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + updated: [ACCOUNT_ID], + }); + expect(bulkSetRead).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + ids: [ACCOUNT_ID], + read, + }); + }); +}); + +describe('star_message and unstar_message', () => { + const SECOND_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; + + it.each([ + ['handleStarMessage', true], + ['handleUnstarMessage', false], + ])('runs %s through bounded single-message writes and excludes no-ops', async ( + handlerName, + starred, + ) => { + setStarred + .mockResolvedValueOnce({ ok: true, is_starred: starred, updated: true }) + .mockResolvedValueOnce({ ok: true, is_starred: starred, updated: false }); + + const result = await mailboxTools[handlerName]({ + message_ids: [ACCOUNT_ID, SECOND_ID], + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + updated: [ACCOUNT_ID], + }); + expect(runInBatches).toHaveBeenCalledWith( + [ACCOUNT_ID, SECOND_ID], + 3, + expect.any(Function), + ); + expect(setStarred).toHaveBeenNthCalledWith(1, deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + starred, + }); + expect(setStarred).toHaveBeenNthCalledWith(2, deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: SECOND_ID, + starred, + }); + }); +}); + +describe.each([ + ['handleMarkRead', bulkSetRead], + ['handleMarkUnread', bulkSetRead], + ['handleStarMessage', setStarred], + ['handleUnstarMessage', setStarred], +])('%s explicit-id validation', (handlerName, service) => { + it.each([ + [{}, 'message_ids must contain at least one id'], + [{ message_ids: [] }, 'message_ids must contain at least one id'], + [{ + message_ids: Array.from({ length: 501 }, () => ACCOUNT_ID), + }, 'Too many ids — maximum 500 per request'], + [{ message_ids: ['not-a-uuid'] }, 'Invalid message id format'], + ])('rejects invalid ids before any service call', async (args, error) => { + const result = await mailboxTools[handlerName](args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(service).not.toHaveBeenCalled(); + }); +}); + +describe('mark_spam and mark_not_spam', () => { + it.each([ + ['handleMarkSpam', markSpam, 'Junk', 110], + ['handleMarkNotSpam', markNotSpam, 'INBOX', null], + ])('%s forwards scope and projects the destination receipt', async ( + handlerName, + service, + folder, + newUid, + ) => { + service.mockResolvedValue({ + ok: true, + status: 200, + body: { + ok: true, + folder, + newUid, + alreadyInFolder: false, + }, + }); + + const result = await mailboxTools[handlerName]({ + message_id: ACCOUNT_ID, + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + folder, + new_uid: newUid, + already_in_folder: false, + }); + expect(service).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + }); + }); + + it.each([ + ['handleMarkSpam', markSpam], + ['handleMarkNotSpam', markNotSpam], + ])('%s rejects a missing or malformed id before the service call', async ( + handlerName, + service, + ) => { + for (const [args, error] of [ + [{}, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid' }, 'Invalid message id format'], + ]) { + const result = await mailboxTools[handlerName](args, scope, deps); + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + } + expect(service).not.toHaveBeenCalled(); + }); + + it('preserves a spam service refusal as an MCP error', async () => { + markSpam.mockResolvedValue({ + ok: false, + status: 422, + error: 'No spam folder configured for this account', + }); + + const result = await mailboxTools.handleMarkSpam({ + message_id: ACCOUNT_ID, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'No spam folder configured for this account', + ); + }); +}); + +describe('snooze_message', () => { + it('reports the whole-conversation move count and sibling ids', async () => { + const siblingId = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; + snoozeConversation.mockResolvedValue({ + ok: true, + movedCount: 2, + movedIds: [ACCOUNT_ID, siblingId], + folder: 'Snoozed', + }); + const until = new Date(Date.now() + 60_000).toISOString(); + + const result = await mailboxTools.handleSnoozeMessage({ + message_id: ACCOUNT_ID, + until, + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + moved_count: 2, + sibling_ids: [siblingId], + folder: 'Snoozed', + }); + expect(snoozeConversation).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + until: new Date(until), + }); + }); + + it.each([ + [{ until: new Date(Date.now() + 60_000).toISOString() }, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid', until: new Date(Date.now() + 60_000).toISOString() }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID }, 'until is required'], + [{ message_id: ACCOUNT_ID, until: 'not-a-date' }, 'until must be a valid ISO date'], + [{ message_id: ACCOUNT_ID, until: new Date(Date.now() - 60_000).toISOString() }, 'until must be in the future'], + [{ message_id: ACCOUNT_ID, until: new Date(Date.now() + 31 * 86_400_000).toISOString() }, 'until must be within 30 days'], + ])('rejects invalid arguments before calling the snooze service', async (args, error) => { + const result = await mailboxTools.handleSnoozeMessage(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(snoozeConversation).not.toHaveBeenCalled(); + }); +}); + +describe('unsnooze_message', () => { + it.each([ + [undefined, false], + [true, true], + ])('restores the conversation with mark_unread=%s', async ( + markUnreadArg, + markUnread, + ) => { + unsnoozeConversation.mockResolvedValue({ + ok: true, + restored: 2, + folder: 'INBOX', + }); + const args = { message_id: ACCOUNT_ID }; + if (markUnreadArg !== undefined) args.mark_unread = markUnreadArg; + + const result = await mailboxTools.handleUnsnoozeMessage(args, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + restored: 2, + folder: 'INBOX', + }); + expect(unsnoozeConversation).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + markUnread, + }); + }); + + it.each([ + [{}, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid' }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID, mark_unread: 'yes' }, 'mark_unread must be a boolean'], + ])('rejects invalid arguments before calling the unsnooze service', async (args, error) => { + const result = await mailboxTools.handleUnsnoozeMessage(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(unsnoozeConversation).not.toHaveBeenCalled(); + }); +}); + +describe('set_category', () => { + it('sets one validated category in the enabled-account scope', async () => { + setCategory.mockResolvedValue({ ok: true, category: 'newsletter' }); + + const result = await mailboxTools.handleSetCategory({ + message_id: ACCOUNT_ID, + category: 'newsletter', + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + category: 'newsletter', + }); + expect(setCategory).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + category: 'newsletter', + }); + }); + + it.each([ + [{ category: 'primary' }, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid', category: 'primary' }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID }, 'Invalid category'], + [{ message_id: ACCOUNT_ID, category: 'other' }, 'Invalid category'], + ])('rejects invalid arguments before the category service', async (args, error) => { + const result = await mailboxTools.handleSetCategory(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(setCategory).not.toHaveBeenCalled(); + }); +}); + +describe('gtd_classify', () => { + it.each([ + [false, gtdClassify, { ok: true, folder: 'Todo' }, { + ok: true, + state: 'todo', + folder: 'Todo', + }], + [true, gtdUnclassify, { + ok: true, + removed: true, + folder: 'Todo', + }, { + ok: true, + state: 'todo', + folder: 'Todo', + removed: true, + }], + ])('dispatches remove=%s to the correct scoped GTD action', async ( + remove, + service, + serviceResult, + receipt, + ) => { + service.mockResolvedValue(serviceResult); + + const result = await mailboxTools.handleGtdClassify({ + message_id: ACCOUNT_ID, + state: 'todo', + remove, + }, scope, deps); + + expect(JSON.parse(result.content[0].text)).toEqual(receipt); + expect(service).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + messageId: ACCOUNT_ID, + state: 'todo', + }); + }); + + it.each([ + [{ state: 'todo' }, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid', state: 'todo' }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID }, 'Unknown GTD state: undefined'], + [{ message_id: ACCOUNT_ID, state: 'inbox' }, 'Unknown GTD state: inbox'], + [{ message_id: ACCOUNT_ID, state: 'todo', remove: 'yes' }, 'remove must be a boolean'], + ])('rejects invalid arguments before either GTD classify service', async (args, error) => { + const result = await mailboxTools.handleGtdClassify(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(gtdClassify).not.toHaveBeenCalled(); + expect(gtdUnclassify).not.toHaveBeenCalled(); + }); +}); + +describe('gtd_done', () => { + it('preserves archive failure as a non-error partial-success receipt', async () => { + gtdDone.mockResolvedValue({ + ok: true, + removed: ['Watch'], + archived: false, + noArchiveFolder: false, + archiveFailed: true, + }); + + const result = await mailboxTools.handleGtdDone({ + message_id: ACCOUNT_ID, + states: ['watch'], + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(JSON.parse(result.content[0].text)).toEqual({ + ok: true, + removed: ['Watch'], + archived: false, + no_archive_folder: false, + archive_failed: true, + }); + expect(gtdDone).toHaveBeenCalledWith(deps.imapManager, { + userId: scope.userId, + accountIds: scope.accountIds, + id: ACCOUNT_ID, + states: ['watch'], + }); + }); + + it('defaults omitted states to all', async () => { + gtdDone.mockResolvedValue({ + ok: true, + removed: [], + archived: false, + noArchiveFolder: true, + archiveFailed: false, + }); + + await mailboxTools.handleGtdDone({ message_id: ACCOUNT_ID }, scope, deps); + + expect(gtdDone).toHaveBeenCalledWith( + deps.imapManager, + expect.objectContaining({ states: 'all' }), + ); + }); + + it.each([ + [{}, 'message_id parameter is required'], + [{ message_id: 'not-a-uuid' }, 'Invalid message id format'], + [{ message_id: ACCOUNT_ID, states: [] }, 'states must be a non-empty array'], + [{ message_id: ACCOUNT_ID, states: ['inbox'] }, 'Unknown GTD state: inbox'], + ])('rejects invalid arguments before gtd_done', async (args, error) => { + const result = await mailboxTools.handleGtdDone(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(error); + expect(gtdDone).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/mcp/messageTools.js b/backend/src/mcp/messageTools.js new file mode 100644 index 00000000..96cc186f --- /dev/null +++ b/backend/src/mcp/messageTools.js @@ -0,0 +1,548 @@ +// The msgvault message/aggregate/deletion tool handlers. Every read goes through +// engineAdapter (all SQL lives there); handlers only shape and page. Byte offsets +// throughout are UTF-8 bytes into the raw body (msgvault wire contract). +import { + getMessage, rowToMessageDetail, listMessages, listAccounts, getTotalStats, aggregate, searchByDomains, + getMessageSummariesByIDs, stageDeletion, resolveAccountScope, messageInScope, countEnforceableQueryPredicates, +} from './engineAdapter.js'; +import { parseQuery } from '../services/search/queryParser.js'; +import { bodyByteSliceRange, contextWindow, findTermMatches } from './bodyMatch.js'; +import { jsonResult, errorResult } from './result.js'; +import { newPaginatedResponse, newPaginatedResponseNoTotal, toRFC3339, wireSummary } from './envelope.js'; +import { + searchLimitArg, offsetArg, + queryParseErrorMessage, unsupportedSearchOperatorMessage, HYBRID_RANKING_WINDOW, +} from './searchTools.js'; +import { collectStats } from './vectorStats.js'; +import { translateVectorError } from './vectorErrors.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; // phase-4 public face +import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; // phase 3 +import { matchesInMessage } from '../services/embeddings/chunkmatch.js'; // phase-5-owned + +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); +const NON_IDEMPOTENT_WRITE_ANNOTATIONS = annotations(); + +const DEFAULT_BODY_CHARS = 2000; +const MAX_BODY_CHARS = 4000; +const MAX_LIMIT = 1000; + +// msgvault limitArg: absent/non-number → default; negative/NaN → 0; clamp to 1000. +// (Distinct from searchLimitArg's 20/50 search clamp.) +function limitArg(args, key, def) { + const raw = args[key]; + if (typeof raw !== 'number') return def; + if (Number.isNaN(raw) || raw < 0) return 0; + if (raw > MAX_LIMIT) return MAX_LIMIT; + return Math.trunc(raw); +} + +// msgvault getDateArg (handlers.go:264-275): an optional after/before arg is a +// strict YYYY-MM-DD; anything else errors at the handler instead of leaking a +// raw Postgres cast error to the wire. Non-strings/empty are "no filter", and +// JS Date rollover (2024-02-31 → Mar 1) is rejected via the Y/M/D round-trip +// (Go time.Parse errors "day out of range"). Returns { value } or { error }. +function dateArg(args, key) { + const v = args[key]; + if (typeof v !== 'string' || v === '') return { value: undefined }; + const err = { error: `invalid ${key} date "${v}": expected YYYY-MM-DD` }; + const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(v); + if (!m) return err; + const [y, mo, d] = [Number(m[1]), Number(m[2]), Number(m[3])]; + const t = new Date(Date.UTC(y, mo - 1, d)); + if (t.getUTCFullYear() !== y || t.getUTCMonth() !== mo - 1 || t.getUTCDate() !== d) return err; + return { value: v }; +} + +export const getMessageDef = { + name: 'get_message', + description: + 'Get message details including recipients, labels, attachments, and a slice of the message body. ' + + 'Returns plain text when available; HTML-only messages return a body_html slice with body_format=html. ' + + 'Body paging mirrors search pagination: body_length=total bytes, offset=where this chunk starts, body_returned=bytes in this chunk, has_more=more body follows. ' + + 'To read sequentially: call again with offset += body_returned. ' + + 'To jump to a known match location: use center_at= to center the window on that location. ' + + 'Note: snippet is pre-stored source metadata (may be empty for non-Gmail sources).', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + id: { type: 'string', description: 'Message ID' }, + offset: { type: 'number', description: 'Byte offset from the start of the selected body to begin reading (default 0). Ignored when center_at is provided.' }, + center_at: { type: 'number', description: 'Byte offset from the start of the selected body to center the window on. Takes precedence over offset.' }, + max_chars: { type: 'number', description: 'Maximum selected-body bytes to return (default 2000, max 4000). Values above 4000 are clamped to 4000; zero or negative values use the default.' }, + body_format: { type: 'string', enum: ['auto', 'text', 'html'], description: 'Which body representation to page: auto (default, plain text when available, HTML fallback), text, or html.' }, + full_body: { type: 'boolean', description: 'Return the complete selected body in one response, ignoring offset, center_at, and max_chars. Use only when the full content is explicitly needed.' }, + }, + required: ['id'], + }, +}; + +export async function handleGetMessage(args, scope) { + const id = args.id; + if (!id || typeof id !== 'string') return errorResult('id parameter is required'); + const row = await getMessage(id, scope.accountIds); + if (!row) return errorResult('message not found'); + const detail = rowToMessageDetail(row); + + let maxChars = Number(args.max_chars); + if (!Number.isFinite(maxChars) || maxChars <= 0) maxChars = DEFAULT_BODY_CHARS; + else if (maxChars > MAX_BODY_CHARS) maxChars = MAX_BODY_CHARS; + + const requested = args.body_format || 'auto'; + let full = detail.body_text; + let bodyFormat = 'text'; + if (requested === 'auto') { + if (!full && detail.body_html) { full = detail.body_html; bodyFormat = 'html'; } + } else if (requested === 'html') { + full = detail.body_html; bodyFormat = 'html'; + } else if (requested !== 'text') { + return errorResult('body_format must be one of auto, text, html'); + } + + const buf = Buffer.from(full || '', 'utf8'); + const bodyLen = buf.length; + + let start, end; + if (args.full_body === true) { + start = 0; end = bodyLen; + } else if (Number.isFinite(Number(args.center_at)) && Number(args.center_at) >= 0) { + [start, end] = contextWindow(bodyLen, Math.trunc(Number(args.center_at)), 0, maxChars); + } else { + start = Math.min(Number.isFinite(Number(args.offset)) ? Math.trunc(Number(args.offset)) : 0, bodyLen); + end = Math.min(start + maxChars, bodyLen); + } + + const { text: slice, adjStart, adjEnd } = bodyByteSliceRange(buf, start, end); + const returnedBytes = Buffer.byteLength(slice, 'utf8'); + + return jsonResult({ + id: detail.id, + source_message_id: detail.source_message_id, + conversation_id: detail.conversation_id, + source_conversation_id: detail.source_conversation_id, + subject: detail.subject, + message_type: detail.message_type, + snippet: detail.snippet, + sent_at: toRFC3339(detail.sent_at), + size_estimate: detail.size_estimate, + has_attachments: detail.has_attachments, + from: detail.from, + to: detail.to, + cc: detail.cc, + bcc: detail.bcc, + body_text: bodyFormat === 'html' ? '' : slice, + body_html: bodyFormat === 'html' ? slice : '', + body_format: bodyFormat, + body_length: bodyLen, + body_returned: returnedBytes, + offset: adjStart, + has_more: adjEnd < bodyLen, + labels: detail.labels, + attachments: detail.attachments, + }); +} + +export const listMessagesDef = { + name: 'list_messages', + description: + 'List messages with optional filters, newest-first. ' + + "Pass conversation_id to enumerate a thread's messages, then call get_message(id) per message to read bodies — " + + 'there is deliberately no bulk body fetch, to avoid loading huge threads into the context window. ' + + 'Paginate with offset/limit (default limit 20, max 50). Response: data, total, returned, offset, has_more. ' + + 'total=-1 because the full count is not computed; use has_more for paging.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + from: { type: 'string', description: 'Filter by sender email address' }, + to: { type: 'string', description: 'Filter by recipient email address' }, + label: { type: 'string', description: 'Filter by Gmail label' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + has_attachment: { type: 'boolean', description: 'Only messages with attachments' }, + conversation_id: { type: 'string', description: 'Filter by conversation/thread ID' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + }, + }, +}; + +export async function handleListMessages(args, scope) { + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const limit = searchLimitArg(args); + const offset = offsetArg(args); + // Over-fetch one to compute has_more without a count query (msgvault limit+1). + let rows = await listMessages({ + accountIds: acc.accountIds, + from: args.from, to: args.to, label: args.label, + hasAttachment: args.has_attachment === true, + after: after.value, before: before.value, + conversationId: args.conversation_id, + limit: limit + 1, offset, + }); + const hasMore = rows.length > limit; + if (hasMore) rows = rows.slice(0, limit); + return jsonResult(newPaginatedResponseNoTotal(rows.map(wireSummary), offset, hasMore)); +} + +export const getStatsDef = { + name: 'get_stats', + description: 'Get archive overview: total messages, size, attachment count, and accounts.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { type: 'object', properties: {} }, +}; + +// A broken vector sub-query must never blank the whole stats response (msgvault +// best-effort semantics); on any failure the vector_search field is simply omitted. +async function safeCollectStats(accountIds) { + try { return await collectStats(accountIds); } catch { return null; } +} + +export async function handleGetStats(_args, scope) { + const stats = await getTotalStats(scope.accountIds); + const accounts = await listAccounts(scope.accountIds); + const resp = { stats, accounts }; + const vs = await safeCollectStats(scope.accountIds); + if (vs) resp.vector_search = vs; // omitempty: present only when vector search is enabled + return jsonResult(resp); +} + +const AGGREGATE_GROUP_BY = ['sender', 'recipient', 'domain', 'label', 'time']; + +export const aggregateDef = { + name: 'aggregate', + description: + 'Get grouped statistics (top senders, recipients, domains, labels, or message volume by calendar year). ' + + 'Returns a JSON array of objects with fields Key, Count, TotalSize, AttachmentSize, AttachmentCount, and TotalUnique.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + group_by: { type: 'string', enum: ['sender', 'recipient', 'domain', 'label', 'time'], description: 'Dimension to group by. When \'time\', buckets are by calendar year only (Key is a year string like "2024").' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 50)' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + }, + required: ['group_by'], + }, +}; + +export async function handleAggregate(args, scope) { + const groupBy = args.group_by || ''; + if (!groupBy) return errorResult('group_by parameter is required'); + if (!AGGREGATE_GROUP_BY.includes(groupBy)) return errorResult('invalid group_by: ' + groupBy); + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const rows = await aggregate(groupBy, { + accountIds: acc.accountIds, + after: after.value, before: before.value, + limit: limitArg(args, 'limit', 50), + }); + return jsonResult(rows); // raw array, no envelope (msgvault parity) +} + +export const searchByDomainsDef = { + name: 'search_by_domains', + description: 'Find emails where any participant (from, to, or cc) belongs to one of the given domains. Useful for finding all communication with a company regardless of direction.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + domains: { type: 'string', description: "Comma-separated domain names (e.g. 'gobright.com,ascentae.com')" }, + limit: { type: 'number', description: 'Maximum results to return (default 100)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + }, + required: ['domains'], + }, +}; + +export async function handleSearchByDomains(args, scope) { + const domainsStr = (args.domains || '').trim(); + if (!domainsStr) return errorResult('domains is required'); + const domains = domainsStr.split(',').map((d) => d.trim()).filter(Boolean); + if (!domains.length) return errorResult('at least one domain is required'); + const limit = limitArg(args, 'limit', 100); + const offset = limitArg(args, 'offset', 0); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const results = await searchByDomains(domains, after.value, before.value, limit, offset, scope.accountIds); + return jsonResult(results.map(wireSummary)); // raw array (msgvault parity) +} + +export const findSimilarMessagesDef = { + name: 'find_similar_messages', + description: 'Find messages whose embeddings are closest to the given message. Requires vector search to be configured and an active index generation.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + message_id: { type: 'string', description: 'Seed message ID; its embedding is used as the query vector' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + message_type: { type: 'string', description: 'Restrict results to one message type, such as email, sms, mms, fbmessenger, or calendar_event' }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + has_attachment: { type: 'boolean', description: 'Only messages with attachments' }, + }, + required: ['message_id'], + }, +}; + +function findSimilarError(message) { + const error = new Error(message); + error.name = 'FindSimilarError'; + return error; +} + +export async function findSimilarSummaries(seedId, { + accountIds, + limit = 20, + after, + before, + hasAttachment = false, +} = {}) { + const { generation } = await resolveActiveGenerationFromConfig(); + + // loadVector is not account-aware, so scope membership must be established + // before the vector is loaded (and before its existence can be observed). + if (!(await messageInScope(seedId, accountIds))) { + throw findSimilarError('message not found'); + } + + let seed; + try { + seed = await loadVector(seedId); + } catch (error) { + throw findSimilarError(`load seed vector: ${error.message}`); + } + + const filter = { accountIds }; + if (after) filter.after = after; + if (before) filter.before = before; + if (hasAttachment) filter.hasAttachment = true; + + // annSearch is generation-specific and rank-ordered. Over-fetch one so the + // seed can be removed without shortening the requested result set. + const hits = await annSearch(generation.id, seed, limit + 1, { filter }); + const ids = []; + for (const hit of hits) { + if (hit.messageId === seedId) continue; + if (ids.length >= limit) break; + ids.push(hit.messageId); + } + + const messages = await getMessageSummariesByIDs(ids, accountIds); + return { generation, messages }; +} + +export async function handleFindSimilarMessages(args, scope) { + const seedId = args.message_id; + if (!seedId || typeof seedId !== 'string') return errorResult('message_id parameter is required'); + let limit = Number(args.limit); + if (!Number.isFinite(limit) || limit < 1) limit = 20; + // msgvault clamps to the hybrid page cap (handlers.go:885-888). + if (limit > HYBRID_RANKING_WINDOW) limit = HYBRID_RANKING_WINDOW; + + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + const messageType = typeof args.message_type === 'string' ? args.message_type.trim().toLowerCase() : ''; + + try { + const result = await findSimilarSummaries(seedId, { + accountIds: acc.accountIds, + limit, + after: after.value, + before: before.value, + hasAttachment: args.has_attachment === true, + }); + let messages = result.messages; + // msgvault applies message_type inside the vector backend filter + // (handlers.go:1042-1044); Mailflow's annSearch filter has no such leg, so + // the advertised filter is applied on the hydrated summaries (every + // Mailflow message is 'email' today — a non-email filter returns zero). + if (messageType) messages = messages.filter((m) => m.message_type === messageType); + return jsonResult({ + seed_message_id: seedId, + returned: messages.length, + generation: { + id: result.generation.id, + model: result.generation.model, + dimension: result.generation.dimension, + fingerprint: result.generation.fingerprint, + state: result.generation.state, + }, + messages: messages.map(wireSummary), + }); + } catch (err) { + if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + if (err.name === 'FindSimilarError') return errorResult(err.message); + throw err; + } +} + +export const searchInMessageDef = { + name: 'search_in_message', + description: + 'Find matches within one message body. Default mode=keyword finds literal term occurrences. ' + + 'mode=vector scores each embedded chunk by semantic similarity to the query (best first, with score on each match). ' + + 'Keyword matches include raw-body char_offset and line. Vector matches always include snippet and score; char_offset and line may be omitted after preprocessing. ' + + 'Use a present char_offset with get_message center_at to read a larger window around the match.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + id: { type: 'string', description: 'Message ID' }, + query: { type: 'string', description: 'Search query (keyword term, or semantic query when mode=vector)' }, + limit: { type: 'number', description: 'Maximum matches to return (default 10)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + mode: { type: 'string', enum: ['keyword', 'vector'], description: 'Search mode: keyword (default, literal term) or vector (semantic chunk scoring)' }, + min_score: { type: 'number', description: 'Minimum chunk similarity score (0–1) when mode=vector (default 0)' }, + }, + required: ['id', 'query'], + }, +}; + +export async function handleSearchInMessage(args, scope) { + const id = args.id; + if (!id || typeof id !== 'string') return errorResult('id parameter is required'); + const q = (args.query || '').trim(); + if (!q) return errorResult('query parameter is required'); + const limit = limitArg(args, 'limit', 10); + const offset = limitArg(args, 'offset', 0); + const mode = args.mode || 'keyword'; + + if (mode === 'vector') { + try { + const minScore = Number.isFinite(Number(args.min_score)) ? Number(args.min_score) : 0; + // matchesInMessage is scope-aware: it resolves the message under + // scope.accountIds and throws VectorUnavailableError on stock Postgres. + const all = await matchesInMessage(id, q, minScore, { accountIds: scope.accountIds }); + const page = all.slice(offset, offset + limit); + return jsonResult(newPaginatedResponse(page, all.length, offset)); + } catch (err) { + if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + throw err; + } + } + if (mode !== 'keyword') return errorResult(`invalid mode "${mode}": must be keyword (default) or vector`); + + const row = await getMessage(id, scope.accountIds); + if (!row) return errorResult('message not found'); + const all = findTermMatches(row.body_text || '', q); // byte-offset keyword matches (real total) + const page = all.slice(offset, offset + limit); + return jsonResult(newPaginatedResponse(page, all.length, offset)); +} + +export const stageDeletionDef = { + name: 'stage_deletion', + description: "Stage messages for deletion. Use EITHER 'query' (Gmail-style search) OR structured filters (from, domain, label, etc.), not both. Does NOT delete immediately - execution is a separate, explicitly-authorized step.", + annotations: NON_IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + query: { type: 'string', description: "Gmail-style search query (e.g. 'from:linkedin subject:job alert'). Cannot be combined with structured filters." }, + from: { type: 'string', description: 'Filter by sender email address' }, + domain: { type: 'string', description: "Filter by sender domain (e.g. 'linkedin.com')" }, + label: { type: 'string', description: "Filter by Gmail label (e.g. 'CATEGORY_PROMOTIONS')" }, + after: { type: 'string', description: 'Only messages after this date (YYYY-MM-DD)' }, + before: { type: 'string', description: 'Only messages before this date (YYYY-MM-DD)' }, + has_attachment: { type: 'boolean', description: 'Only messages with attachments' }, + }, + }, +}; + +// A parsed query that survives validation but produces zero enforceable +// predicates would stage the entire (capped) mailbox — refuse it. Parse errors +// and unsupported operators are already rejected before this runs (msgvault +// ordering), so the only remaining cause is all-discarded terms. +function noEnforceableFiltersMessage() { + return 'query produced no enforceable filters (all query terms were discarded as too short, ' + + 'punctuation-only, or negation-only); refusing to stage deletions'; +} + +export async function handleStageDeletion(args, scope) { + const q = (args.query || '').trim(); + const hasQuery = q !== ''; + const structured = !!(args.from || args.domain || args.label || args.has_attachment || args.after || args.before); + if (hasQuery && structured) return errorResult("use either 'query' or structured filters (from, domain, label, etc.), not both"); + if (!hasQuery && !structured) return errorResult("must provide either 'query' or at least one filter (from, domain, label, after, before, has_attachment)"); + const after = dateArg(args, 'after'); + if (after.error) return errorResult(after.error); + const before = dateArg(args, 'before'); + if (before.error) return errorResult(before.error); + + const parsed = hasQuery ? parseQuery(q) : null; + if (parsed) { + // Parse-value errors and unsupported operators reject the staging query + // outright (msgvault handlers.go:1818-1821). Deletion is where silent + // widening bites hardest: dropping `label:promotions` from + // `invoice label:promotions` would stage a SUPERSET of what was asked. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + // Guard the "stage EVERYTHING" hazard: a query whose tokens are ALL + // discarded (negation-only, sub-2-char/punctuation-only terms) leaves only + // account+liveness in the WHERE. Refuse before staging. + if (countEnforceableQueryPredicates(parsed) === 0) { + return errorResult(noEnforceableFiltersMessage()); + } + } + + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + + const { batchId, messageCount } = await stageDeletion({ + userId: scope.userId, accountIds: acc.accountIds, + parsed, + from: args.from, domain: args.domain, label: args.label, + hasAttachment: args.has_attachment === true, after: after.value, before: before.value, + description: hasQuery ? `query: ${q}`.slice(0, 50) : 'filter', + }); + if (!messageCount) return errorResult('no messages match the specified criteria'); + return jsonResult({ + batch_id: batchId, + message_count: messageCount, + // Wire literal is msgvault's manifest.StatusPending ("pending", + // deletion/manifest.go:25, surfaced at handlers.go:1932). The DB row keeps + // Mailflow's internal 'staged' state (engineAdapter.stageDeletion). + status: 'pending', + next_step: `POST /api/mcp-deletions/${batchId}/execute to soft-delete, or DELETE /api/mcp-deletions/${batchId} to cancel`, + }); +} diff --git a/backend/src/mcp/messageTools.test.js b/backend/src/mcp/messageTools.test.js new file mode 100644 index 00000000..072a176a --- /dev/null +++ b/backend/src/mcp/messageTools.test.js @@ -0,0 +1,490 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('./engineAdapter.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getMessage: vi.fn(), listMessages: vi.fn(), listAccounts: vi.fn(), getTotalStats: vi.fn(), aggregate: vi.fn(), searchByDomains: vi.fn(), + getMessageSummariesByIDs: vi.fn(), stageDeletion: vi.fn(), + resolveAccountScope: vi.fn(), messageInScope: vi.fn(), + }; +}); +vi.mock('./vectorStats.js', () => ({ collectStats: vi.fn() })); +vi.mock('../services/embeddings/hybrid.js', () => ({ resolveActiveGenerationFromConfig: vi.fn() })); +vi.mock('../services/embeddings/vectorStore.js', () => ({ loadVector: vi.fn(), annSearch: vi.fn() })); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchesInMessage: vi.fn() })); +import * as adapter from './engineAdapter.js'; +import { resolveAccountScope, messageInScope } from './engineAdapter.js'; +import { collectStats } from './vectorStats.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; +import { loadVector, annSearch } from '../services/embeddings/vectorStore.js'; +import { matchesInMessage } from '../services/embeddings/chunkmatch.js'; +import { ALL_SCOPES } from './auth.js'; + +// The one vector-availability gate returns { cfg, generation }; a VectorUnavailableError +// (name + reason) thrown from it is what every degraded/disabled path surfaces. +class VUE extends Error { constructor(r) { super(r); this.name = 'VectorUnavailableError'; this.reason = r; } } +import { + handleGetMessage, handleListMessages, handleGetStats, handleAggregate, + handleSearchByDomains, handleFindSimilarMessages, handleSearchInMessage, handleStageDeletion, +} from './messageTools.js'; + +const scope = { userId: 'u', accountIds: ['acc-1'], scopes: ALL_SCOPES }; +const detailRow = { + id: 'm1', account_id: 'acc-1', message_id: '', thread_id: 't', subject: 'S', snippet: '', + from_email: 'a@b.com', from_name: 'A', to_addresses: [], cc_addresses: [], date: new Date('2024-01-01T00:00:00Z'), + has_attachments: false, attachments: [], flags: [], folder: 'INBOX', + body_text: 'café — meeting notes and a much longer body '.repeat(100), body_html: '', +}; + +beforeEach(() => { + adapter.getMessage.mockReset(); + resolveAccountScope.mockReset(); + messageInScope.mockReset(); + // Defaults: no account narrowing; seed messages are in scope unless a test says otherwise. + resolveAccountScope.mockImplementation(async (account, ids) => ({ accountIds: ids })); + messageInScope.mockResolvedValue(true); +}); + +describe('get_message', () => { + it('404s a missing message with a string id', async () => { + adapter.getMessage.mockResolvedValue(null); + const r = await handleGetMessage({ id: 'nope' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('message not found'); + }); + + it('pages by BYTES, never splitting a rune, with correct body_length', async () => { + adapter.getMessage.mockResolvedValue(detailRow); + const total = Buffer.byteLength(detailRow.body_text, 'utf8'); + const r = await handleGetMessage({ id: 'm1', max_chars: 20 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_length).toBe(total); + expect(b.offset).toBe(0); + expect(b.body_returned).toBeLessThanOrEqual(20); + expect(b.has_more).toBe(true); + // returned slice is valid UTF-8 (no replacement char from a split é) + expect(b.body_text).not.toContain('�'); + expect(b.body_format).toBe('text'); + expect(b.id).toBe('m1'); // uuid string + // scoped read + expect(adapter.getMessage).toHaveBeenCalledWith('m1', ['acc-1']); + }); + + it('center_at round-trips a byte offset into a window around it', async () => { + adapter.getMessage.mockResolvedValue(detailRow); + const r = await handleGetMessage({ id: 'm1', center_at: 200, max_chars: 100 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.offset).toBeLessThanOrEqual(200); + expect(b.offset + b.body_returned).toBeGreaterThanOrEqual(200); + }); + + it('clamps max_chars to 4000', async () => { + adapter.getMessage.mockResolvedValue({ ...detailRow, body_text: 'z'.repeat(9000) }); + const r = await handleGetMessage({ id: 'm1', max_chars: 99999 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_returned).toBe(4000); + }); + + it('full_body returns the whole body ignoring paging', async () => { + adapter.getMessage.mockResolvedValue({ ...detailRow, body_text: 'z'.repeat(5000) }); + const r = await handleGetMessage({ id: 'm1', full_body: true }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_returned).toBe(5000); + expect(b.has_more).toBe(false); + }); + + it('emits sent_at as RFC3339 without milliseconds (Go wire format)', async () => { + adapter.getMessage.mockResolvedValue(detailRow); + const b = JSON.parse((await handleGetMessage({ id: 'm1' }, scope)).content[0].text); + expect(b.sent_at).toBe('2024-01-01T00:00:00Z'); + }); + + it('html-only message pages the html body with body_format=html under auto', async () => { + adapter.getMessage.mockResolvedValue({ ...detailRow, body_text: '', body_html: '

hello world

' }); + const r = await handleGetMessage({ id: 'm1' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.body_format).toBe('html'); + expect(b.body_html).toContain('hello'); + expect(b.body_text).toBe(''); + }); +}); + +describe('list_messages', () => { + beforeEach(() => { adapter.listMessages.mockReset(); adapter.listAccounts.mockReset(); }); + + it('returns a newest-first envelope with total=-1 and pages via has_more (limit+1 over-fetch)', async () => { + // 21 rows for a default limit of 20 -> has_more true, sliced to 20. + adapter.listMessages.mockResolvedValue(Array.from({ length: 21 }, (_, i) => ({ id: `m${i}` }))); + const r = await handleListMessages({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.total).toBe(-1); + expect(b.returned).toBe(20); + expect(b.has_more).toBe(true); + expect(adapter.listMessages).toHaveBeenCalledWith(expect.objectContaining({ accountIds: ['acc-1'], limit: 21, offset: 0 })); + }); + + it('threads a string conversation_id filter through to listMessages', async () => { + adapter.listMessages.mockResolvedValue([]); + await handleListMessages({ conversation_id: 'tid-1' }, scope); + expect(adapter.listMessages).toHaveBeenCalledWith(expect.objectContaining({ conversationId: 'tid-1' })); + }); + + it('resolves an account email to its id and narrows the scope', async () => { + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + adapter.listMessages.mockResolvedValue([]); + await handleListMessages({ account: 'c@d.com' }, scope); + expect(resolveAccountScope).toHaveBeenCalledWith('c@d.com', ['acc-1']); + expect(adapter.listMessages).toHaveBeenCalledWith(expect.objectContaining({ accountIds: ['acc-2'] })); + }); + + it('404-style errors an unknown account (msgvault getAccountID parity)', async () => { + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleListMessages({ account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('account not found: nope@x.com'); + }); + + it('rejects malformed after/before dates before touching the engine (msgvault getDateArg, handlers.go:265-275)', async () => { + for (const [key, bad] of [['after', '13/45/2024'], ['before', '2024-02-31']]) { + const r = await handleListMessages({ [key]: bad }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe(`invalid ${key} date "${bad}": expected YYYY-MM-DD`); + } + expect(adapter.listMessages).not.toHaveBeenCalled(); + }); + + it('emits sent_at as RFC3339 without milliseconds (Go wire format)', async () => { + adapter.listMessages.mockResolvedValue([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const r = await handleListMessages({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.data[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('get_stats', () => { + beforeEach(() => { adapter.getTotalStats.mockReset(); adapter.listAccounts.mockReset(); collectStats.mockReset(); }); + + const stats = { MessageCount: 10, TotalSize: 500, AttachmentCount: 2, AttachmentSize: 0, LabelCount: 3, AccountCount: 1, ActiveMessageCount: 10, SourceDeletedMessageCount: 0 }; + const accounts = [{ ID: 'acc-1', SourceType: 'imap', Identifier: 'a@b.com', DisplayName: 'Work' }]; + + it('omits vector_search when vector search is disabled (collectStats null)', async () => { + adapter.getTotalStats.mockResolvedValue(stats); + adapter.listAccounts.mockResolvedValue(accounts); + collectStats.mockResolvedValue(null); + const r = await handleGetStats({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.stats).toEqual(stats); + expect(b.accounts).toEqual(accounts); + expect(b).not.toHaveProperty('vector_search'); + expect(adapter.getTotalStats).toHaveBeenCalledWith(['acc-1']); + }); + + it('includes the StatsView when vector search is enabled', async () => { + adapter.getTotalStats.mockResolvedValue(stats); + adapter.listAccounts.mockResolvedValue(accounts); + const vs = { enabled: true, active_generation: null, missing_embeddings_total: 4 }; + collectStats.mockResolvedValue(vs); + const r = await handleGetStats({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.vector_search).toEqual(vs); + }); + + it('survives a broken vector sub-query (omits vector_search, keeps stats)', async () => { + adapter.getTotalStats.mockResolvedValue(stats); + adapter.listAccounts.mockResolvedValue(accounts); + collectStats.mockRejectedValue(new Error('boom')); + const r = await handleGetStats({}, scope); + const b = JSON.parse(r.content[0].text); + expect(b.stats).toEqual(stats); + expect(b).not.toHaveProperty('vector_search'); + }); +}); + +describe('aggregate', () => { + beforeEach(() => { adapter.aggregate.mockReset(); adapter.listAccounts.mockReset(); }); + + it('requires group_by', async () => { + const r = await handleAggregate({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('group_by parameter is required'); + }); + + it('rejects an invalid group_by', async () => { + const r = await handleAggregate({ group_by: 'colour' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid group_by: colour'); + }); + + it('returns the raw AggregateRow array (no envelope), default limit 50, scoped', async () => { + const rows = [{ Key: 'a@b.com', Count: 3, TotalSize: 10, AttachmentSize: 0, AttachmentCount: 1, TotalUnique: 5 }]; + adapter.aggregate.mockResolvedValue(rows); + const r = await handleAggregate({ group_by: 'sender' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b).toEqual(rows); + expect(adapter.aggregate).toHaveBeenCalledWith('sender', expect.objectContaining({ accountIds: ['acc-1'], limit: 50 })); + }); + + it('rejects malformed after/before dates instead of leaking a raw PG error (msgvault getDateArg)', async () => { + const r = await handleAggregate({ group_by: 'sender', after: 'notadate' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid after date "notadate": expected YYYY-MM-DD'); + expect(adapter.aggregate).not.toHaveBeenCalled(); + }); +}); + +describe('search_by_domains', () => { + beforeEach(() => adapter.searchByDomains.mockReset()); + + it('requires domains', async () => { + const r = await handleSearchByDomains({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('domains is required'); + }); + + it('splits/trims the CSV, default limit 100, scoped, returns the raw array', async () => { + const rows = [{ id: 'm1', from_email: 'x@gobright.com' }]; + adapter.searchByDomains.mockResolvedValue(rows); + const r = await handleSearchByDomains({ domains: ' gobright.com , ascentae.com ' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b).toEqual(rows); + expect(adapter.searchByDomains).toHaveBeenCalledWith(['gobright.com', 'ascentae.com'], undefined, undefined, 100, 0, ['acc-1']); + }); + + it('rejects malformed after/before dates and emits sent_at as RFC3339 (Go wire format)', async () => { + const bad = await handleSearchByDomains({ domains: 'x.com', before: '2024-1-1' }, scope); + expect(bad.isError).toBe(true); + expect(bad.content[0].text).toBe('invalid before date "2024-1-1": expected YYYY-MM-DD'); + expect(adapter.searchByDomains).not.toHaveBeenCalled(); + + adapter.searchByDomains.mockResolvedValue([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const ok = await handleSearchByDomains({ domains: 'x.com' }, scope); + expect(JSON.parse(ok.content[0].text)[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('find_similar_messages', () => { + beforeEach(() => { + resolveActiveGenerationFromConfig.mockReset(); loadVector.mockReset(); annSearch.mockReset(); + adapter.getMessageSummariesByIDs.mockReset(); adapter.listAccounts.mockReset(); + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 }, + generation: { id: 3, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + }); + + it('returns vector_not_enabled when embeddings are disabled (config check before the resolver)', async () => { + resolveActiveGenerationFromConfig.mockRejectedValueOnce(new VUE('vector_not_enabled')); + const r = await handleFindSimilarMessages({ message_id: 'seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('vector_not_enabled: vector search is not configured on this server'); + }); + + it('maps a VectorUnavailableError reason from the resolver to its wire string', async () => { + resolveActiveGenerationFromConfig.mockRejectedValue(new VUE('no_active_generation')); + const r = await handleFindSimilarMessages({ message_id: 'seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^no_active_generation:/); + }); + + it('excludes the seed, hydrates in rank order via annSearch(gen.id, ...), scoped', async () => { + resolveActiveGenerationFromConfig.mockResolvedValue({ + cfg: { enabled: true, model: 'm', dimension: 2, preprocess: {}, maxInputChars: 100 }, + generation: { id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }, + }); + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([ + { messageId: 'm1', score: 0.9, rank: 1 }, + { messageId: 'seed', score: 1, rank: 0 }, + { messageId: 'm2', score: 0.7, rank: 2 }, + ]); + adapter.getMessageSummariesByIDs.mockResolvedValue([{ id: 'm1' }, { id: 'm2' }]); + const r = await handleFindSimilarMessages({ message_id: 'seed', limit: 20 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.seed_message_id).toBe('seed'); + expect(b.returned).toBe(2); + expect(b.generation).toEqual({ id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }); + expect(b.messages).toEqual([{ id: 'm1' }, { id: 'm2' }]); + // real annSearch takes the generation ID and the {filter} with accountIds + expect(annSearch).toHaveBeenCalledWith(3, [0.1, 0.2], 21, { filter: { accountIds: ['acc-1'] } }); + expect(adapter.getMessageSummariesByIDs).toHaveBeenCalledWith(['m1', 'm2'], ['acc-1']); + }); + + it('reports a readable error when the seed has no embedding', async () => { + loadVector.mockRejectedValue(new Error('no embedding for message seed')); + const r = await handleFindSimilarMessages({ message_id: 'seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('load seed vector: no embedding for message seed'); + }); + + it('rejects a foreign/out-of-scope seed id without loading its vector (owner-scope isolation)', async () => { + messageInScope.mockResolvedValue(false); // seed belongs to another user + const r = await handleFindSimilarMessages({ message_id: 'foreign-seed' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('message not found'); + expect(loadVector).not.toHaveBeenCalled(); + expect(annSearch).not.toHaveBeenCalled(); + expect(messageInScope).toHaveBeenCalledWith('foreign-seed', ['acc-1']); + }); + + it('applies the advertised message_type filter to hydrated results (msgvault handlers.go:1042-1044)', async () => { + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([{ messageId: 'm1', score: 0.9, rank: 1 }]); + adapter.getMessageSummariesByIDs.mockResolvedValue([{ id: 'm1', message_type: 'email' }]); + const none = JSON.parse((await handleFindSimilarMessages({ message_id: 'seed', message_type: 'sms' }, scope)).content[0].text); + expect(none.messages).toEqual([]); + expect(none.returned).toBe(0); + const all = JSON.parse((await handleFindSimilarMessages({ message_id: 'seed', message_type: 'email' }, scope)).content[0].text); + expect(all.messages).toHaveLength(1); + }); + + it('clamps limit to the hybrid page cap (msgvault handlers.go:885-888)', async () => { + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([]); + adapter.getMessageSummariesByIDs.mockResolvedValue([]); + await handleFindSimilarMessages({ message_id: 'seed', limit: 5000 }, scope); + expect(annSearch).toHaveBeenCalledWith(3, [0.1, 0.2], 101, expect.anything()); // 100 + 1 seed over-fetch + }); + + it('rejects malformed after/before dates and emits sent_at as RFC3339 (Go wire format)', async () => { + const bad = await handleFindSimilarMessages({ message_id: 'seed', after: '2024-13-45' }, scope); + expect(bad.isError).toBe(true); + expect(bad.content[0].text).toBe('invalid after date "2024-13-45": expected YYYY-MM-DD'); + expect(annSearch).not.toHaveBeenCalled(); + + loadVector.mockResolvedValue([0.1, 0.2]); + annSearch.mockResolvedValue([{ messageId: 'm1', score: 0.9, rank: 1 }]); + adapter.getMessageSummariesByIDs.mockResolvedValue([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const ok = JSON.parse((await handleFindSimilarMessages({ message_id: 'seed' }, scope)).content[0].text); + expect(ok.messages[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('search_in_message', () => { + const body = 'café note — the budget is set here'; + const detail = { id: 'm1', account_id: 'acc-1', message_id: '', thread_id: 't', subject: 'S', snippet: '', from_email: 'a@b.com', from_name: 'A', to_addresses: [], cc_addresses: [], date: new Date('2024-01-01T00:00:00Z'), has_attachments: false, attachments: [], flags: [], folder: 'INBOX', body_text: body, body_html: '' }; + beforeEach(() => { adapter.getMessage.mockReset(); matchesInMessage.mockReset(); }); + + it('keyword mode returns a byte char_offset + real total; the offset round-trips into get_message center_at', async () => { + adapter.getMessage.mockResolvedValue(detail); + const r = await handleSearchInMessage({ id: 'm1', query: 'budget' }, scope); + const b = JSON.parse(r.content[0].text); + const byteOffset = Buffer.from(body, 'utf8').indexOf(Buffer.from('budget', 'utf8')); + expect(b.total).toBe(1); + expect(b.data[0].char_offset).toBe(byteOffset); // BYTE offset (not code-point index) + expect(b.data[0]).not.toHaveProperty('score'); // keyword = no score + const g = JSON.parse((await handleGetMessage({ id: 'm1', center_at: byteOffset, max_chars: 100 }, scope)).content[0].text); + expect(g.offset).toBeLessThanOrEqual(byteOffset); + expect(g.offset + g.body_returned).toBeGreaterThanOrEqual(byteOffset); + }); + + it('404s a missing message in keyword mode', async () => { + adapter.getMessage.mockResolvedValue(null); + const r = await handleSearchInMessage({ id: 'x', query: 'q' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('message not found'); + }); + + it('vector mode maps VectorUnavailableError to vector_not_enabled (stock Postgres)', async () => { + class VUE extends Error { constructor(r) { super(r); this.name = 'VectorUnavailableError'; this.reason = r; } } + matchesInMessage.mockRejectedValue(new VUE('vector_not_enabled')); + const r = await handleSearchInMessage({ id: 'm1', query: 'travel', mode: 'vector' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('vector_not_enabled: vector search is not configured on this server'); + }); + + it('vector mode pages scored matches from matchesInMessage (scoped)', async () => { + matchesInMessage.mockResolvedValue([{ snippet: 'a', score: 0.9 }, { snippet: 'b', score: 0.8 }]); + const r = await handleSearchInMessage({ id: 'm1', query: 'travel', mode: 'vector', limit: 1 }, scope); + const b = JSON.parse(r.content[0].text); + expect(b.total).toBe(2); + expect(b.returned).toBe(1); + expect(b.has_more).toBe(true); + expect(matchesInMessage).toHaveBeenCalledWith('m1', 'travel', 0, { accountIds: ['acc-1'] }); + }); + + it('rejects an unknown mode', async () => { + const r = await handleSearchInMessage({ id: 'm1', query: 'q', mode: 'fuzzy' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid mode "fuzzy": must be keyword (default) or vector'); + }); +}); + +describe('stage_deletion', () => { + beforeEach(() => adapter.stageDeletion.mockReset()); + + it('rejects query + structured filters together', async () => { + const r = await handleStageDeletion({ query: 'from:x', from: 'y@z.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe("use either 'query' or structured filters (from, domain, label, etc.), not both"); + }); + + it('rejects neither query nor filters', async () => { + const r = await handleStageDeletion({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe("must provide either 'query' or at least one filter (from, domain, label, after, before, has_attachment)"); + }); + + it('rejects an all-whitespace query with no structured filters (empty/missing query guard)', async () => { + const r = await handleStageDeletion({ query: ' ' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe("must provide either 'query' or at least one filter (from, domain, label, after, before, has_attachment)"); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('rejects a query with an unsupported operator via the taxonomy error — never stages the whole mailbox', async () => { + const r = await handleStageDeletion({ query: 'label:promotions' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: label: /); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('refuses a negation-only / sub-2-char / punctuation-only query (all tokens discarded)', async () => { + for (const query of ['-newsletter', 'a', '!!!']) { + adapter.stageDeletion.mockReset(); + const r = await handleStageDeletion({ query }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('query produced no enforceable filters (all query terms were discarded as too short, punctuation-only, or negation-only); refusing to stage deletions'); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + } + }); + + it('rejects a mixed query too — dropping label: would stage a SUPERSET of what was asked (msgvault handlers.go:1818-1821)', async () => { + const r = await handleStageDeletion({ query: 'invoice label:promotions' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: label: /); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('returns parser value errors verbatim (msgvault q.Err front-door rule)', async () => { + const r = await handleStageDeletion({ query: 'invoice older_than:xyz' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid value "xyz" for older_than: — expected a relative age like 7d, 2w, 1m, or 1y'); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('rejects malformed structured after/before dates (msgvault getDateArg, handlers.go:1793-1800)', async () => { + const r = await handleStageDeletion({ after: '13/45/2024' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid after date "13/45/2024": expected YYYY-MM-DD'); + expect(adapter.stageDeletion).not.toHaveBeenCalled(); + }); + + it('errors when no messages match', async () => { + adapter.stageDeletion.mockResolvedValue({ batchId: null, messageCount: 0 }); + const r = await handleStageDeletion({ from: 'linkedin.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('no messages match the specified criteria'); + }); + + it('stages a batch with wire status "pending" (msgvault manifest.StatusPending, manifest.go:25) — never soft-deletes', async () => { + adapter.stageDeletion.mockResolvedValue({ batchId: 'batch-9', messageCount: 5 }); + const r = await handleStageDeletion({ domain: 'linkedin.com' }, scope); + const b = JSON.parse(r.content[0].text); + expect(b).toEqual({ + batch_id: 'batch-9', message_count: 5, status: 'pending', + next_step: 'POST /api/mcp-deletions/batch-9/execute to soft-delete, or DELETE /api/mcp-deletions/batch-9 to cancel', + }); + // batch is scoped to the token user (cross-user isolation) and never flips is_deleted here + expect(adapter.stageDeletion).toHaveBeenCalledWith(expect.objectContaining({ userId: 'u', accountIds: ['acc-1'], domain: 'linkedin.com' })); + }); +}); diff --git a/backend/src/mcp/messageTriage.integration.test.js b/backend/src/mcp/messageTriage.integration.test.js new file mode 100644 index 00000000..394cfa96 --- /dev/null +++ b/backend/src/mcp/messageTriage.integration.test.js @@ -0,0 +1,158 @@ +import { randomUUID } from 'crypto'; +import { readFile } from 'fs/promises'; +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from '../services/embeddings/testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const integrationDescribe = DSN ? describe : describe.skip; +const migrationsUrl = new URL('../../migrations/', import.meta.url); + +describe('message triage migration sources', () => { + it('keeps table creation transactional and the feed index in a later concurrent migration', async () => { + const tableSql = await readFile(new URL('0050_message_triage.sql', migrationsUrl), 'utf8').catch(() => ''); + const feedSql = await readFile(new URL('0052_message_triage_feed_index.sql', migrationsUrl), 'utf8').catch(() => ''); + + expect(tableSql).toContain('CREATE TABLE IF NOT EXISTS message_triage'); + expect(tableSql).not.toMatch(/^--\s*no-transaction\b/i); + expect(tableSql).not.toContain('CREATE INDEX CONCURRENTLY'); + expect(feedSql).toMatch(/^--\s*no-transaction\b/i); + expect(feedSql).toContain('DROP INDEX CONCURRENTLY IF EXISTS idx_messages_triage_feed'); + expect(feedSql).toContain('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_triage_feed'); + }); +}); + +integrationDescribe('message_triage migration', () => { + let client; + const users = []; + + beforeAll(async () => { + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + }); + + afterAll(async () => { + await client.end(); + }); + + beforeEach(() => { + users.length = 0; + }); + + afterEach(async () => { + for (const userId of users) await cleanupAccount(client, userId); + }); + + async function account(label) { + const seeded = await seedAccount(client, label); + users.push(seeded.userId); + return seeded; + } + + async function token(userId) { + const result = await client.query( + `INSERT INTO api_tokens (user_id, token_hash, name) + VALUES ($1, $2, 'message-triage-it') + RETURNING id`, + [userId, `triage-it-${randomUUID()}`], + ); + return result.rows[0].id; + } + + async function triage({ userId, accountId, tokenId = null, header = `<${randomUUID()}@example.com>` }) { + const result = await client.query( + `INSERT INTO message_triage + (user_id, account_id, message_id_header, token_id) + VALUES ($1, $2, $3, $4) + RETURNING id`, + [userId, accountId, header, tokenId], + ); + return { id: result.rows[0].id, header }; + } + + it('has the required columns and constraints', async () => { + const columns = await client.query( + `SELECT column_name, is_nullable, column_default + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'message_triage' + ORDER BY ordinal_position`, + ); + expect(columns.rows.map(row => row.column_name)).toEqual([ + 'id', + 'user_id', + 'account_id', + 'message_id_header', + 'triaged_at', + 'action', + 'note', + 'source', + 'token_id', + ]); + expect(columns.rows.find(row => row.column_name === 'message_id_header')?.is_nullable).toBe('NO'); + expect(columns.rows.find(row => row.column_name === 'source')?.column_default).toContain("'mcp'"); + + const constraints = await client.query( + `SELECT c.contype, pg_get_constraintdef(c.oid) AS definition + FROM pg_constraint c + WHERE c.conrelid = 'message_triage'::regclass`, + ); + const definitions = constraints.rows.map(row => row.definition); + expect(definitions).toContain('PRIMARY KEY (id)'); + expect(definitions).toContain('UNIQUE (account_id, message_id_header)'); + expect(definitions).toContain('FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE'); + expect(definitions).toContain('FOREIGN KEY (account_id) REFERENCES email_accounts(id) ON DELETE CASCADE'); + expect(definitions).toContain('FOREIGN KEY (token_id) REFERENCES api_tokens(id) ON DELETE SET NULL'); + expect(definitions.some(definition => definition.includes('message_id_header') && definition.includes('FOREIGN KEY'))).toBe(false); + }); + + it('enforces uniqueness by account and Message-ID header', async () => { + const { userId, accountId } = await account('triage-unique'); + const first = await triage({ userId, accountId }); + + await expect(triage({ userId, accountId, header: first.header })).rejects.toMatchObject({ code: '23505' }); + }); + + it('cascades when an email account is deleted', async () => { + const { userId, accountId } = await account('triage-account-cascade'); + const row = await triage({ userId, accountId }); + + await client.query('DELETE FROM email_accounts WHERE id = $1', [accountId]); + + const result = await client.query('SELECT id FROM message_triage WHERE id = $1', [row.id]); + expect(result.rows).toEqual([]); + }); + + it('cascades when a user is deleted', async () => { + const { userId, accountId } = await account('triage-user-cascade'); + const row = await triage({ userId, accountId }); + + await client.query('DELETE FROM users WHERE id = $1', [userId]); + users.splice(users.indexOf(userId), 1); + + const result = await client.query('SELECT id FROM message_triage WHERE id = $1', [row.id]); + expect(result.rows).toEqual([]); + }); + + it('sets token_id to null when the API token is deleted', async () => { + const { userId, accountId } = await account('triage-token-null'); + const tokenId = await token(userId); + const row = await triage({ userId, accountId, tokenId }); + + await client.query('DELETE FROM api_tokens WHERE id = $1', [tokenId]); + + const result = await client.query('SELECT token_id FROM message_triage WHERE id = $1', [row.id]); + expect(result.rows[0].token_id).toBeNull(); + }); + + it('has the partial inbox feed index', async () => { + const result = await client.query( + `SELECT indexdef + FROM pg_indexes + WHERE schemaname = 'public' AND indexname = 'idx_messages_triage_feed'`, + ); + expect(result.rows).toHaveLength(1); + expect(result.rows[0].indexdef).toContain('(account_id, date, message_id)'); + expect(result.rows[0].indexdef).toContain("folder = 'INBOX'"); + expect(result.rows[0].indexdef).toContain('(is_deleted = false)'); + }); +}); diff --git a/backend/src/mcp/result.js b/backend/src/mcp/result.js new file mode 100644 index 00000000..f416103a --- /dev/null +++ b/backend/src/mcp/result.js @@ -0,0 +1,9 @@ +// Mirror msgvault's mcp.NewToolResultText / mcp.NewToolResultError: every tool +// returns a single text content block; errors additionally set isError. +export function jsonResult(obj) { + return { content: [{ type: 'text', text: JSON.stringify(obj) }] }; +} + +export function errorResult(msg) { + return { content: [{ type: 'text', text: msg }], isError: true }; +} diff --git a/backend/src/mcp/result.test.js b/backend/src/mcp/result.test.js new file mode 100644 index 00000000..a9e118b0 --- /dev/null +++ b/backend/src/mcp/result.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect } from 'vitest'; +import { jsonResult, errorResult } from './result.js'; +import { HANDLERS, TOOL_DEFS } from './tools.js'; + +describe('result helpers', () => { + it('jsonResult wraps a JSON string in a text content block', () => { + expect(jsonResult({ ok: true })).toEqual({ + content: [{ type: 'text', text: '{"ok":true}' }], + }); + }); + it('errorResult marks isError and carries the message verbatim', () => { + expect(errorResult('boom')).toEqual({ + content: [{ type: 'text', text: 'boom' }], + isError: true, + }); + }); +}); + +describe('ping tool', () => { + it('is registered with a JSON-schema input', () => { + const def = TOOL_DEFS.find((t) => t.name === 'ping'); + expect(def).toBeTruthy(); + expect(def.inputSchema).toEqual({ type: 'object', properties: {} }); + }); + it('echoes pong and is scope-agnostic', async () => { + const r = await HANDLERS.ping({}, { userId: 'u', accountIds: [] }); + expect(r.content[0].text).toBe('{"pong":true}'); + }); +}); diff --git a/backend/src/mcp/searchTools.js b/backend/src/mcp/searchTools.js new file mode 100644 index 00000000..d602dbaa --- /dev/null +++ b/backend/src/mcp/searchTools.js @@ -0,0 +1,435 @@ +import { parseQuery } from '../services/search/queryParser.js'; +import { search } from '../services/search/searchService.js'; +import { jsonResult, errorResult } from './result.js'; +import { newPaginatedResponse, newPaginatedResponseNoTotal, wireSummary } from './envelope.js'; +import { extractContextChar } from './bodyMatch.js'; +import { translateVectorError } from './vectorErrors.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; + +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); + +// The search seam returns raw REST-shaped rows (subject/from/date/snippet/… under a +// column set REST froze); MCP must emit msgvault MessageSummary. We hydrate the hit +// ids through engineAdapter (recipients, thread_id, attachments — data the ranked row +// set doesn't carry) so every search tool speaks the same wire shape as the message +// tools. Order is preserved by getMessageSummariesByIDs; ids are already in scope. +// wireSummary re-formats sent_at to Go RFC3339 (no millis) at the wire. +async function hydrateSummaries(messages, accountIds) { + const ids = (messages || []).map((m) => m.id); + const summaries = await getMessageSummariesByIDs(ids, accountIds); + return new Map(summaries.map((s) => [s.id, wireSummary(s)])); +} + +// msgvault front doors reject queries whose known operators carry unparseable +// values (q.Err(), internal/search/parser.go:45-52; returned verbatim at +// handlers.go:373-375) instead of silently dropping the filter and returning +// wider-than-requested results. parsed.errors carries the verbatim messages; +// errors.Join separates with newlines. +export function queryParseErrorMessage(parsed) { + const errs = (parsed && parsed.errors) || []; + return errs.length ? errs.join('\n') : ''; +} + +// Port of unsupportedSearchOperatorMessage (msgvault handlers.go:408-428) with +// per-operator reasons. msgvault's parser-level unsupported set is only +// list:/list-id: (Gmail-only, parser.go:270-273); Mailflow additionally cannot +// serve label:/l:, bcc:, larger:, smaller: — msgvault supports those four, but +// Mailflow's messages schema stores no labels, BCC recipients, or byte sizes +// (documented divergence). The taxonomy prefix is verbatim. +const UNSUPPORTED_OPERATOR_REASONS = { + list: 'is Gmail-only syntax (this server does not index List-ID); use supported operators instead', + 'list-id': 'is Gmail-only syntax (this server does not index List-ID); use supported operators instead', + label: 'is not supported on this server (Mailflow stores no Gmail labels)', + bcc: 'is not supported on this server (Mailflow does not store BCC recipients)', + larger: 'is not supported on this server (Mailflow does not store message sizes)', + smaller: 'is not supported on this server (Mailflow does not store message sizes)', +}; + +// list:/list-id: live in msgvault's parser-level unsupported set; Mailflow's +// queryParser (Wave D's file) currently leaves them as literal free-text +// terms, so recognize them here at the handler seam. +// TODO(seam): consolidate into queryParser's unsupported set with Wave D. +function unsupportedListOperators(parsed) { + const out = []; + for (const t of (parsed && parsed.terms) || []) { + const m = /^(list|list-id):/i.exec(t.value || ''); + if (m) out.push(m[1].toLowerCase()); + } + return out; +} + +export function unsupportedSearchOperatorMessage(parsed) { + const names = []; + const seen = new Set(); + const push = (name) => { if (name && !seen.has(name)) { seen.add(name); names.push(name); } }; + for (const u of (parsed && parsed.unsupported) || []) push(u.key); + for (const n of unsupportedListOperators(parsed)) push(n); + if (!names.length) return ''; + // Group operators sharing a reason, msgvault-style ("name:, name2: "), + // preserving first-appearance order. + const groups = []; + const byReason = new Map(); + for (const n of names) { + const reason = UNSUPPORTED_OPERATOR_REASONS[n] || 'is not supported on this server'; + let g = byReason.get(reason); + if (!g) { g = { reason, ops: [] }; byReason.set(reason, g); groups.push(g); } + g.ops.push(`${n}:`); + } + return 'unsupported_search_operator: ' + + groups.map((g) => `${g.ops.join(', ')} ${g.reason}`).join('; '); +} + +// msgvault clamps hybrid paging to [vector.search].max_page_size_hybrid +// (default 50; vector/config.go:196-205,339-342, enforced at handlers.go:651-668). +// Mailflow has no config knob; its effective ranking window is the fused +// per-signal candidate cap (hybrid.js K_PER_SIGNAL = 100) — offsets past it +// can only return silently-empty pages, so reject them the msgvault way. +// TODO(seam): read this from hybrid.js if Wave D exports the per-signal cap. +export const HYBRID_RANKING_WINDOW = 100; + +// Tool descriptions/schemas follow msgvault internal/mcp/server.go, with the +// Mailflow divergences spelled out in the text: no label:/bcc:/larger:/smaller: +// (schema lacks the data — msgvault supports them), negation IS supported +// (msgvault does not), free-text ordering is relevance-ranked (D5), and +// semantic hits carry at most 1 excerpt. README D6: only id-bearing fields +// diverge to UUID strings — search tools have none. +const SEARCH_METADATA_OPERATOR_DOC = + 'Supported operators: from:, to:, cc:, subject:, has:attachment, ' + + 'before:/after: (YYYY-MM-DD), older_than:/newer_than: (e.g. 7d, 2w, 1m, 1y). ' + + 'Bare domains on from:/to: match any address at that domain. Multiple terms are ANDed. ' + + 'Rejected as unsupported on this server (divergence from msgvault, which supports them): ' + + 'label: (or l:), bcc:, larger:, smaller: — Mailflow stores no labels, BCC recipients, or message sizes; ' + + 'list:/list-id: are Gmail-only and also rejected. ' + + 'Negation with a leading - (e.g. -from:alice, -invoice) IS supported (divergence from msgvault); ' + + 'OR and parentheses grouping are not.'; +const SEARCH_METADATA_FREETEXT_DOC = + 'Free text matches subject, snippet, and sender/recipient metadata only (not bodies). ' + + 'Use search_message_bodies for body keywords or semantic_search_messages for vector/hybrid search.'; +const SEARCH_METADATA_PAGINATION_DOC = + 'Free-text results are relevance-ranked (divergence from msgvault); filter-only queries ' + + 'are ordered newest-first (by sent date). There is no sort parameter — ' + + 'use before:/after: to scope a date range. ' + + 'Paginate with offset/limit (default limit 20, max 50). ' + + 'Response: data, total, returned, offset, has_more.'; + +export const searchMetadataDef = { + name: 'search_metadata', + description: + 'Search message metadata using a subset of Gmail query syntax (not full Gmail compatibility). ' + + SEARCH_METADATA_OPERATOR_DOC + ' ' + SEARCH_METADATA_FREETEXT_DOC + ' ' + + SEARCH_METADATA_PAGINATION_DOC + + 'For body keywords use search_message_bodies; for vector/hybrid search use semantic_search_messages.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: "Search query (e.g. 'from:alice subject:meeting after:2024-01-01'). See tool description for supported operators and limitations." }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + }, + required: ['query'], + }, +}; + +const DEFAULT_SEARCH_LIMIT = 20; +const MAX_SEARCH_LIMIT = 50; +export function searchLimitArg(args) { + const v = Number(args.limit); + if (!Number.isFinite(v) || v <= 0) return DEFAULT_SEARCH_LIMIT; + return Math.min(Math.trunc(v), MAX_SEARCH_LIMIT); +} +export function offsetArg(args) { + const v = Number(args.offset); + if (!Number.isFinite(v) || v < 0) return 0; + return Math.trunc(v); +} + +export async function handleSearchMetadata(args, scope) { + const query = (args.query || '').trim(); + if (!query) return errorResult('query parameter is required'); + const parsed = parseQuery(query); + // msgvault ordering (handlers.go:372-378): parse-value errors verbatim, + // then unsupported operators — never silently widen the result set. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + const limit = searchLimitArg(args); + const offset = offsetArg(args); + // Narrow to a single account when `account` is given (msgvault getAccountID) — + // searchService trusts a pre-resolved accountIds as-is, so the narrowing must + // happen here (it never re-reads the `account` email). + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + const result = await search({ + mode: 'lexical', scope: 'metadata', + rawQuery: query, parsed, + accountIds: acc.accountIds, + limit, offset, + }); + const byId = await hydrateSummaries(result.messages, acc.accountIds); + const data = (result.messages || []).map((m) => byId.get(m.id)).filter(Boolean); + // total is ALWAYS present on this envelope (msgvault SearchFastCount is a + // real count, handlers.go:400-405). The seam omits it on degenerate queries + // whose terms were all dropped — those return zero rows, so total is 0. + const total = Number.isFinite(result.total) ? result.total : 0; + return jsonResult(newPaginatedResponse(data, total, offset)); +} + +export const searchMessageBodiesDef = { + name: 'search_message_bodies', + description: + 'Keyword full-text search over message bodies. ' + + 'Returns messages whose body text contains the query terms, relevance-ranked, ' + + 'each with matches — up to 5 excerpt snippets centered on matched terms. ' + + 'Backend excerpts may omit char_offset and line when efficient source locations are unavailable; use search_in_message when exact locations are needed. ' + + 'When matches_truncated is true on a hit, more than 5 excerpts matched — use search_in_message or get_message to read the full body. ' + + 'Known Gmail operators (from:, subject:, etc.) apply as metadata filters only and do not satisfy the free-text requirement. ' + + 'Filter-only queries such as from:alice are rejected — use search_metadata for filter-only queries. ' + + 'Unrecognized word:value tokens (e.g. RXD2:V2) are treated as literal body text, not filters. ' + + 'Query syntax: space-separated words are ANDed (each must appear somewhere in the body); ' + + 'a double-quoted phrase is one exact phrase (e.g. "RXD2 V2"); negation with a leading - excludes a term ' + + '(divergence from msgvault); OR is not supported. ' + + SEARCH_METADATA_OPERATOR_DOC + ' ' + + 'Results are relevance-ranked, best lexical match first (divergence from msgvault, which orders newest-first). ' + + 'Paginate with offset/limit (default limit 20, max 50). Response: data, returned, offset, has_more. ' + + 'Body search does not return a total; use has_more to detect more pages.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: 'Body search query with at least one free-text term (bare word or quoted phrase). Gmail operators (from:, subject:, etc.) are metadata filters, not body search — subject:test alone is rejected; combine with body terms (from:alice budget) or use search_metadata for filter-only queries. Unrecognized word:value tokens (RXD2:V2) are literal text. Space-separated words are ANDed; double quotes match an exact phrase; a leading - negates a term; OR unsupported.' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + }, + required: ['query'], + }, +}; + +const EMPTY_GENERATION = { id: 0, model: '', dimension: 0, fingerprint: '', state: '' }; +const MAX_CONTEXT_SNIPPETS = 5; +const SEARCH_CONTEXT_CHARS = 300; + +function freeTextTerms(parsed) { + return (parsed.terms || []).filter((t) => !t.negate).map((t) => t.value); +} + +// hybridScoreBreakdown omitempty parity (msgvault handlers.go:563-572): all +// signal fields are pointer-typed with omitempty — rrf is omitted in +// mode=vector (one signal, nothing to fuse) and subject_boosted when false. +// The seam's explain object always carries rrf + a boolean subject_boosted. +function wireScore(score, mode) { + const out = {}; + if (mode === 'hybrid' && score.rrf != null) out.rrf = score.rrf; + if (score.bm25 != null) out.bm25 = score.bm25; + if (score.vector != null) out.vector = score.vector; + if (score.subject_boosted) out.subject_boosted = true; + return out; +} + +export async function handleSearchMessageBodies(args, scope) { + const query = (args.query || '').trim(); + if (!query) return errorResult('query parameter is required'); + // Keyword-only tool: an explicit vector/hybrid mode is rejected, not + // ignored (msgvault handlers.go:442-457 — same two wordings). + const mode = args.mode || 'keyword'; + if (mode === 'vector' || mode === 'hybrid') { + return errorResult( + `invalid mode "${mode}": search_message_bodies is keyword-only; use semantic_search_messages for vector or hybrid search`, + ); + } + if (mode !== 'keyword') { + return errorResult( + `invalid mode "${mode}": search_message_bodies only supports keyword search; use semantic_search_messages for vector or hybrid search`, + ); + } + const parsed = parseQuery(query); + // msgvault ordering (handlers.go:459-465): parse errors, then unsupported + // operators, before the free-text requirement. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + const terms = freeTextTerms(parsed); + if (!terms.length) { + return errorResult( + 'search_message_bodies requires at least one free-text term (bare word or quoted phrase); ' + + 'Gmail operators such as from: or subject: are metadata filters and do not count — ' + + 'use search_metadata for filter-only queries', + ); + } + const limit = searchLimitArg(args); + const offset = offsetArg(args); + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + // Over-fetch one to compute has_more without a count query (msgvault limit+1). + const result = await search({ + mode: 'lexical', scope: 'body', + rawQuery: query, parsed, + accountIds: acc.accountIds, + limit: limit + 1, offset, + }); + let hits = result.messages || []; + const hasMore = hits.length > limit; + if (hasMore) hits = hits.slice(0, limit); + + const byId = await hydrateSummaries(hits, acc.accountIds); + const data = hits.map((m) => { + const summary = byId.get(m.id); + if (!summary) return null; + const snippets = extractContextChar(m.body_text || '', terms, SEARCH_CONTEXT_CHARS) || []; + const capped = snippets.slice(0, MAX_CONTEXT_SNIPPETS); + // Go omitempty parity (msgvault handlers.go:339-341): matches is omitted + // when empty and matches_truncated when false — never emitted as []/false. + const item = { ...summary }; + if (capped.length) item.matches = capped.map((snippet) => ({ snippet })); + if (snippets.length > MAX_CONTEXT_SNIPPETS) item.matches_truncated = true; + return item; + }).filter(Boolean); + + return jsonResult({ + ...newPaginatedResponseNoTotal(data, offset, hasMore), + mode: 'keyword', + pool_saturated: false, + generation: EMPTY_GENERATION, + }); +} + +export const semanticSearchMessagesDef = { + name: 'semantic_search_messages', + description: + 'Semantic (embedding) search over each preprocessed message subject and body. ' + + 'Returns messages ranked by similarity to the query — there is no exact total, so page on has_more. ' + + 'Each hit includes matches — at most 1 best-matching embedded subject/body chunk excerpt with a score (divergence from msgvault, which returns up to 5). ' + + 'Vector char_offset and line locations may be omitted because preprocessing usually prevents exact raw-body mapping; use snippet terms with search_in_message keyword mode when navigation is needed. ' + + 'min_score filters chunk excerpts only; it does not remove or reorder ranked messages. ' + + 'Requires at least one free-text term (used to embed); filter-only queries must use search_metadata. ' + + 'Known Gmail operators (from:, subject:, etc.) apply as metadata filters only. ' + + SEARCH_METADATA_OPERATOR_DOC + ' ' + + 'mode=vector for pure semantic search or mode=hybrid to fuse BM25 and vector ranking via RRF. ' + + 'Paginate with offset/limit (default limit 20, max 50). Response: data, returned, offset, has_more, mode, pool_saturated, generation. ' + + 'total is not available; use has_more to page.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: 'Free-text query to embed (requires at least one free-text term). Gmail operators are metadata filters, not body search; combine with body terms or use search_metadata for filter-only queries.' }, + account: { type: 'string', description: 'Filter by account email address (use get_stats to list available accounts)' }, + limit: { type: 'number', description: 'Maximum results to return (default 20)' }, + offset: { type: 'number', description: 'Number of results to skip for pagination (default 0)' }, + mode: { type: 'string', enum: ['vector', 'hybrid'], description: 'Search mode: vector (semantic only) or hybrid (BM25 + vector fused via RRF). Defaults to hybrid when omitted.' }, + explain: { type: 'boolean', description: 'Include per-signal scores in the response (for debugging or ranking inspection)' }, + min_score: { type: 'number', description: 'Minimum chunk similarity score for included match excerpts (default 0); does not filter ranked messages' }, + }, + required: ['query'], + }, +}; + +export async function handleSemanticSearchMessages(args, scope) { + const query = (args.query || '').trim(); + if (!query) return errorResult('query parameter is required'); + + const mode = args.mode || 'hybrid'; + if (mode !== 'vector' && mode !== 'hybrid') { + return errorResult( + `invalid mode "${mode}": must be vector or hybrid (default hybrid); use search_message_bodies for keyword search`, + ); + } + + const parsed = parseQuery(query); + // msgvault ordering (handlers.go:552-558): parse errors, then unsupported + // operators, before the free-text requirement. + const parseErr = queryParseErrorMessage(parsed); + if (parseErr) return errorResult(parseErr); + const unsupportedMsg = unsupportedSearchOperatorMessage(parsed); + if (unsupportedMsg) return errorResult(unsupportedMsg); + const terms = freeTextTerms(parsed); + if (!terms.length) { + return errorResult( + `missing_free_text: mode=${mode} requires at least one free-text term; use search_metadata for filter-only queries`, + ); + } + + const limit = searchLimitArg(args); + const offset = offsetArg(args); + // Offsets past the ranked window cannot be served — reject them instead of + // returning silently-empty pages (msgvault handlers.go:656-663 wording). + if (offset >= HYBRID_RANKING_WINDOW) { + return errorResult( + `pagination_limit: offset ${offset} exceeds hybrid ranking window (max ${HYBRID_RANKING_WINDOW}); ` + + 'use search_metadata or search_message_bodies for deeper pagination', + ); + } + const explain = args.explain === true; + const minScore = Number.isFinite(Number(args.min_score)) ? Number(args.min_score) : 0; + + const acc = await resolveAccountScope(args.account, scope.accountIds); + if (acc.error) return errorResult(acc.error); + + // strictVector: the seam rethrows VectorUnavailableError (msgvault taxonomy) + // instead of the REST silent-lexical fallback. + let result; + try { + result = await search({ + mode, scope: 'body', strictVector: true, + rawQuery: query, parsed, + accountIds: acc.accountIds, + limit, offset, explain, minScore, + }); + } catch (err) { + if (err.name === 'VectorUnavailableError') return errorResult(translateVectorError(err.reason)); + if (err.name === 'MissingFreeTextError') { + // The seam applies stricter term hygiene (sub-2-char / punctuation-only + // tokens never embed) than the raw-terms pre-check above; surface the + // same msgvault wording (handlers.go:631-635) instead of letting it + // escape the tool call as `internal error: missing_free_text`. + return errorResult( + `missing_free_text: mode=${mode} requires at least one free-text term; use search_metadata for filter-only queries`, + ); + } + throw err; + } + + // Phase 4 surfaces best_chunk per hit (chunk_index + code-point char_start/char_end + // into preprocessed text + score). Phase 5 owns the snippet + raw-body byte offsets: + // build the wire matches[] (≤1 excerpt — documented divergence from msgvault's ≤5) + // from best_chunk via chunkmatch.matchFromChunk. + const byId = await hydrateSummaries(result.messages, acc.accountIds); + const data = (await Promise.all((result.messages || []).map(async (m) => { + const summary = byId.get(m.id); + if (!summary) return null; + const item = { ...summary }; + if (m.best_chunk) { + const match = await matchFromChunk(m.id, m.best_chunk, { accountIds: acc.accountIds }); + if (match && match.score >= minScore) item.matches = [match]; + } + if (explain && m.score) item.score = wireScore(m.score, mode); + return item; + }))).filter(Boolean); + + return jsonResult({ + ...newPaginatedResponseNoTotal(data, offset, result.page?.hasMore || false), + mode, + pool_saturated: result.pool_saturated || false, + generation: result.generation || EMPTY_GENERATION, + }); +} diff --git a/backend/src/mcp/searchTools.test.js b/backend/src/mcp/searchTools.test.js new file mode 100644 index 00000000..367da772 --- /dev/null +++ b/backend/src/mcp/searchTools.test.js @@ -0,0 +1,390 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../services/search/queryParser.js', () => ({ parseQuery: vi.fn() })); +vi.mock('../services/search/searchService.js', () => ({ search: vi.fn() })); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchFromChunk: vi.fn(), matchesInMessage: vi.fn() })); +// The search seam returns raw rows; MCP hydrates hit ids into MessageSummary shape. +vi.mock('./engineAdapter.js', () => ({ getMessageSummariesByIDs: vi.fn(), resolveAccountScope: vi.fn() })); +import { parseQuery } from '../services/search/queryParser.js'; +import { search } from '../services/search/searchService.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; +import { ALL_SCOPES } from './auth.js'; +import { handleSearchMetadata, handleSearchMessageBodies, handleSemanticSearchMessages } from './searchTools.js'; + +class VectorUnavailableError extends Error { + constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } +} + +const scope = { userId: 'u1', accountIds: ['acc-1'], scopes: ALL_SCOPES }; +function payload(r) { return JSON.parse(r.content[0].text); } +// Hydration echoes each requested id as a minimal summary unless a test overrides it. +function echoSummaries(rows) { + getMessageSummariesByIDs.mockImplementation(async (ids) => ids.map((id) => rows.find((r) => r.id === id)).filter(Boolean)); +} + +beforeEach(() => { + parseQuery.mockReset(); search.mockReset(); matchFromChunk.mockReset(); getMessageSummariesByIDs.mockReset(); + resolveAccountScope.mockReset(); + // Default: no `account` narrowing — pass the token's full scope through. + resolveAccountScope.mockImplementation(async (account, ids) => ({ accountIds: ids })); +}); + +describe('search_metadata', () => { + it('requires a query', async () => { + const r = await handleSearchMetadata({}, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('query parameter is required'); + }); + + it('hydrates hits into the msgvault envelope with a real total, scoped to the token accounts', async () => { + parseQuery.mockReturnValue({ filters: [{ key: 'from', value: 'alice' }], terms: [], unsupported: [] }); + search.mockResolvedValue({ + messages: [{ id: 'm1' }], // raw seam row — only the id is load-bearing for hydration + total: 5, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: true }, + }); + echoSummaries([{ id: 'm1', subject: 'Hi', to: [{ Email: 'c@d.com', Name: 'C' }] }]); + const r = await handleSearchMetadata({ query: 'from:alice' }, scope); + const body = payload(r); + expect(body).toEqual({ data: [{ id: 'm1', subject: 'Hi', to: [{ Email: 'c@d.com', Name: 'C' }] }], total: 5, returned: 1, offset: 0, has_more: true }); + // scope + parsed handed to the seam; MCP never builds SQL; hydration is scoped + expect(search).toHaveBeenCalledWith(expect.objectContaining({ mode: 'lexical', scope: 'metadata', accountIds: ['acc-1'], limit: 20, offset: 0 })); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-1']); + }); + + it('clamps limit to 50', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + search.mockResolvedValue({ messages: [], total: 0, mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + getMessageSummariesByIDs.mockResolvedValue([]); + await handleSearchMetadata({ query: 'x', limit: 999 }, scope); + expect(search.mock.calls[0][0].limit).toBe(50); + }); + + it('narrows scope to a resolved account id (the account arg is no longer a silent no-op)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + search.mockResolvedValue({ messages: [{ id: 'm1' }], total: 1, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + echoSummaries([{ id: 'm1', subject: 'Hi' }]); + await handleSearchMetadata({ query: 'x', account: 'work@x.com' }, scope); + expect(resolveAccountScope).toHaveBeenCalledWith('work@x.com', ['acc-1']); + expect(search.mock.calls[0][0].accountIds).toEqual(['acc-2']); // narrowed, not the full token scope + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-2']); // hydration narrowed too + }); + + it('rejects an unknown account without querying (msgvault getAccountID parity)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleSearchMetadata({ query: 'x', account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('account not found: nope@x.com'); + expect(search).not.toHaveBeenCalled(); + }); + + it('returns parser value errors verbatim instead of silently widening (msgvault q.Err, handlers.go:373-375)', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [], unsupported: [], + errors: ['invalid value "xyz" for older_than: — expected a relative age like 7d, 2w, 1m, or 1y'], + }); + const r = await handleSearchMetadata({ query: 'older_than:xyz' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('invalid value "xyz" for older_than: — expected a relative age like 7d, 2w, 1m, or 1y'); + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects schema-unsupported operators with the unsupported_search_operator taxonomy (handlers.go:408-428)', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [{ value: 'x', negate: false }], + unsupported: [{ key: 'label', token: 'label:promos' }, { key: 'larger', token: 'larger:5M' }], + errors: [], + }); + const r = await handleSearchMetadata({ query: 'x label:promos larger:5M' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: label: /); + expect(r.content[0].text).toContain('larger:'); + expect(search).not.toHaveBeenCalled(); + }); + + it('treats literal list:/list-id: terms as unsupported (msgvault parser set, parser.go:270-273, hoisted to the handler)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'list-id:foo', negate: false }], unsupported: [], errors: [] }); + const r = await handleSearchMetadata({ query: 'list-id:foo' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: list-id: is Gmail-only syntax/); + expect(search).not.toHaveBeenCalled(); + }); + + it('always carries a numeric total, even when the seam omits it (degenerate all-terms-dropped query)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'a', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); // no total key + getMessageSummariesByIDs.mockResolvedValue([]); + const body = payload(await handleSearchMetadata({ query: 'a' }, scope)); + expect(body.total).toBe(0); + expect(body.has_more).toBe(false); + }); + + it('emits sent_at as RFC3339 without milliseconds (Go wire format)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ messages: [{ id: 'm1' }], total: 1, mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + echoSummaries([{ id: 'm1', sent_at: '2024-01-01T00:00:00.000Z' }]); + const body = payload(await handleSearchMetadata({ query: 'x' }, scope)); + expect(body.data[0].sent_at).toBe('2024-01-01T00:00:00Z'); + }); +}); + +describe('search_message_bodies', () => { + it('rejects filter-only queries (no free-text term)', async () => { + parseQuery.mockReturnValue({ filters: [{ key: 'from', value: 'alice' }], terms: [], unsupported: [] }); + const r = await handleSearchMessageBodies({ query: 'from:alice' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toContain('requires at least one free-text term'); + }); + + it('returns keyword envelope: total -1, mode keyword, empty generation, snippet-only matches on a hydrated summary', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + search.mockResolvedValue({ + messages: [{ id: 'm1', body_text: 'the budget is approved. budget notes.' }], + mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false }, + }); + echoSummaries([{ id: 'm1', subject: 'Q3', from_email: 'a@b.com' }]); + const r = await handleSearchMessageBodies({ query: 'budget' }, scope); + const body = JSON.parse(r.content[0].text); + expect(body.total).toBe(-1); + expect(body.mode).toBe('keyword'); + expect(body.pool_saturated).toBe(false); + expect(body.generation).toEqual({ id: 0, model: '', dimension: 0, fingerprint: '', state: '' }); + expect(body.data[0].subject).toBe('Q3'); // hydrated summary, not the raw row + expect(body.data[0]).not.toHaveProperty('body_text'); // body is not leaked to the wire + expect(body.data[0].matches[0]).toHaveProperty('snippet'); + expect(body.data[0].matches[0]).not.toHaveProperty('char_offset'); // keyword body = snippet only + }); + + it('narrows scope to a resolved account id', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + search.mockResolvedValue({ messages: [{ id: 'm1', body_text: 'budget' }], mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + echoSummaries([{ id: 'm1', subject: 'Q3' }]); + await handleSearchMessageBodies({ query: 'budget', account: 'work@x.com' }, scope); + expect(search.mock.calls[0][0].accountIds).toEqual(['acc-2']); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-2']); + }); + + it('rejects an unknown account', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleSearchMessageBodies({ query: 'budget', account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects explicit mode=vector/hybrid — keyword-only tool (msgvault handlers.go:447-452)', async () => { + for (const mode of ['vector', 'hybrid']) { + const r = await handleSearchMessageBodies({ query: 'budget', mode }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe( + `invalid mode "${mode}": search_message_bodies is keyword-only; use semantic_search_messages for vector or hybrid search`, + ); + } + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects unknown modes with the only-supports wording (msgvault handlers.go:453-457)', async () => { + const r = await handleSearchMessageBodies({ query: 'budget', mode: 'fuzzy' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe( + 'invalid mode "fuzzy": search_message_bodies only supports keyword search; use semantic_search_messages for vector or hybrid search', + ); + }); + + it('accepts an explicit mode=keyword', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false } }); + getMessageSummariesByIDs.mockResolvedValue([]); + const r = await handleSearchMessageBodies({ query: 'budget', mode: 'keyword' }, scope); + expect(r.isError).toBeFalsy(); + }); + + it('returns parser value errors and unsupported operators before the free-text check (msgvault ordering)', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [], unsupported: [{ key: 'bcc', token: 'bcc:x@y.com' }], errors: [], + }); + const r = await handleSearchMessageBodies({ query: 'bcc:x@y.com' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^unsupported_search_operator: bcc: /); + + parseQuery.mockReturnValue({ + filters: [], terms: [], unsupported: [], + errors: ['invalid value "5X" for smaller: — expected a size like 5M, 100K, or 1G'], + }); + const r2 = await handleSearchMessageBodies({ query: 'smaller:5X' }, scope); + expect(r2.content[0].text).toBe('invalid value "5X" for smaller: — expected a size like 5M, 100K, or 1G'); + expect(search).not.toHaveBeenCalled(); + }); + + it('OMITS matches/matches_truncated when empty/false and emits matches_truncated past 5 excerpts (Go omitempty, handlers.go:339-341)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }); + // m1: term absent from the body → no snippets; m2: 7 occurrences spaced + // past the 300-byte context window (no merge) → 5 capped + truncated. + const spread = Array.from({ length: 7 }, () => 'budget').join(' filler'.repeat(120)); + search.mockResolvedValue({ + messages: [{ id: 'm1', body_text: 'nothing relevant here' }, { id: 'm2', body_text: spread }], + mode: 'lexical', page: { offset: 0, limit: 20, hasMore: false }, + }); + echoSummaries([{ id: 'm1', subject: 'A' }, { id: 'm2', subject: 'B' }]); + const body = payload(await handleSearchMessageBodies({ query: 'budget' }, scope)); + expect(body.data[0]).not.toHaveProperty('matches'); + expect(body.data[0]).not.toHaveProperty('matches_truncated'); + expect(body.data[1].matches).toHaveLength(5); + expect(body.data[1].matches_truncated).toBe(true); + }); +}); + +describe('semantic_search_messages', () => { + it('rejects filter-only queries with missing_free_text', async () => { + parseQuery.mockReturnValue({ filters: [{ key: 'from', value: 'alice' }], terms: [], unsupported: [] }); + const r = await handleSemanticSearchMessages({ query: 'from:alice' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/^missing_free_text: mode=hybrid/); + }); + + it('rejects mode=keyword', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [] }); + const r = await handleSemanticSearchMessages({ query: 'x', mode: 'keyword' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toMatch(/invalid mode "keyword"/); + }); + + it('returns vector_not_enabled when the seam is strict-unavailable', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + search.mockRejectedValue(new VectorUnavailableError('vector_not_enabled')); + const r = await handleSemanticSearchMessages({ query: 'travel plans' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('vector_not_enabled: vector search is not configured on this server'); + }); + + it('builds matches from best_chunk via matchFromChunk on a hydrated summary, with mode/pool_saturated/generation and per-signal scores when explain', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + matchFromChunk.mockResolvedValue({ char_offset: 42, snippet: 'flights', line: 3, score: 0.9 }); + search.mockResolvedValue({ + // Real hybrid/vector seam shape: keyed on message_id, with the additive + // `id` alias searchService now emits (mode-invariant with the lexical path). + messages: [{ message_id: 'm1', id: 'm1', + best_chunk: { chunk_index: 2, char_start: 100, char_end: 180, score: 0.9 }, + score: { rrf: 0.5, bm25: 1.2, vector: 0.8, subject_boosted: true } }], + mode: 'hybrid', page: { offset: 0, limit: 20, hasMore: false }, + pool_saturated: true, + generation: { id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const r = await handleSemanticSearchMessages({ query: 'travel plans', explain: true }, scope); + const body = JSON.parse(r.content[0].text); + expect(body.total).toBe(-1); + expect(body.mode).toBe('hybrid'); + expect(body.pool_saturated).toBe(true); + expect(body.generation).toEqual({ id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }); + expect(body.data[0].subject).toBe('Trip'); // hydrated summary + expect(body.data[0].score).toEqual({ rrf: 0.5, bm25: 1.2, vector: 0.8, subject_boosted: true }); + expect(body.data[0].matches).toEqual([{ char_offset: 42, snippet: 'flights', line: 3, score: 0.9 }]); + expect(body.data[0]).not.toHaveProperty('best_chunk'); + expect(matchFromChunk).toHaveBeenCalledWith('m1', { chunk_index: 2, char_start: 100, char_end: 180, score: 0.9 }, { accountIds: scope.accountIds }); + }); + + it('drops the excerpt when its score is below min_score (ranking unaffected)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + matchFromChunk.mockResolvedValue({ snippet: 'weak', score: 0.1 }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', best_chunk: { chunk_index: 0, char_start: 10, char_end: 20, score: 0.1 } }], + mode: 'vector', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const r = await handleSemanticSearchMessages({ query: 'travel', mode: 'vector', min_score: 0.5 }, scope); + const body = JSON.parse(r.content[0].text); + expect(body.data[0]).not.toHaveProperty('matches'); // excerpt dropped, message still returned + expect(body.mode).toBe('vector'); + }); + + it('narrows scope to a resolved account id (search + hydration + matchFromChunk)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ accountIds: ['acc-2'] }); + matchFromChunk.mockResolvedValue({ snippet: 'x', score: 0.9 }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', best_chunk: { chunk_index: 0, char_start: 0, char_end: 5, score: 0.9 } }], + mode: 'hybrid', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + await handleSemanticSearchMessages({ query: 'travel', account: 'work@x.com' }, scope); + expect(search.mock.calls[0][0].accountIds).toEqual(['acc-2']); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['m1'], ['acc-2']); + expect(matchFromChunk).toHaveBeenCalledWith('m1', expect.any(Object), { accountIds: ['acc-2'] }); + }); + + it('rejects an unknown account', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [] }); + resolveAccountScope.mockResolvedValue({ error: 'account not found: nope@x.com' }); + const r = await handleSemanticSearchMessages({ query: 'travel', account: 'nope@x.com' }, scope); + expect(r.isError).toBe(true); + expect(search).not.toHaveBeenCalled(); + }); + + it('returns parser value errors verbatim and unsupported operators as taxonomy errors', async () => { + parseQuery.mockReturnValue({ + filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [], + errors: ['invalid value "13" for newer_than: — expected a relative age like 7d, 2w, 1m, or 1y'], + }); + const r = await handleSemanticSearchMessages({ query: 'travel newer_than:13' }, scope); + expect(r.content[0].text).toBe('invalid value "13" for newer_than: — expected a relative age like 7d, 2w, 1m, or 1y'); + + parseQuery.mockReturnValue({ + filters: [], terms: [{ value: 'travel', negate: false }], + unsupported: [{ key: 'smaller', token: 'smaller:1M' }], errors: [], + }); + const r2 = await handleSemanticSearchMessages({ query: 'travel smaller:1M' }, scope); + expect(r2.content[0].text).toMatch(/^unsupported_search_operator: smaller: /); + expect(search).not.toHaveBeenCalled(); + }); + + it('rejects offsets past the hybrid ranking window with pagination_limit (msgvault handlers.go:656-663)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'x', negate: false }], unsupported: [], errors: [] }); + const r = await handleSemanticSearchMessages({ query: 'x', offset: 100 }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe( + 'pagination_limit: offset 100 exceeds hybrid ranking window (max 100); use search_metadata or search_message_bodies for deeper pagination', + ); + expect(search).not.toHaveBeenCalled(); + }); + + it('maps a seam MissingFreeTextError to the missing_free_text result — never `internal error:` (msgvault handlers.go:631-635)', async () => { + // 'a' survives the handler's raw-terms pre-check, but the seam's stricter + // hygiene (sub-2-char tokens do not embed) throws MissingFreeTextError. + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'a', negate: false }], unsupported: [], errors: [] }); + class MissingFreeTextError extends Error { constructor() { super('missing_free_text'); this.name = 'MissingFreeTextError'; } } + search.mockRejectedValue(new MissingFreeTextError()); + const r = await handleSemanticSearchMessages({ query: 'a' }, scope); + expect(r.isError).toBe(true); + expect(r.content[0].text).toBe('missing_free_text: mode=hybrid requires at least one free-text term; use search_metadata for filter-only queries'); + }); + + it('explain omits rrf in mode=vector and subject_boosted when false (Go omitempty, handlers.go:563-572)', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', score: { rrf: 0.5, vector: 0.8, subject_boosted: false } }], + mode: 'vector', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const body = payload(await handleSemanticSearchMessages({ query: 'travel', mode: 'vector', explain: true }, scope)); + expect(body.data[0].score).toEqual({ vector: 0.8 }); // no rrf (one signal), no false subject_boosted + }); + + it('explain keeps rrf in mode=hybrid but still omits a false subject_boosted', async () => { + parseQuery.mockReturnValue({ filters: [], terms: [{ value: 'travel', negate: false }], unsupported: [], errors: [] }); + search.mockResolvedValue({ + messages: [{ message_id: 'm1', id: 'm1', score: { rrf: 0.5, bm25: 1.2, vector: 0.8, subject_boosted: false } }], + mode: 'hybrid', page: { offset: 0, limit: 20, hasMore: false }, pool_saturated: false, + generation: { id: 1, model: 'm', dimension: 2, fingerprint: 'fp', state: 'active' }, + }); + echoSummaries([{ id: 'm1', subject: 'Trip' }]); + const body = payload(await handleSemanticSearchMessages({ query: 'travel', explain: true }, scope)); + expect(body.data[0].score).toEqual({ rrf: 0.5, bm25: 1.2, vector: 0.8 }); + }); +}); diff --git a/backend/src/mcp/semanticSearchIds.regression.test.js b/backend/src/mcp/semanticSearchIds.regression.test.js new file mode 100644 index 00000000..a71a815a --- /dev/null +++ b/backend/src/mcp/semanticSearchIds.regression.test.js @@ -0,0 +1,89 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Regression for the live MCP bug: `semantic_search_messages` returned 0 because +// hybrid/vector seam hits carry `message_id` (fusedSearch DISPLAY_COLS), not `id`, +// so the handler's `byId.get(m.id)` hydration mapped every hit to null. +// +// Unlike searchTools.test.js (which mocks the searchService seam directly), this +// test drives the REAL seam and mocks only the DEEPER dependency (hybridSearch) +// with the REAL fused-row field inventory the running container exposed +// (message_id/uid/folder/…/rrf_score/best_char_*). That exercises the actual +// `id`-alias fix in searchService plus the handler's hydration + excerpt + explain +// assembly end-to-end: it fails (returned:0) without the seam fix and passes with it. +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn(), pool: {} })); +vi.mock('../services/embeddings/hybrid.js', () => ({ + hybridSearch: vi.fn(), + isLexicalFallback: () => false, + MissingFreeTextError: class MissingFreeTextError extends Error {}, + VectorUnavailableError: class VectorUnavailableError extends Error { + constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } + }, + resolveActiveGeneration: vi.fn(), +})); +vi.mock('../services/embeddings/chunkmatch.js', () => ({ matchFromChunk: vi.fn(), matchesInMessage: vi.fn() })); +vi.mock('./engineAdapter.js', () => ({ getMessageSummariesByIDs: vi.fn(), resolveAccountScope: vi.fn() })); + +import { hybridSearch } from '../services/embeddings/hybrid.js'; +import { matchFromChunk } from '../services/embeddings/chunkmatch.js'; +import { getMessageSummariesByIDs, resolveAccountScope } from './engineAdapter.js'; +import { handleSemanticSearchMessages } from './searchTools.js'; + +const scope = { userId: 'u1', accountIds: ['acc-1'] }; +const payload = (r) => JSON.parse(r.content[0].text); + +// The exact key inventory the deployed container returned for a hybrid/vector hit — +// keyed on message_id, with NO `id` (that is the bug the seam fix repairs). +function realFusedHit(overrides = {}) { + return { + message_id: 'uuid-1', uid: 42, folder: 'INBOX', + subject: 'Run failed: CD deploy', from_name: 'GitHub', from_email: 'notifications@github.com', + date: new Date('2026-07-15T00:00:00Z'), snippet: 'deployment failing', is_read: false, + is_starred: false, has_attachments: false, account_id: 'acc-1', + account_name: 'Work', account_email: 'me@work.com', account_color: '#fff', + rrf_score: 0.033, bm25_score: 1.2, vector_score: 0.88, subject_boosted: false, + best_chunk_index: 0, best_char_start: 5, best_char_end: 40, + ...overrides, + }; +} + +beforeEach(() => { + hybridSearch.mockReset(); matchFromChunk.mockReset(); + getMessageSummariesByIDs.mockReset(); resolveAccountScope.mockReset(); + resolveAccountScope.mockImplementation(async (account, ids) => ({ accountIds: ids })); + // Hydration echoes each requested id back as a minimal summary. + getMessageSummariesByIDs.mockImplementation(async (ids) => + ids.map((id) => ({ id, subject: 'Run failed: CD deploy', from_email: 'notifications@github.com' }))); +}); + +describe('semantic_search_messages end-to-end over the real seam (message_id → id alias)', () => { + for (const mode of ['hybrid', 'vector']) { + it(`mode=${mode}: hydrates real message_id-keyed seam hits into data (was returned:0)`, async () => { + hybridSearch.mockResolvedValue({ + hits: [realFusedHit()], + poolSaturated: false, + generation: { id: 3, model: 'text-embedding-3-small', dimension: 1536, fingerprint: 'fp', state: 'active' }, + }); + matchFromChunk.mockResolvedValue({ char_offset: 5, snippet: 'deploy', score: 0.88 }); + + const r = await handleSemanticSearchMessages({ query: 'server deployment failing', mode, explain: true }, scope); + const body = payload(r); + + expect(r.isError).toBeFalsy(); + expect(body.returned).toBe(1); // the bug returned 0 + expect(body.mode).toBe(mode); + expect(body.data[0].id).toBe('uuid-1'); // hydrated via the aliased id + expect(body.data[0].subject).toBe('Run failed: CD deploy'); + // explain surfaces the real per-signal fields with Go omitempty parity + // (msgvault handlers.go:563-572): rrf only fuses in hybrid, and a false + // subject_boosted is omitted from the wire. + expect(body.data[0].score).toEqual( + mode === 'hybrid' ? { rrf: 0.033, bm25: 1.2, vector: 0.88 } : { bm25: 1.2, vector: 0.88 }, + ); + // matches assembled from best_chunk, keyed on the aliased id + expect(body.data[0].matches).toEqual([{ char_offset: 5, snippet: 'deploy', score: 0.88 }]); + expect(getMessageSummariesByIDs).toHaveBeenCalledWith(['uuid-1'], ['acc-1']); + expect(matchFromChunk).toHaveBeenCalledWith( + 'uuid-1', { chunk_index: 0, char_start: 5, char_end: 40, score: 0.88 }, { accountIds: ['acc-1'] }); + }); + } +}); diff --git a/backend/src/mcp/sendTools.js b/backend/src/mcp/sendTools.js new file mode 100644 index 00000000..88a70e91 --- /dev/null +++ b/backend/src/mcp/sendTools.js @@ -0,0 +1,484 @@ +import { + deleteMessageRow, + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + getOutboxRowByMessageId, + getUserPreferences, +} from './accountAdapter.js'; +import { newPaginatedResponseNoTotal, toRFC3339 } from './envelope.js'; +import { errorResult, jsonResult } from './result.js'; +import { buildWriteReceipt, writeError } from './writeResult.js'; +import { normalizeRecipients } from '../services/mail/addresses.js'; +import { resolveFromIdentity } from '../services/mail/identity.js'; + +const annotations = (readOnlyHint, destructiveHint, idempotentHint) => ({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, +}); + +const attachmentSchema = { + type: 'array', + items: { + type: 'object', + required: ['filename', 'content'], + properties: { + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }, +}; + +const undoSendSecondsSchema = { + type: 'integer', + minimum: 0, + maximum: 120, + description: "Cancellation window in seconds (max 120). Defaults to the user's undo-send preference.", +}; + +const idempotencyKeySchema = { + type: 'string', + description: 'Stable key; a retry with the same key returns the original result instead of sending twice.', +}; + +export const sendEmailDef = { + name: 'send_email', + description: 'Send an email. When undo_send_seconds > 0 the message is QUEUED and can be cancelled with unsend_email until send_at; when 0 it is delivered immediately. Returns a receipt of exactly what was sent.', + inputSchema: { + type: 'object', + required: ['account', 'to'], + properties: { + account: { type: 'string' }, + to: { type: 'array', items: { type: 'string' } }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string' }, + body_html: { type: 'string' }, + alias: { + type: 'string', + description: 'Send-as alias email; must be configured on the account (hard error if not)', + }, + priority: { type: 'string', enum: ['high', 'normal', 'low'] }, + attachments: attachmentSchema, + undo_send_seconds: undoSendSecondsSchema, + idempotency_key: idempotencyKeySchema, + }, + }, + annotations: annotations(false, true, false), +}; + +export const sendDraftDef = { + name: 'send_draft', + description: 'Reads the draft via `getDraftRow`, reconstructs the compose input (recipients from `to_addresses`/`cc_addresses`, body from `body_text`/`body_html`, threading from `in_reply_to`/`thread_references`), sends, and **deletes the draft only after delivery succeeds** (or after enqueue succeeds, with a `delete_draft_on_send` flag on the outbox payload the worker honors). Errors `draft_not_found` when the uid is absent or not a draft.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + account: { type: 'string' }, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + undo_send_seconds: undoSendSecondsSchema, + idempotency_key: idempotencyKeySchema, + }, + }, + annotations: annotations(false, true, false), +}; + +export const unsendEmailDef = { + name: 'unsend_email', + description: 'Cancel a queued email before it is delivered. Only works while the message is still in its undo window (see the send_at returned by send_email).', + inputSchema: { + type: 'object', + required: ['outbox_id'], + properties: { + outbox_id: { type: 'string' }, + }, + }, + annotations: annotations(false, false, true), +}; + +export const listOutboxDef = { + name: 'list_outbox', + description: 'List emails still queued in the undo-send outbox. Entries can be cancelled with unsend_email before send_at.', + inputSchema: { + type: 'object', + properties: {}, + }, + annotations: annotations(true, false, true), +}; + +export const recallEmailDef = { + name: 'recall_email', + description: "Best-effort recall of an already-sent message. SMTP CANNOT retract delivered mail — recipients already have it. This tool (1) cancels the send if it is still queued, otherwise (2) deletes your Sent copy and (3) prepares a 'please disregard' follow-up DRAFT addressed to the original recipients, which it never sends automatically.", + inputSchema: { + type: 'object', + properties: { + message_id: { + type: 'string', + description: 'Id of the sent message (from search or the Sent folder)', + }, + outbox_id: { + type: 'string', + description: "Alternative: a queued message's outbox id", + }, + delete_sent_copy: { type: 'boolean', default: true }, + draft_followup: { type: 'boolean', default: true }, + followup_note: { + type: 'string', + description: "Body of the follow-up draft; defaults to a short 'please disregard' note.", + }, + }, + }, + annotations: annotations(false, true, false), +}; + +function attachmentInput(attachment) { + return { + filename: attachment.filename, + content: attachment.content, + contentType: attachment.content_type ?? attachment.contentType, + }; +} + +function addressString(address) { + if (typeof address === 'string') return address; + if (!address?.email) return ''; + return address.name ? `${address.name} <${address.email}>` : address.email; +} + +function addressStrings(value) { + return (Array.isArray(value) ? value : []).map(addressString).filter(Boolean); +} + +function normalizedRecipients(args) { + try { + const normalized = { + to: normalizeRecipients(args.to, 'to'), + cc: normalizeRecipients(args.cc || [], 'cc'), + bcc: normalizeRecipients(args.bcc || [], 'bcc'), + }; + if (normalized.to.length + normalized.cc.length + normalized.bcc.length > 100) { + throw Object.assign(new Error('Too many recipients (max 100)'), { + code: 'too_many_recipients', + }); + } + return normalized; + } catch (err) { + err.code ||= 'invalid_recipient'; + throw err; + } +} + +function sendResult(result, subject) { + if (result.queued) { + return jsonResult(buildWriteReceipt({ subject }, { + queued: true, + outboxId: result.outboxId, + sendAt: result.sendAt, + undoSeconds: result.undoSeconds, + note: 'Cancel with unsend_email before send_at.', + })); + } + return jsonResult(buildWriteReceipt(result.receipt, { sent: true })); +} + +function errorFrom(err) { + if (err?.code) return writeError(err.code, err.message); + return writeError('invalid_arguments', err?.message || 'send failed'); +} + +export async function handleSendEmail(args, scope, deps = {}) { + if (!deps.sendService || !deps.outboxService?.normalizeUndoWindow) { + return writeError('unsupported', 'send tools require sendService and outboxService'); + } + try { + const account = await getAccountByEmail(args.account, scope.accountIds); + if (account?.error) return errorResult(account.error); + let identity; + if (args.alias) { + identity = await resolveFromIdentity(account, { aliasEmail: args.alias }, deps); + } + const recipients = normalizedRecipients(args); + const preferences = await getUserPreferences(scope.userId); + const undoSeconds = deps.outboxService.normalizeUndoWindow( + args.undo_send_seconds, + preferences.undoSendSeconds, + ); + const bodyIsHtml = args.body_html !== undefined; + const result = await deps.sendService.sendOrEnqueue({ + userId: scope.userId, + account, + aliasId: identity?.aliasId, + aliasEmail: args.alias, + ...recipients, + subject: args.subject || '', + body: bodyIsHtml ? args.body_html : (args.body || ''), + bodyIsHtml, + priority: args.priority, + attachments: (args.attachments || []).map(attachmentInput), + undoSeconds, + idempotencyKey: args.idempotency_key, + }, deps); + return sendResult(result, args.subject || ''); + } catch (err) { + return errorFrom(err); + } +} + +export async function handleSendDraft(args, scope, deps = {}) { + if ( + !deps.sendService || + !deps.draftService || + !deps.outboxService?.normalizeUndoWindow + ) { + return writeError( + 'unsupported', + 'send_draft requires sendService, draftService, and outboxService', + ); + } + try { + const account = await getAccountByEmail(args.account, scope.accountIds); + if (account?.error) return errorResult(account.error); + const folder = args.folder || account?.folder_mappings?.drafts || 'Drafts'; + const draft = await getDraftRow(account.id, folder, args.draft_uid); + if (!draft) return writeError('draft_not_found', args.draft_uid); + const aliasEmail = draft.from_email && draft.from_email !== account.email_address + ? draft.from_email + : undefined; + const identity = aliasEmail + ? await resolveFromIdentity(account, { aliasEmail }, deps) + : undefined; + const preferences = await getUserPreferences(scope.userId); + const undoSeconds = deps.outboxService.normalizeUndoWindow( + args.undo_send_seconds, + preferences.undoSendSeconds, + ); + const bodyIsHtml = Boolean(draft.body_html); + const result = await deps.sendService.sendOrEnqueue({ + userId: scope.userId, + account, + aliasId: identity?.aliasId, + aliasEmail, + to: addressStrings(draft.to_addresses), + cc: addressStrings(draft.cc_addresses), + bcc: addressStrings(draft.bcc_addresses), + subject: draft.subject || '', + body: bodyIsHtml ? draft.body_html : (draft.body_text || ''), + bodyIsHtml, + inReplyTo: draft.in_reply_to || undefined, + references: draft.thread_references || undefined, + undoSeconds, + idempotencyKey: args.idempotency_key, + ...(undoSeconds + ? { deleteDraftOnSend: { uid: draft.uid, folder: draft.folder } } + : {}), + }, deps); + if (!result.queued) { + try { + await deps.draftService.deleteDraft({ + account, + uid: draft.uid, + folder: draft.folder, + }, deps); + } catch (err) { + console.error('Draft cleanup after send failed:', err.message); + } + } + return sendResult(result, draft.subject || ''); + } catch (err) { + return errorFrom(err); + } +} + +function cancelledOutbox(outboxId, row) { + return { + cancelled: true, + outbox_id: outboxId, + subject: row?.subject || '', + to: row?.to_preview || [], + }; +} + +function recalledOutbox(outboxId, row) { + return { + recalled: 'cancelled_before_send', + outbox_id: outboxId, + subject: row?.subject || '', + to: row?.to_preview || [], + }; +} + +function cancelFailure(outboxId, reason) { + if (reason === 'already_sent') { + return writeError( + 'already_sent', + `message ${outboxId} is no longer pending and cannot be unsent`, + ); + } + return writeError('outbox_not_found', outboxId); +} + +async function pendingOutboxById(outboxId, scope, deps) { + if (!deps.outboxService?.listPending) return null; + const rows = await deps.outboxService.listPending({ userId: scope.userId }, deps); + return rows.find(row => row.id === outboxId) || null; +} + +async function cancelOutbox(outboxId, row, scope, deps, resultBuilder) { + const result = await deps.outboxService.cancel({ + id: outboxId, + userId: scope.userId, + }, deps); + if (result.cancelled || result.reason === 'cancelled') { + return jsonResult(resultBuilder(outboxId, row)); + } + return cancelFailure(outboxId, result.reason); +} + +export async function handleUnsendEmail(args, scope, deps = {}) { + if (!deps.outboxService?.cancel) { + return writeError('unsupported', 'unsend_email requires outboxService'); + } + try { + const row = await pendingOutboxById(args.outbox_id, scope, deps); + return await cancelOutbox( + args.outbox_id, + row, + scope, + deps, + cancelledOutbox, + ); + } catch (err) { + return errorFrom(err); + } +} + +function wireOutboxRow(row) { + const sendAt = row?.send_at instanceof Date + ? row.send_at.toISOString() + : row?.send_at; + return { + ...row, + ...(sendAt !== undefined ? { send_at: toRFC3339(sendAt) } : {}), + }; +} + +export async function handleListOutbox(_args, scope, deps = {}) { + if (!deps.outboxService?.listPending) { + return writeError('unsupported', 'list_outbox requires outboxService'); + } + try { + const rows = await deps.outboxService.listPending({ userId: scope.userId }, deps); + return jsonResult(newPaginatedResponseNoTotal( + rows.map(wireOutboxRow), + 0, + false, + )); + } catch (err) { + return errorFrom(err); + } +} + +function followupSubject(subject) { + const value = subject || ''; + return /^re:/i.test(value) ? value : `Re: ${value}`; +} + +async function recallDelivered(args, message, account, scope, deps) { + const deleteSentCopy = args.delete_sent_copy !== false; + const draftFollowup = args.draft_followup !== false; + if (deleteSentCopy && !deps.imapManager?.permanentDeleteMessage) { + return writeError('unsupported', 'recall_email requires imapManager'); + } + if (draftFollowup && !deps.draftService?.saveDraft) { + return writeError('unsupported', 'recall_email requires draftService'); + } + + if (deleteSentCopy) { + await deps.imapManager.permanentDeleteMessage( + account, + message.uid, + message.folder, + ); + await deleteMessageRow(message.account_id, message.uid, message.folder); + } + + let followupDraft; + if (draftFollowup) { + const saved = await deps.draftService.saveDraft({ + userId: scope.userId, + account, + to: addressStrings(message.to_addresses), + cc: addressStrings(message.cc_addresses), + bcc: [], + subject: followupSubject(message.subject), + body: args.followup_note ?? 'Please disregard my previous email.', + bodyIsHtml: false, + attachments: [], + inReplyTo: message.message_id, + }, deps); + followupDraft = { + draft_uid: saved.uid, + folder: saved.folder, + }; + } + + return jsonResult({ + recalled: 'not_possible', + note: 'SMTP cannot retract a delivered message. Recipients already received it; deleting your Sent copy does not affect their mailboxes.', + sent_copy_deleted: deleteSentCopy, + ...(followupDraft ? { followup_draft: followupDraft } : {}), + }); +} + +export async function handleRecallEmail(args, scope, deps = {}) { + if (!deps.outboxService?.cancel) { + return writeError('unsupported', 'recall_email requires outboxService'); + } + try { + if (!args.outbox_id && !args.message_id) { + return writeError( + 'invalid_arguments', + 'recall_email requires message_id or outbox_id', + ); + } + + if (args.outbox_id) { + const row = await pendingOutboxById(args.outbox_id, scope, deps); + return await cancelOutbox( + args.outbox_id, + row, + scope, + deps, + recalledOutbox, + ); + } + + const pending = await getOutboxRowByMessageId( + args.message_id, + scope.userId, + ); + if (pending) { + return await cancelOutbox( + pending.id, + pending, + scope, + deps, + recalledOutbox, + ); + } + + const message = await getComposeSource(args.message_id, scope.accountIds); + if (!message) return writeError('message_not_found', args.message_id); + const account = await getAccountRow(message.account_id, scope.accountIds); + if (!account) return writeError('account_not_found', message.account_id); + return recallDelivered(args, message, account, scope, deps); + } catch (err) { + return errorFrom(err); + } +} diff --git a/backend/src/mcp/sendTools.test.js b/backend/src/mcp/sendTools.test.js new file mode 100644 index 00000000..cbed9938 --- /dev/null +++ b/backend/src/mcp/sendTools.test.js @@ -0,0 +1,1024 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./accountAdapter.js', () => ({ + deleteMessageRow: vi.fn(), + getAccountByEmail: vi.fn(), + getAccountRow: vi.fn(), + getComposeSource: vi.fn(), + getDraftRow: vi.fn(), + getOutboxRowByMessageId: vi.fn(), + getUserPreferences: vi.fn(), +})); +vi.mock('../services/mail/identity.js', () => ({ + resolveFromIdentity: vi.fn(), +})); + +import { + deleteMessageRow, + getAccountByEmail, + getAccountRow, + getComposeSource, + getDraftRow, + getOutboxRowByMessageId, + getUserPreferences, +} from './accountAdapter.js'; +import * as sendTools from './sendTools.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; +import { resolveFromIdentity } from '../services/mail/identity.js'; + +const scope = { + userId: 'user-1', + accountIds: ['account-1'], + scopes: ['send'], +}; +const account = { + id: 'account-1', + email_address: 'sender@example.com', + sender_name: 'Sender', + folder_mappings: { drafts: 'Drafts' }, +}; +const identity = { + fromName: 'Sender', + fromEmail: 'sender@example.com', + fromReplyTo: null, + signature: null, + aliasId: null, +}; +const immediateReceipt = { + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + messageId: '', + sentCopySaved: true, + folder: 'Sent', +}; + +function deps(overrides = {}) { + return { + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: immediateReceipt, + }), + }, + outboxService: { + normalizeUndoWindow: vi.fn((requested, preference) => requested ?? preference ?? 0), + cancel: vi.fn().mockResolvedValue({ cancelled: true }), + listPending: vi.fn().mockResolvedValue([]), + }, + draftService: { + deleteDraft: vi.fn().mockResolvedValue({ ok: true }), + saveDraft: vi.fn().mockResolvedValue({ + uid: 57, + folder: 'Drafts', + messageId: '', + }), + }, + imapManager: { + permanentDeleteMessage: vi.fn().mockResolvedValue(undefined), + }, + ...overrides, + }; +} + +function payload(result) { + return JSON.parse(result.content[0].text); +} + +beforeEach(() => { + vi.clearAllMocks(); + getAccountByEmail.mockResolvedValue(account); + getAccountRow.mockResolvedValue(account); + getComposeSource.mockResolvedValue(null); + getOutboxRowByMessageId.mockResolvedValue(null); + getUserPreferences.mockResolvedValue({}); + deleteMessageRow.mockResolvedValue(undefined); + resolveFromIdentity.mockResolvedValue(identity); +}); + +describe('send tool definitions and registration', () => { + it('publishes the plan schemas, descriptions, annotations, scopes, and handlers', () => { + expect(sendTools.sendEmailDef).toEqual({ + name: 'send_email', + description: 'Send an email. When undo_send_seconds > 0 the message is QUEUED and can be cancelled with unsend_email until send_at; when 0 it is delivered immediately. Returns a receipt of exactly what was sent.', + inputSchema: { + type: 'object', + required: ['account', 'to'], + properties: { + account: { type: 'string' }, + to: { type: 'array', items: { type: 'string' } }, + cc: { type: 'array', items: { type: 'string' } }, + bcc: { type: 'array', items: { type: 'string' } }, + subject: { type: 'string' }, + body: { type: 'string' }, + body_html: { type: 'string' }, + alias: { + type: 'string', + description: 'Send-as alias email; must be configured on the account (hard error if not)', + }, + priority: { type: 'string', enum: ['high', 'normal', 'low'] }, + attachments: { + type: 'array', + items: { + type: 'object', + required: ['filename', 'content'], + properties: { + filename: { type: 'string' }, + content: { type: 'string', description: 'base64' }, + content_type: { type: 'string' }, + }, + }, + }, + undo_send_seconds: { + type: 'integer', + minimum: 0, + maximum: 120, + description: "Cancellation window in seconds (max 120). Defaults to the user's undo-send preference.", + }, + idempotency_key: { + type: 'string', + description: 'Stable key; a retry with the same key returns the original result instead of sending twice.', + }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, + }, + }); + + expect(sendTools.sendDraftDef).toEqual({ + name: 'send_draft', + description: 'Reads the draft via `getDraftRow`, reconstructs the compose input (recipients from `to_addresses`/`cc_addresses`, body from `body_text`/`body_html`, threading from `in_reply_to`/`thread_references`), sends, and **deletes the draft only after delivery succeeds** (or after enqueue succeeds, with a `delete_draft_on_send` flag on the outbox payload the worker honors). Errors `draft_not_found` when the uid is absent or not a draft.', + inputSchema: { + type: 'object', + required: ['account', 'draft_uid'], + properties: { + account: { type: 'string' }, + draft_uid: { type: 'integer' }, + folder: { type: 'string' }, + undo_send_seconds: { + type: 'integer', + minimum: 0, + maximum: 120, + description: "Cancellation window in seconds (max 120). Defaults to the user's undo-send preference.", + }, + idempotency_key: { + type: 'string', + description: 'Stable key; a retry with the same key returns the original result instead of sending twice.', + }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, + }, + }); + + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const [name, handler] of [ + ['send_email', sendTools.handleSendEmail], + ['send_draft', sendTools.handleSendDraft], + ]) { + expect(defs.get(name)).toBeTruthy(); + expect(TOOL_SCOPES[name]).toBe('send'); + expect(HANDLERS[name]).toBe(handler); + } + }); + + it('publishes and registers the unsend, outbox-list, and recall tools', () => { + expect(sendTools.unsendEmailDef).toEqual({ + name: 'unsend_email', + description: 'Cancel a queued email before it is delivered. Only works while the message is still in its undo window (see the send_at returned by send_email).', + inputSchema: { + type: 'object', + required: ['outbox_id'], + properties: { + outbox_id: { type: 'string' }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + }); + expect(sendTools.listOutboxDef).toEqual({ + name: 'list_outbox', + description: 'List emails still queued in the undo-send outbox. Entries can be cancelled with unsend_email before send_at.', + inputSchema: { type: 'object', properties: {} }, + annotations: { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + }); + expect(sendTools.recallEmailDef).toEqual({ + name: 'recall_email', + description: "Best-effort recall of an already-sent message. SMTP CANNOT retract delivered mail — recipients already have it. This tool (1) cancels the send if it is still queued, otherwise (2) deletes your Sent copy and (3) prepares a 'please disregard' follow-up DRAFT addressed to the original recipients, which it never sends automatically.", + inputSchema: { + type: 'object', + properties: { + message_id: { + type: 'string', + description: 'Id of the sent message (from search or the Sent folder)', + }, + outbox_id: { + type: 'string', + description: "Alternative: a queued message's outbox id", + }, + delete_sent_copy: { type: 'boolean', default: true }, + draft_followup: { type: 'boolean', default: true }, + followup_note: { + type: 'string', + description: "Body of the follow-up draft; defaults to a short 'please disregard' note.", + }, + }, + }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, + }, + }); + + const defs = new Map(TOOL_DEFS.map(def => [def.name, def])); + for (const [name, requiredScope, handler] of [ + ['unsend_email', 'send', sendTools.handleUnsendEmail], + ['list_outbox', 'read', sendTools.handleListOutbox], + ['recall_email', 'send', sendTools.handleRecallEmail], + ]) { + expect(defs.get(name)).toBeTruthy(); + expect(TOOL_SCOPES[name]).toBe(requiredScope); + expect(HANDLERS[name]).toBe(handler); + } + }); +}); + +describe('send_email', () => { + it('does not reveal an out-of-scope account and never calls the service', async () => { + const service = deps(); + getAccountByEmail.mockResolvedValue({ error: 'account_not_found: foreign@example.com' }); + + const result = await sendTools.handleSendEmail({ + account: 'foreign@example.com', + to: ['recipient@example.com'], + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account_not_found: foreign@example.com'); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('hard-fails an unknown alias and preserves its stable error code', async () => { + const service = deps(); + resolveFromIdentity.mockRejectedValue( + Object.assign(new Error('Alias not found'), { code: 'alias_not_found' }), + ); + + const result = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + alias: 'unknown@example.com', + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('alias_not_found: Alias not found'); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('maps malformed recipients to invalid_recipient before sending', async () => { + const service = deps(); + + const result = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['victim@example.com\r\nBcc: attacker@example.com'], + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toMatch(/^invalid_recipient: /); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('rejects more than 100 total recipients as too_many_recipients', async () => { + const service = deps(); + + const result = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: Array.from({ length: 99 }, (_, i) => `to-${i}@example.com`), + cc: ['copy@example.com'], + bcc: ['blind@example.com'], + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'too_many_recipients: Too many recipients (max 100)', + ); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('sends immediately with normalized input and returns the exact write receipt', async () => { + const service = deps(); + const aliasIdentity = { + ...identity, + fromName: 'Team', + fromEmail: 'team@example.com', + aliasId: 'alias-1', + }; + resolveFromIdentity.mockResolvedValue(aliasIdentity); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 60 }); + + const wireResult = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: [' Recipient '], + cc: [], + bcc: [], + subject: 'Subject', + body: 'Plain fallback', + body_html: '

Body

', + alias: 'team@example.com', + priority: 'high', + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + content_type: 'text/plain', + }], + undo_send_seconds: 0, + idempotency_key: 'send-1', + }, scope, service); + expect(wireResult.isError).not.toBe(true); + const result = payload(wireResult); + + expect(getAccountByEmail).toHaveBeenCalledWith('sender@example.com', ['account-1']); + expect(resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasEmail: 'team@example.com' }, + service, + ); + expect(getUserPreferences).toHaveBeenCalledWith('user-1'); + expect(service.outboxService.normalizeUndoWindow).toHaveBeenCalledWith(0, 60); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith({ + userId: 'user-1', + account, + aliasId: 'alias-1', + aliasEmail: 'team@example.com', + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Subject', + body: '

Body

', + bodyIsHtml: true, + priority: 'high', + attachments: [{ + filename: 'note.txt', + content: 'aGVsbG8=', + contentType: 'text/plain', + }], + undoSeconds: 0, + idempotencyKey: 'send-1', + }, service); + expect(result).toEqual({ + sent: true, + message_id: '', + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + sent_copy_saved: true, + folder: 'Sent', + }); + }); + + it('uses the preference undo window and returns the exact queued result shape', async () => { + const sendAt = new Date('2026-07-28T10:00:30.000Z'); + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + queued: true, + outboxId: 'outbox-1', + sendAt, + undoSeconds: 30, + }), + }, + }); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 30 }); + + const wireResult = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + subject: 'Subject', + idempotency_key: 'queued-1', + }, scope, service); + + expect(wireResult.isError).not.toBe(true); + expect(service.outboxService.normalizeUndoWindow).toHaveBeenCalledWith(undefined, 30); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + undoSeconds: 30, + idempotencyKey: 'queued-1', + }), + service, + ); + expect(payload(wireResult)).toEqual({ + queued: true, + outbox_id: 'outbox-1', + send_at: '2026-07-28T10:00:30Z', + undo_seconds: 30, + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [], + note: 'Cancel with unsend_email before send_at.', + }); + }); + + it('defaults the undo window to zero when no argument or preference exists', async () => { + const service = deps(); + + await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + }, scope, service); + + expect(service.outboxService.normalizeUndoWindow).toHaveBeenCalledWith( + undefined, + undefined, + ); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ undoSeconds: 0 }), + service, + ); + }); + + it.each([ + ['invalid_recipient', 'Recipient is invalid'], + ['too_many_recipients', 'Too many recipients'], + ])('preserves an underlying %s service error', async (code, message) => { + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockRejectedValue( + Object.assign(new Error(message), { code }), + ), + }, + }); + + const result = await sendTools.handleSendEmail({ + account: 'sender@example.com', + to: ['recipient@example.com'], + }, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(`${code}: ${message}`); + }); +}); + +describe('send_draft', () => { + it('returns draft_not_found for an absent folder and uid match', async () => { + const service = deps(); + getDraftRow.mockResolvedValue(null); + + const result = await sendTools.handleSendDraft({ + account: 'sender@example.com', + draft_uid: 404, + folder: 'Other Drafts', + }, scope, service); + + expect(getDraftRow).toHaveBeenCalledWith('account-1', 'Other Drafts', 404); + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('draft_not_found: 404'); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(service.draftService.deleteDraft).not.toHaveBeenCalled(); + }); + + it('reconstructs and immediately sends a draft before deleting it', async () => { + const service = deps(); + const draft = { + uid: 7, + folder: 'Drafts', + from_email: 'team@example.com', + to_addresses: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc_addresses: [{ name: '', email: 'copy@example.com' }], + bcc_addresses: [{ name: '', email: 'blind@example.com' }], + subject: 'Draft subject', + body_text: 'Plain fallback', + body_html: '

Draft body

', + in_reply_to: '', + thread_references: ' ', + }; + getDraftRow.mockResolvedValue(draft); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 60 }); + resolveFromIdentity.mockResolvedValue({ + ...identity, + fromName: 'Team', + fromEmail: 'team@example.com', + aliasId: 'alias-1', + }); + + const wireResult = await sendTools.handleSendDraft({ + account: 'sender@example.com', + draft_uid: 7, + undo_send_seconds: 0, + idempotency_key: 'draft-7', + }, scope, service); + + expect(wireResult.isError).not.toBe(true); + expect(resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasEmail: 'team@example.com' }, + service, + ); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith({ + userId: 'user-1', + account, + aliasId: 'alias-1', + aliasEmail: 'team@example.com', + to: ['Recipient '], + cc: ['copy@example.com'], + bcc: ['blind@example.com'], + subject: 'Draft subject', + body: '

Draft body

', + bodyIsHtml: true, + inReplyTo: '', + references: ' ', + undoSeconds: 0, + idempotencyKey: 'draft-7', + }, service); + expect(service.draftService.deleteDraft).toHaveBeenCalledWith({ + account, + uid: 7, + folder: 'Drafts', + }, service); + expect( + service.sendService.sendOrEnqueue.mock.invocationCallOrder[0], + ).toBeLessThan(service.draftService.deleteDraft.mock.invocationCallOrder[0]); + expect(payload(wireResult)).toEqual({ + sent: true, + message_id: '', + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'note.txt', size: 5 }], + sent_copy_saved: true, + folder: 'Sent', + }); + }); + + it('queues a draft with a delete-on-send marker and does not delete synchronously', async () => { + const sendAt = new Date('2026-07-28T10:00:30.000Z'); + const service = deps({ + sendService: { + sendOrEnqueue: vi.fn().mockResolvedValue({ + queued: true, + outboxId: 'outbox-draft-7', + sendAt, + undoSeconds: 30, + }), + }, + }); + getDraftRow.mockResolvedValue({ + uid: 7, + folder: 'Drafts', + from_email: 'sender@example.com', + to_addresses: [{ name: '', email: 'recipient@example.com' }], + cc_addresses: [], + bcc_addresses: [], + subject: 'Queued draft', + body_text: 'Queued body', + body_html: '', + in_reply_to: null, + thread_references: null, + }); + getUserPreferences.mockResolvedValue({ undoSendSeconds: 30 }); + + const wireResult = await sendTools.handleSendDraft({ + account: 'sender@example.com', + draft_uid: 7, + }, scope, service); + + expect(wireResult.isError).not.toBe(true); + expect(service.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + undoSeconds: 30, + deleteDraftOnSend: { uid: 7, folder: 'Drafts' }, + }), + service, + ); + expect(service.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(payload(wireResult)).toEqual({ + queued: true, + outbox_id: 'outbox-draft-7', + send_at: '2026-07-28T10:00:30Z', + undo_seconds: 30, + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Queued draft', + attachments: [], + note: 'Cancel with unsend_email before send_at.', + }); + }); + + it('keeps a successful immediate send successful when draft cleanup fails', async () => { + const service = deps(); + service.draftService.deleteDraft.mockRejectedValue(new Error('IMAP delete failed')); + getDraftRow.mockResolvedValue({ + uid: 7, + folder: 'Drafts', + from_email: 'sender@example.com', + to_addresses: [{ name: '', email: 'recipient@example.com' }], + cc_addresses: [], + bcc_addresses: [], + subject: 'Draft subject', + body_text: 'Draft body', + body_html: '', + }); + vi.spyOn(console, 'error').mockImplementation(() => {}); + + const result = await sendTools.handleSendDraft({ + account: 'sender@example.com', + draft_uid: 7, + undo_send_seconds: 0, + }, scope, service); + + expect(result.isError).not.toBe(true); + expect(console.error).toHaveBeenCalledWith( + 'Draft cleanup after send failed:', + 'IMAP delete failed', + ); + }); +}); + +describe('queued draft payload transport', () => { + it('preserves deleteDraftOnSend in the credential-free outbox payload', async () => { + const { sendOrEnqueue } = await import('../services/sendService.js'); + const outboxService = { + enqueue: vi.fn().mockResolvedValue({ + outbox_id: 'outbox-draft-7', + send_at: new Date('2026-07-28T10:00:30.000Z'), + undo_seconds: 30, + }), + }; + const service = { + outboxService, + resolveFromIdentity: vi.fn().mockResolvedValue(identity), + randomBytes: vi.fn(() => Buffer.alloc(16, 1)), + }; + + await sendOrEnqueue({ + userId: 'user-1', + account, + to: ['recipient@example.com'], + cc: [], + bcc: [], + subject: 'Queued draft', + body: 'Queued body', + bodyIsHtml: false, + undoSeconds: 30, + deleteDraftOnSend: { uid: 7, folder: 'Drafts' }, + }, service); + + expect(outboxService.enqueue).toHaveBeenCalledWith( + expect.objectContaining({ + payload: expect.objectContaining({ + deleteDraftOnSend: { uid: 7, folder: 'Drafts' }, + }), + }), + service, + ); + }); +}); + +describe('unsend_email', () => { + it('cancels a pending outbox row and returns its prefetched metadata', async () => { + const service = deps(); + service.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: new Date('2026-07-28T10:00:30.000Z'), + }]); + + const result = await sendTools.handleUnsendEmail( + { outbox_id: 'outbox-1' }, + scope, + service, + ); + + expect(service.outboxService.listPending).toHaveBeenCalledWith( + { userId: 'user-1' }, + service, + ); + expect(service.outboxService.cancel).toHaveBeenCalledWith( + { id: 'outbox-1', userId: 'user-1' }, + service, + ); + expect(payload(result)).toEqual({ + cancelled: true, + outbox_id: 'outbox-1', + subject: 'Queued subject', + to: ['recipient@example.com'], + }); + }); + + it('returns already_sent when the undo window has closed', async () => { + const service = deps(); + service.outboxService.cancel.mockResolvedValue({ + cancelled: false, + reason: 'already_sent', + }); + + const result = await sendTools.handleUnsendEmail( + { outbox_id: 'outbox-sent' }, + scope, + service, + ); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'already_sent: message outbox-sent is no longer pending and cannot be unsent', + ); + }); + + it('returns outbox_not_found for an absent or out-of-scope id', async () => { + const service = deps(); + service.outboxService.cancel.mockResolvedValue({ + cancelled: false, + reason: 'not_found', + }); + + const result = await sendTools.handleUnsendEmail( + { outbox_id: 'outbox-missing' }, + scope, + service, + ); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('outbox_not_found: outbox-missing'); + }); + + it('treats an already-cancelled row as an idempotent success', async () => { + const service = deps(); + service.outboxService.cancel.mockResolvedValue({ + cancelled: false, + reason: 'cancelled', + }); + + const result = await sendTools.handleUnsendEmail( + { outbox_id: 'outbox-cancelled' }, + scope, + service, + ); + + expect(payload(result)).toEqual({ + cancelled: true, + outbox_id: 'outbox-cancelled', + subject: '', + to: [], + }); + }); +}); + +describe('list_outbox', () => { + it('returns pending rows in a no-total pagination envelope', async () => { + const service = deps(); + service.outboxService.listPending.mockResolvedValue([{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: new Date('2026-07-28T10:00:30.000Z'), + }]); + + const result = await sendTools.handleListOutbox({}, scope, service); + + expect(service.outboxService.listPending).toHaveBeenCalledWith( + { userId: 'user-1' }, + service, + ); + expect(payload(result)).toEqual({ + data: [{ + id: 'outbox-1', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + send_at: '2026-07-28T10:00:30Z', + }], + total: -1, + returned: 1, + offset: 0, + has_more: false, + }); + }); + + it('degrades to unsupported without outboxService.listPending', async () => { + const result = await sendTools.handleListOutbox({}, scope, {}); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe( + 'unsupported: list_outbox requires outboxService', + ); + }); +}); + +describe('recall_email', () => { + const delivered = { + id: 'message-1', + account_id: 'account-1', + uid: 42, + folder: 'Sent', + message_id: '', + subject: 'Project update', + to_addresses: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc_addresses: [{ name: '', email: 'copy@example.com' }], + }; + + it('cancels a pending row found by message id before touching a Sent copy', async () => { + const service = deps(); + getOutboxRowByMessageId.mockResolvedValue({ + id: 'outbox-1', + message_id: '', + subject: 'Queued subject', + to_preview: ['recipient@example.com'], + }); + + const result = await sendTools.handleRecallEmail( + { message_id: '' }, + scope, + service, + ); + + expect(getOutboxRowByMessageId).toHaveBeenCalledWith( + '', + 'user-1', + ); + expect(service.outboxService.cancel).toHaveBeenCalledWith( + { id: 'outbox-1', userId: 'user-1' }, + service, + ); + expect(payload(result)).toEqual({ + recalled: 'cancelled_before_send', + outbox_id: 'outbox-1', + subject: 'Queued subject', + to: ['recipient@example.com'], + }); + expect(getComposeSource).not.toHaveBeenCalled(); + expect(service.imapManager.permanentDeleteMessage).not.toHaveBeenCalled(); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + }); + + it('deletes the Sent copy and creates, but never sends, a threaded follow-up draft', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(delivered); + + const result = await sendTools.handleRecallEmail( + { + message_id: 'message-1', + followup_note: 'Please disregard the previous update.', + }, + scope, + service, + ); + + expect(getComposeSource).toHaveBeenCalledWith('message-1', ['account-1']); + expect(getAccountRow).toHaveBeenCalledWith('account-1', ['account-1']); + expect(service.imapManager.permanentDeleteMessage).toHaveBeenCalledWith( + account, + 42, + 'Sent', + ); + expect(deleteMessageRow).toHaveBeenCalledWith('account-1', 42, 'Sent'); + expect(service.draftService.saveDraft).toHaveBeenCalledWith({ + userId: 'user-1', + account, + to: ['Recipient '], + cc: ['copy@example.com'], + bcc: [], + subject: 'Re: Project update', + body: 'Please disregard the previous update.', + bodyIsHtml: false, + attachments: [], + inReplyTo: '', + }, service); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(payload(result)).toEqual({ + recalled: 'not_possible', + note: 'SMTP cannot retract a delivered message. Recipients already received it; deleting your Sent copy does not affect their mailboxes.', + sent_copy_deleted: true, + followup_draft: { + draft_uid: 57, + folder: 'Drafts', + }, + }); + }); + + it('keeps the Sent copy when delete_sent_copy is false', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(delivered); + + const result = await sendTools.handleRecallEmail( + { message_id: 'message-1', delete_sent_copy: false }, + scope, + service, + ); + + expect(service.imapManager.permanentDeleteMessage).not.toHaveBeenCalled(); + expect(deleteMessageRow).not.toHaveBeenCalled(); + expect(service.draftService.saveDraft).toHaveBeenCalled(); + expect(payload(result)).toMatchObject({ + recalled: 'not_possible', + sent_copy_deleted: false, + followup_draft: { draft_uid: 57, folder: 'Drafts' }, + }); + }); + + it('does not create a follow-up when draft_followup is false', async () => { + const service = deps(); + getComposeSource.mockResolvedValue(delivered); + + const result = await sendTools.handleRecallEmail( + { message_id: 'message-1', draft_followup: false }, + scope, + service, + ); + + expect(service.imapManager.permanentDeleteMessage).toHaveBeenCalled(); + expect(deleteMessageRow).toHaveBeenCalled(); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + expect(service.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(payload(result)).toEqual({ + recalled: 'not_possible', + note: 'SMTP cannot retract a delivered message. Recipients already received it; deleting your Sent copy does not affect their mailboxes.', + sent_copy_deleted: true, + }); + }); + + it('returns message_not_found without destructive effects', async () => { + const service = deps(); + + const result = await sendTools.handleRecallEmail( + { message_id: 'message-missing' }, + scope, + service, + ); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message_not_found: message-missing'); + expect(service.imapManager.permanentDeleteMessage).not.toHaveBeenCalled(); + expect(deleteMessageRow).not.toHaveBeenCalled(); + expect(service.draftService.saveDraft).not.toHaveBeenCalled(); + }); +}); + +describe('missing dependencies', () => { + it.each([ + ['send_email/sendService', sendTools.handleSendEmail, { + account: 'sender@example.com', + to: ['recipient@example.com'], + }, { + outboxService: { normalizeUndoWindow: vi.fn() }, + }], + ['send_draft/sendService', sendTools.handleSendDraft, { + account: 'sender@example.com', + draft_uid: 7, + }, { + draftService: { deleteDraft: vi.fn() }, + outboxService: { normalizeUndoWindow: vi.fn() }, + }], + ['send_draft/draftService', sendTools.handleSendDraft, { + account: 'sender@example.com', + draft_uid: 7, + }, { + sendService: { sendOrEnqueue: vi.fn() }, + outboxService: { normalizeUndoWindow: vi.fn() }, + }], + ['unsend_email/outboxService', sendTools.handleUnsendEmail, { + outbox_id: 'outbox-1', + }, {}], + ['recall_email/outboxService', sendTools.handleRecallEmail, { + message_id: 'message-1', + }, {}], + ])('%s degrades to unsupported', async (_name, handler, args, service) => { + const result = await handler(args, scope, service); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toMatch(/^unsupported: /); + }); +}); diff --git a/backend/src/mcp/server.js b/backend/src/mcp/server.js new file mode 100644 index 00000000..e5d0c0e2 --- /dev/null +++ b/backend/src/mcp/server.js @@ -0,0 +1,273 @@ +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { ListToolsRequestSchema, CallToolRequestSchema } from '@modelcontextprotocol/sdk/types.js'; +import express from 'express'; +import { hasScope, mcpBearerAuth } from './auth.js'; +import { TOOL_DEFS, TOOL_SCOPES, HANDLERS } from './tools.js'; +import { errorResult } from './result.js'; +import { listRules } from './accountAdapter.js'; + +// Attachments are base64-inlined in MCP tool arguments, so this parser needs the +// same headroom as the REST send/draft routes. index.js mounts it before the +// global 1 MB parser. +export function mcpBodyLimit() { + return express.json({ limit: process.env.MCP_BODY_LIMIT || '35mb' }); +} + +export function entityTooLargeResponse(err, path) { + if (err?.type !== 'entity.too.large') return null; + if (path.startsWith('/mcp')) { + return { + status: 413, + body: { + jsonrpc: '2.0', + error: { code: -32000, message: 'Request too large (max 35 MB).' }, + id: null, + }, + }; + } + return { + status: 413, + body: { + error: 'Request too large. Total attachment size must not exceed 25 MB.', + }, + }; +} + +// --- Origin validation (DNS-rebinding protection) -------------------------------- +// The MCP Streamable HTTP spec REQUIRES servers to validate the Origin header so a +// malicious web page cannot drive a local MCP endpoint from a victim's browser via +// DNS rebinding. Our SDK (@modelcontextprotocol/sdk 1.29) can enforce this in the +// transport (`enableDnsRebindingProtection` + `allowedOrigins` on +// WebStandardStreamableHTTPServerTransport) but ships with it DISABLED by default +// (GHSA-w48q-cv73-mx4w) and compares origins by exact string match. We enforce at +// the Express layer instead: one owner, normalized (URL.origin, lowercased) +// comparison, and rejection happens before bearer auth ever touches the database. +// +// Policy (mirrors websocket.js): a request with NO Origin header passes — MCP +// clients are non-browser processes and send none. A request WITH an Origin must +// resolve to an allowlisted origin: APP_URL, FRONTEND_URL, any localhost / +// 127.0.0.1 / [::1] origin on any port (a DNS-rebinding page always presents the +// attacker's hostname as Origin, never localhost), or an operator-supplied extra +// via MCP_ALLOWED_ORIGINS (comma-separated URLs, e.g. "https://lan-host:8087"). +const LOCAL_HOSTNAMES = new Set(['localhost', '127.0.0.1', '[::1]']); + +export function buildAllowedOrigins(env = process.env) { + const allowed = new Set(); + const add = (raw) => { + if (!raw || !raw.trim()) return; + try { allowed.add(new URL(raw.trim()).origin.toLowerCase()); } + catch { /* malformed allowlist entry — skip it rather than fail the boot */ } + }; + add(env.APP_URL); + add(env.FRONTEND_URL); + for (const entry of (env.MCP_ALLOWED_ORIGINS || '').split(',')) add(entry); + return allowed; +} + +export function mcpOriginGuard(allowed = buildAllowedOrigins()) { + return (req, res, next) => { + const raw = req.get('Origin'); + if (!raw) return next(); // non-browser MCP client + try { + const url = new URL(raw); + if (allowed.has(url.origin.toLowerCase()) || LOCAL_HOSTNAMES.has(url.hostname.toLowerCase())) { + return next(); + } + } catch { /* unparseable Origin (including the literal "null") — reject below */ } + // Same JSON-RPC error envelope the SDK transport uses for its own HTTP-level + // rejections (webStandardStreamableHttp.js createJsonErrorResponse). + res.status(403).json({ + jsonrpc: '2.0', + error: { code: -32000, message: `Origin not allowed: ${raw}` }, + id: null, + }); + }; +} + +// --- Per-token tool-call rate limits ---------------------------------------------- +// REST search is limited per user (routes/search.js); the MCP surface reuses the same +// in-memory bucket pattern, keyed by the api_tokens row id — NOT the client IP, since +// many agents legitimately share one egress. Only tools/call requests count, so the +// initialize / tools-list handshake a stateless client repeats is never throttled. +// Over-limit requests get an HTTP 429 with Retry-After BEFORE the transport, carrying +// the SDK's JSON-RPC error envelope shape. Read/write/send/settings have independent +// per-minute budgets, and send also has a per-token daily cap. +export function countToolCalls(body) { + if (Array.isArray(body)) return body.filter((m) => m && m.method === 'tools/call').length; + return body && body.method === 'tools/call' ? 1 : 0; +} + +const RATE_CLASSES = ['read', 'write', 'send', 'settings']; + +const MCP_RUNTIME_DEPENDENCY_KEYS = [ + 'imapManager', + 'refreshMicrosoftToken', + 'redisClient', + 'sendService', + 'outboxService', + 'draftService', + 'composeSessionService', + 'composeSessionLifecycle', + 'query', + 'withTransaction', + 'broadcast', +]; + +export function createMcpRuntimeDependencies(deps = {}) { + const runtime = {}; + for (const name of MCP_RUNTIME_DEPENDENCY_KEYS) { + if (deps[name] == null) throw new Error(`MCP runtime dependency missing: ${name}`); + runtime[name] = deps[name]; + } + return runtime; +} + +export function classifyToolCalls(body) { + const counts = { read: 0, write: 0, send: 0, settings: 0 }; + const messages = Array.isArray(body) ? body : [body]; + for (const message of messages) { + if (!message || message.method !== 'tools/call') continue; + const required = TOOL_SCOPES[message.params?.name]; + // Array scopes use AND semantics at authorization. For rate limiting, the + // first-listed scope is the primary/most-restrictive class. No current tool + // has an array scope; this pins the forward-looking tie-break explicitly. + const primary = Array.isArray(required) ? required[0] : required; + const rateClass = RATE_CLASSES.includes(primary) ? primary : 'read'; + counts[rateClass]++; + } + return counts; +} + +function positiveEnv(name, fallback) { + const value = Number(process.env[name]); + return value > 0 ? value : fallback; +} + +export function createMcpRateLimiter({ limits, windowMs = 60_000, now = Date.now } = {}) { + const maximums = { + read: limits?.read ?? positiveEnv('MCP_RATE_LIMIT_PER_MIN', 60), + write: limits?.write ?? positiveEnv('MCP_WRITE_RATE_LIMIT_PER_MIN', 30), + send: limits?.send ?? positiveEnv('MCP_SEND_RATE_LIMIT_PER_MIN', 10), + settings: limits?.settings ?? positiveEnv('MCP_SETTINGS_RATE_LIMIT_PER_MIN', 10), + }; + const dailySendMax = positiveEnv('MCP_SEND_DAILY_CAP', 200); + const dayMs = 24 * 60 * 60 * 1000; + const buckets = new Map(); + const sweeper = setInterval(() => { + const t = now(); + for (const [k, b] of buckets) if (t >= b.resetAt) buckets.delete(k); + }, windowMs); + sweeper.unref?.(); // observability sweeper must never keep the process alive + + const bucketFor = (key, duration, t) => { + let bucket = buckets.get(key); + if (!bucket || t >= bucket.resetAt) { + bucket = { count: 0, resetAt: t + duration }; + buckets.set(key, bucket); + } + return bucket; + }; + + const reject = (req, res, bucket, max, rateClass, period, t) => { + res.setHeader('Retry-After', Math.ceil((bucket.resetAt - t) / 1000)); + return res.status(429).json({ + jsonrpc: '2.0', + error: { + code: -32000, + message: `Rate limit exceeded: at most ${max} ${rateClass} tool calls per ${period} per token`, + }, + id: Array.isArray(req.body) ? null : (req.body?.id ?? null), + }); + }; + + return (req, res, next) => { + const calls = classifyToolCalls(req.body); + if (!RATE_CLASSES.some((rateClass) => calls[rateClass] > 0)) return next(); + const t = now(); + const admissions = []; + for (const rateClass of RATE_CLASSES) { + if (!calls[rateClass]) continue; + const bucket = bucketFor(`${req.mcpTokenId}:${rateClass}`, windowMs, t); + if (bucket.count + calls[rateClass] > maximums[rateClass]) { + return reject(req, res, bucket, maximums[rateClass], rateClass, 'minute', t); + } + admissions.push([bucket, calls[rateClass]]); + } + + if (calls.send) { + const daily = bucketFor(`${req.mcpTokenId}:send:day`, dayMs, t); + if (daily.count + calls.send > dailySendMax) { + return reject(req, res, daily, dailySendMax, 'send', 'day', t); + } + admissions.push([daily, calls.send]); + } + + // Admit only after every affected bucket passes, so a rejected batch consumes + // no budget in any class. + for (const [bucket, count] of admissions) bucket.count += count; + next(); + }; +} + +// Build a fresh Server bound to one request's scope. Stateless: no session store, +// one Server+transport per HTTP request, matching msgvault's daemon-less posture. +export function buildServer(scope, deps = {}) { + const server = new Server( + { name: 'mailflow', version: '1.0.0' }, + { capabilities: { tools: {} } }, + ); + + server.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: TOOL_DEFS.filter((definition) => hasScope(scope, TOOL_SCOPES[definition.name])), + })); + + server.setRequestHandler(CallToolRequestSchema, async (req) => { + const name = req.params.name; + const handler = HANDLERS[name]; + if (!handler) return errorResult(`unknown tool: ${name}`); + const requiredScope = TOOL_SCOPES[name]; + if (!requiredScope) { + return errorResult(`permission_denied: tool "${name}" has no scope classification`); + } + if (!hasScope(scope, requiredScope)) { + return errorResult( + `permission_denied: tool "${name}" requires the "${requiredScope}" scope; ` + + `this token has [${(scope.scopes || []).join(', ')}]`, + ); + } + try { + return await handler(req.params.arguments || {}, scope, deps); + } catch { + // Tool-level failures flow as isError results, not JSON-RPC errors, + // so clients get a stable result without private backend details. + return errorResult('internal error'); + } + }); + + return server; +} + +export function mountMcp(app, deps = {}) { + // get_triage_context's matched-rules section needs a read-only rules loader; + // the adapter's scoped listRules is the natural default, overridable in tests. + const resolvedDeps = { loadInboxRules: listRules, ...deps }; + const handle = async (req, res) => { + const scope = { ...req.mcpScope, tokenId: req.mcpTokenId }; + const server = buildServer(scope, resolvedDeps); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); + res.on('close', () => { transport.close(); server.close(); }); + await server.connect(transport); + // Global express.json already parsed the body; hand it to the transport. + await transport.handleRequest(req, res, req.body); + }; + + const originGuard = mcpOriginGuard(buildAllowedOrigins()); + const rateLimiter = createMcpRateLimiter(); + + // Tool calls only arrive as POST bodies; GET (SSE open) and DELETE (session + // teardown) carry none, so the rate limiter guards POST alone. + app.post('/mcp', originGuard, mcpBearerAuth, rateLimiter, handle); + app.get('/mcp', originGuard, mcpBearerAuth, handle); + app.delete('/mcp', originGuard, mcpBearerAuth, handle); +} diff --git a/backend/src/mcp/server.test.js b/backend/src/mcp/server.test.js new file mode 100644 index 00000000..376e7429 --- /dev/null +++ b/backend/src/mcp/server.test.js @@ -0,0 +1,299 @@ +import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from 'vitest'; +import express from 'express'; +import { createServer } from 'http'; + +vi.mock('../services/db.js', () => ({ query: vi.fn(), withTransaction: vi.fn() })); +import { query } from '../services/db.js'; +import { ALL_SCOPES, hashToken } from './auth.js'; +import { createMcpRuntimeDependencies, mountMcp } from './server.js'; + +let server, base; + +const COMPOSE_TOOL_NAMES = [ + 'list_compose_sessions', + 'get_compose_session', + 'create_compose_session', + 'update_compose_session', + 'minimize_compose_session', + 'restore_compose_session', + 'add_compose_attachment', + 'remove_compose_attachment', + 'close_compose_session', + 'discard_compose_session', + 'send_compose_session', +]; + +const composeSessionService = { + createComposeSession: vi.fn(), +}; + +beforeAll(async () => { + const app = express(); + app.use(express.json()); + mountMcp(app, { composeSessionService }); + server = createServer(app); + await new Promise((r) => server.listen(0, r)); + base = `http://127.0.0.1:${server.address().port}`; +}); +afterAll(() => new Promise((r) => server.close(r))); + +beforeEach(() => { + composeSessionService.createComposeSession.mockReset().mockResolvedValue({ + id: 'session-1', + slot: 1, + revision: 1, + presentationState: 'expanded', + }); +}); + +// Every authed request: token lookup -> last_used_at update -> resolveScope. +function primeAuth( + userId = 'user-1', + accountIds = ['acc-1'], + scopes = ALL_SCOPES, + tokenId = 'tok', +) { + query + .mockResolvedValueOnce({ rows: [{ id: tokenId, user_id: userId, scopes }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: accountIds.map((id) => ({ id })) }); +} + +async function rpc(method, params, { + auth = true, + scopes = ALL_SCOPES, + tokenId = 'tok', +} = {}) { + query.mockReset(); + if (auth) primeAuth('user-1', ['acc-1'], scopes, tokenId); + const headers = { 'Content-Type': 'application/json', Accept: 'application/json, text/event-stream' }; + if (auth) headers.Authorization = 'Bearer mcp_good'; + const res = await fetch(`${base}/mcp`, { + method: 'POST', + headers, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }), + }); + return res; +} + +async function rpcPayload(response) { + const body = await response.text(); + const data = body.split(/\r?\n/).find(line => line.startsWith('data:')); + return JSON.parse(data ? data.slice('data:'.length).trim() : body); +} + +describe('/mcp transport', () => { + it('rejects unauthenticated requests with 401', async () => { + const res = await rpc('tools/list', {}, { auth: false }); + expect(res.status).toBe(401); + }); + + it('completes initialize', async () => { + const res = await rpc('initialize', { + protocolVersion: '2025-06-18', + capabilities: {}, + clientInfo: { name: 'test', version: '0' }, + }); + expect(res.status).toBe(200); + const text = await res.text(); + expect(text).toContain('"serverInfo"'); + expect(text).toContain('mailflow'); + }); + + it('lists the ping tool', async () => { + const res = await rpc('tools/list', {}); + const text = await res.text(); + expect(text).toContain('"ping"'); + }); + + it('calls ping and returns pong', async () => { + const res = await rpc('tools/call', { name: 'ping', arguments: {} }); + const text = await res.text(); + expect(text).toContain('{\\"pong\\":true}'); + }); + + it('verifies the token by hash, not plaintext', async () => { + await rpc('tools/list', {}); + expect(query.mock.calls[0][1]).toEqual([hashToken('mcp_good')]); + }); + + it.each([ + ['read', ['read'], COMPOSE_TOOL_NAMES.slice(0, 2)], + ['write', ['write'], COMPOSE_TOOL_NAMES.slice(0, 10)], + ['send', ['send'], [ + 'list_compose_sessions', + 'get_compose_session', + 'send_compose_session', + ]], + ])('lists only permitted compose-session tools for a %s token', async ( + _label, + scopes, + expected, + ) => { + const response = await rpc('tools/list', {}, { scopes }); + const payload = await rpcPayload(response); + const listed = payload.result.tools + .map(({ name }) => name) + .filter(name => COMPOSE_TOOL_NAMES.includes(name)); + expect(listed).toEqual(expected); + }); + + it.each([ + ['read', ['read'], 'create_compose_session', {}, 'write'], + ['write', ['write'], 'send_compose_session', { + slot: 1, + expected_revision: 1, + }, 'send'], + ['send', ['send'], 'update_compose_session', { + slot: 1, + expected_revision: 1, + }, 'write'], + ])('returns permission_denied for a %s token calling a forbidden compose tool', async ( + _label, + scopes, + name, + args, + required, + ) => { + const response = await rpc('tools/call', { name, arguments: args }, { scopes }); + const payload = await rpcPayload(response); + expect(payload.result.isError).toBe(true); + expect(payload.result.content[0].text) + .toContain(`permission_denied: tool "${name}" requires the "${required}" scope`); + }); + + it('returns stable unsupported when an authorized compose dependency is absent', async () => { + const response = await rpc('tools/call', { + name: 'list_compose_sessions', + arguments: {}, + }, { scopes: ['read'] }); + const payload = await rpcPayload(response); + expect(payload.result).toEqual({ + content: [{ + type: 'text', + text: 'unsupported: compose session tools require composeSessionService', + }], + isError: true, + }); + }); + + it('returns stable unsupported when an authorized lifecycle dependency is absent', async () => { + const response = await rpc('tools/call', { + name: 'send_compose_session', + arguments: { slot: 1, expected_revision: 1 }, + }, { scopes: ['send'] }); + const payload = await rpcPayload(response); + expect(payload.result).toEqual({ + content: [{ + type: 'text', + text: 'unsupported: compose session tools require composeSessionLifecycle', + }], + isError: true, + }); + }); + + it('propagates the authenticated token id into compose service client ids', async () => { + const response = await rpc('tools/call', { + name: 'create_compose_session', + arguments: { slot: 1 }, + }, { scopes: ['write'], tokenId: 'tok-write' }); + const payload = await rpcPayload(response); + expect(payload.result.isError).toBeUndefined(); + expect(composeSessionService.createComposeSession).toHaveBeenCalledWith( + expect.objectContaining({ + userId: 'user-1', + requestedSlot: 1, + clientId: 'mcp:tok-write', + }), + expect.any(Object), + ); + }); + + it('keeps unexpected compose service details out of tool results', async () => { + composeSessionService.createComposeSession.mockRejectedValueOnce( + new Error('private database detail'), + ); + const response = await rpc('tools/call', { + name: 'create_compose_session', + arguments: {}, + }, { scopes: ['write'] }); + const payload = await rpcPayload(response); + expect(payload.result).toEqual({ + content: [{ type: 'text', text: 'internal error' }], + isError: true, + }); + expect(JSON.stringify(payload)).not.toContain('private database detail'); + }); +}); + +describe('index MCP bootstrap', () => { + it('supplies the real compose services and their complete runtime dependency graph', async () => { + const [ + composeSessionService, + composeSessionLifecycle, + sendService, + outboxService, + draftService, + db, + redis, + ] = await Promise.all([ + import('../services/composeSessionService.js'), + import('../services/composeSessionLifecycle.js'), + import('../services/sendService.js'), + import('../services/outboxService.js'), + import('../services/draftService.js'), + vi.importActual('../services/db.js'), + import('../services/redis.js'), + ]); + for (const method of [ + 'listComposeSessions', + 'getComposeSession', + 'createComposeSession', + 'patchComposeSession', + 'setComposePresentation', + 'addComposeAttachment', + 'removeComposeAttachment', + ]) { + expect(composeSessionService[method], `composeSessionService.${method}`) + .toBeTypeOf('function'); + } + for (const method of [ + 'closeComposeSession', + 'discardComposeSession', + 'sendComposeSession', + ]) { + expect(composeSessionLifecycle[method], `composeSessionLifecycle.${method}`) + .toBeTypeOf('function'); + } + + const imapManager = { broadcast: vi.fn() }; + const refreshMicrosoftToken = vi.fn(); + const broadcast = (event, userId) => imapManager.broadcast(event, userId); + const inputs = { + imapManager, + refreshMicrosoftToken, + redisClient: redis.redisClient, + sendService, + outboxService, + draftService, + composeSessionService, + composeSessionLifecycle, + query: db.query, + withTransaction: db.withTransaction, + broadcast, + }; + const dependencies = createMcpRuntimeDependencies(inputs); + expect(dependencies).toEqual(inputs); + + const event = { type: 'compose_sessions_updated' }; + dependencies.broadcast(event, 'user-1'); + expect(imapManager.broadcast).toHaveBeenCalledWith(event, 'user-1'); + + for (const name of Object.keys(inputs)) { + const incomplete = { ...inputs }; + delete incomplete[name]; + expect(() => createMcpRuntimeDependencies(incomplete), name) + .toThrow(`MCP runtime dependency missing: ${name}`); + } + }); +}); diff --git a/backend/src/mcp/serverGuards.test.js b/backend/src/mcp/serverGuards.test.js new file mode 100644 index 00000000..cd4c38ff --- /dev/null +++ b/backend/src/mcp/serverGuards.test.js @@ -0,0 +1,580 @@ +import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest'; +import express from 'express'; +import { createServer } from 'http'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('./accountAdapter.js', async (orig) => { + const actual = await orig(); + return { ...actual, runRules: vi.fn() }; +}); +import { query } from '../services/db.js'; +import { runRules } from './accountAdapter.js'; +import { ALL_SCOPES } from './auth.js'; +import { + buildAllowedOrigins, buildServer, mcpBodyLimit, mcpOriginGuard, + classifyToolCalls, countToolCalls, createMcpRateLimiter, + entityTooLargeResponse, mountMcp, +} from './server.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; + +function mockReq({ origin, body, tokenId } = {}) { + return { + get: (h) => (h.toLowerCase() === 'origin' ? origin : undefined), + body, + mcpTokenId: tokenId, + }; +} + +function mockRes() { + const res = { statusCode: 200, headers: {}, body: null }; + res.setHeader = (k, v) => { res.headers[k] = v; }; + res.status = (c) => { res.statusCode = c; return res; }; + res.json = (b) => { res.body = b; return res; }; + return res; +} + +async function withMcpClient(scope, callback, deps = {}) { + const server = buildServer(scope, deps); + const client = new Client({ name: 'scope-test', version: '1.0.0' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + try { + return await callback(client); + } finally { + await client.close(); + } +} + +describe('MCP dependency injection', () => { + it('passes the buildServer dependency bag to a three-argument tool handler', async () => { + const scope = { userId: 'u', accountIds: ['acc-1'], scopes: ['read'] }; + const deps = { marker: Symbol('deps') }; + const handler = vi.fn(async () => ({ + content: [{ type: 'text', text: JSON.stringify({ ok: true }) }], + })); + HANDLERS.__dependency_probe = handler; + TOOL_SCOPES.__dependency_probe = 'read'; + + try { + await withMcpClient(scope, async (client) => { + await client.callTool({ + name: '__dependency_probe', + arguments: { value: 42 }, + }); + }, deps); + expect(handler).toHaveBeenCalledWith({ value: 42 }, scope, deps); + } finally { + delete HANDLERS.__dependency_probe; + delete TOOL_SCOPES.__dependency_probe; + } + }); + + it('mounts both with a dependency bag and without one', () => { + expect(() => mountMcp(express(), { marker: true })).not.toThrow(); + expect(() => mountMcp(express())).not.toThrow(); + }); +}); + +describe('tool scope classifications', () => { + it('classifies every listed tool and handler', () => { + const definedNames = TOOL_DEFS.map(({ name }) => name).sort(); + expect(Object.keys(HANDLERS).sort()).toEqual(definedNames); + expect(Object.keys(TOOL_SCOPES).sort()).toEqual(definedNames); + }); +}); + +describe('tool scope enforcement', () => { + it('omits tools from tools/list when the token lacks their scope', async () => { + await withMcpClient({ userId: 'u', accountIds: [], scopes: ['read'] }, async (client) => { + const { tools } = await client.listTools(); + expect(tools.map(({ name }) => name)).toContain('ping'); + expect(tools.map(({ name }) => name)).not.toContain('stage_deletion'); + }); + }); + + it('refuses tools/call when the token lacks the required scope', async () => { + await withMcpClient({ userId: 'u', accountIds: [], scopes: ['read'] }, async (client) => { + const result = await client.callTool({ + name: 'stage_deletion', + arguments: { from: 'sender@example.com' }, + }); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain( + 'permission_denied: tool "stage_deletion" requires the "write" scope; this token has [read]', + ); + }); + }); + + it('refuses tools/call when the tool has no scope classification', async () => { + const requiredScope = TOOL_SCOPES.ping; + delete TOOL_SCOPES.ping; + try { + await withMcpClient({ userId: 'u', accountIds: [], scopes: ['read'] }, async (client) => { + const result = await client.callTool({ name: 'ping', arguments: {} }); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain( + 'permission_denied: tool "ping" has no scope classification', + ); + }); + } finally { + TOOL_SCOPES.ping = requiredScope; + } + }); + + it('allows tools/call when the token has the required scope', async () => { + await withMcpClient({ userId: 'u', accountIds: [], scopes: ['read'] }, async (client) => { + const result = await client.callTool({ name: 'ping', arguments: {} }); + expect(JSON.parse(result.content[0].text)).toEqual({ pong: true }); + }); + }); + + it('requires both settings and write before run_rules reaches its adapter', async () => { + const deps = { imapManager: { marker: 'injected' } }; + runRules.mockReset().mockResolvedValue({ processed: 0, matched: 0 }); + + await withMcpClient({ + userId: 'u', + accountIds: ['acc-1'], + scopes: ['settings'], + }, async (client) => { + const result = await client.callTool({ + name: 'run_rules', + arguments: {}, + }); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain('permission_denied'); + expect(runRules).not.toHaveBeenCalled(); + }, deps); + + await withMcpClient({ + userId: 'u', + accountIds: ['acc-1'], + scopes: ['settings', 'write', 'read'], + }, async (client) => { + const result = await client.callTool({ + name: 'run_rules', + arguments: {}, + }); + expect(result.isError).toBeUndefined(); + expect(JSON.parse(result.content[0].text)).toEqual({ + processed: 0, + matched: 0, + }); + expect(runRules).toHaveBeenCalledTimes(1); + expect(runRules).toHaveBeenCalledWith({ + userId: 'u', + accountIds: ['acc-1'], + imapManager: deps.imapManager, + }); + }, deps); + }); +}); + +describe('buildAllowedOrigins', () => { + it('derives normalized origins from APP_URL, FRONTEND_URL, and MCP_ALLOWED_ORIGINS', () => { + const allowed = buildAllowedOrigins({ + APP_URL: 'https://Mail.Example.com/', // trailing slash + case normalize away + FRONTEND_URL: 'http://localhost:5173', + MCP_ALLOWED_ORIGINS: 'https://lan-host:8087, http://mail.internal', + }); + expect(allowed).toEqual(new Set([ + 'https://mail.example.com', + 'http://localhost:5173', + 'https://lan-host:8087', + 'http://mail.internal', + ])); + }); + + it('skips malformed and empty entries instead of throwing', () => { + const allowed = buildAllowedOrigins({ APP_URL: 'not a url', MCP_ALLOWED_ORIGINS: ' ,, ' }); + expect(allowed.size).toBe(0); + }); +}); + +describe('mcpOriginGuard', () => { + const guard = mcpOriginGuard(buildAllowedOrigins({ APP_URL: 'https://mail.example.com' })); + + it('passes requests with no Origin header (non-browser MCP clients)', () => { + const next = vi.fn(); + guard(mockReq(), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('passes an allowlisted Origin', () => { + const next = vi.fn(); + guard(mockReq({ origin: 'https://mail.example.com' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('normalizes Origin casing before comparing', () => { + const next = vi.fn(); + guard(mockReq({ origin: 'HTTPS://MAIL.EXAMPLE.COM' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('passes localhost variants on any port', () => { + for (const origin of ['http://localhost:8087', 'http://127.0.0.1:3000', 'http://[::1]:8087', 'https://localhost']) { + const next = vi.fn(); + guard(mockReq({ origin }), mockRes(), next); + expect(next, origin).toHaveBeenCalledTimes(1); + } + }); + + it('rejects a non-allowlisted Origin with a 403 JSON-RPC error', () => { + const next = vi.fn(); + const res = mockRes(); + guard(mockReq({ origin: 'http://evil.example' }), res, next); + expect(next).not.toHaveBeenCalled(); + expect(res.statusCode).toBe(403); + expect(res.body).toEqual({ + jsonrpc: '2.0', + error: { code: -32000, message: 'Origin not allowed: http://evil.example' }, + id: null, + }); + }); + + it('rejects an attacker hostname that merely resolves to this host (DNS rebinding)', () => { + const res = mockRes(); + guard(mockReq({ origin: 'http://rebind.attacker.net:8087' }), res, vi.fn()); + expect(res.statusCode).toBe(403); + }); + + it('rejects unparseable Origins, including the literal "null"', () => { + for (const origin of ['null', 'not a url']) { + const res = mockRes(); + guard(mockReq({ origin }), res, vi.fn()); + expect(res.statusCode, origin).toBe(403); + } + }); +}); + +describe('mcpBodyLimit', () => { + it('parses a 2 MB MCP body before the global 1 MB parser', async () => { + const app = express(); + app.use('/mcp', mcpBodyLimit()); + app.use(express.json({ limit: '1mb' })); + app.post('/mcp', (req, res) => res.json({ size: req.body.payload.length })); + const server = createServer(app); + await new Promise((resolve) => server.listen(0, resolve)); + + try { + const payload = 'x'.repeat(2 * 1024 * 1024); + const response = await fetch(`http://127.0.0.1:${server.address().port}/mcp`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ payload }), + }); + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ size: payload.length }); + } finally { + await new Promise((resolve) => server.close(resolve)); + } + }); +}); + +describe('entityTooLargeResponse', () => { + it('returns a JSON-RPC-shaped MCP 413 response', () => { + expect(entityTooLargeResponse( + { type: 'entity.too.large' }, + '/mcp', + )).toEqual({ + status: 413, + body: { + jsonrpc: '2.0', + error: { code: -32000, message: 'Request too large (max 35 MB).' }, + id: null, + }, + }); + }); + + it('preserves the existing REST attachment 413 response', () => { + expect(entityTooLargeResponse( + { type: 'entity.too.large' }, + '/api/mail/send', + )).toEqual({ + status: 413, + body: { + error: 'Request too large. Total attachment size must not exceed 25 MB.', + }, + }); + }); + + it('returns null for other errors so Express can forward them', () => { + expect(entityTooLargeResponse(new Error('boom'), '/mcp')).toBeNull(); + }); +}); + +describe('countToolCalls', () => { + it('counts a single tools/call body as 1 and other methods as 0', () => { + expect(countToolCalls({ jsonrpc: '2.0', method: 'tools/call', id: 1 })).toBe(1); + expect(countToolCalls({ jsonrpc: '2.0', method: 'initialize', id: 1 })).toBe(0); + expect(countToolCalls({ jsonrpc: '2.0', method: 'tools/list', id: 1 })).toBe(0); + expect(countToolCalls(undefined)).toBe(0); + }); + + it('counts tools/call entries inside a batch array', () => { + expect(countToolCalls([ + { method: 'tools/call' }, { method: 'notifications/initialized' }, { method: 'tools/call' }, + ])).toBe(2); + }); +}); + +describe('classifyToolCalls', () => { + it('counts tool calls by the tool scope class and ignores handshake methods', () => { + TOOL_SCOPES.__send_probe = 'send'; + TOOL_SCOPES.__settings_probe = 'settings'; + try { + expect(classifyToolCalls([ + { method: 'initialize' }, + { method: 'tools/call', params: { name: 'ping' } }, + { method: 'tools/call', params: { name: 'stage_deletion' } }, + { method: 'tools/call', params: { name: '__send_probe' } }, + { method: 'tools/call', params: { name: '__settings_probe' } }, + ])).toEqual({ read: 1, write: 1, send: 1, settings: 1 }); + } finally { + delete TOOL_SCOPES.__send_probe; + delete TOOL_SCOPES.__settings_probe; + } + }); + + it('uses the first listed scope as an array-scoped tool primary class', () => { + TOOL_SCOPES.__array_probe = ['send', 'write']; + try { + expect(classifyToolCalls({ + method: 'tools/call', + params: { name: '__array_probe' }, + })).toEqual({ read: 0, write: 0, send: 1, settings: 0 }); + } finally { + delete TOOL_SCOPES.__array_probe; + } + }); + + it('charges unknown tool names to read and returns zeros for non-call bodies', () => { + expect(classifyToolCalls({ + method: 'tools/call', + params: { name: 'typo_tool' }, + })).toEqual({ read: 1, write: 0, send: 0, settings: 0 }); + expect(classifyToolCalls({ method: 'tools/list' })) + .toEqual({ read: 0, write: 0, send: 0, settings: 0 }); + expect(classifyToolCalls(undefined)) + .toEqual({ read: 0, write: 0, send: 0, settings: 0 }); + }); +}); + +describe('createMcpRateLimiter', () => { + let clock; + const now = () => clock; + const call = (name = 'ping', id = 1) => ({ + jsonrpc: '2.0', + method: 'tools/call', + params: { name }, + id, + }); + const limits = { read: 1, write: 1, send: 1, settings: 1 }; + + beforeEach(() => { + clock = 1_000_000; + TOOL_SCOPES.__send_probe = 'send'; + TOOL_SCOPES.__settings_probe = 'settings'; + }); + afterEach(() => { + delete TOOL_SCOPES.__send_probe; + delete TOOL_SCOPES.__settings_probe; + vi.unstubAllEnvs(); + }); + + it.each([ + ['read', 'ping'], + ['write', 'stage_deletion'], + ['send', '__send_probe'], + ['settings', '__settings_probe'], + ])('enforces the %s per-minute bucket independently', (rateClass, toolName) => { + const limiter = createMcpRateLimiter({ limits, now }); + const first = vi.fn(); + limiter(mockReq({ body: call(toolName), tokenId: 'tok-1' }), mockRes(), first); + expect(first).toHaveBeenCalledTimes(1); + + const res = mockRes(); + limiter(mockReq({ body: call(toolName, 42), tokenId: 'tok-1' }), res, vi.fn()); + expect(res.statusCode).toBe(429); + expect(res.headers['Retry-After']).toBe(60); + expect(res.body).toEqual({ + jsonrpc: '2.0', + error: { + code: -32000, + message: `Rate limit exceeded: at most 1 ${rateClass} tool calls per minute per token`, + }, + id: 42, + }); + }); + + it('keys buckets per token, not globally', () => { + const limiter = createMcpRateLimiter({ limits, now }); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); + const next = vi.fn(); + limiter(mockReq({ body: call(), tokenId: 'tok-2' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); // a different token has its own budget + }); + + it('never throttles the initialize/tools-list handshake', () => { + const limiter = createMcpRateLimiter({ limits, now }); + for (const method of ['initialize', 'notifications/initialized', 'tools/list', 'tools/list']) { + const next = vi.fn(); + limiter(mockReq({ body: { jsonrpc: '2.0', method, id: 1 }, tokenId: 'tok-1' }), mockRes(), next); + expect(next, method).toHaveBeenCalledTimes(1); + } + }); + + it('resets the budget after the window elapses', () => { + const limiter = createMcpRateLimiter({ limits, now }); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), vi.fn()); + const blocked = mockRes(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), blocked, vi.fn()); + expect(blocked.statusCode).toBe(429); + clock += 60_001; + const next = vi.fn(); + limiter(mockReq({ body: call(), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('reads every default class limit from its environment variable', () => { + vi.stubEnv('MCP_RATE_LIMIT_PER_MIN', '1'); + vi.stubEnv('MCP_WRITE_RATE_LIMIT_PER_MIN', '1'); + vi.stubEnv('MCP_SEND_RATE_LIMIT_PER_MIN', '1'); + vi.stubEnv('MCP_SETTINGS_RATE_LIMIT_PER_MIN', '1'); + const limiter = createMcpRateLimiter({ now }); + for (const toolName of ['ping', 'stage_deletion', '__send_probe', '__settings_probe']) { + limiter(mockReq({ body: call(toolName), tokenId: 'tok-1' }), mockRes(), vi.fn()); + const res = mockRes(); + limiter(mockReq({ body: call(toolName), tokenId: 'tok-1' }), res, vi.fn()); + expect(res.statusCode).toBe(429); + } + }); + + it('rejects an over-budget batch atomically with a null JSON-RPC id', () => { + const limiter = createMcpRateLimiter({ limits, now }); + const batch = [ + call('ping', 1), + call('stage_deletion', 2), + call('stage_deletion', 3), + ]; + const res = mockRes(); + limiter(mockReq({ body: batch, tokenId: 'tok-1' }), res, vi.fn()); + expect(res.statusCode).toBe(429); + expect(res.body.id).toBeNull(); + expect(res.body.error.message).toContain('write tool calls'); + + // Rejection admitted none of the batch, including its otherwise-valid read. + const next = vi.fn(); + limiter(mockReq({ body: call('ping'), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('enforces and resets the 24-hour send cap alongside the minute bucket', () => { + vi.stubEnv('MCP_SEND_DAILY_CAP', '2'); + const limiter = createMcpRateLimiter({ + limits: { ...limits, send: 10 }, + now, + }); + for (let i = 0; i < 2; i++) { + const next = vi.fn(); + limiter(mockReq({ body: call('__send_probe'), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + } + + const blocked = mockRes(); + limiter(mockReq({ body: call('__send_probe', 9), tokenId: 'tok-1' }), blocked, vi.fn()); + expect(blocked.statusCode).toBe(429); + expect(blocked.headers['Retry-After']).toBe(86_400); + expect(blocked.body.error.message) + .toBe('Rate limit exceeded: at most 2 send tool calls per day per token'); + + clock += 86_400_001; + const next = vi.fn(); + limiter(mockReq({ body: call('__send_probe'), tokenId: 'tok-1' }), mockRes(), next); + expect(next).toHaveBeenCalledTimes(1); + }); +}); + +// --- End-to-end through the live Express mount (same harness as server.test.js) --- +describe('mounted /mcp guards', () => { + const servers = []; + afterAll(async () => { + for (const s of servers) await new Promise((r) => s.close(r)); + }); + + async function mountApp(env) { + for (const [k, v] of Object.entries(env)) vi.stubEnv(k, v); + const app = express(); + app.use(express.json()); + mountMcp(app); // captures env-derived allowlist + limit at mount time + vi.unstubAllEnvs(); + const server = createServer(app); + servers.push(server); + await new Promise((r) => server.listen(0, r)); + return `http://127.0.0.1:${server.address().port}`; + } + + // Every authed request: token lookup -> last_used_at update -> resolveScope. + function primeAuth() { + query + .mockResolvedValueOnce({ rows: [{ id: 'tok', user_id: 'user-1', scopes: ALL_SCOPES }] }) + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }); + } + + async function rpc(base, method, params, { origin } = {}) { + query.mockReset(); + primeAuth(); + const headers = { + 'Content-Type': 'application/json', + Accept: 'application/json, text/event-stream', + Authorization: 'Bearer mcp_good', + }; + if (origin) headers.Origin = origin; + return fetch(`${base}/mcp`, { + method: 'POST', + headers, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }), + }); + } + + it('403s a cross-origin browser request before auth, passes the app origin and no-Origin clients', async () => { + const base = await mountApp({ APP_URL: 'https://mail.example.com' }); + + query.mockReset(); + const evil = await fetch(`${base}/mcp`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Origin: 'http://evil.example' }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list', params: {} }), + }); + expect(evil.status).toBe(403); + expect((await evil.json()).error.code).toBe(-32000); + expect(query).not.toHaveBeenCalled(); // rejected before the token ever hits the DB + + const sameOrigin = await rpc(base, 'tools/list', {}, { origin: 'https://mail.example.com' }); + expect(sameOrigin.status).toBe(200); + + const headless = await rpc(base, 'tools/list', {}); + expect(headless.status).toBe(200); + }); + + it('429s tool calls over the per-token budget but leaves tools/list untouched', async () => { + const base = await mountApp({ MCP_RATE_LIMIT_PER_MIN: '1' }); + + const first = await rpc(base, 'tools/call', { name: 'ping', arguments: {} }); + expect(first.status).toBe(200); + + const second = await rpc(base, 'tools/call', { name: 'ping', arguments: {} }); + expect(second.status).toBe(429); + expect(second.headers.get('Retry-After')).toMatch(/^\d+$/); + const body = await second.json(); + expect(body.error.message).toMatch(/rate limit/i); + + const list = await rpc(base, 'tools/list', {}); + expect(list.status).toBe(200); + }); +}); diff --git a/backend/src/mcp/tools.js b/backend/src/mcp/tools.js new file mode 100644 index 00000000..c0a3734e --- /dev/null +++ b/backend/src/mcp/tools.js @@ -0,0 +1,319 @@ +import { jsonResult } from './result.js'; +import { + searchMetadataDef, handleSearchMetadata, + searchMessageBodiesDef, handleSearchMessageBodies, + semanticSearchMessagesDef, handleSemanticSearchMessages, +} from './searchTools.js'; +import { + getMessageDef, handleGetMessage, + listMessagesDef, handleListMessages, + getStatsDef, handleGetStats, + aggregateDef, handleAggregate, + searchByDomainsDef, handleSearchByDomains, + findSimilarMessagesDef, handleFindSimilarMessages, + searchInMessageDef, handleSearchInMessage, + stageDeletionDef, handleStageDeletion, +} from './messageTools.js'; +import { + createDraftDef, handleCreateDraft, + updateDraftDef, handleUpdateDraft, + listDraftsDef, handleListDrafts, + getDraftDef, handleGetDraft, + deleteDraftDef, handleDeleteDraft, +} from './draftTools.js'; +import { + sendEmailDef, handleSendEmail, + sendDraftDef, handleSendDraft, + unsendEmailDef, handleUnsendEmail, + listOutboxDef, handleListOutbox, + recallEmailDef, handleRecallEmail, +} from './sendTools.js'; +import { + replyEmailDef, handleReplyEmail, + replyAllEmailDef, handleReplyAllEmail, + forwardEmailDef, handleForwardEmail, +} from './composeTools.js'; +import { + listFoldersDef, handleListFolders, + createFolderDef, handleCreateFolder, + renameFolderDef, handleRenameFolder, + deleteFolderDef, handleDeleteFolder, + moveMessagesDef, handleMoveMessages, + archiveMessagesDef, handleArchiveMessages, + trashMessagesDef, handleTrashMessages, + markReadDef, handleMarkRead, + markUnreadDef, handleMarkUnread, + starMessageDef, handleStarMessage, + unstarMessageDef, handleUnstarMessage, + markSpamDef, handleMarkSpam, + markNotSpamDef, handleMarkNotSpam, + snoozeMessageDef, handleSnoozeMessage, + unsnoozeMessageDef, handleUnsnoozeMessage, + setCategoryDef, handleSetCategory, + gtdClassifyDef, handleGtdClassify, + gtdDoneDef, handleGtdDone, +} from './mailboxTools.js'; +import { + markTriagedDef, handleMarkTriaged, + triageInboxDef, handleTriageInbox, + getTriageContextDef, handleGetTriageContext, +} from './triageTools.js'; +import { + listAccountsDef, handleListAccounts, + addAccountDef, handleAddAccount, + updateAccountSettingsDef, handleUpdateAccountSettings, + testAccountConnectionDef, handleTestAccountConnection, + createAliasDef, handleCreateAlias, + updateAliasDef, handleUpdateAlias, + deleteAliasDef, handleDeleteAlias, + listRulesDef, handleListRules, + createRuleDef, handleCreateRule, + updateRuleDef, handleUpdateRule, + deleteRuleDef, handleDeleteRule, + runRulesDef, handleRunRules, +} from './accountTools.js'; +import { + listComposeSessionsDef, handleListComposeSessions, + getComposeSessionDef, handleGetComposeSession, + createComposeSessionDef, handleCreateComposeSession, + updateComposeSessionDef, handleUpdateComposeSession, + minimizeComposeSessionDef, handleMinimizeComposeSession, + restoreComposeSessionDef, handleRestoreComposeSession, + addComposeAttachmentDef, handleAddComposeAttachment, + removeComposeAttachmentDef, handleRemoveComposeAttachment, + closeComposeSessionDef, handleCloseComposeSession, + discardComposeSessionDef, handleDiscardComposeSession, + sendComposeSessionDef, handleSendComposeSession, +} from './composeSessionTools.js'; + +// TOOL_DEFS drives tools/list; HANDLERS drives tools/call. Slices 09 and 10 +// append to both. Keep names, descriptions, and inputSchema field names verbatim +// from internal/mcp/server.go (README D6: id-bearing fields diverge to strings). +export const TOOL_DEFS = [ + { + name: 'ping', + description: 'Health check: returns {"pong":true}. Proves transport + auth round-trip.', + annotations: { + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + inputSchema: { type: 'object', properties: {} }, + }, + searchMetadataDef, + searchMessageBodiesDef, + semanticSearchMessagesDef, + getMessageDef, + listMessagesDef, + getStatsDef, + aggregateDef, + searchByDomainsDef, + findSimilarMessagesDef, + searchInMessageDef, + stageDeletionDef, + createDraftDef, + updateDraftDef, + listDraftsDef, + getDraftDef, + deleteDraftDef, + sendEmailDef, + sendDraftDef, + unsendEmailDef, + listOutboxDef, + recallEmailDef, + replyEmailDef, + replyAllEmailDef, + forwardEmailDef, + listFoldersDef, + createFolderDef, + renameFolderDef, + deleteFolderDef, + moveMessagesDef, + archiveMessagesDef, + trashMessagesDef, + markReadDef, + markUnreadDef, + starMessageDef, + unstarMessageDef, + markSpamDef, + markNotSpamDef, + snoozeMessageDef, + unsnoozeMessageDef, + setCategoryDef, + gtdClassifyDef, + gtdDoneDef, + markTriagedDef, + triageInboxDef, + getTriageContextDef, + listAccountsDef, + addAccountDef, + updateAccountSettingsDef, + testAccountConnectionDef, + createAliasDef, + updateAliasDef, + deleteAliasDef, + listRulesDef, + createRuleDef, + updateRuleDef, + deleteRuleDef, + runRulesDef, + listComposeSessionsDef, + getComposeSessionDef, + createComposeSessionDef, + updateComposeSessionDef, + minimizeComposeSessionDef, + restoreComposeSessionDef, + addComposeAttachmentDef, + removeComposeAttachmentDef, + closeComposeSessionDef, + discardComposeSessionDef, + sendComposeSessionDef, +]; + +// Name → required scope. Keep this map separate from TOOL_DEFS because those +// definitions are serialized verbatim onto the tools/list wire response. +export const TOOL_SCOPES = { + ping: 'read', + search_metadata: 'read', + search_message_bodies: 'read', + semantic_search_messages: 'read', + get_message: 'read', + list_messages: 'read', + get_stats: 'read', + aggregate: 'read', + search_by_domains: 'read', + find_similar_messages: 'read', + search_in_message: 'read', + stage_deletion: 'write', + create_draft: 'write', + update_draft: 'write', + list_drafts: 'read', + get_draft: 'read', + delete_draft: 'write', + send_email: 'send', + send_draft: 'send', + unsend_email: 'send', + list_outbox: 'read', + recall_email: 'send', + reply_email: 'send', + reply_all_email: 'send', + forward_email: 'send', + list_folders: 'read', + create_folder: 'write', + rename_folder: 'write', + delete_folder: 'write', + move_messages: 'write', + archive_messages: 'write', + trash_messages: 'write', + mark_read: 'write', + mark_unread: 'write', + star_message: 'write', + unstar_message: 'write', + mark_spam: 'write', + mark_not_spam: 'write', + snooze_message: 'write', + unsnooze_message: 'write', + set_category: 'write', + gtd_classify: 'write', + gtd_done: 'write', + mark_triaged: 'write', + triage_inbox: 'read', + get_triage_context: 'read', + list_accounts: 'read', + add_account: 'settings', + update_account_settings: 'settings', + test_account_connection: 'settings', + create_alias: 'settings', + update_alias: 'settings', + delete_alias: 'settings', + list_rules: 'settings', + create_rule: 'settings', + update_rule: 'settings', + delete_rule: 'settings', + run_rules: ['settings', 'write'], + list_compose_sessions: 'read', + get_compose_session: 'read', + create_compose_session: 'write', + update_compose_session: 'write', + minimize_compose_session: 'write', + restore_compose_session: 'write', + add_compose_attachment: 'write', + remove_compose_attachment: 'write', + close_compose_session: 'write', + discard_compose_session: 'write', + send_compose_session: 'send', +}; + +export const HANDLERS = { + // eslint-disable-next-line no-unused-vars + ping: async (_args, _scope) => jsonResult({ pong: true }), + search_metadata: (a, s) => handleSearchMetadata(a, s), + search_message_bodies: (a, s) => handleSearchMessageBodies(a, s), + semantic_search_messages: (a, s) => handleSemanticSearchMessages(a, s), + get_message: (a, s) => handleGetMessage(a, s), + list_messages: (a, s) => handleListMessages(a, s), + get_stats: (a, s) => handleGetStats(a, s), + aggregate: (a, s) => handleAggregate(a, s), + search_by_domains: (a, s) => handleSearchByDomains(a, s), + find_similar_messages: (a, s) => handleFindSimilarMessages(a, s), + search_in_message: (a, s) => handleSearchInMessage(a, s), + stage_deletion: (a, s) => handleStageDeletion(a, s), + create_draft: handleCreateDraft, + update_draft: handleUpdateDraft, + list_drafts: handleListDrafts, + get_draft: handleGetDraft, + delete_draft: handleDeleteDraft, + send_email: handleSendEmail, + send_draft: handleSendDraft, + unsend_email: handleUnsendEmail, + list_outbox: handleListOutbox, + recall_email: handleRecallEmail, + reply_email: handleReplyEmail, + reply_all_email: handleReplyAllEmail, + forward_email: handleForwardEmail, + list_folders: handleListFolders, + create_folder: handleCreateFolder, + rename_folder: handleRenameFolder, + delete_folder: handleDeleteFolder, + move_messages: handleMoveMessages, + archive_messages: handleArchiveMessages, + trash_messages: handleTrashMessages, + mark_read: handleMarkRead, + mark_unread: handleMarkUnread, + star_message: handleStarMessage, + unstar_message: handleUnstarMessage, + mark_spam: handleMarkSpam, + mark_not_spam: handleMarkNotSpam, + snooze_message: handleSnoozeMessage, + unsnooze_message: handleUnsnoozeMessage, + set_category: handleSetCategory, + gtd_classify: handleGtdClassify, + gtd_done: handleGtdDone, + mark_triaged: handleMarkTriaged, + triage_inbox: handleTriageInbox, + get_triage_context: handleGetTriageContext, + list_accounts: handleListAccounts, + add_account: handleAddAccount, + update_account_settings: handleUpdateAccountSettings, + test_account_connection: handleTestAccountConnection, + create_alias: handleCreateAlias, + update_alias: handleUpdateAlias, + delete_alias: handleDeleteAlias, + list_rules: handleListRules, + create_rule: handleCreateRule, + update_rule: handleUpdateRule, + delete_rule: handleDeleteRule, + run_rules: handleRunRules, + list_compose_sessions: handleListComposeSessions, + get_compose_session: handleGetComposeSession, + create_compose_session: handleCreateComposeSession, + update_compose_session: handleUpdateComposeSession, + minimize_compose_session: handleMinimizeComposeSession, + restore_compose_session: handleRestoreComposeSession, + add_compose_attachment: handleAddComposeAttachment, + remove_compose_attachment: handleRemoveComposeAttachment, + close_compose_session: handleCloseComposeSession, + discard_compose_session: handleDiscardComposeSession, + send_compose_session: handleSendComposeSession, +}; diff --git a/backend/src/mcp/tools.test.js b/backend/src/mcp/tools.test.js new file mode 100644 index 00000000..f3b79837 --- /dev/null +++ b/backend/src/mcp/tools.test.js @@ -0,0 +1,131 @@ +import { describe, expect, it } from 'vitest'; +import { + addComposeAttachmentDef, + closeComposeSessionDef, + createComposeSessionDef, + discardComposeSessionDef, + getComposeSessionDef, + handleAddComposeAttachment, + handleCloseComposeSession, + handleCreateComposeSession, + handleDiscardComposeSession, + handleGetComposeSession, + handleListComposeSessions, + handleMinimizeComposeSession, + handleRemoveComposeAttachment, + handleRestoreComposeSession, + handleSendComposeSession, + handleUpdateComposeSession, + listComposeSessionsDef, + minimizeComposeSessionDef, + removeComposeAttachmentDef, + restoreComposeSessionDef, + sendComposeSessionDef, + updateComposeSessionDef, +} from './composeSessionTools.js'; +import { HANDLERS, TOOL_DEFS, TOOL_SCOPES } from './tools.js'; + +const VALID_SCOPES = new Set(['read', 'write', 'send', 'settings']); +const ANNOTATION_KEYS = [ + 'readOnlyHint', + 'destructiveHint', + 'idempotentHint', + 'openWorldHint', +]; + +const COMPOSE_DEFINITIONS = [ + listComposeSessionsDef, + getComposeSessionDef, + createComposeSessionDef, + updateComposeSessionDef, + minimizeComposeSessionDef, + restoreComposeSessionDef, + addComposeAttachmentDef, + removeComposeAttachmentDef, + closeComposeSessionDef, + discardComposeSessionDef, + sendComposeSessionDef, +]; + +const COMPOSE_SCOPES = { + list_compose_sessions: 'read', + get_compose_session: 'read', + create_compose_session: 'write', + update_compose_session: 'write', + minimize_compose_session: 'write', + restore_compose_session: 'write', + add_compose_attachment: 'write', + remove_compose_attachment: 'write', + close_compose_session: 'write', + discard_compose_session: 'write', + send_compose_session: 'send', +}; + +const COMPOSE_HANDLERS = { + list_compose_sessions: handleListComposeSessions, + get_compose_session: handleGetComposeSession, + create_compose_session: handleCreateComposeSession, + update_compose_session: handleUpdateComposeSession, + minimize_compose_session: handleMinimizeComposeSession, + restore_compose_session: handleRestoreComposeSession, + add_compose_attachment: handleAddComposeAttachment, + remove_compose_attachment: handleRemoveComposeAttachment, + close_compose_session: handleCloseComposeSession, + discard_compose_session: handleDiscardComposeSession, + send_compose_session: handleSendComposeSession, +}; + +describe('MCP tool registry invariants', () => { + it('appends every compose-session definition exactly once in the approved order', () => { + const names = TOOL_DEFS.map(({ name }) => name); + const composeNames = COMPOSE_DEFINITIONS.map(({ name }) => name); + + expect(names.slice(-composeNames.length)).toEqual(composeNames); + for (const definition of COMPOSE_DEFINITIONS) { + expect(TOOL_DEFS.filter(item => item === definition), definition.name).toHaveLength(1); + expect(names.filter(name => name === definition.name), definition.name).toHaveLength(1); + } + }); + + it('uses the exact approved compose-session scope map', () => { + expect(Object.fromEntries( + Object.keys(COMPOSE_SCOPES).map(name => [name, TOOL_SCOPES[name]]), + )).toEqual(COMPOSE_SCOPES); + }); + + it('wires every compose-session name directly to its approved handler', () => { + for (const [name, handler] of Object.entries(COMPOSE_HANDLERS)) { + expect(HANDLERS[name], name).toBe(handler); + } + }); + + it('gives every tool definition all four boolean annotation hints', () => { + for (const definition of TOOL_DEFS) { + expect(definition.annotations, definition.name).toBeTypeOf('object'); + for (const key of ANNOTATION_KEYS) { + expect(definition.annotations, `${definition.name}.${key}`).toHaveProperty(key); + expect(typeof definition.annotations[key], `${definition.name}.${key}`).toBe('boolean'); + } + } + }); + + it('classifies every definition with one or more valid scopes', () => { + for (const definition of TOOL_DEFS) { + const required = TOOL_SCOPES[definition.name]; + if (Array.isArray(required)) { + expect(required.length, definition.name).toBeGreaterThanOrEqual(1); + for (const scope of required) { + expect(VALID_SCOPES.has(scope), `${definition.name}: ${scope}`).toBe(true); + } + } else { + expect(VALID_SCOPES.has(required), `${definition.name}: ${required}`).toBe(true); + } + } + }); + + it('keeps definitions, scope classifications, and handlers in exact three-way sync', () => { + const definitions = TOOL_DEFS.map(({ name }) => name).sort(); + expect(Object.keys(TOOL_SCOPES).sort()).toEqual(definitions); + expect(Object.keys(HANDLERS).sort()).toEqual(definitions); + }); +}); diff --git a/backend/src/mcp/triageAdapter.integration.test.js b/backend/src/mcp/triageAdapter.integration.test.js new file mode 100644 index 00000000..329e5800 --- /dev/null +++ b/backend/src/mcp/triageAdapter.integration.test.js @@ -0,0 +1,145 @@ +import { randomUUID } from 'crypto'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from '../services/embeddings/testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const integrationDescribe = DSN ? describe : describe.skip; + +integrationDescribe('triageAdapter', () => { + let client; + let adapter; + let db; + let userId; + let accountId; + let nextUid = 700000; + + beforeAll(async () => { + const url = new URL(DSN); + process.env.DB_HOST = url.hostname; + process.env.DB_PORT = url.port; + process.env.DB_NAME = url.pathname.slice(1); + process.env.DB_USER = url.username; + process.env.DB_PASSWORD = url.password; + + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + ({ userId, accountId } = await seedAccount(client, 'triage-adapter')); + adapter = await import('./triageAdapter.js'); + db = await import('../services/db.js'); + }); + + afterAll(async () => { + await cleanupAccount(client, userId); + await client.end(); + await db.pool.end(); + }); + + async function insertMessage({ + date, + header = `<${randomUUID()}@example.com>`, + folder = 'INBOX', + isRead = false, + threadId = randomUUID(), + }) { + const result = await client.query( + `INSERT INTO messages + (account_id, uid, folder, message_id, subject, from_email, date, is_read, thread_id) + VALUES ($1, $2, $3, $4, $5, 'sender@example.com', $6, $7, $8) + RETURNING id, message_id`, + [accountId, nextUid++, folder, header, `message ${nextUid}`, date, isRead, threadId], + ); + return result.rows[0]; + } + + it('keyset-pages a seeded inbox without duplicates or gaps under a mid-stream insert', async () => { + const expectedIds = []; + for (let index = 0; index < 60; index++) { + const message = await insertMessage({ + date: new Date(Date.UTC(2026, 0, 1, 0, index)).toISOString(), + header: ``, + }); + expectedIds.push(message.id); + } + + const seen = []; + let page = await adapter.listTriageCandidates({ + accountIds: [accountId], + limit: 20, + unreadOnly: true, + }); + seen.push(...page.rows.map(row => row.id)); + + const inserted = await insertMessage({ + date: new Date(Date.UTC(2026, 0, 1, 0, 25, 30)).toISOString(), + header: '', + }); + expectedIds.push(inserted.id); + + while (page.hasMore) { + page = await adapter.listTriageCandidates({ + accountIds: [accountId], + cursor: page.cursor, + limit: 20, + unreadOnly: true, + }); + seen.push(...page.rows.map(row => row.id)); + } + + expect(new Set(seen).size).toBe(seen.length); + expect(new Set(seen)).toEqual(new Set(expectedIds)); + }); + + it('marks idempotently and distinguishes INSERT from conflict UPDATE with xmax=0', async () => { + const message = await insertMessage({ date: '2026-02-01T00:00:00.000Z' }); + const args = { + userId, + accountIds: [accountId], + messageIds: [message.id], + action: 'archived', + note: 'integration', + tokenId: null, + }; + + await expect(adapter.markTriaged(args)).resolves.toMatchObject({ + marked: 1, + newly_marked: 1, + already_triaged: 0, + }); + await expect(adapter.markTriaged(args)).resolves.toMatchObject({ + marked: 1, + newly_marked: 0, + already_triaged: 1, + }); + }); + + it('skips null headers and cascades checkpoints when the account is deleted', async () => { + const noHeader = await insertMessage({ + date: '2026-02-02T00:00:00.000Z', + header: null, + }); + const durable = await insertMessage({ date: '2026-02-03T00:00:00.000Z' }); + + await expect(adapter.markTriaged({ + userId, + accountIds: [accountId], + messageIds: [noHeader.id], + })).resolves.toMatchObject({ + marked: 0, + skipped: [{ id: noHeader.id, reason: 'no_message_id_header' }], + }); + + await adapter.markTriaged({ + userId, + accountIds: [accountId], + messageIds: [durable.id], + }); + await client.query('DELETE FROM email_accounts WHERE id = $1', [accountId]); + + const result = await client.query( + 'SELECT id FROM message_triage WHERE account_id = $1', + [accountId], + ); + expect(result.rows).toEqual([]); + }); +}); diff --git a/backend/src/mcp/triageAdapter.js b/backend/src/mcp/triageAdapter.js new file mode 100644 index 00000000..1314c2bf --- /dev/null +++ b/backend/src/mcp/triageAdapter.js @@ -0,0 +1,342 @@ +// SQL owner for the inbox-triage MCP surface. Handlers validate and shape wire +// envelopes; this adapter keeps every read/checkpoint query scoped to accountIds. +import { query } from '../services/db.js'; + +function decodeCursor(cursor) { + if (!cursor) return null; + try { + const parsed = JSON.parse(Buffer.from(cursor, 'base64').toString('utf8')); + if ( + !parsed + || typeof parsed.d !== 'string' + || !parsed.d + || typeof parsed.h !== 'string' + || !parsed.h + ) { + throw new Error('invalid shape'); + } + return parsed; + } catch { + throw new Error('invalid triage cursor'); + } +} + +function encodeCursor(row) { + if (!row) return null; + const date = row.date instanceof Date ? row.date.toISOString() : row.date; + return Buffer.from(JSON.stringify({ d: date, h: row.message_id }), 'utf8').toString('base64'); +} + +export async function listTriageCandidates({ + accountIds, + cursor, + limit = 25, + unreadOnly = true, + includeTriaged = false, + categories, + since, +}) { + const pageLimit = Number(limit); + if (!accountIds?.length) return { rows: [], hasMore: false, cursor: null }; + + const params = [accountIds]; + const bind = value => { + params.push(value); + return `$${params.length}`; + }; + const where = [ + 'm.account_id = ANY($1)', + "m.folder = 'INBOX'", + 'm.is_deleted = false', + 'm.message_id IS NOT NULL', + ]; + + if (unreadOnly) where.push('m.is_read = false'); + if (!includeTriaged) { + where.push(`NOT EXISTS ( + SELECT 1 + FROM message_triage mt + WHERE mt.account_id = m.account_id + AND mt.message_id_header = m.message_id + )`); + } + if (categories?.length) { + where.push(`COALESCE(m.category, 'primary') = ANY(${bind(categories)})`); + } + if (since) where.push(`m.date >= ${bind(since)}`); + + const decodedCursor = decodeCursor(cursor); + if (decodedCursor) { + const dateParam = bind(decodedCursor.d); + const headerParam = bind(decodedCursor.h); + where.push(`(m.date, m.message_id) > (${dateParam}, ${headerParam})`); + } + + const sql = ` + WITH sender_history AS ( + SELECT + lower(mh.from_email) AS sender_email, + COUNT(*)::int AS received_count, + MIN(mh.date) AS first_received, + MAX(mh.date) AS last_received + FROM messages mh + WHERE mh.account_id = ANY($1) + AND mh.is_deleted = false + AND mh.from_email IS NOT NULL + GROUP BY lower(mh.from_email) + ) + SELECT + m.id, + m.account_id, + a.email_address AS account, + m.message_id, + m.thread_key AS conversation_id, + m.subject, + m.snippet, + m.from_email, + m.from_name, + m.date, + m.is_read, + m.is_starred, + m.has_attachments, + COALESCE(m.category, 'primary') AS category, + COALESCE(m.is_bulk, false) AS is_bulk, + (m.list_unsubscribe IS NOT NULL) AS has_unsubscribe, + m.spam_verdict, + thread_state.message_count AS thread_message_count, + thread_state.last_activity AS thread_last_activity, + thread_state.i_replied, + COALESCE(sh.received_count, 0)::int AS received_count, + sh.first_received, + sh.last_received, + c.id AS contact_id, + c.display_name AS contact_name, + COALESCE(c.send_count, 0)::int AS send_count, + c.last_sent, + c.is_auto, + (c.id IS NOT NULL) AS contact_known + FROM messages m + JOIN email_accounts a ON a.id = m.account_id + LEFT JOIN LATERAL ( + SELECT + COUNT(*)::int AS message_count, + MAX(t.date) AS last_activity, + EXISTS ( + SELECT 1 + FROM messages sent + WHERE sent.account_id = m.account_id + AND sent.thread_key = m.thread_key + AND sent.is_deleted = false + AND ( + sent.folder = COALESCE(a.folder_mappings->>'sent', '') + OR sent.folder ~* '(^|[./ ])sent($|[./ ])' + OR EXISTS ( + SELECT 1 + FROM folders sf + WHERE sf.account_id = sent.account_id + AND sf.path = sent.folder + AND sf.special_use = '\\Sent' + ) + ) + ) AS i_replied + FROM messages t + WHERE t.account_id = m.account_id + AND t.thread_key = m.thread_key + AND t.is_deleted = false + ) thread_state ON true + LEFT JOIN sender_history sh ON sh.sender_email = lower(m.from_email) + LEFT JOIN contacts c + ON c.user_id = a.user_id + AND c.primary_email = lower(m.from_email) + WHERE ${where.join('\n AND ')} + ORDER BY m.date ASC, m.message_id ASC + LIMIT ${bind(pageLimit + 1)} + `; + + const result = await query(sql, params); + const hasMore = result.rows.length > pageLimit; + const rows = result.rows.slice(0, pageLimit); + return { + rows, + hasMore, + cursor: encodeCursor(rows.at(-1)), + }; +} + +// Backlog total for triage_inbox's counts.untriaged_unread: unread INBOX +// messages not yet checkpointed in message_triage, regardless of paging. +export async function countUntriagedUnread(accountIds) { + if (!accountIds?.length) return 0; + const { rows } = await query( + `SELECT COUNT(*) AS total + FROM messages m + WHERE m.account_id = ANY($1) + AND m.folder = 'INBOX' + AND m.is_deleted = false + AND m.message_id IS NOT NULL + AND m.is_read = false + AND NOT EXISTS ( + SELECT 1 + FROM message_triage mt + WHERE mt.account_id = m.account_id + AND mt.message_id_header = m.message_id + )`, + [accountIds], + ); + return parseInt(rows[0]?.total, 10) || 0; +} + +export async function senderHistory(fromEmail, accountIds) { + if (!fromEmail || !accountIds?.length) return null; + const { rows } = await query( + `WITH sender_history AS ( + SELECT + COUNT(*)::int AS received_count, + MIN(m.date) AS first_received, + MAX(m.date) AS last_received + FROM messages m + WHERE m.account_id = ANY($1) + AND m.is_deleted = false + AND lower(m.from_email) = lower($2) + ) + SELECT + sh.received_count, + sh.first_received, + sh.last_received, + c.id AS contact_id, + c.display_name AS contact_name, + c.primary_email, + COALESCE(c.send_count, 0)::int AS send_count, + c.last_sent, + c.is_auto, + (c.id IS NOT NULL) AS contact_known + FROM sender_history sh + LEFT JOIN contacts c + ON c.user_id IN ( + SELECT a.user_id + FROM email_accounts a + WHERE a.id = ANY($1) + ) + AND c.primary_email = lower($2) + LIMIT 1`, + [accountIds, fromEmail], + ); + return rows[0] || null; +} + +// Resolved Sent folder for disposition classification: the account's explicit +// folder_mappings.sent wins, else the IMAP \Sent special-use folder. Mirrors +// services/mail/sentCopy.js resolveSentFolder but keyed by accountId so the +// signals path never needs a full account row. +export async function sentFolderForAccount(accountId) { + if (!accountId) return null; + const { rows } = await query( + `SELECT COALESCE( + a.folder_mappings->>'sent', + (SELECT f.path FROM folders f + WHERE f.account_id = a.id AND f.special_use = '\\Sent' + LIMIT 1) + ) AS path + FROM email_accounts a + WHERE a.id = $1`, + [accountId], + ); + return rows[0]?.path || null; +} + +export async function triageActionsForMessages(pairs) { + if (!pairs?.length) return []; + const { rows } = await query( + `SELECT + mt.account_id, + mt.message_id_header, + mt.action, + mt.triaged_at + FROM unnest($1::uuid[], $2::text[]) AS input(account_id, message_id_header) + JOIN message_triage mt + ON mt.account_id = input.account_id + AND mt.message_id_header = input.message_id_header`, + [ + pairs.map(pair => pair.accountId), + pairs.map(pair => pair.messageIdHeader), + ], + ); + return rows; +} + +export async function resolveHeadersForIds(ids, accountIds) { + if (!ids?.length || !accountIds?.length) return []; + const { rows } = await query( + `SELECT id, account_id, message_id AS message_id_header + FROM messages + WHERE id = ANY($1::uuid[]) + AND account_id = ANY($2::uuid[])`, + [ids, accountIds], + ); + return rows; +} + +export async function markTriaged({ + userId, + accountIds, + messageIds, + action = null, + note = null, + tokenId = null, +}) { + const resolved = await resolveHeadersForIds(messageIds, accountIds); + const byId = new Map(resolved.map(row => [row.id, row])); + const skipped = []; + const durableByKey = new Map(); + + for (const id of messageIds || []) { + const row = byId.get(id); + if (!row) { + skipped.push({ id, reason: 'not_found_or_out_of_scope' }); + } else if (!row.message_id_header) { + skipped.push({ id, reason: 'no_message_id_header' }); + } else { + durableByKey.set(`${row.account_id}\0${row.message_id_header}`, row); + } + } + + const durable = [...durableByKey.values()]; + if (!durable.length) { + return { + ok: true, + marked: 0, + newly_marked: 0, + already_triaged: 0, + skipped, + }; + } + + const { rows } = await query( + `INSERT INTO message_triage + (user_id, account_id, message_id_header, action, note, source, token_id) + SELECT $1, input.account_id, input.message_id_header, $4, $5, 'mcp', $6 + FROM unnest($2::uuid[], $3::text[]) AS input(account_id, message_id_header) + ON CONFLICT (account_id, message_id_header) DO UPDATE + SET triaged_at = NOW(), + action = EXCLUDED.action, + note = EXCLUDED.note + RETURNING account_id, message_id_header, (xmax = 0) AS inserted`, + [ + userId, + durable.map(row => row.account_id), + durable.map(row => row.message_id_header), + action, + note, + tokenId, + ], + ); + + const newlyMarked = rows.filter(row => row.inserted === true).length; + return { + ok: true, + marked: rows.length, + newly_marked: newlyMarked, + already_triaged: rows.length - newlyMarked, + skipped, + }; +} diff --git a/backend/src/mcp/triageAdapter.test.js b/backend/src/mcp/triageAdapter.test.js new file mode 100644 index 00000000..da6a5646 --- /dev/null +++ b/backend/src/mcp/triageAdapter.test.js @@ -0,0 +1,344 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); + +import { query } from '../services/db.js'; + +const adapterPromise = import('./triageAdapter.js').catch(() => null); + +function cursor(date = '2026-07-28T08:00:00.000Z', header = '') { + return Buffer.from(JSON.stringify({ d: date, h: header }), 'utf8').toString('base64'); +} + +describe('listTriageCandidates', () => { + beforeEach(() => query.mockReset()); + + it('uses one scoped, enriched inbox query and excludes triaged messages by default', async () => { + const adapter = await adapterPromise; + expect(adapter?.listTriageCandidates).toBeTypeOf('function'); + query.mockResolvedValueOnce({ rows: [] }); + + await adapter.listTriageCandidates({ + accountIds: ['acc-1'], + limit: 25, + unreadOnly: true, + includeTriaged: false, + }); + + expect(query).toHaveBeenCalledTimes(1); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($1)'); + expect(sql).toContain("m.folder = 'INBOX'"); + expect(sql).toContain('m.is_deleted = false'); + expect(sql).toContain('m.message_id IS NOT NULL'); + expect(sql).toContain('m.is_read = false'); + expect(sql).toMatch(/NOT EXISTS\s*\(\s*SELECT 1\s+FROM message_triage mt/i); + expect(sql).toContain('LEFT JOIN LATERAL'); + expect(sql).toContain('t.thread_key = m.thread_key'); + expect(sql).toContain('AS i_replied'); + expect(sql).toContain('WITH sender_history AS'); + expect(sql).toContain('LEFT JOIN sender_history'); + expect(sql).toContain('LEFT JOIN contacts'); + expect(sql).toContain('received_count'); + expect(sql).not.toContain('receive_count'); + expect(sql).toContain('ORDER BY m.date ASC, m.message_id ASC'); + expect(params[0]).toEqual(['acc-1']); + expect(params.at(-1)).toBe(26); + }); + + it('adds the tuple keyset predicate only when a cursor is supplied', async () => { + const adapter = await adapterPromise; + query.mockResolvedValue({ rows: [] }); + + await adapter.listTriageCandidates({ accountIds: ['acc-1'], limit: 10 }); + expect(query.mock.calls[0][0]).not.toMatch(/\(m\.date,\s*m\.message_id\)\s*>/); + + query.mockClear(); + const date = '2026-07-28T08:00:00.000Z'; + const header = ''; + await adapter.listTriageCandidates({ + accountIds: ['acc-1'], + cursor: cursor(date, header), + limit: 10, + }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/\(m\.date,\s*m\.message_id\)\s*>\s*\(\$\d+,\s*\$\d+\)/); + expect(params).toContain(date); + expect(params).toContain(header); + }); + + it('honors includeTriaged, unreadOnly, categories, and since without losing scope', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ rows: [] }); + + await adapter.listTriageCandidates({ + accountIds: ['acc-1', 'acc-2'], + limit: 5, + unreadOnly: false, + includeTriaged: true, + categories: ['primary', 'newsletter'], + since: '2026-07-01T00:00:00.000Z', + }); + + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($1)'); + expect(sql).not.toContain('FROM message_triage mt'); + expect(sql).not.toContain('m.is_read = false'); + expect(sql).toContain("COALESCE(m.category, 'primary') = ANY"); + expect(sql).toMatch(/m\.date >= \$\d+/); + expect(params).toContainEqual(['primary', 'newsletter']); + expect(params).toContain('2026-07-01T00:00:00.000Z'); + }); + + it('returns limit rows plus hasMore and a header-based cursor from limit+1 results', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ + rows: [ + { id: 'm1', date: new Date('2026-07-01T00:00:00Z'), message_id: '' }, + { id: 'm2', date: new Date('2026-07-02T00:00:00Z'), message_id: '' }, + { id: 'm3', date: new Date('2026-07-03T00:00:00Z'), message_id: '' }, + ], + }); + + const page = await adapter.listTriageCandidates({ accountIds: ['acc-1'], limit: 2 }); + + expect(page.rows.map(row => row.id)).toEqual(['m1', 'm2']); + expect(page.hasMore).toBe(true); + expect(JSON.parse(Buffer.from(page.cursor, 'base64').toString('utf8'))).toEqual({ + d: '2026-07-02T00:00:00.000Z', + h: '', + }); + }); +}); + +describe('senderHistory', () => { + beforeEach(() => query.mockReset()); + + it('returns computed receive history and the matching scoped contact in one query', async () => { + const adapter = await adapterPromise; + const row = { + received_count: 4, + first_received: '2026-01-01T00:00:00Z', + last_received: '2026-07-01T00:00:00Z', + send_count: 2, + last_sent: '2026-06-01T00:00:00Z', + is_auto: false, + }; + query.mockResolvedValueOnce({ rows: [row] }); + + await expect(adapter.senderHistory('Sender@Example.com', ['acc-1'])).resolves.toEqual(row); + + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($1)'); + expect(sql).toContain('COUNT(*)::int AS received_count'); + expect(sql).toContain('LEFT JOIN contacts'); + expect(sql).toContain('c.primary_email = lower($2)'); + expect(params).toEqual([['acc-1'], 'Sender@Example.com']); + }); +}); + +describe('resolveHeadersForIds', () => { + beforeEach(() => query.mockReset()); + + it('resolves message row ids to durable headers within account scope', async () => { + const adapter = await adapterPromise; + const rows = [{ id: 'm1', account_id: 'acc-1', message_id_header: '' }]; + query.mockResolvedValueOnce({ rows }); + + await expect(adapter.resolveHeadersForIds(['m1'], ['acc-1'])).resolves.toEqual(rows); + + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('id = ANY($1'); + expect(sql).toContain('account_id = ANY($2'); + expect(sql).toContain('message_id AS message_id_header'); + expect(params).toEqual([['m1'], ['acc-1']]); + }); +}); + +describe('triageActionsForMessages', () => { + beforeEach(() => query.mockReset()); + + it('reads actions for account/header pairs with one pairwise scoped query', async () => { + const adapter = await adapterPromise; + const pairs = [ + { accountId: 'acc-1', messageIdHeader: '' }, + { accountId: 'acc-2', messageIdHeader: '' }, + ]; + const rows = [ + { + account_id: 'acc-1', + message_id_header: '', + action: 'archived', + triaged_at: '2026-07-28T08:00:00Z', + }, + ]; + query.mockResolvedValueOnce({ rows }); + + await expect(adapter.triageActionsForMessages(pairs)).resolves.toEqual(rows); + + expect(query).toHaveBeenCalledTimes(1); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch( + /FROM unnest\(\$1::uuid\[\], \$2::text\[\]\) AS input\(account_id, message_id_header\)/, + ); + expect(sql).toContain('JOIN message_triage mt'); + expect(sql).toContain('mt.account_id = input.account_id'); + expect(sql).toContain('mt.message_id_header = input.message_id_header'); + expect(sql).toContain('mt.action'); + expect(sql).toContain('mt.triaged_at'); + expect(params).toEqual([ + ['acc-1', 'acc-2'], + ['', ''], + ]); + }); + + it('returns no actions without querying when the pair list is empty', async () => { + const adapter = await adapterPromise; + + await expect(adapter.triageActionsForMessages([])).resolves.toEqual([]); + + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('sentFolderForAccount', () => { + beforeEach(() => query.mockReset()); + + it('resolves the mapped sent folder with a \\Sent special-use fallback in one query', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ rows: [{ path: 'Custom/Outgoing' }] }); + + await expect(adapter.sentFolderForAccount('acc-1')).resolves.toBe('Custom/Outgoing'); + + expect(query).toHaveBeenCalledTimes(1); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain("a.folder_mappings->>'sent'"); + expect(sql).toContain("f.special_use = '\\Sent'"); + expect(sql).toContain('WHERE a.id = $1'); + expect(params).toEqual(['acc-1']); + }); + + it('returns null without querying when no account id is given', async () => { + const adapter = await adapterPromise; + + await expect(adapter.sentFolderForAccount(null)).resolves.toBeNull(); + + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('markTriaged', () => { + beforeEach(() => query.mockReset()); + + it('skips a null-header id without issuing an UPSERT for it', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ + rows: [{ id: 'm-null', account_id: 'acc-1', message_id_header: null }], + }); + + const result = await adapter.markTriaged({ + userId: 'user-1', + accountIds: ['acc-1'], + messageIds: ['m-null'], + action: 'left', + note: null, + tokenId: 'token-1', + }); + + expect(query).toHaveBeenCalledTimes(1); + expect(result).toEqual({ + ok: true, + marked: 0, + newly_marked: 0, + already_triaged: 0, + skipped: [{ id: 'm-null', reason: 'no_message_id_header' }], + }); + }); + + it('uses xmax=0 to distinguish new checkpoints from updated checkpoints', async () => { + const adapter = await adapterPromise; + query + .mockResolvedValueOnce({ + rows: [ + { id: 'm-new', account_id: 'acc-1', message_id_header: '' }, + { id: 'm-old', account_id: 'acc-1', message_id_header: '' }, + ], + }) + .mockResolvedValueOnce({ + rows: [ + { account_id: 'acc-1', message_id_header: '', inserted: true }, + { account_id: 'acc-1', message_id_header: '', inserted: false }, + ], + }); + + const result = await adapter.markTriaged({ + userId: 'user-1', + accountIds: ['acc-1'], + messageIds: ['m-new', 'm-old'], + action: 'archived', + note: 'morning pass', + tokenId: 'token-1', + }); + + expect(result).toEqual({ + ok: true, + marked: 2, + newly_marked: 1, + already_triaged: 1, + skipped: [], + }); + const [sql, params] = query.mock.calls[1]; + expect(sql).toContain('ON CONFLICT (account_id, message_id_header) DO UPDATE'); + expect(sql).toContain('triaged_at = NOW()'); + expect(sql).toContain('action = EXCLUDED.action'); + expect(sql).toContain('note = EXCLUDED.note'); + expect(sql).toContain('RETURNING'); + expect(sql).toContain('(xmax = 0) AS inserted'); + expect(params).toEqual([ + 'user-1', + ['acc-1', 'acc-1'], + ['', ''], + 'archived', + 'morning pass', + 'token-1', + ]); + }); + + it('reports unresolved or out-of-scope ids as skipped', async () => { + const adapter = await adapterPromise; + query.mockResolvedValueOnce({ rows: [] }); + + const result = await adapter.markTriaged({ + userId: 'user-1', + accountIds: ['acc-1'], + messageIds: ['foreign-id'], + }); + + expect(result.skipped).toEqual([{ id: 'foreign-id', reason: 'not_found_or_out_of_scope' }]); + expect(query).toHaveBeenCalledTimes(1); + }); + + it('deduplicates folder-copy rows that share one durable checkpoint key', async () => { + const adapter = await adapterPromise; + query + .mockResolvedValueOnce({ + rows: [ + { id: 'm-inbox', account_id: 'acc-1', message_id_header: '' }, + { id: 'm-label', account_id: 'acc-1', message_id_header: '' }, + ], + }) + .mockResolvedValueOnce({ + rows: [{ account_id: 'acc-1', message_id_header: '', inserted: true }], + }); + + const result = await adapter.markTriaged({ + userId: 'user-1', + accountIds: ['acc-1'], + messageIds: ['m-inbox', 'm-label'], + }); + + expect(result).toMatchObject({ marked: 1, newly_marked: 1, already_triaged: 0 }); + expect(query.mock.calls[1][1][1]).toEqual(['acc-1']); + expect(query.mock.calls[1][1][2]).toEqual(['']); + }); +}); diff --git a/backend/src/mcp/triageProbes.js b/backend/src/mcp/triageProbes.js new file mode 100644 index 00000000..a41db53b --- /dev/null +++ b/backend/src/mcp/triageProbes.js @@ -0,0 +1,70 @@ +import { EmbeddingClient } from '../services/embeddings/client.js'; + +// Fixed v1 probes: they are not user-tunable. Raw cosine scores are not +// calibrated across requests or users; only ranking candidates within one +// response is meaningful, so callers must not treat them as global thresholds. +export const TRIAGE_PROBES = { + urgent: 'urgent, needs action today, deadline, time sensitive', + needs_reply: 'a direct question addressed to me that expects a reply', + financial: 'invoice, payment due, receipt, billing, subscription charge', + scheduling: 'meeting request, calendar invite, reschedule, availability', + bulk: 'newsletter, marketing promotion, unsubscribe, mass mailing', +}; + +const vectorsByFingerprint = new Map(); + +export async function getTriageProbeVectors(cfg, generation) { + const fingerprint = generation?.fingerprint; + if (!fingerprint) throw new Error('embedding generation fingerprint is required'); + if (vectorsByFingerprint.has(fingerprint)) { + return vectorsByFingerprint.get(fingerprint); + } + + const pending = (async () => { + const names = Object.keys(TRIAGE_PROBES); + const vectors = await new EmbeddingClient(cfg).embed(Object.values(TRIAGE_PROBES)); + if (!Array.isArray(vectors) || vectors.length !== names.length) { + throw new Error(`embedder returned ${vectors?.length} probe vectors, want ${names.length}`); + } + return Object.fromEntries(names.map((name, index) => [name, vectors[index]])); + })(); + vectorsByFingerprint.set(fingerprint, pending); + + try { + return await pending; + } catch (error) { + if (vectorsByFingerprint.get(fingerprint) === pending) { + vectorsByFingerprint.delete(fingerprint); + } + throw error; + } +} + +function cosineSimilarity(left, right) { + if (!Array.isArray(left) || !Array.isArray(right)) { + throw new Error('cosine similarity requires vector arrays'); + } + if (left.length !== right.length) { + throw new Error(`probe vector dimension mismatch: candidate=${left.length}, probe=${right.length}`); + } + + let dot = 0; + let leftSquared = 0; + let rightSquared = 0; + for (let index = 0; index < left.length; index++) { + dot += left[index] * right[index]; + leftSquared += left[index] * left[index]; + rightSquared += right[index] * right[index]; + } + if (leftSquared === 0 || rightSquared === 0) return 0; + return dot / (Math.sqrt(leftSquared) * Math.sqrt(rightSquared)); +} + +export function scoreTriageProbes(candidateVector, probeVectors) { + return Object.fromEntries( + Object.keys(TRIAGE_PROBES).map(name => [ + name, + cosineSimilarity(candidateVector, probeVectors?.[name]), + ]), + ); +} diff --git a/backend/src/mcp/triageProbes.test.js b/backend/src/mcp/triageProbes.test.js new file mode 100644 index 00000000..4d935361 --- /dev/null +++ b/backend/src/mcp/triageProbes.test.js @@ -0,0 +1,96 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const clientMocks = vi.hoisted(() => ({ + construct: vi.fn(), + embed: vi.fn(), +})); + +vi.mock('../services/embeddings/client.js', () => ({ + EmbeddingClient: class { + constructor(cfg) { + clientMocks.construct(cfg); + } + + embed(inputs) { + return clientMocks.embed(inputs); + } + }, +})); + +const probesPromise = import('./triageProbes.js').catch(() => null); + +const cfg = { + endpoint: 'https://embeddings.example.test/v1', + apiKey: 'secret', + model: 'embed-model', + dimension: 2, +}; + +const vectors = [ + [1, 0], + [0.8, 0.2], + [0, 1], + [-0.5, 0.5], + [-1, 0], +]; + +describe('getTriageProbeVectors', () => { + beforeEach(() => { + clientMocks.construct.mockReset(); + clientMocks.embed.mockReset(); + }); + + it('embeds all five fixed probes in one client call', async () => { + const probes = await probesPromise; + expect(probes?.getTriageProbeVectors).toBeTypeOf('function'); + clientMocks.embed.mockResolvedValueOnce(vectors); + + const result = await probes.getTriageProbeVectors(cfg, { fingerprint: 'fp-all-five' }); + + expect(clientMocks.construct).toHaveBeenCalledOnce(); + expect(clientMocks.construct).toHaveBeenCalledWith(cfg); + expect(clientMocks.embed).toHaveBeenCalledOnce(); + expect(clientMocks.embed).toHaveBeenCalledWith(Object.values(probes.TRIAGE_PROBES)); + expect(result).toEqual(Object.fromEntries( + Object.keys(probes.TRIAGE_PROBES).map((name, index) => [name, vectors[index]]), + )); + }); + + it('reuses one cached result for a fingerprint and re-embeds for a new generation', async () => { + const probes = await probesPromise; + clientMocks.embed + .mockResolvedValueOnce(vectors) + .mockResolvedValueOnce(vectors.map(vector => [...vector].reverse())); + + const first = await probes.getTriageProbeVectors(cfg, { fingerprint: 'fp-cache-a' }); + const second = await probes.getTriageProbeVectors(cfg, { fingerprint: 'fp-cache-a' }); + + expect(second).toBe(first); + expect(clientMocks.embed).toHaveBeenCalledTimes(1); + + const changed = await probes.getTriageProbeVectors(cfg, { fingerprint: 'fp-cache-b' }); + expect(changed).not.toBe(first); + expect(clientMocks.embed).toHaveBeenCalledTimes(2); + }); +}); + +describe('scoreTriageProbes', () => { + it('ranks a hand-built candidate closer to the aligned probe', async () => { + const probes = await probesPromise; + expect(probes?.scoreTriageProbes).toBeTypeOf('function'); + + const scores = probes.scoreTriageProbes([1, 0], { + urgent: [0.9, 0.1], + needs_reply: [0.5, 0.5], + financial: [-1, 0], + scheduling: [0.2, 0.8], + bulk: [0, 1], + }); + + expect(Object.keys(scores)).toEqual(Object.keys(probes.TRIAGE_PROBES)); + expect(scores.urgent).toBeGreaterThan(scores.needs_reply); + expect(scores.needs_reply).toBeGreaterThan(scores.bulk); + expect(scores.bulk).toBeGreaterThan(scores.financial); + expect(scores.urgent).toBeCloseTo(0.9939, 4); + }); +}); diff --git a/backend/src/mcp/triageTools.js b/backend/src/mcp/triageTools.js new file mode 100644 index 00000000..26fc6ef2 --- /dev/null +++ b/backend/src/mcp/triageTools.js @@ -0,0 +1,757 @@ +import { + countUntriagedUnread, + listTriageCandidates, + markTriaged, + senderHistory, + sentFolderForAccount, + triageActionsForMessages, +} from './triageAdapter.js'; +import { + getMessage, + getMessageSummariesByIDs, + listMessages, + resolveAccountScope, +} from './engineAdapter.js'; +import { findSimilarSummaries } from './messageTools.js'; +import { errorResult, jsonResult } from './result.js'; +import { toRFC3339, wireSummary } from './envelope.js'; +import { translateVectorError } from './vectorErrors.js'; +import { resolveActiveGenerationFromConfig } from '../services/embeddings/hybrid.js'; +import { annSearch, loadVector } from '../services/embeddings/vectorStore.js'; +import { runInBatches } from '../services/mailbox/batch.js'; +import { + getTriageProbeVectors, + scoreTriageProbes, +} from './triageProbes.js'; +import { + resolveAllSpamPaths, + resolveAllTrashPaths, + resolveArchiveFolder, +} from '../utils/mailUtils.js'; +import { getGtdFolderSet } from '../services/gtdConfig.js'; +import { matchingRules, toRuleMessage } from '../services/inboxRules.js'; +import { areValidUUIDs } from '../utils/validation.js'; + +function annotations({ + readOnlyHint = false, + destructiveHint = false, + idempotentHint = false, +} = {}) { + return Object.freeze({ + readOnlyHint, + destructiveHint, + idempotentHint, + openWorldHint: false, + }); +} + +const IDEMPOTENT_WRITE_ANNOTATIONS = annotations({ idempotentHint: true }); +const READ_ONLY_ANNOTATIONS = annotations({ + readOnlyHint: true, + idempotentHint: true, +}); + +const messageIdsSchema = { + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, +}; + +function messageIdsArg(args) { + const ids = args.message_ids; + if (!Array.isArray(ids) || ids.length === 0) { + return { error: 'message_ids must contain at least one id' }; + } + if (ids.length > 500) return { error: 'Too many ids — maximum 500 per request' }; + if (!areValidUUIDs(ids)) return { error: 'Invalid message id format' }; + return { value: ids }; +} + +export const markTriagedDef = { + name: 'mark_triaged', + description: + 'Checkpoint messages as triaged so future triage_inbox runs skip them. Idempotent: re-marking updates the timestamp and action rather than erroring. ' + + "Call mark_triaged before a move/archive, or use the move receipt's new_id, because marking after a move with the stale id silently resolves to skipped.", + annotations: IDEMPOTENT_WRITE_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_ids'], + properties: { + message_ids: messageIdsSchema, + action: { type: 'string' }, + note: { type: 'string', maxLength: 500 }, + }, + }, +}; + +export const triageInboxDef = { + name: 'triage_inbox', + description: + 'One-call inbox triage feed across scoped accounts, oldest first, enriched with category, sender history, and thread state. ' + + 'The cursor is for paging only; the message_triage table (via NOT EXISTS) guarantees correctness and prevents duplicate skipping, so re-running with no cursor is safe and cheap. ' + + 'Call mark_triaged after acting so later no-cursor runs skip checkpointed messages. ' + + 'Optional similar-message and fixed v1 urgency-probe signals are included only at limit 25 or below. Raw cosine scores are not calibrated: rank within this response, not threshold across responses.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + properties: { + account: { type: 'string' }, + cursor: { type: 'string' }, + limit: { type: 'number', minimum: 1, maximum: 50, default: 25 }, + unread_only: { type: 'boolean', default: true }, + include_triaged: { type: 'boolean', default: false }, + categories: { type: 'array', items: { type: 'string' } }, + since: { type: 'string' }, + include_signals: { type: 'boolean', default: true }, + }, + }, +}; + +export const getTriageContextDef = { + name: 'get_triage_context', + description: + 'Get independently degradable thread, sender-history, similar-message, and matched-rule context for one scoped message. ' + + 'Matched rules are report only: this tool never executes rule actions, and body/header rules are reported as unevaluated when those fields are not loaded.', + annotations: READ_ONLY_ANNOTATIONS, + inputSchema: { + type: 'object', + required: ['message_id'], + properties: { + message_id: { type: 'string' }, + thread_limit: { type: 'number', minimum: 1, maximum: 50 }, + similar_limit: { type: 'number', minimum: 1, maximum: 50 }, + }, + }, +}; + +function dateArg(args, key) { + const value = args[key]; + if (typeof value !== 'string' || value === '') return { value: undefined }; + const error = { error: `invalid ${key} date "${value}": expected YYYY-MM-DD` }; + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value); + if (!match) return error; + const [year, month, day] = [ + Number(match[1]), + Number(match[2]), + Number(match[3]), + ]; + const parsed = new Date(Date.UTC(year, month - 1, day)); + if ( + parsed.getUTCFullYear() !== year + || parsed.getUTCMonth() !== month - 1 + || parsed.getUTCDate() !== day + ) { + return error; + } + return { value }; +} + +function triageLimit(args) { + const raw = args.limit; + if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 1) return 25; + return Math.min(Math.trunc(raw), 50); +} + +function wireDate(value) { + const iso = value instanceof Date ? value.toISOString() : value; + return toRFC3339(iso); +} + +function wireCandidate(row, includeSignals) { + const item = { + id: row.id, + message_id: row.message_id, + account: row.account, + conversation_id: row.conversation_id, + subject: row.subject, + snippet: row.snippet, + from_email: row.from_email, + from_name: row.from_name, + date: wireDate(row.date), + is_read: !!row.is_read, + is_starred: !!row.is_starred, + has_attachments: !!row.has_attachments, + category: row.category, + is_bulk: !!row.is_bulk, + has_unsubscribe: !!row.has_unsubscribe, + spam_verdict: row.spam_verdict ?? null, + thread: { + message_count: Number(row.thread_message_count || 0), + last_activity: wireDate(row.thread_last_activity), + i_replied: !!row.i_replied, + }, + contact: { + known: !!row.contact_known, + send_count: Number(row.send_count || 0), + last_sent: wireDate(row.last_sent), + received_count: Number(row.received_count || 0), + first_received: wireDate(row.first_received), + last_received: wireDate(row.last_received), + }, + }; + if (includeSignals) item.signals = { similar: [], probes: {} }; + return item; +} + +const SIMILAR_SIGNAL_LIMIT = 5; + +function emptySignals(reason) { + const signals = { similar: [], probes: {} }; + if (reason) signals.reason = reason; + return signals; +} + +function messageFolder(summary) { + return Array.isArray(summary?.labels) && typeof summary.labels[0] === 'string' + ? summary.labels[0] + : ''; +} + +function hasFlag(summary, flag) { + const wanted = flag.toLowerCase(); + return (Array.isArray(summary?.labels) ? summary.labels : []) + .some(label => typeof label === 'string' && label.toLowerCase() === wanted); +} + +// Name-based fallback only — used when the account has neither a sent mapping +// nor a \Sent special-use folder for sentFolderForAccount to resolve. +function isSentFolder(folder) { + return /(^|[./ ])sent($|[./ ])/i.test(folder || ''); +} + +async function resolvedFolderClasses(accountId, cache) { + if (!accountId) { + return { + archive: null, + trash: new Set(), + spam: new Set(), + gtd: new Set(), + }; + } + if (!cache.has(accountId)) { + const pending = (async () => { + const [archive, trash, spam, gtd, sent] = await Promise.allSettled([ + resolveArchiveFolder(accountId), + resolveAllTrashPaths(accountId), + resolveAllSpamPaths(accountId), + getGtdFolderSet(accountId), + sentFolderForAccount(accountId), + ]); + return { + archive: archive.status === 'fulfilled' ? archive.value : null, + trash: trash.status === 'fulfilled' ? trash.value : new Set(), + spam: spam.status === 'fulfilled' ? spam.value : new Set(), + gtd: gtd.status === 'fulfilled' ? gtd.value : new Set(), + sent: sent.status === 'fulfilled' ? sent.value : null, + }; + })(); + cache.set(accountId, pending); + } + return cache.get(accountId); +} + +function folderClass(folder, resolved) { + if (!folder) return 'unknown'; + if (folder.toLowerCase() === 'inbox') return 'inbox'; + if (resolved.trash?.has(folder)) return 'trashed'; + if (resolved.spam?.has(folder)) return 'spam'; + if (resolved.sent ? folder === resolved.sent : isSentFolder(folder)) return 'sent'; + if (resolved.archive && folder === resolved.archive) return 'archived'; + if (resolved.gtd?.has(folder)) return 'labelled'; + return 'labelled'; +} + +async function similarWithDisposition( + candidate, + vector, + generation, + accountIds, + folderCache, +) { + const hits = await annSearch( + generation.id, + vector, + SIMILAR_SIGNAL_LIMIT + 1, + { + filter: { + accountIds, + before: candidate.date, + }, + }, + ); + const keptHits = hits + .filter(hit => hit.messageId !== candidate.id) + .slice(0, SIMILAR_SIGNAL_LIMIT); + const hitIds = keptHits.map(hit => hit.messageId); + const summaries = await getMessageSummariesByIDs(hitIds, accountIds); + const scoreById = new Map(keptHits.map(hit => [hit.messageId, hit.score])); + const classified = await Promise.all(summaries.map(async summary => { + const resolved = await resolvedFolderClasses(summary.source_id, folderCache); + return { + summary, + folderClass: folderClass(messageFolder(summary), resolved), + }; + })); + const repliedThreads = new Set( + classified + .filter(entry => entry.folderClass === 'sent') + .map(entry => entry.summary.conversation_id) + .filter(Boolean), + ); + + return classified.map(({ summary, folderClass: classification }) => ({ + ...wireSummary(summary), + score: scoreById.get(summary.id), + disposition: { + folder_class: classification, + was_read: hasFlag(summary, '\\Seen'), + was_starred: hasFlag(summary, '\\Flagged'), + was_replied: !!( + summary.conversation_id + && repliedThreads.has(summary.conversation_id) + ), + }, + })); +} + +async function candidateSignals( + candidate, + { + generation, + probeVectors, + accountIds, + folderCache, + }, +) { + let vector; + try { + vector = await loadVector(candidate.id); + } catch { + return emptySignals('not_embedded'); + } + + let probes = {}; + let probeFailed = false; + if (probeVectors) { + try { + probes = scoreTriageProbes(vector, probeVectors); + } catch { + probeFailed = true; + } + } + + try { + const similar = await similarWithDisposition( + candidate, + vector, + generation, + accountIds, + folderCache, + ); + const signals = { similar, probes }; + if (probeFailed) signals.reason = 'probe_error'; + return signals; + } catch (error) { + return { + similar: [], + probes, + reason: error?.message || 'error', + }; + } +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +} + +function probeCalibration(signalsByCandidate) { + const byProbe = new Map(); + for (const signals of signalsByCandidate) { + for (const [name, score] of Object.entries(signals?.probes || {})) { + if (!Number.isFinite(score)) continue; + if (!byProbe.has(name)) byProbe.set(name, []); + byProbe.get(name).push(score); + } + } + return Object.fromEntries( + [...byProbe].map(([name, values]) => [ + name, + { + min: Math.min(...values), + median: median(values), + max: Math.max(...values), + }, + ]), + ); +} + +function triageMessageKey(accountId, messageIdHeader) { + return `${accountId}\0${messageIdHeader}`; +} + +async function pageSignals(rows, accountIds) { + if (!rows.length) { + return { + signals: [], + available: true, + reason: null, + calibration: {}, + }; + } + + let cfg; + let generation; + try { + ({ cfg, generation } = await resolveActiveGenerationFromConfig()); + } catch (error) { + return { + signals: rows.map(() => emptySignals()), + available: false, + reason: error?.name === 'VectorUnavailableError' + ? translateVectorError(error.reason) + : 'error', + calibration: {}, + }; + } + + let probeVectors = null; + try { + probeVectors = await getTriageProbeVectors(cfg, generation); + } catch { + // Similar-message dispositions remain useful when only probe embedding fails. + } + + const folderCache = new Map(); + const settled = await runInBatches(rows, 4, candidate => candidateSignals( + candidate, + { + generation, + probeVectors, + accountIds, + folderCache, + }, + )); + const signals = settled.map(result => ( + result.status === 'fulfilled' + ? result.value + : emptySignals(result.reason?.message || 'error') + )); + const pairsByKey = new Map(); + for (const signal of signals) { + for (const summary of signal.similar) { + if (!summary.source_message_id) continue; + const pair = { + accountId: summary.source_id, + messageIdHeader: summary.source_message_id, + }; + pairsByKey.set(triageMessageKey(pair.accountId, pair.messageIdHeader), pair); + } + } + + let actionRows = []; + try { + actionRows = await triageActionsForMessages([...pairsByKey.values()]); + } catch { + // Similar-message signals remain useful when checkpoint history is unavailable. + } + const actionByKey = new Map(actionRows.map(row => [ + triageMessageKey(row.account_id, row.message_id_header), + row.action, + ])); + const enrichedSignals = signals.map(signal => ({ + ...signal, + similar: signal.similar.map(summary => ({ + ...summary, + disposition: { + ...summary.disposition, + triage_action: actionByKey.get( + triageMessageKey(summary.source_id, summary.source_message_id), + ) ?? null, + }, + })), + })); + return { + signals: enrichedSignals, + available: true, + reason: null, + calibration: probeCalibration(enrichedSignals), + }; +} + +function contextLimit(raw) { + if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 1) return 20; + return Math.min(Math.trunc(raw), 50); +} + +function sectionError(error) { + return { + available: false, + reason: 'error', + detail: error?.message || String(error), + }; +} + +async function threadSection(seed, accountIds, limit) { + try { + if (!seed.thread_id) { + return { available: false, reason: 'conversation_not_available' }; + } + const messages = await listMessages({ + accountIds, + conversationId: seed.thread_id, + limit, + }); + return { + available: true, + messages: messages.map(wireSummary), + }; + } catch (error) { + return sectionError(error); + } +} + +async function senderHistorySection(seed, accountIds) { + try { + const history = await senderHistory(seed.from_email, accountIds); + return { available: true, history }; + } catch (error) { + return sectionError(error); + } +} + +async function similarSection(seedId, accountIds, limit) { + try { + const result = await findSimilarSummaries(seedId, { + accountIds, + limit, + }); + return { + available: true, + generation: result.generation, + messages: result.messages.map(wireSummary), + }; + } catch (error) { + if (error?.name === 'VectorUnavailableError') { + return { + available: false, + reason: translateVectorError(error.reason), + }; + } + return sectionError(error); + } +} + +function arrayField(value) { + if (Array.isArray(value)) return value; + if (typeof value !== 'string') return []; + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } +} + +function normalizedRule(rule) { + return { + ...rule, + conditions: arrayField(rule?.conditions), + actions: arrayField(rule?.actions), + }; +} + +function reportMatchingRules(rules, seed) { + const message = toRuleMessage(seed); + const reports = []; + for (const rawRule of Array.isArray(rules) ? rules : []) { + const rule = normalizedRule(rawRule); + const needsUnavailableField = rule.conditions.some( + condition => condition?.field === 'body' || condition?.field === 'header', + ); + if (needsUnavailableField) { + reports.push({ + id: rule.id, + name: rule.name, + evaluated: false, + reason: 'body_not_loaded', + actions: rule.actions, + }); + continue; + } + const match = matchingRules([rule], message)[0]; + if (match) reports.push(match); + } + return reports; +} + +async function matchedRulesSection(seed, scope, deps) { + if (typeof deps?.loadInboxRules !== 'function') { + return { + available: false, + reason: 'rules_loader_unavailable', + }; + } + try { + const loaded = await deps.loadInboxRules({ + userId: scope.userId, + accountId: seed.account_id, + accountIds: scope.accountIds, + }); + const rules = Array.isArray(loaded) ? loaded : loaded?.rows; + return { + available: true, + rules: reportMatchingRules(rules, seed), + }; + } catch (error) { + return sectionError(error); + } +} + +export async function handleMarkTriaged(args, scope) { + const ids = messageIdsArg(args); + if (ids.error) return errorResult(ids.error); + if (args.action !== undefined && typeof args.action !== 'string') { + return errorResult('action must be a string'); + } + if (args.note !== undefined && typeof args.note !== 'string') { + return errorResult('note must be a string'); + } + if (args.note?.length > 500) { + return errorResult('note must be at most 500 characters'); + } + + const result = await markTriaged({ + userId: scope.userId, + accountIds: scope.accountIds, + messageIds: ids.value, + action: args.action ?? null, + note: args.note ?? null, + }); + return jsonResult(result); +} + +export async function handleTriageInbox(args, scope) { + if (args.cursor !== undefined && typeof args.cursor !== 'string') { + return errorResult('cursor must be a string'); + } + if ( + args.categories !== undefined + && ( + !Array.isArray(args.categories) + || args.categories.some(category => typeof category !== 'string') + ) + ) { + return errorResult('categories must be an array of strings'); + } + + const since = dateArg(args, 'since'); + if (since.error) return errorResult(since.error); + const account = await resolveAccountScope(args.account, scope.accountIds); + if (account.error) return errorResult(account.error); + + const limit = triageLimit(args); + const includeSignals = args.include_signals !== false; + try { + const [page, untriagedUnread] = await Promise.all([ + listTriageCandidates({ + accountIds: account.accountIds, + cursor: args.cursor, + limit, + unreadOnly: args.unread_only !== false, + includeTriaged: args.include_triaged === true, + categories: args.categories, + since: since.value, + }), + countUntriagedUnread(account.accountIds), + ]); + let signalState; + if (!includeSignals) { + signalState = { + signals: [], + available: false, + reason: 'disabled', + calibration: {}, + }; + } else if (limit > 25) { + signalState = { + signals: page.rows.map(() => emptySignals()), + available: false, + reason: 'limit_exceeds_25', + calibration: {}, + }; + } else { + signalState = await pageSignals(page.rows, account.accountIds); + } + + const items = page.rows.map((row, index) => { + const item = wireCandidate(row, includeSignals); + if (includeSignals) { + item.signals = signalState.signals[index] || emptySignals(); + } + return item; + }); + return jsonResult({ + items, + cursor: page.cursor, + has_more: page.hasMore, + counts: { + untriaged_unread: untriagedUnread, + returned: items.length, + }, + signals_available: signalState.available, + signals_reason: signalState.reason, + probe_calibration: signalState.calibration, + }); + } catch (error) { + if (error?.message === 'invalid triage cursor') { + return errorResult(error.message); + } + throw error; + } +} + +export async function handleGetTriageContext(args, scope, deps = {}) { + const seedId = args.message_id; + if (!seedId || typeof seedId !== 'string') { + return errorResult('message_id parameter is required'); + } + + let seed; + try { + seed = await getMessage(seedId, scope.accountIds); + } catch (error) { + const unavailable = { + available: false, + reason: 'seed_lookup_failed', + detail: error?.message || String(error), + }; + return jsonResult({ + message_id: seedId, + thread: { ...unavailable }, + sender_history: { ...unavailable }, + similar: { ...unavailable }, + matched_rules: { ...unavailable }, + }); + } + if (!seed) return errorResult('message not found'); + + const [thread, senderHistoryResult, similar, matchedRules] = await Promise.all([ + threadSection(seed, scope.accountIds, contextLimit(args.thread_limit)), + senderHistorySection(seed, scope.accountIds), + similarSection(seedId, scope.accountIds, contextLimit(args.similar_limit)), + matchedRulesSection(seed, scope, deps), + ]); + + return jsonResult({ + message_id: seedId, + thread, + sender_history: senderHistoryResult, + similar, + matched_rules: matchedRules, + }); +} diff --git a/backend/src/mcp/triageTools.test.js b/backend/src/mcp/triageTools.test.js new file mode 100644 index 00000000..308bbd5d --- /dev/null +++ b/backend/src/mcp/triageTools.test.js @@ -0,0 +1,1117 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; + +vi.mock('./triageAdapter.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + countUntriagedUnread: vi.fn(async () => 0), + listTriageCandidates: vi.fn(), + markTriaged: vi.fn(), + senderHistory: vi.fn(), + sentFolderForAccount: vi.fn(async () => null), + triageActionsForMessages: vi.fn(), + }; +}); +vi.mock('./engineAdapter.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getMessage: vi.fn(), + getMessageSummariesByIDs: vi.fn(), + listMessages: vi.fn(), + resolveAccountScope: vi.fn(), + }; +}); +vi.mock('../services/embeddings/hybrid.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + resolveActiveGenerationFromConfig: vi.fn(), + }; +}); +vi.mock('../services/embeddings/vectorStore.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + annSearch: vi.fn(), + loadVector: vi.fn(), + }; +}); +vi.mock('../services/mailbox/batch.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + runInBatches: vi.fn(actual.runInBatches), + }; +}); +vi.mock('./triageProbes.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getTriageProbeVectors: vi.fn(), + scoreTriageProbes: vi.fn(), + }; +}); +vi.mock('../utils/mailUtils.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + resolveArchiveFolder: vi.fn(), + resolveAllTrashPaths: vi.fn(), + resolveAllSpamPaths: vi.fn(), + }; +}); +vi.mock('../services/gtdConfig.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + getGtdFolderSet: vi.fn(), + }; +}); +vi.mock('./messageTools.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + findSimilarSummaries: vi.fn(), + }; +}); +vi.mock('../services/inboxRules.js', async (orig) => { + const actual = await orig(); + return { + ...actual, + matchingRules: vi.fn(actual.matchingRules), + toRuleMessage: vi.fn(actual.toRuleMessage), + }; +}); + +const triageAdapter = await import('./triageAdapter.js'); +const engineAdapter = await import('./engineAdapter.js'); +const hybrid = await import('../services/embeddings/hybrid.js'); +const vectorStore = await import('../services/embeddings/vectorStore.js'); +const batch = await import('../services/mailbox/batch.js'); +const triageProbes = await import('./triageProbes.js'); +const mailUtils = await import('../utils/mailUtils.js'); +const gtdConfig = await import('../services/gtdConfig.js'); +const messageTools = await import('./messageTools.js'); +const inboxRules = await import('../services/inboxRules.js'); +const triageTools = await import('./triageTools.js').catch(() => ({})); +const registeredTools = await import('./tools.js').catch(() => ({})); + +const MESSAGE_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const scope = { + userId: 'user-1', + accountIds: ['bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'], + scopes: ['read', 'write'], +}; +const deps = { ignored: true }; +const defaultProbeScores = { + urgent: 0.2, + needs_reply: 0.1, + financial: 0.05, + scheduling: 0.15, + bulk: 0.3, +}; + +function jsonOf(result) { + return JSON.parse(result.content[0].text); +} + +beforeEach(() => { + vi.clearAllMocks(); + engineAdapter.resolveAccountScope.mockImplementation(async (_account, accountIds) => ({ + accountIds, + })); + engineAdapter.getMessageSummariesByIDs.mockReset().mockResolvedValue([]); + triageAdapter.triageActionsForMessages.mockReset().mockResolvedValue([]); + hybrid.resolveActiveGenerationFromConfig.mockReset().mockResolvedValue({ + cfg: { enabled: true }, + generation: { + id: 7, + fingerprint: 'fingerprint-1', + }, + }); + vectorStore.loadVector.mockReset().mockResolvedValue([1, 0]); + vectorStore.annSearch.mockReset().mockResolvedValue([]); + triageProbes.getTriageProbeVectors.mockReset().mockResolvedValue({ + urgent: [1, 0], + }); + triageProbes.scoreTriageProbes.mockReset().mockReturnValue(defaultProbeScores); + mailUtils.resolveArchiveFolder.mockReset().mockResolvedValue('Archive'); + mailUtils.resolveAllTrashPaths.mockReset().mockResolvedValue(new Set(['Trash'])); + mailUtils.resolveAllSpamPaths.mockReset().mockResolvedValue(new Set(['Junk'])); + gtdConfig.getGtdFolderSet.mockReset().mockResolvedValue(new Set(['Todo'])); +}); + +describe('mark_triaged definition and registration', () => { + it('registers an idempotent write tool with the move-ordering warning', () => { + const def = registeredTools.TOOL_DEFS?.find(entry => entry.name === 'mark_triaged'); + + expect(def?.inputSchema.required).toEqual(['message_ids']); + expect(def?.inputSchema.properties.message_ids).toEqual({ + type: 'array', + items: { type: 'string' }, + minItems: 1, + maxItems: 500, + }); + expect(def?.annotations).toEqual({ + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }); + expect(def?.description).toMatch(/before (?:a )?move\/archive/i); + expect(def?.description).toMatch(/new_id/); + expect(def?.description).toMatch(/stale id.*skipped/i); + expect(registeredTools.TOOL_SCOPES?.mark_triaged).toBe('write'); + expect(registeredTools.HANDLERS?.mark_triaged).toBe(triageTools.handleMarkTriaged); + }); +}); + +describe('mark_triaged handler', () => { + it('passes scoped ids and optional receipt metadata to the adapter without inventing tokenId', async () => { + const receipt = { + ok: true, + marked: 1, + newly_marked: 1, + already_triaged: 0, + skipped: [], + }; + triageAdapter.markTriaged.mockResolvedValue(receipt); + + const result = await triageTools.handleMarkTriaged({ + message_ids: [MESSAGE_ID], + action: 'archived', + note: 'Morning pass', + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual(receipt); + expect(triageAdapter.markTriaged).toHaveBeenCalledWith({ + userId: scope.userId, + accountIds: scope.accountIds, + messageIds: [MESSAGE_ID], + action: 'archived', + note: 'Morning pass', + }); + }); + + it('passes through the adapter skip reason for a message without a Message-ID header', async () => { + const receipt = { + ok: true, + marked: 0, + newly_marked: 0, + already_triaged: 0, + skipped: [{ id: MESSAGE_ID, reason: 'no_message_id_header' }], + }; + triageAdapter.markTriaged.mockResolvedValue(receipt); + + const result = await triageTools.handleMarkTriaged({ + message_ids: [MESSAGE_ID], + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual(receipt); + }); + + it('passes through an idempotent re-mark receipt under already_triaged', async () => { + const receipt = { + ok: true, + marked: 1, + newly_marked: 0, + already_triaged: 1, + skipped: [], + }; + triageAdapter.markTriaged.mockResolvedValue(receipt); + + const result = await triageTools.handleMarkTriaged({ + message_ids: [MESSAGE_ID], + action: 'left', + }, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual(receipt); + }); + + it.each([ + [{}, 'message_ids must contain at least one id'], + [{ message_ids: [] }, 'message_ids must contain at least one id'], + [{ message_ids: Array.from({ length: 501 }, () => MESSAGE_ID) }, 'Too many ids — maximum 500 per request'], + [{ message_ids: ['not-a-uuid'] }, 'Invalid message id format'], + [{ message_ids: [MESSAGE_ID], action: 42 }, 'action must be a string'], + [{ message_ids: [MESSAGE_ID], note: 42 }, 'note must be a string'], + [{ message_ids: [MESSAGE_ID], note: 'x'.repeat(501) }, 'note must be at most 500 characters'], + ])('rejects invalid arguments before checkpointing', async (args, message) => { + const result = await triageTools.handleMarkTriaged(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(message); + expect(triageAdapter.markTriaged).not.toHaveBeenCalled(); + }); +}); + +const candidateRow = { + id: MESSAGE_ID, + account_id: scope.accountIds[0], + message_id: '', + account: 'me@example.com', + conversation_id: 'thread-1', + subject: 'Quarterly planning', + snippet: 'Could you review the agenda?', + from_email: 'alice@example.com', + from_name: 'Alice', + date: new Date('2026-07-28T06:11:00.123Z'), + is_read: false, + is_starred: true, + has_attachments: true, + category: 'primary', + is_bulk: false, + has_unsubscribe: false, + spam_verdict: null, + thread_message_count: 3, + thread_last_activity: new Date('2026-07-28T07:12:00.456Z'), + i_replied: true, + contact_known: true, + send_count: 12, + last_sent: new Date('2026-07-27T10:00:00.789Z'), + received_count: 47, + first_received: new Date('2025-01-01T00:00:00.111Z'), + last_received: new Date('2026-07-28T06:11:00.123Z'), +}; + +describe('triage_inbox definition and registration', () => { + it('registers an idempotent read tool and documents cursor correctness', () => { + const def = registeredTools.TOOL_DEFS?.find(entry => entry.name === 'triage_inbox'); + + expect(def?.annotations).toEqual({ + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }); + expect(def?.inputSchema.properties).toEqual(expect.objectContaining({ + account: { type: 'string' }, + cursor: { type: 'string' }, + limit: expect.objectContaining({ maximum: 50 }), + unread_only: expect.objectContaining({ type: 'boolean', default: true }), + include_triaged: expect.objectContaining({ type: 'boolean', default: false }), + categories: { type: 'array', items: { type: 'string' } }, + since: { type: 'string' }, + include_signals: expect.objectContaining({ type: 'boolean', default: true }), + })); + expect(def?.description).toMatch(/cursor is for paging only/i); + expect(def?.description).toMatch(/message_triage.*NOT EXISTS/i); + expect(def?.description).toMatch(/re-running with no cursor is safe and cheap/i); + expect(def?.description).toMatch(/raw cosine.*not calibrated/i); + expect(def?.description).toMatch(/rank.*not threshold/i); + expect(def?.description).toMatch(/fixed.*v1/i); + expect(registeredTools.TOOL_SCOPES?.triage_inbox).toBe('read'); + expect(registeredTools.HANDLERS?.triage_inbox).toBe(triageTools.handleTriageInbox); + }); +}); + +describe('triage_inbox handler', () => { + it('maps one enriched adapter row to the snake_case feed envelope', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: 'cursor-one', + }); + triageAdapter.countUntriagedUnread.mockResolvedValueOnce(118); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + + expect(result.isError).toBeUndefined(); + expect(jsonOf(result)).toEqual({ + items: [{ + id: MESSAGE_ID, + message_id: '', + account: 'me@example.com', + conversation_id: 'thread-1', + subject: 'Quarterly planning', + snippet: 'Could you review the agenda?', + from_email: 'alice@example.com', + from_name: 'Alice', + date: '2026-07-28T06:11:00Z', + is_read: false, + is_starred: true, + has_attachments: true, + category: 'primary', + is_bulk: false, + has_unsubscribe: false, + spam_verdict: null, + thread: { + message_count: 3, + last_activity: '2026-07-28T07:12:00Z', + i_replied: true, + }, + contact: { + known: true, + send_count: 12, + last_sent: '2026-07-27T10:00:00Z', + received_count: 47, + first_received: '2025-01-01T00:00:00Z', + last_received: '2026-07-28T06:11:00Z', + }, + signals: { similar: [], probes: defaultProbeScores }, + }], + cursor: 'cursor-one', + has_more: false, + counts: { untriaged_unread: 118, returned: 1 }, + signals_available: true, + signals_reason: null, + probe_calibration: { + urgent: { min: 0.2, median: 0.2, max: 0.2 }, + needs_reply: { min: 0.1, median: 0.1, max: 0.1 }, + financial: { min: 0.05, median: 0.05, max: 0.05 }, + scheduling: { min: 0.15, median: 0.15, max: 0.15 }, + bulk: { min: 0.3, median: 0.3, max: 0.3 }, + }, + }); + expect(triageAdapter.listTriageCandidates).toHaveBeenCalledWith({ + accountIds: scope.accountIds, + cursor: undefined, + limit: 25, + unreadOnly: true, + includeTriaged: false, + categories: undefined, + since: undefined, + }); + }); + + it('round-trips the opaque returned cursor into the next adapter call', async () => { + triageAdapter.listTriageCandidates + .mockResolvedValueOnce({ rows: [candidateRow], hasMore: true, cursor: 'opaque-page-1' }) + .mockResolvedValueOnce({ rows: [], hasMore: false, cursor: null }); + + const first = jsonOf(await triageTools.handleTriageInbox({ include_signals: false }, scope, deps)); + const second = jsonOf(await triageTools.handleTriageInbox({ + cursor: first.cursor, + include_signals: false, + }, scope, deps)); + + expect(first.cursor).toBe('opaque-page-1'); + expect(second.cursor).toBeNull(); + expect(triageAdapter.listTriageCandidates).toHaveBeenLastCalledWith( + expect.objectContaining({ cursor: 'opaque-page-1' }), + ); + }); + + it('surfaces adapter limit+1 paging as has_more while returning only the page rows', async () => { + const rows = Array.from({ length: 25 }, (_, index) => ({ + ...candidateRow, + id: `message-${index}`, + })); + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows, + hasMore: true, + cursor: 'next-page', + }); + + const body = jsonOf(await triageTools.handleTriageInbox({ + limit: 25, + include_signals: false, + }, scope, deps)); + + expect(body.items).toHaveLength(25); + expect(body.counts.returned).toBe(25); + expect(body.has_more).toBe(true); + expect(body.cursor).toBe('next-page'); + expect(triageAdapter.listTriageCandidates).toHaveBeenCalledWith( + expect.objectContaining({ limit: 25 }), + ); + }); + + it('turns a malformed adapter cursor failure into a clean errorResult', async () => { + triageAdapter.listTriageCandidates.mockRejectedValue(new Error('invalid triage cursor')); + + const result = await triageTools.handleTriageInbox({ cursor: 'not-a-cursor' }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('invalid triage cursor'); + }); + + it('narrows account scope and threads feed filters with a max-50 limit', async () => { + engineAdapter.resolveAccountScope.mockResolvedValue({ accountIds: ['account-2'] }); + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [], + hasMore: false, + cursor: null, + }); + + await triageTools.handleTriageInbox({ + account: 'other@example.com', + limit: 99, + unread_only: false, + include_triaged: true, + categories: ['newsletter', 'promotion'], + since: '2026-07-01', + include_signals: false, + }, scope, deps); + + expect(engineAdapter.resolveAccountScope).toHaveBeenCalledWith( + 'other@example.com', + scope.accountIds, + ); + expect(triageAdapter.listTriageCandidates).toHaveBeenCalledWith({ + accountIds: ['account-2'], + cursor: undefined, + limit: 50, + unreadOnly: false, + includeTriaged: true, + categories: ['newsletter', 'promotion'], + since: '2026-07-01', + }); + }); + + it('reports explicitly disabled signals without attaching per-item placeholders', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + + const body = jsonOf(await triageTools.handleTriageInbox({ + include_signals: false, + }, scope, deps)); + + expect(body.signals_available).toBe(false); + expect(body.signals_reason).toBe('disabled'); + expect(body.items[0]).not.toHaveProperty('signals'); + expect(hybrid.resolveActiveGenerationFromConfig).not.toHaveBeenCalled(); + expect(vectorStore.loadVector).not.toHaveBeenCalled(); + }); + + it('silently disables signals above the 25-item cost bound', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + + const result = await triageTools.handleTriageInbox({ + limit: 30, + }, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.signals_available).toBe(false); + expect(body.signals_reason).toBe('limit_exceeds_25'); + expect(body.items[0].signals).toEqual({ similar: [], probes: {} }); + expect(hybrid.resolveActiveGenerationFromConfig).not.toHaveBeenCalled(); + expect(vectorStore.loadVector).not.toHaveBeenCalled(); + }); + + it('computes similar dispositions at concurrency 4 and calibrates page probe scores', async () => { + const secondCandidate = { + ...candidateRow, + id: 'candidate-two', + message_id: '', + date: new Date('2026-07-28T08:00:00.000Z'), + }; + const rows = [candidateRow, secondCandidate]; + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows, + hasMore: false, + cursor: null, + }); + vectorStore.loadVector.mockImplementation(async id => ( + id === MESSAGE_ID ? [0.2] : [0.8] + )); + triageProbes.scoreTriageProbes.mockImplementation(vector => ({ + urgent: vector[0], + bulk: 1 - vector[0], + })); + vectorStore.annSearch.mockImplementation(async (_generationId, _vector, _k, options) => [ + { + messageId: options.filter.before.getUTCHours() === 6 + ? MESSAGE_ID + : 'candidate-two', + score: 1, + }, + { messageId: 'past-archived', score: 0.9 }, + { messageId: 'past-sent', score: 0.8 }, + ]); + const hydrated = { + 'past-archived': { + id: 'past-archived', + source_id: scope.accountIds[0], + source_message_id: 'past-archived', + conversation_id: 'past-thread', + subject: 'Past archived message', + sent_at: '2026-06-01T10:00:00.123Z', + labels: ['Archive', '\\Seen', '\\Flagged'], + }, + 'past-sent': { + id: 'past-sent', + source_id: scope.accountIds[0], + source_message_id: 'past-sent', + conversation_id: 'past-thread', + subject: 'My reply', + sent_at: '2026-06-01T11:00:00.456Z', + labels: ['Sent'], + }, + }; + engineAdapter.getMessageSummariesByIDs.mockImplementation(async ids => ( + ids.map(id => hydrated[id]).filter(Boolean) + )); + triageAdapter.triageActionsForMessages.mockResolvedValue([{ + account_id: scope.accountIds[0], + message_id_header: 'past-archived', + action: 'archived', + triaged_at: '2026-07-28T08:00:00Z', + }]); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.signals_available).toBe(true); + expect(body.signals_reason).toBeNull(); + expect(body.probe_calibration).toEqual({ + urgent: { min: 0.2, median: 0.5, max: 0.8 }, + bulk: { min: 0.19999999999999996, median: 0.5, max: 0.8 }, + }); + expect(batch.runInBatches).toHaveBeenCalledWith(rows, 4, expect.any(Function)); + expect(vectorStore.annSearch).toHaveBeenCalledWith( + 7, + [0.2], + 6, + { + filter: { + accountIds: scope.accountIds, + before: candidateRow.date, + }, + }, + ); + expect(engineAdapter.getMessageSummariesByIDs).toHaveBeenCalledWith( + ['past-archived', 'past-sent'], + scope.accountIds, + ); + expect(body.items[0].signals.probes).toEqual({ urgent: 0.2, bulk: 0.8 }); + expect(body.items[0].signals.similar[0]).toEqual(expect.objectContaining({ + id: 'past-archived', + sent_at: '2026-06-01T10:00:00Z', + score: 0.9, + disposition: { + folder_class: 'archived', + was_read: true, + was_starred: true, + was_replied: true, + triage_action: 'archived', + }, + })); + expect(body.items[0].signals.similar[1].disposition).toEqual(expect.objectContaining({ + folder_class: 'sent', + triage_action: null, + })); + expect(triageAdapter.triageActionsForMessages).toHaveBeenCalledTimes(1); + expect(triageAdapter.triageActionsForMessages).toHaveBeenCalledWith([ + { accountId: scope.accountIds[0], messageIdHeader: 'past-archived' }, + { accountId: scope.accountIds[0], messageIdHeader: 'past-sent' }, + ]); + expect(mailUtils.resolveArchiveFolder).toHaveBeenCalledTimes(1); + expect(mailUtils.resolveAllTrashPaths).toHaveBeenCalledTimes(1); + expect(mailUtils.resolveAllSpamPaths).toHaveBeenCalledTimes(1); + expect(gtdConfig.getGtdFolderSet).toHaveBeenCalledTimes(1); + expect(triageAdapter.sentFolderForAccount).toHaveBeenCalledTimes(1); + }); + + it('prefers the resolved sent folder over the name heuristic when one exists', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + triageAdapter.sentFolderForAccount.mockResolvedValue('Custom/Outgoing'); + vectorStore.annSearch.mockResolvedValue([ + { messageId: 'past-outgoing', score: 0.9 }, + { messageId: 'past-sent-items', score: 0.8 }, + ]); + engineAdapter.getMessageSummariesByIDs.mockResolvedValue([ + { + id: 'past-outgoing', + source_id: scope.accountIds[0], + source_message_id: 'past-outgoing', + subject: 'Reply from the mapped folder', + sent_at: '2026-06-01T10:00:00.123Z', + labels: ['Custom/Outgoing'], + }, + { + id: 'past-sent-items', + source_id: scope.accountIds[0], + source_message_id: 'past-sent-items', + subject: 'Name-matches sent but is not the resolved folder', + sent_at: '2026-06-01T11:00:00.456Z', + labels: ['Sent Items'], + }, + ]); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + const byId = Object.fromEntries( + body.items[0].signals.similar.map(entry => [entry.id, entry]), + ); + expect(byId['past-outgoing'].disposition.folder_class).toBe('sent'); + expect(byId['past-sent-items'].disposition.folder_class).toBe('labelled'); + }); + + it('degrades a rejected page triage-action lookup to null dispositions', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + vectorStore.annSearch.mockResolvedValue([ + { messageId: 'past-archived', score: 0.9 }, + { messageId: 'past-sent', score: 0.8 }, + ]); + engineAdapter.getMessageSummariesByIDs.mockResolvedValue([ + { + id: 'past-archived', + source_id: scope.accountIds[0], + source_message_id: 'past-archived', + subject: 'Past archived message', + sent_at: '2026-06-01T10:00:00.123Z', + labels: ['Archive'], + }, + { + id: 'past-sent', + source_id: scope.accountIds[0], + source_message_id: 'past-sent', + subject: 'My reply', + sent_at: '2026-06-01T11:00:00.456Z', + labels: ['Sent'], + }, + ]); + triageAdapter.triageActionsForMessages.mockRejectedValue( + new Error('triage lookup unavailable'), + ); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(triageAdapter.triageActionsForMessages).toHaveBeenCalledTimes(1); + expect(body.items[0].signals.similar).toHaveLength(2); + expect(body.items[0].signals.similar.every( + similar => similar.disposition.triage_action === null, + )).toBe(true); + }); + + it('degrades one unembedded candidate without affecting the rest of the page', async () => { + const missingCandidate = { + ...candidateRow, + id: 'missing-vector', + message_id: '', + }; + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow, missingCandidate], + hasMore: false, + cursor: null, + }); + vectorStore.loadVector.mockImplementation(async id => { + if (id === 'missing-vector') throw new Error('no embedding for message'); + return [1, 0]; + }); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.signals_available).toBe(true); + expect(body.items[0].signals).toEqual({ + similar: [], + probes: defaultProbeScores, + }); + expect(body.items[1].signals).toEqual({ + similar: [], + probes: {}, + reason: 'not_embedded', + }); + }); + + it('turns page-level VectorUnavailableError into empty signals without isError', async () => { + triageAdapter.listTriageCandidates.mockResolvedValue({ + rows: [candidateRow], + hasMore: false, + cursor: null, + }); + const error = new Error('no active generation'); + error.name = 'VectorUnavailableError'; + error.reason = 'no_active_generation'; + hybrid.resolveActiveGenerationFromConfig.mockRejectedValue(error); + + const result = await triageTools.handleTriageInbox({}, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.signals_available).toBe(false); + expect(body.signals_reason).toBe( + 'no_active_generation: vector search has no active index yet; wait for the embedding worker to finish an initial build', + ); + expect(body.items[0].signals).toEqual({ similar: [], probes: {} }); + expect(vectorStore.loadVector).not.toHaveBeenCalled(); + }); + + it.each([ + [{ since: '2026-02-30' }, 'invalid since date "2026-02-30": expected YYYY-MM-DD'], + [{ since: '07/28/2026' }, 'invalid since date "07/28/2026": expected YYYY-MM-DD'], + [{ categories: 'primary' }, 'categories must be an array of strings'], + [{ categories: ['primary', 42] }, 'categories must be an array of strings'], + [{ cursor: 42 }, 'cursor must be a string'], + ])('rejects malformed filters before reading the feed', async (args, message) => { + const result = await triageTools.handleTriageInbox(args, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe(message); + expect(triageAdapter.listTriageCandidates).not.toHaveBeenCalled(); + }); + + it('returns an unknown-account error without widening scope', async () => { + engineAdapter.resolveAccountScope.mockResolvedValue({ + error: 'account not found: missing@example.com', + }); + + const result = await triageTools.handleTriageInbox({ + account: 'missing@example.com', + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('account not found: missing@example.com'); + expect(triageAdapter.listTriageCandidates).not.toHaveBeenCalled(); + }); +}); + +const contextSeed = { + id: MESSAGE_ID, + account_id: scope.accountIds[0], + uid: 10, + folder: 'INBOX', + message_id: '', + thread_id: 'thread-1', + subject: 'Quarterly planning', + snippet: 'Could you review the agenda?', + from_email: 'alice@example.com', + from_name: 'Alice', + to_addresses: [{ email: 'me@example.com', name: 'Me' }], + date: new Date('2026-07-28T06:11:00.123Z'), + has_attachments: false, + is_read: false, +}; +const threadSummary = { + id: 'thread-message-1', + subject: 'Re: Quarterly planning', + sent_at: '2026-07-28T05:00:00.123Z', +}; +const similarSummary = { + id: 'similar-message-1', + subject: 'Prior quarterly planning', + sent_at: '2026-04-01T05:00:00.456Z', +}; +const senderHistoryRow = { + received_count: 8, + first_received: '2025-01-01T00:00:00Z', + last_received: '2026-07-28T06:11:00Z', + contact_id: 'contact-1', + contact_name: 'Alice', + primary_email: 'alice@example.com', + send_count: 3, + last_sent: '2026-07-20T12:00:00Z', + is_auto: false, + contact_known: true, +}; +const generation = { + id: 7, + model: 'embed-model', + dimension: 3, + fingerprint: 'fingerprint-1', + state: 'active', +}; + +function contextDeps(rules = []) { + return { loadInboxRules: vi.fn().mockResolvedValue(rules) }; +} + +function arrangeContextSuccess() { + engineAdapter.getMessage.mockResolvedValue(contextSeed); + engineAdapter.listMessages.mockResolvedValue([threadSummary]); + triageAdapter.senderHistory.mockResolvedValue(senderHistoryRow); + messageTools.findSimilarSummaries.mockResolvedValue({ + generation, + messages: [similarSummary], + }); +} + +describe('get_triage_context definition and registration', () => { + it('registers an idempotent read-only report tool', () => { + const def = registeredTools.TOOL_DEFS?.find( + entry => entry.name === 'get_triage_context', + ); + + expect(def?.inputSchema.required).toEqual(['message_id']); + expect(def?.inputSchema.properties).toEqual({ + message_id: { type: 'string' }, + thread_limit: { type: 'number', minimum: 1, maximum: 50 }, + similar_limit: { type: 'number', minimum: 1, maximum: 50 }, + }); + expect(def?.annotations).toEqual({ + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }); + expect(def?.description).toMatch(/report only/i); + expect(def?.description).toMatch(/never executes rule actions/i); + expect(registeredTools.TOOL_SCOPES?.get_triage_context).toBe('read'); + expect(registeredTools.HANDLERS?.get_triage_context) + .toBe(triageTools.handleGetTriageContext); + }); +}); + +describe('get_triage_context handler', () => { + it('returns thread, sender, similar, and report-only rule sections', async () => { + arrangeContextSuccess(); + const rules = [ + { + id: 'rule-subject', + name: 'Planning', + condition_logic: 'AND', + conditions: [{ + field: 'subject', + operator: 'contains', + value: 'planning', + }], + actions: [{ type: 'star' }], + }, + { + id: 'rule-body', + name: 'Body-only rule', + condition_logic: 'AND', + conditions: [{ + field: 'body', + operator: 'contains', + value: 'deadline', + }], + actions: [{ type: 'archive' }], + }, + { + id: 'rule-header', + name: 'Header-only rule', + condition_logic: 'AND', + conditions: [{ + field: 'header', + headerName: 'List-Id', + operator: 'contains', + value: 'newsletter', + }], + actions: [{ type: 'mark_read' }], + }, + ]; + const loaderDeps = contextDeps(rules); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + thread_limit: 12, + similar_limit: 6, + }, scope, loaderDeps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body).toEqual({ + message_id: MESSAGE_ID, + thread: { + available: true, + messages: [{ + ...threadSummary, + sent_at: '2026-07-28T05:00:00Z', + }], + }, + sender_history: { + available: true, + history: senderHistoryRow, + }, + similar: { + available: true, + generation, + messages: [{ + ...similarSummary, + sent_at: '2026-04-01T05:00:00Z', + }], + }, + matched_rules: { + available: true, + rules: [ + { + id: 'rule-subject', + name: 'Planning', + would_match: true, + actions: [{ type: 'star' }], + }, + { + id: 'rule-body', + name: 'Body-only rule', + evaluated: false, + reason: 'body_not_loaded', + actions: [{ type: 'archive' }], + }, + { + id: 'rule-header', + name: 'Header-only rule', + evaluated: false, + reason: 'body_not_loaded', + actions: [{ type: 'mark_read' }], + }, + ], + }, + }); + expect(engineAdapter.getMessage).toHaveBeenCalledWith( + MESSAGE_ID, + scope.accountIds, + ); + expect(engineAdapter.listMessages).toHaveBeenCalledWith({ + accountIds: scope.accountIds, + conversationId: 'thread-1', + limit: 12, + }); + expect(triageAdapter.senderHistory).toHaveBeenCalledWith( + 'alice@example.com', + scope.accountIds, + ); + expect(messageTools.findSimilarSummaries).toHaveBeenCalledWith( + MESSAGE_ID, + { accountIds: scope.accountIds, limit: 6 }, + ); + expect(loaderDeps.loadInboxRules).toHaveBeenCalledWith({ + userId: scope.userId, + accountId: scope.accountIds[0], + accountIds: scope.accountIds, + }); + }); + + it.each([ + ['thread', () => engineAdapter.listMessages.mockRejectedValue(new Error('thread down'))], + ['sender_history', () => triageAdapter.senderHistory.mockRejectedValue(new Error('sender down'))], + ['similar', () => messageTools.findSimilarSummaries.mockRejectedValue(new Error('similar down'))], + ['matched_rules', (_deps) => _deps.loadInboxRules.mockRejectedValue(new Error('rules down'))], + ])('degrades only the %s section and never sets isError', async (failedSection, fail) => { + arrangeContextSuccess(); + const loaderDeps = contextDeps([]); + fail(loaderDeps); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, loaderDeps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body[failedSection]).toEqual({ + available: false, + reason: 'error', + detail: expect.any(String), + }); + for (const section of ['thread', 'sender_history', 'similar', 'matched_rules']) { + if (section !== failedSection) expect(body[section].available).toBe(true); + } + }); + + it('translates VectorUnavailableError inside similar without failing the tool', async () => { + arrangeContextSuccess(); + const error = new Error('no active generation'); + error.name = 'VectorUnavailableError'; + error.reason = 'no_active_generation'; + messageTools.findSimilarSummaries.mockRejectedValue(error); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, contextDeps([])); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.similar).toEqual({ + available: false, + reason: 'no_active_generation: vector search has no active index yet; wait for the embedding worker to finish an initial build', + }); + expect(body.thread.available).toBe(true); + expect(body.sender_history.available).toBe(true); + expect(body.matched_rules.available).toBe(true); + }); + + it('degrades only matched_rules when no read-only loader is importable', async () => { + arrangeContextSuccess(); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + expect(body.matched_rules).toEqual({ + available: false, + reason: 'rules_loader_unavailable', + }); + expect(body.thread.available).toBe(true); + expect(body.sender_history.available).toBe(true); + expect(body.similar.available).toBe(true); + }); + + it('returns a clean input error before any section work when message_id is absent', async () => { + const result = await triageTools.handleGetTriageContext({}, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message_id parameter is required'); + expect(engineAdapter.getMessage).not.toHaveBeenCalled(); + }); + + it('returns message not found before starting enrichment sections', async () => { + engineAdapter.getMessage.mockResolvedValue(null); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, deps); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toBe('message not found'); + expect(engineAdapter.listMessages).not.toHaveBeenCalled(); + expect(triageAdapter.senderHistory).not.toHaveBeenCalled(); + expect(messageTools.findSimilarSummaries).not.toHaveBeenCalled(); + }); + + it('degrades all sections without isError when the scoped seed lookup fails', async () => { + engineAdapter.getMessage.mockRejectedValue(new Error('message store down')); + + const result = await triageTools.handleGetTriageContext({ + message_id: MESSAGE_ID, + }, scope, deps); + const body = jsonOf(result); + + expect(result.isError).toBeUndefined(); + for (const section of ['thread', 'sender_history', 'similar', 'matched_rules']) { + expect(body[section]).toEqual({ + available: false, + reason: 'seed_lookup_failed', + detail: 'message store down', + }); + } + }); +}); + +describe('mock-drift guard: mocked seams exist on their real modules', () => { + it.each([ + ['triageAdapter', () => triageAdapter, './triageAdapter.js'], + ['engineAdapter', () => engineAdapter, './engineAdapter.js'], + ['messageTools', () => messageTools, './messageTools.js'], + ['inboxRules', () => inboxRules, '../services/inboxRules.js'], + ['hybrid', () => hybrid, '../services/embeddings/hybrid.js'], + ['vectorStore', () => vectorStore, '../services/embeddings/vectorStore.js'], + ['batch', () => batch, '../services/mailbox/batch.js'], + ['triageProbes', () => triageProbes, './triageProbes.js'], + ['mailUtils', () => mailUtils, '../utils/mailUtils.js'], + ['gtdConfig', () => gtdConfig, '../services/gtdConfig.js'], + ])('%s mock surface matches the real module', async (_name, getMock, path) => { + const real = await vi.importActual(path); + expect(mockSurfaceDrift(getMock(), real)).toEqual([]); + }); +}); diff --git a/backend/src/mcp/vectorErrors.js b/backend/src/mcp/vectorErrors.js new file mode 100644 index 00000000..13b658d5 --- /dev/null +++ b/backend/src/mcp/vectorErrors.js @@ -0,0 +1,18 @@ +// Code prefixes are verbatim from msgvault (handlers.go translateVectorErr). +// Remediation hints are Mailflow-specific (msgvault references a CLI we don't +// ship) — a documented divergence; golden diffing asserts the prefix only. +const MESSAGES = { + vector_not_enabled: 'vector_not_enabled: vector search is not configured on this server', + index_stale: 'index_stale: the vector index does not match the configured model; reconfigure embeddings to rebuild', + index_building: 'index_building: the initial vector index is still being built', + no_active_generation: 'no_active_generation: vector search has no active index yet; wait for the embedding worker to finish an initial build', + // msgvault handlers.go:225-229; remediation names Mailflow's knob (the + // embedding client timeout) instead of msgvault's [vector.embeddings].timeout TOML. + embedding_timeout: 'embedding_timeout: the embedding endpoint did not respond in time; retry, or raise the embedding client timeout in settings', +}; + +export const VECTOR_ERROR_CODES = Object.keys(MESSAGES); + +export function translateVectorError(reason) { + return MESSAGES[reason] || MESSAGES.vector_not_enabled; +} diff --git a/backend/src/mcp/vectorErrors.test.js b/backend/src/mcp/vectorErrors.test.js new file mode 100644 index 00000000..334a07cd --- /dev/null +++ b/backend/src/mcp/vectorErrors.test.js @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest'; +import { translateVectorError } from './vectorErrors.js'; + +describe('translateVectorError', () => { + it('maps each reason to its verbatim code prefix', () => { + expect(translateVectorError('vector_not_enabled')).toMatch(/^vector_not_enabled: /); + expect(translateVectorError('index_stale')).toMatch(/^index_stale: /); + expect(translateVectorError('index_building')).toMatch(/^index_building: /); + expect(translateVectorError('no_active_generation')).toMatch(/^no_active_generation: /); + expect(translateVectorError('embedding_timeout')).toMatch(/^embedding_timeout: /); + }); + it('index_stale matches msgvault prefix-verbatim (handlers.go:206-210 — no inserted "embedding")', () => { + expect(translateVectorError('index_stale')) + .toMatch(/^index_stale: the vector index does not match the configured model; /); + }); + it('embedding_timeout ports msgvault wording (handlers.go:225-229)', () => { + expect(translateVectorError('embedding_timeout')) + .toMatch(/^embedding_timeout: the embedding endpoint did not respond in time; retry, /); + }); + it('falls back to vector_not_enabled for an unknown reason', () => { + expect(translateVectorError('mystery')).toMatch(/^vector_not_enabled: /); + }); +}); diff --git a/backend/src/mcp/vectorStats.js b/backend/src/mcp/vectorStats.js new file mode 100644 index 00000000..dc90f2e3 --- /dev/null +++ b/backend/src/mcp/vectorStats.js @@ -0,0 +1,75 @@ +import { query } from '../services/db.js'; +import * as generations from '../services/embeddings/generations.js'; // phase 3 + +// Epoch SECONDS (index_generations.activated_at/started_at, bigint — node-pg +// returns it as a string) → RFC3339 UTC string, matching msgvault CollectStats +// (vector/stats.go:146-153: time.Unix(sec,0).UTC().Format(time.RFC3339) — no +// sub-second digits, unlike Date.toISOString()). "" for a missing/zero/ +// unparsable epoch; callers omit the field entirely then (Go omitempty on +// activated_at/started_at, stats.go:47,:56). +function epochToISO(sec) { + const n = Number(sec); + if (!Number.isFinite(n) || n <= 0) return ''; + return new Date(n * 1000).toISOString().replace('.000Z', 'Z'); +} + +// Live-message count still needing embedding, SCOPED to the caller's accounts +// (documented divergence from msgvault's single-archive count — avoids a cross-user +// cardinality leak). Frozen invariant: `embed_gen IS NULL ⟺ needs embedding` — +// createGeneration resets every live row's stamp to NULL on a rebuild, so the count +// is generation-agnostic and the old `OR embed_gen <> gen` arm was dead/double-counting. +async function missingCount(accountIds) { + if (!accountIds || !accountIds.length) return 0; + const { rows } = await query( + `SELECT COUNT(*)::bigint AS n FROM messages + WHERE account_id = ANY($1) AND is_deleted = false AND embed_gen IS NULL`, + [accountIds], + ); + return Number(rows[0].n); +} + +// Port of vector.CollectStats. Returns null when vector search is disabled (the +// generation queries throw on stock Postgres without the vector schema); an +// enabled archive with no active generation yet reports active_generation: null. +// Generation metadata is archive-global; missing_embeddings_total is account-scoped. +export async function collectStats(accountIds) { + let active; + try { + active = await generations.activeGeneration(); // null = none yet; throw = disabled + } catch { + return null; + } + + // Sub-query failures degrade to partial data (msgvault stats.go:69-78: one + // broken sub-query never blanks the whole stats envelope) — every leg below + // catches and falls back rather than throwing the block away. + const out = { enabled: true, active_generation: null, missing_embeddings_total: 0 }; + if (active) { + const messageCount = await generations.chunkCount(active.id).catch(() => 0); + const activatedAt = epochToISO(active.activatedAt); + out.active_generation = { + id: active.id, model: active.model, dimension: active.dimension, + fingerprint: active.fingerprint, state: active.state, + ...(activatedAt ? { activated_at: activatedAt } : {}), // omitempty (stats.go:47) + message_count: messageCount, + }; + } + + const building = await generations.buildingGeneration().catch(() => null); + if (building) { + // A rebuild in flight is the actionable coverage target; active-generation + // top-ups are frozen until activation (msgvault CollectStats semantics). + const done = await generations.chunkCount(building.id).catch(() => 0); + const pending = await missingCount(accountIds).catch(() => 0); + const startedAt = epochToISO(building.startedAt); + out.building_generation = { + id: building.id, model: building.model, dimension: building.dimension, + ...(startedAt ? { started_at: startedAt } : {}), // omitempty (stats.go:56) + progress: { done, total: done + pending }, + }; + out.missing_embeddings_total = pending; + } else if (active) { + out.missing_embeddings_total = await missingCount(accountIds).catch(() => 0); + } + return out; +} diff --git a/backend/src/mcp/vectorStats.test.js b/backend/src/mcp/vectorStats.test.js new file mode 100644 index 00000000..cbc093ee --- /dev/null +++ b/backend/src/mcp/vectorStats.test.js @@ -0,0 +1,98 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('../services/embeddings/generations.js', () => ({ + activeGeneration: vi.fn(), buildingGeneration: vi.fn(), chunkCount: vi.fn(), +})); +import { query } from '../services/db.js'; +import * as generations from '../services/embeddings/generations.js'; +import { collectStats } from './vectorStats.js'; +import { mockSurfaceDrift } from '../testSupport/mockSurface.js'; + +beforeEach(() => { query.mockReset(); generations.activeGeneration.mockReset(); generations.buildingGeneration.mockReset(); generations.chunkCount.mockReset(); }); + +describe('mock-drift guard', () => { + it('every mocked generations key exists as a function on the real module', async () => { + // Regression guard: chunkCount was mocked here (and in goldenParity) while the + // real generations.js never implemented it — collectStats threw live. + const real = await vi.importActual('../services/embeddings/generations.js'); + expect(mockSurfaceDrift(generations, real)).toEqual([]); + }); +}); + +describe('collectStats', () => { + it('returns null when vector search is disabled', async () => { + generations.activeGeneration.mockRejectedValue(new Error('disabled')); + expect(await collectStats(['a'])).toBeNull(); + }); + + it('reports the active generation and a scoped missing count', async () => { + // activatedAt is epoch SECONDS (bigint) as generationByState now returns it; + // the wire field is RFC3339 UTC WITHOUT sub-second digits (msgvault + // vector/stats.go:146-153 formatTime uses time.RFC3339, never millis). + // 1704067200 = 2024-01-01T00:00:00Z. + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 1536, fingerprint: 'fp', state: 'active', activatedAt: 1704067200 }); + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(1000); + query.mockResolvedValueOnce({ rows: [{ n: '7' }] }); // missing count + const vs = await collectStats(['acc-1']); + expect(vs.enabled).toBe(true); + expect(vs.active_generation).toEqual({ id: 2, model: 'm', dimension: 1536, fingerprint: 'fp', state: 'active', activated_at: '2024-01-01T00:00:00Z', message_count: 1000 }); + expect(vs.missing_embeddings_total).toBe(7); + // chunkCount is keyed by generation id (real signature). The missing count is scoped to + // accountIds and keys off `embed_gen IS NULL` only (frozen invariant — no dead OR arm). + expect(generations.chunkCount).toHaveBeenCalledWith(2); + expect(query.mock.calls[0][1]).toEqual([['acc-1']]); + const sql = query.mock.calls[0][0]; + expect(sql).toMatch(/embed_gen IS NULL/); + expect(sql).not.toMatch(/embed_gen\s*<>/); + }); + + it('enabled with no active generation yet (first build) reports a null active_generation', async () => { + generations.activeGeneration.mockResolvedValue(null); + generations.buildingGeneration.mockResolvedValue(null); + const vs = await collectStats([]); + expect(vs).toEqual({ enabled: true, active_generation: null, missing_embeddings_total: 0 }); + expect(query).not.toHaveBeenCalled(); // empty scope skips the count query + }); + + it('OMITS activated_at when the epoch is absent or zero (msgvault omitempty, stats.go:47)', async () => { + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 8, fingerprint: 'fp', state: 'active' }); // no activatedAt + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(3); + query.mockResolvedValueOnce({ rows: [{ n: '0' }] }); + const vs = await collectStats(['acc-1']); + expect(vs.active_generation).not.toHaveProperty('activated_at'); + }); + + it('reports a building generation with scoped progress and freezes missing on the build target', async () => { + // startedAt is epoch SECONDS → RFC3339 (no millis) wire. 1706745600 = 2024-02-01T00:00:00Z. + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 8, fingerprint: 'fp', state: 'active' }); + generations.buildingGeneration.mockResolvedValue({ id: 3, model: 'm2', dimension: 8, startedAt: 1706745600 }); + generations.chunkCount.mockImplementation(async (id) => (id === 2 ? 100 : 40)); // active done, building done + query.mockResolvedValueOnce({ rows: [{ n: '10' }] }); // missing for building gen 3 + const vs = await collectStats(['acc-1']); + expect(vs.building_generation).toEqual({ id: 3, model: 'm2', dimension: 8, started_at: '2024-02-01T00:00:00Z', progress: { done: 40, total: 50 } }); + expect(vs.missing_embeddings_total).toBe(10); // building coverage is the actionable target + expect(query.mock.calls[0][1]).toEqual([['acc-1']]); // generation-agnostic (embed_gen IS NULL) + }); + + it('OMITS started_at when the building epoch is absent (msgvault omitempty, stats.go:56)', async () => { + generations.activeGeneration.mockResolvedValue(null); + generations.buildingGeneration.mockResolvedValue({ id: 3, model: 'm2', dimension: 8 }); // no startedAt + generations.chunkCount.mockResolvedValue(40); + query.mockResolvedValueOnce({ rows: [{ n: '10' }] }); + const vs = await collectStats(['acc-1']); + expect(vs.building_generation).not.toHaveProperty('started_at'); + }); + + it('degrades to partial data when the missing-count sub-query fails (msgvault stats.go:69-78 best-effort)', async () => { + generations.activeGeneration.mockResolvedValue({ id: 2, model: 'm', dimension: 8, fingerprint: 'fp', state: 'active', activatedAt: 1704067200 }); + generations.buildingGeneration.mockResolvedValue(null); + generations.chunkCount.mockResolvedValue(3); + query.mockRejectedValueOnce(new Error('relation vanished')); // missingCount blows up + const vs = await collectStats(['acc-1']); + expect(vs).not.toBeNull(); // one broken sub-query must not blank the whole block + expect(vs.active_generation.id).toBe(2); + expect(vs.missing_embeddings_total).toBe(0); + }); +}); diff --git a/backend/src/mcp/writeResult.js b/backend/src/mcp/writeResult.js new file mode 100644 index 00000000..724352ba --- /dev/null +++ b/backend/src/mcp/writeResult.js @@ -0,0 +1,79 @@ +import { toRFC3339 } from './envelope.js'; +import { errorResult } from './result.js'; + +export const WRITE_ERROR_CODES = [ + 'account_not_found', + 'alias_not_found', + 'message_not_found', + 'draft_not_found', + 'outbox_not_found', + 'invalid_recipient', + 'too_many_recipients', + 'attachment_too_large', + 'no_drafts_folder', + 'no_sent_folder', + 'smtp_failed', + 'already_sent', + 'compose_session_not_found', + 'compose_session_limit', + 'compose_slot_occupied', + 'compose_draft_claimed', + 'compose_conflict', + 'compose_operation_in_progress', + 'attachment_limit', + 'invalid_arguments', + 'unsupported', +]; + +const CAMEL_TO_WIRE = { + draftUid: 'draft_uid', + inReplyTo: 'in_reply_to', + messageId: 'message_id', + outboxId: 'outbox_id', + recipientsComputed: 'recipients_computed', + sendAt: 'send_at', + sentCopySaved: 'sent_copy_saved', + undoSeconds: 'undo_seconds', +}; + +function wireTime(value) { + if (value instanceof Date) return toRFC3339(value.toISOString()); + return toRFC3339(value); +} + +function copyResultFields(target, fields) { + for (const [key, value] of Object.entries(fields || {})) { + if (value === undefined) continue; + const wireKey = CAMEL_TO_WIRE[key] || key; + target[wireKey] = wireKey === 'send_at' ? wireTime(value) : value; + } +} + +export function buildWriteReceipt(receipt = {}, resultFields = {}) { + const result = {}; + copyResultFields(result, resultFields); + + for (const key of ['messageId', 'inReplyTo', 'references']) { + if (receipt[key] !== undefined) copyResultFields(result, { [key]: receipt[key] }); + } + + result.from = receipt.from || {}; + result.to = receipt.to || []; + result.cc = receipt.cc || []; + result.bcc = receipt.bcc || []; + result.subject = receipt.subject || ''; + result.attachments = (receipt.attachments || []).map(attachment => ({ + filename: attachment.filename, + size: attachment.size, + ...(attachment.source !== undefined ? { source: attachment.source } : {}), + })); + + for (const key of ['sentCopySaved', 'folder']) { + if (receipt[key] !== undefined) copyResultFields(result, { [key]: receipt[key] }); + } + return result; +} + +export function writeError(code, detail) { + return errorResult(`${code}: ${detail}`); +} diff --git a/backend/src/mcp/writeResult.test.js b/backend/src/mcp/writeResult.test.js new file mode 100644 index 00000000..69a461af --- /dev/null +++ b/backend/src/mcp/writeResult.test.js @@ -0,0 +1,125 @@ +import { describe, expect, it } from 'vitest'; +import { + WRITE_ERROR_CODES, + buildWriteReceipt, + writeError, +} from './writeResult.js'; + +const COMPOSE_WRITE_ERROR_CODES = [ + 'compose_session_not_found', + 'compose_session_limit', + 'compose_slot_occupied', + 'compose_draft_claimed', + 'compose_conflict', + 'compose_operation_in_progress', + 'attachment_limit', +]; + +describe('buildWriteReceipt', () => { + it('matches the documented immediate-send receipt exactly', () => { + expect(buildWriteReceipt({ + from: { name: 'A', email: 'a@b.com' }, + to: [{ name: '', email: 'x@y.com' }], + cc: [], + bcc: [], + subject: 'Quarterly report', + attachments: [{ filename: 'q.pdf', size: 8123 }], + messageId: '', + sentCopySaved: true, + folder: 'Sent', + }, { + sent: true, + })).toEqual({ + sent: true, + message_id: '', + from: { name: 'A', email: 'a@b.com' }, + to: [{ name: '', email: 'x@y.com' }], + cc: [], + bcc: [], + subject: 'Quarterly report', + attachments: [{ filename: 'q.pdf', size: 8123 }], + sent_copy_saved: true, + folder: 'Sent', + }); + }); + + it('matches the documented queued-send receipt exactly', () => { + expect(buildWriteReceipt({ + subject: 'Quarterly report', + }, { + queued: true, + outboxId: 'outbox-1', + sendAt: '2026-07-28T10:00:30.000Z', + undoSeconds: 30, + note: 'Cancel with unsend_email before send_at.', + })).toEqual({ + queued: true, + outbox_id: 'outbox-1', + send_at: '2026-07-28T10:00:30Z', + undo_seconds: 30, + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Quarterly report', + attachments: [], + note: 'Cancel with unsend_email before send_at.', + }); + }); + + it('re-keys result-specific service fields and preserves attachment source metadata', () => { + expect(buildWriteReceipt({ + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Re: Subject', + attachments: [{ filename: 'deck.pdf', size: 2144000, source: 'forwarded' }], + messageId: '', + sentCopySaved: true, + inReplyTo: '', + references: ' ', + }, { + sent: true, + recipientsComputed: { + reply_target: 'sender@example.com', + excluded_self: ['me@example.com'], + }, + })).toEqual({ + sent: true, + message_id: '', + in_reply_to: '', + references: ' ', + recipients_computed: { + reply_target: 'sender@example.com', + excluded_self: ['me@example.com'], + }, + from: {}, + to: [], + cc: [], + bcc: [], + subject: 'Re: Subject', + attachments: [{ filename: 'deck.pdf', size: 2144000, source: 'forwarded' }], + sent_copy_saved: true, + }); + }); +}); + +describe('writeError', () => { + it('registers every stable compose-session write error exactly once in order', () => { + const offset = WRITE_ERROR_CODES.indexOf(COMPOSE_WRITE_ERROR_CODES[0]); + expect(WRITE_ERROR_CODES.slice(offset, offset + COMPOSE_WRITE_ERROR_CODES.length)) + .toEqual(COMPOSE_WRITE_ERROR_CODES); + for (const code of COMPOSE_WRITE_ERROR_CODES) { + expect(WRITE_ERROR_CODES.filter(item => item === code), code).toHaveLength(1); + } + }); + + it.each(WRITE_ERROR_CODES)('%s is emitted as a stable isError prefix', (code) => { + const result = writeError(code, 'detail'); + expect(result).toEqual({ + content: [{ type: 'text', text: `${code}: detail` }], + isError: true, + }); + }); +}); diff --git a/backend/src/routes/accounts.js b/backend/src/routes/accounts.js index 8484afde..be509deb 100644 --- a/backend/src/routes/accounts.js +++ b/backend/src/routes/accounts.js @@ -3,62 +3,27 @@ import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; import { imapManager } from '../index.js'; import { encrypt } from '../services/encryption.js'; -import { sanitizeSignature } from '../services/emailSanitizer.js'; +import { hasHeaderInjectionChars, sanitizeSignature } from '../services/emailSanitizer.js'; import { validateHost } from '../services/hostValidation.js'; import { getConnectionPolicy } from '../services/connectionPolicy.js'; import { invalidateGtdConfigCache, sanitizeGtdFoldersDetailed, findGtdFolderCollisions, DEFAULT_GTD_FOLDERS } from '../services/gtdConfig.js'; import { invalidateOwnerAddressesCache } from '../services/gtdTransitions.js'; -import { createKeyedSerializer } from '../utils/keyedSerializer.js'; - -// Serialize an account's reconnect triggers so a rapid settings change (e.g. a -// gtd_enabled double-toggle) can't fire two overlapping disconnect→connect chains — -// connectAccount's in-progress guard would drop the second and leave the GTD sync -// tick armed inconsistently with the final DB value. Queued per account id. -const reconnectQueue = createKeyedSerializer(); - -const ALLOWED_IMAP_PORTS = new Set([143, 993]); -const ALLOWED_SMTP_PORTS = new Set([465, 587]); - -function validatePort(port, allowed) { - const n = Number(port); - if (!Number.isInteger(n) || n < 1 || n > 65535) { - return `Port ${port} is not a valid port number`; - } - // When private/local hosts are explicitly allowed (e.g. Proton Mail Bridge on 1143/1025), - // skip the whitelist — the operator has already opted into unrestricted host access. - if (process.env.ALLOW_PRIVATE_IMAP_HOSTS === 'true') return null; - if (!allowed.has(n)) { - return `Port ${port} is not allowed. Allowed: ${[...allowed].join(', ')}`; - } - return null; -} - -// Reject strings that contain characters that could inject extra email headers. -function hasHeaderInjectionChars(str) { - return typeof str === 'string' && /[\r\n\0]/.test(str); -} +import { startAccountBodyBackfill } from '../services/bodyBackfill.js'; +import { providerProfile } from '../services/imapManager.js'; +import { upsertJob } from '../services/backgroundJobs.js'; +import { safeAccount } from '../services/accountFields.js'; +import { + ALLOWED_IMAP_PORTS, + ALLOWED_SMTP_PORTS, + createAccount, + reconcileConnectionState, + validatePort, +} from '../services/accountService.js'; +import { testConnection } from '../services/connectionTest.js'; const router = Router(); router.use(requireAuth); -// Fields safe to return to the client — matches the GET list, excludes credentials and tokens -const SAFE_FIELDS = [ - 'id', 'name', 'sender_name', 'email_address', 'color', 'protocol', - 'imap_host', 'imap_port', 'imap_skip_tls_verify', - 'smtp_host', 'smtp_port', 'smtp_tls', - 'auth_user', 'oauth_provider', 'enabled', - 'include_in_unified_inbox', - 'last_sync', 'sync_error', 'sort_order', 'folder_mappings', - 'signature', 'created_at', 'categorization_enabled', - 'gtd_enabled', 'gtd_folders', -]; -function safeAccount(row) { - const obj = Object.fromEntries(SAFE_FIELDS.map(k => [k, row[k]])); - // Sanitize on read so legacy values stored before the write-time sanitizer are safe - if (obj.signature) obj.signature = sanitizeSignature(obj.signature); - return obj; -} - router.get('/', async (req, res) => { const result = await query( `SELECT id, name, sender_name, email_address, color, protocol, imap_host, imap_port, imap_tls, imap_skip_tls_verify, @@ -96,64 +61,9 @@ router.get('/', async (req, res) => { }); router.post('/', async (req, res) => { - const { - name, sender_name = null, email_address, color = '#6366f1', protocol = 'imap', - imap_host, imap_port = 993, imap_skip_tls_verify = false, - smtp_host, smtp_port = 587, smtp_tls = 'STARTTLS', - auth_user, auth_pass, - oauth_provider, oauth_access_token, oauth_refresh_token, - signature = null - } = req.body; - - if (!name || !email_address) return res.status(400).json({ error: 'Name and email required' }); - if (hasHeaderInjectionChars(name) || hasHeaderInjectionChars(email_address)) { - return res.status(400).json({ error: 'Name and email address cannot contain control characters' }); - } - if (sender_name && hasHeaderInjectionChars(sender_name)) { - return res.status(400).json({ error: 'Sender name cannot contain control characters' }); - } - - const policy = await getConnectionPolicy(); - - if (imap_host) { - const err = (await validateHost(imap_host, { allowPrivate: policy.allowPrivateHosts })) - || (!policy.allowNonstandardPorts && validatePort(imap_port, ALLOWED_IMAP_PORTS)); - if (err) return res.status(400).json({ error: `IMAP: ${err}` }); - } - if (smtp_host) { - const err = (await validateHost(smtp_host, { allowPrivate: policy.allowPrivateHosts })) - || (!policy.allowNonstandardPorts && validatePort(smtp_port, ALLOWED_SMTP_PORTS)); - if (err) return res.status(400).json({ error: `SMTP: ${err}` }); - } - - try { - const result = await query(` - INSERT INTO email_accounts ( - user_id, name, sender_name, email_address, color, protocol, - imap_host, imap_port, imap_tls, imap_skip_tls_verify, smtp_host, smtp_port, smtp_tls, - auth_user, auth_pass, oauth_provider, oauth_access_token, oauth_refresh_token, - signature - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) - RETURNING * - `, [ - req.session.userId, name, sender_name || null, email_address, color, protocol, - imap_host, imap_port, Number(imap_port) % 1000 === 993, !!imap_skip_tls_verify, smtp_host, smtp_port, smtp_tls, - auth_user, encrypt(auth_pass), oauth_provider, encrypt(oauth_access_token), encrypt(oauth_refresh_token), - sanitizeSignature(signature) || null - ]); - - const account = result.rows[0]; - - // Immediately try to connect — needs full credentials from DB row - if (protocol === 'imap') { - imapManager.connectAccount(account).catch(console.error); - } - - res.json(safeAccount(account)); - } catch (err) { - console.error(err); - res.status(500).json({ error: 'Failed to add account' }); - } + const result = await createAccount({ userId: req.session.userId, fields: req.body }); + if (result.error) return res.status(result.status).json({ error: result.error }); + res.json(result.account); }); router.put('/:id', async (req, res) => { @@ -257,39 +167,12 @@ router.put('/:id', async (req, res) => { if ('gtd_enabled' in updates || 'gtd_folders' in updates) invalidateGtdConfigCache(id); // Sync live IMAP state after DB update (fire-and-forget, non-fatal). - // Toggling gtd_enabled reconnects so the GTD sync tick is armed/torn down (it is - // only decided at connectAccount) and the persistent-connection account object the - // INBOX/send transition hooks close over picks up the new flag. Remapping a GTD state - // to a different folder reconnects for the same reason: connectAccount's syncFolders + - // backfillAllFolders is what pulls the newly-designated folder's existing mail into the - // rail (backfillAllFolders backfills every discovered folder except the provider's - // skipFolderPatterns, which a GTD label folder never matches). - const isDisabling = 'enabled' in updates && !updates.enabled; - const needsReconnect = !isDisabling && ( - 'enabled' in updates || - 'auth_user' in updates || - 'auth_pass' in updates || - 'imap_host' in updates || - 'imap_port' in updates || - 'imap_tls' in updates || - 'imap_skip_tls_verify' in updates || - 'gtd_enabled' in updates || - gtdFoldersChanged - ); - - // Both branches queue through the per-account serializer so overlapping settings - // changes (e.g. a rapid gtd_enabled double-toggle) apply their connection-state - // effects in order, never as two overlapping chains. - if (isDisabling) { - reconnectQueue(id, () => imapManager.disconnectAccount(id)) - .catch(err => console.error(`Failed to disconnect account ${id} after disable:`, err.message)); - } else if (needsReconnect && updated.protocol === 'imap' && updated.enabled) { - reconnectQueue(id, () => - imapManager.disconnectAccount(id) - .then(() => query('SELECT * FROM email_accounts WHERE id = $1', [id])) - .then(r => { if (r.rows.length) return imapManager.connectAccount(r.rows[0]); }) - ).catch(err => console.error(`Failed to reconnect account ${id} after update:`, err.message)); - } + reconcileConnectionState({ + id, + updates, + before: { gtdFoldersChanged }, + updated, + }); }); router.delete('/:id', async (req, res) => { @@ -312,6 +195,16 @@ router.delete('/:id', async (req, res) => { } }); +router.post('/:id/test-connection', async (req, res) => { + const result = await query( + 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', + [req.params.id, req.session.userId] + ); + if (!result.rows.length) return res.status(404).json({ error: 'Account not found' }); + + res.json(await testConnection(result.rows[0])); +}); + router.post('/:id/reconnect', async (req, res) => { const { id } = req.params; const result = await query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [id, req.session.userId]); @@ -424,7 +317,18 @@ router.post('/:id/reindex', async (req, res) => { console.error(`Manual reindex error for ${account.email_address}:`, err.message) ); } - res.json({ ok: true, alreadyRunning }); + + // Also kick a body-materialization pass for allowlisted providers. Fire-and-forget: the + // drainer self-guards against concurrent runs and caps each session, and Gmail/PurelyMail/ + // Microsoft are gated off inside startBodyBackfill. Progress lands in background_jobs. + const bodyBackfillEnabled = providerProfile(account).bodyBackfill; + if (bodyBackfillEnabled) { + startAccountBodyBackfill(account, imapManager, upsertJob).catch(err => + console.error(`Body backfill error for ${account.email_address}:`, err.message) + ); + } + + res.json({ ok: true, alreadyRunning, bodyBackfillEnabled }); } catch (err) { console.error('POST /accounts/:id/reindex error:', err.message); res.status(500).json({ error: 'Failed to start reindex' }); diff --git a/backend/src/routes/accounts.test.js b/backend/src/routes/accounts.test.js new file mode 100644 index 00000000..530e639a --- /dev/null +++ b/backend/src/routes/accounts.test.js @@ -0,0 +1,123 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { query, testConnection } = vi.hoisted(() => ({ + query: vi.fn(), + testConnection: vi.fn(), +})); + +vi.mock('../services/db.js', () => ({ query })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'user-1' }; + next(); + }, +})); +vi.mock('../index.js', () => ({ imapManager: {} })); +vi.mock('../services/connectionTest.js', () => ({ testConnection })); +vi.mock('../services/accountService.js', () => ({ + ALLOWED_IMAP_PORTS: new Set([143, 993]), + ALLOWED_SMTP_PORTS: new Set([465, 587]), + createAccount: vi.fn(), + reconcileConnectionState: vi.fn(), + validatePort: vi.fn(), +})); + +import 'express-async-errors'; +import express from 'express'; +import accountRoutes from './accounts.js'; + +const account = { + id: 'account-1', + user_id: 'user-1', + imap_host: 'imap.example.com', + smtp_host: 'smtp.example.com', +}; + +function buildApp() { + const app = express(); + app.use(express.json()); + app.use('/api/accounts', accountRoutes); + return app; +} + +let server; +let base; + +beforeAll(async () => { + await new Promise(resolve => { + server = buildApp().listen(0, '127.0.0.1', resolve); + }); + base = `http://127.0.0.1:${server.address().port}`; +}); + +afterAll(async () => { + await new Promise(resolve => server.close(resolve)); +}); + +beforeEach(() => { + query.mockReset(); + testConnection.mockReset(); +}); + +async function postTestConnection(id = account.id) { + const response = await fetch(`${base}/api/accounts/${id}/test-connection`, { + method: 'POST', + }); + return { + status: response.status, + body: await response.json(), + }; +} + +describe('POST /api/accounts/:id/test-connection', () => { + it('404s an absent or unowned account', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + const response = await postTestConnection('foreign-account'); + + expect(response).toEqual({ + status: 404, + body: { error: 'Account not found' }, + }); + expect(query).toHaveBeenCalledWith( + 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', + ['foreign-account', 'user-1'] + ); + expect(testConnection).not.toHaveBeenCalled(); + }); + + it('returns both successful probe legs', async () => { + query.mockResolvedValueOnce({ rows: [account] }); + testConnection.mockResolvedValueOnce({ + imap: { ok: true }, + smtp: { ok: true }, + }); + + const response = await postTestConnection(); + + expect(response).toEqual({ + status: 200, + body: { imap: { ok: true }, smtp: { ok: true } }, + }); + expect(testConnection).toHaveBeenCalledWith(account); + }); + + it.each([ + ['IMAP', { + imap: { ok: false, error: 'Authentication failed.' }, + smtp: { ok: true }, + }], + ['SMTP', { + imap: { ok: true }, + smtp: { ok: false, error: 'Could not connect.' }, + }], + ])('returns the per-leg failure shape when %s fails', async (_leg, probeResult) => { + query.mockResolvedValueOnce({ rows: [account] }); + testConnection.mockResolvedValueOnce(probeResult); + + const response = await postTestConnection(); + + expect(response.status).toBe(200); + expect(response.body).toEqual(probeResult); + }); +}); diff --git a/backend/src/routes/ai.buildEmbeddingsConfig.test.js b/backend/src/routes/ai.buildEmbeddingsConfig.test.js new file mode 100644 index 00000000..8505fc8b --- /dev/null +++ b/backend/src/routes/ai.buildEmbeddingsConfig.test.js @@ -0,0 +1,32 @@ +import { describe, it, expect, vi } from 'vitest'; +vi.mock('../services/encryption.js', () => ({ + encrypt: (v) => `enc:${v}`, + decrypt: (v) => (v ? String(v).replace(/^enc:/, '') : v), +})); +import { buildEmbeddingsConfig } from './ai.js'; + +describe('buildEmbeddingsConfig', () => { + it('encrypts a freshly supplied apiKey', () => { + const out = buildEmbeddingsConfig( + { enabled: true, endpoint: 'http://h/v1', apiKey: 'sk-1', model: 'm', dimension: 768 }, + null, + ); + expect(out.apiKey).toBe('enc:sk-1'); + expect(out.dimension).toBe(768); + expect(out.preprocess.stripHTML).toBe(true); + }); + it('keeps the existing key when the masked sentinel is sent back', () => { + const out = buildEmbeddingsConfig( + { apiKey: '••••••••', model: 'm', dimension: 4 }, + { apiKey: 'enc:old' }, + ); + expect(out.apiKey).toBe('enc:old'); + }); + it('preserves an explicit-false preprocess flag', () => { + const out = buildEmbeddingsConfig( + { model: 'm', dimension: 4, preprocess: { stripQuotes: false } }, + null, + ); + expect(out.preprocess.stripQuotes).toBe(false); + }); +}); diff --git a/backend/src/routes/ai.js b/backend/src/routes/ai.js index d1b5f402..f692be8c 100644 --- a/backend/src/routes/ai.js +++ b/backend/src/routes/ai.js @@ -1,6 +1,8 @@ import { Router } from 'express'; import { requireAuth, requireAdmin } from '../middleware/auth.js'; import { query } from '../services/db.js'; +export { buildEmbeddingsConfig } from '../services/aiProvider.js'; +import { isVectorAvailable } from '../services/embeddings/vectorStore.js'; import { deleteAiConfig, getAdminAiConfig, @@ -149,7 +151,13 @@ router.delete('/admin/ai/codex', requireAdmin, async (_req, res) => { router.get('/ai/status', requireAuth, async (_req, res) => { try { - res.json(await getAiStatus()); + const [status, config] = await Promise.all([getAiStatus(), getAdminAiConfig()]); + res.json({ + ...status, + vectorAvailable: isVectorAvailable(), + embeddingsEnabled: config?.embeddings?.enabled === true + && Boolean(config.embeddings.endpoint && config.embeddings.model), + }); } catch (error) { serviceError(res, error, 'Failed to load AI status'); } diff --git a/backend/src/routes/ai.test.js b/backend/src/routes/ai.test.js index 0fe20e69..a69e39f0 100644 --- a/backend/src/routes/ai.test.js +++ b/backend/src/routes/ai.test.js @@ -263,7 +263,11 @@ describe('authenticated AI status and streaming', () => { const authenticated = await request('/api/ai/status', { user: MEMBER }); expect(authenticated.status).toBe(200); expect(await authenticated.json()).toEqual({ - enabled: true, provider: 'api-key', features: { compose: true }, + enabled: true, + provider: 'api-key', + features: { compose: true }, + vectorAvailable: false, + embeddingsEnabled: false, }); }); diff --git a/backend/src/routes/aiEmbeddings.js b/backend/src/routes/aiEmbeddings.js new file mode 100644 index 00000000..e41021e6 --- /dev/null +++ b/backend/src/routes/aiEmbeddings.js @@ -0,0 +1,148 @@ +import { Router } from 'express'; +import { requireAdmin } from '../middleware/auth.js'; +import { resolveEmbedConfig, generationFingerprint } from '../services/embeddings/config.js'; +import { isVectorAvailable } from '../services/embeddings/vectorStore.js'; +import * as store from '../services/embeddings/vectorStore.js'; +import * as generations from '../services/embeddings/generations.js'; +const { createGeneration, buildingGeneration, retireGeneration, BuildingInProgressError } = generations; +import { EmbeddingClient } from '../services/embeddings/client.js'; +import { EmbeddingWorker } from '../services/embeddings/worker.js'; +import { tryAcquireEmbedRun, releaseEmbedRun } from '../services/embeddings/embedRunLock.js'; +import { upsertJob } from '../services/backgroundJobs.js'; +import { query } from '../services/db.js'; + +const router = Router(); + +// Returns an error string when the config cannot start a build, else null. Pure helper. +export function validateBuildConfig(cfg) { + if (!isVectorAvailable()) return 'Vector extension unavailable — semantic search is disabled on this database'; + if (!cfg) return 'Embeddings not configured'; + if (!cfg.enabled) return 'Embeddings are disabled'; + if (!cfg.endpoint) return 'Embeddings endpoint is required'; + if (!cfg.model) return 'Embeddings model is required'; + if (!(cfg.dimension > 0)) return 'Embeddings dimension must be a positive integer'; + return null; +} + +// Probe the embedding endpoint with one input and echo the returned dimension. Pure helper. +export async function probeEmbeddings(client) { + const vecs = await client.embed(['mailflow embeddings connectivity probe']); + return { ok: true, dimension: vecs[0].length }; +} + +// The probe client intentionally omits the dimension expectation (dimension: null +// skips the client's per-vector assertion): Test's whole job is to DISCOVER the +// endpoint's real dimension so the UI can reconcile a wrong saved value +// (reconcileDimension auto-fill). With the assertion in place, a mismatch threw +// before the probed dimension ever reached the response, making that UI path +// unreachable. Worker/query paths keep the strict assertion — they construct +// their clients with cfg.dimension. +export function buildProbeClient(cfg) { + return new EmbeddingClient({ endpoint: cfg.endpoint, apiKey: cfg.apiKey, model: cfg.model, dimension: null }); +} + +router.post('/admin/ai/embeddings/test-embeddings', requireAdmin, async (req, res) => { + const cfg = await resolveEmbedConfig(); + if (!cfg || !cfg.endpoint || !cfg.model || !(cfg.dimension > 0)) { + return res.status(400).json({ error: 'Embeddings endpoint, model, and dimension are required' }); + } + try { + res.json(await probeEmbeddings(buildProbeClient(cfg))); + } catch (err) { + res.status(400).json({ error: err.message }); + } +}); + +// Count live messages still needing embedding under generation `gen` (the build's +// initial "total"). Default collaborator for startEmbeddingBuild. +async function countPending(gen) { + const r = await query( + 'SELECT COUNT(*)::int n FROM messages WHERE (embed_gen IS NULL OR embed_gen <> $1) AND is_deleted = false', [gen], + ); + return r.rows[0].n; +} + +// Fire-and-forget worker run toward coverage. Returns worker.runOnce's promise so the +// caller can chain job-state updates and the single-flight release onto its settlement. +function runWorker(gen, total, cfg) { + const client = new EmbeddingClient({ endpoint: cfg.endpoint, apiKey: cfg.apiKey, model: cfg.model, dimension: cfg.dimension }); + const worker = new EmbeddingWorker({ + // `generations` lets the worker activate this building generation once its + // scan drains to full coverage (the shared activation seam in worker.js). + store, client, generations, preprocessCfg: cfg.preprocess, maxInputChars: cfg.maxInputChars, batchSize: cfg.batchSize, + onProgress: (p) => { upsertJob({ kind: 'embeddings', state: 'running', processed: p.done, total }).catch(() => {}); }, + }); + return worker.runOnce(gen); +} + +// Collaborators for startEmbeddingBuild, bound to the real modules by default and +// overridable in tests (the injection pattern used across the embeddings services). +export const BUILD_DEPS = { + tryAcquireEmbedRun, releaseEmbedRun, + createGeneration, buildingGeneration, retireGeneration, generationFingerprint, + countPending, upsertJob, runWorker, log: console.log, +}; + +// Orchestrates an embeddings (re)build. Returns { status, body } for the route to send. +// +// Ordering matters: createGeneration atomically NULLs every live embed_gen stamp. If an +// embed run were mid-flight when that reset lands, it could re-stamp rows with the OLD +// generation id afterward — and the new generation's scan (embed_gen IS NULL only) would +// never see them, so activation's coverage gate blocks forever. So we take the single- +// flight lock BEFORE createGeneration, making the stamp-reset mutually exclusive with any +// embed run. If the lock is busy we return an honest, retryable 409 and never touch the +// stamps. On every non-success exit the lock is released exactly once; on success the +// fire-and-forget worker chain owns the single release when the run settles. +export async function startEmbeddingBuild(cfg, username, deps = {}) { + const d = { ...BUILD_DEPS, ...deps }; + const fingerprint = d.generationFingerprint(cfg); + + if (!d.tryAcquireEmbedRun()) { + return { status: 409, body: { error: 'An embedding run is in progress — retry in a moment' } }; + } + + let gen, total; + try { + try { + gen = await d.createGeneration(cfg.model, cfg.dimension, fingerprint); + } catch (err) { + // A building generation with a DIFFERENT fingerprint blocks this build. A + // new-fingerprint build supersedes an incomplete old-fingerprint one, so retire + // the stale gen (deletes its rows — generations never mix) and retry once. + if (!(err instanceof BuildingInProgressError)) throw err; + const stale = await d.buildingGeneration(); + if (!stale || stale.fingerprint === fingerprint) throw err; + await d.retireGeneration(stale.id); + d.log(`[admin] ${username} retired stale building gen ${stale.id} (fingerprint ${stale.fingerprint}); superseded by ${fingerprint}`); + gen = await d.createGeneration(cfg.model, cfg.dimension, fingerprint); + } + total = await d.countPending(gen); + await d.upsertJob({ kind: 'embeddings', state: 'running', processed: 0, total }); + } catch (err) { + // Any failure before the worker chain is attached below leaves the lock ours to + // free — release it so a failed start never wedges every future build. + d.releaseEmbedRun(); + return { status: 409, body: { error: err.message } }; + } + + // We already hold the single-flight lock; fire the worker and release exactly once + // when it settles, on every outcome. The request returns immediately. + d.runWorker(gen, total, cfg) + .then((r) => d.upsertJob({ kind: 'embeddings', state: 'done', processed: r.succeeded, total })) + .catch((err) => d.upsertJob({ kind: 'embeddings', state: 'error', processed: 0, total, lastError: err.message })) + .finally(() => d.releaseEmbedRun()) + .catch(() => {}); + d.log(`[admin] ${username} started embeddings build gen ${gen} (${total} pending)`); + return { status: 200, body: { ok: true, generationId: gen, total } }; +} + +router.post('/admin/ai/embeddings/build', requireAdmin, async (req, res) => { + const cfg = await resolveEmbedConfig(); + const invalid = validateBuildConfig(cfg); + if (invalid) return res.status(400).json({ error: invalid }); + + const { status, body } = await startEmbeddingBuild(cfg, req.session.username); + res.status(status).json(body); +}); + +export default router; diff --git a/backend/src/routes/aiEmbeddings.test.js b/backend/src/routes/aiEmbeddings.test.js new file mode 100644 index 00000000..7ce05b84 --- /dev/null +++ b/backend/src/routes/aiEmbeddings.test.js @@ -0,0 +1,149 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +vi.mock('../services/embeddings/vectorStore.js', () => ({ isVectorAvailable: vi.fn(() => true) })); +import { validateBuildConfig, probeEmbeddings, buildProbeClient, startEmbeddingBuild } from './aiEmbeddings.js'; +import { isVectorAvailable } from '../services/embeddings/vectorStore.js'; +import { BuildingInProgressError } from '../services/embeddings/generations.js'; + +const full = { enabled: true, endpoint: 'http://h/v1', model: 'm', dimension: 768, maxInputChars: 32768, batchSize: 32, preprocess: {} }; + +describe('validateBuildConfig', () => { + it('accepts a complete config when vector is available', () => { + expect(validateBuildConfig(full)).toBeNull(); + }); + it('rejects when vector is unavailable', () => { + isVectorAvailable.mockReturnValueOnce(false); + expect(validateBuildConfig(full)).toMatch(/vector/i); + }); + it('rejects an incomplete config', () => { + expect(validateBuildConfig({ ...full, dimension: 0 })).toMatch(/dimension/i); + expect(validateBuildConfig(null)).toMatch(/not configured/i); + }); +}); + +describe('probeEmbeddings', () => { + afterEach(() => vi.unstubAllGlobals()); + + it('echoes the returned dimension', async () => { + const fakeClient = { embed: vi.fn().mockResolvedValue([[0.1, 0.2, 0.3]]) }; + const out = await probeEmbeddings(fakeClient); + expect(out).toEqual({ ok: true, dimension: 3 }); + }); + it('surfaces an embed error', async () => { + const fakeClient = { embed: vi.fn().mockRejectedValue(new Error('connect ECONNREFUSED')) }; + await expect(probeEmbeddings(fakeClient)).rejects.toThrow(/ECONNREFUSED/); + }); + it('returns the endpoint\'s ACTUAL dimension even when the saved config disagrees', async () => { + // Regression pin: with the probe client asserting cfg.dimension, a mismatch threw + // before Test could return the probed value, so the UI's reconcileDimension + // auto-fill was unreachable. The probe client must skip the assertion. + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + status: 200, + headers: { get: () => null }, + json: async () => ({ data: [{ index: 0, embedding: Array(768).fill(0.1) }] }), + text: async () => '', + })); + const out = await probeEmbeddings(buildProbeClient({ ...full, dimension: 1536 })); + expect(out).toEqual({ ok: true, dimension: 768 }); + }); +}); + +// Let the fire-and-forget worker chain (.then/.catch/.finally) settle. +const flush = () => new Promise((r) => setTimeout(r, 0)); + +function makeDeps(overrides = {}) { + return { + tryAcquireEmbedRun: vi.fn(() => true), + releaseEmbedRun: vi.fn(), + createGeneration: vi.fn().mockResolvedValue('gen-1'), + buildingGeneration: vi.fn().mockResolvedValue(null), + retireGeneration: vi.fn().mockResolvedValue(undefined), + generationFingerprint: vi.fn(() => 'fp'), + countPending: vi.fn().mockResolvedValue(5), + upsertJob: vi.fn().mockResolvedValue(undefined), + runWorker: vi.fn().mockResolvedValue({ succeeded: 5 }), + log: vi.fn(), + ...overrides, + }; +} + +describe('startEmbeddingBuild', () => { + it('returns a retryable 409 and never resets stamps when an embed run is active (Fix 1)', async () => { + const deps = makeDeps({ tryAcquireEmbedRun: vi.fn(() => false) }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(409); + expect(res.body.error).toMatch(/in progress/i); + expect(deps.createGeneration).not.toHaveBeenCalled(); // stamp-reset never runs + expect(deps.releaseEmbedRun).not.toHaveBeenCalled(); // never acquired ⇒ nothing to release + }); + + it('acquires the single-flight lock before createGeneration (Fix 1)', async () => { + const deps = makeDeps(); + await startEmbeddingBuild(full, 'admin', deps); + expect(deps.tryAcquireEmbedRun).toHaveBeenCalled(); + expect(deps.createGeneration).toHaveBeenCalledWith('m', 768, 'fp'); + expect(deps.tryAcquireEmbedRun.mock.invocationCallOrder[0]) + .toBeLessThan(deps.createGeneration.mock.invocationCallOrder[0]); + await flush(); + }); + + it('on success fires the worker and releases the lock exactly once (Fix 1)', async () => { + const deps = makeDeps(); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(200); + expect(res.body).toEqual({ ok: true, generationId: 'gen-1', total: 5 }); + expect(deps.runWorker).toHaveBeenCalledWith('gen-1', 5, full); + await flush(); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); + + it('releases the lock and returns 409 when createGeneration fails outright (Fix 1)', async () => { + const deps = makeDeps({ createGeneration: vi.fn().mockRejectedValue(new Error('boom')) }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(409); + expect(res.body.error).toBe('boom'); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + expect(deps.retireGeneration).not.toHaveBeenCalled(); + }); + + it('releases the lock if the build fails after createGeneration but before the worker starts (Fix 1)', async () => { + // A failing countPending would otherwise leave the lock held forever (it is taken + // before createGeneration now), wedging every future build. + const deps = makeDeps({ countPending: vi.fn().mockRejectedValue(new Error('db down')) }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(res.status).toBe(409); + expect(res.body.error).toBe('db down'); + expect(deps.createGeneration).toHaveBeenCalledTimes(1); + expect(deps.runWorker).not.toHaveBeenCalled(); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); + + it('retires a stale different-fingerprint building gen and retries createGeneration once (Fix 2b)', async () => { + const createGeneration = vi.fn() + .mockRejectedValueOnce(new BuildingInProgressError('building fingerprint=old-fp, requested=fp')) + .mockResolvedValueOnce('gen-2'); + const deps = makeDeps({ + createGeneration, + buildingGeneration: vi.fn().mockResolvedValue({ id: 'stale-1', fingerprint: 'old-fp' }), + }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(deps.retireGeneration).toHaveBeenCalledWith('stale-1'); + expect(createGeneration).toHaveBeenCalledTimes(2); + expect(res.status).toBe(200); + expect(res.body.generationId).toBe('gen-2'); + await flush(); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); + + it('surfaces a same-fingerprint BuildingInProgressError as 409 without retiring (Fix 2b)', async () => { + const createGeneration = vi.fn().mockRejectedValue(new BuildingInProgressError('already building')); + const deps = makeDeps({ + createGeneration, + buildingGeneration: vi.fn().mockResolvedValue({ id: 'x', fingerprint: 'fp' }), + }); + const res = await startEmbeddingBuild(full, 'admin', deps); + expect(deps.retireGeneration).not.toHaveBeenCalled(); + expect(createGeneration).toHaveBeenCalledTimes(1); // no infinite retry + expect(res.status).toBe(409); + expect(deps.releaseEmbedRun).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/routes/apiTokens.js b/backend/src/routes/apiTokens.js new file mode 100644 index 00000000..1bfbced8 --- /dev/null +++ b/backend/src/routes/apiTokens.js @@ -0,0 +1,51 @@ +import { Router } from 'express'; +import { query } from '../services/db.js'; +import { requireAuth } from '../middleware/auth.js'; +import { ALL_SCOPES, expandScopes, generateToken, hashToken } from '../mcp/auth.js'; + +const router = Router(); +router.use(requireAuth); + +router.post('/', async (req, res) => { + const name = (req.body?.name || '').trim(); + if (!name) return res.status(400).json({ error: 'name is required' }); + const requestedScopes = Array.isArray(req.body?.scopes) && req.body.scopes.length + ? req.body.scopes + : ['read']; + const unknownScopes = requestedScopes.filter((scope) => !ALL_SCOPES.includes(scope)); + if (unknownScopes.length) { + return res.status(400).json({ error: `unknown scope(s): ${unknownScopes.join(', ')}` }); + } + const scopes = expandScopes(requestedScopes); + const token = generateToken(); + const { rows } = await query( + 'INSERT INTO api_tokens (user_id, token_hash, name, scopes) VALUES ($1, $2, $3, $4) RETURNING id, name, scopes', + [req.session.userId, hashToken(token), name, scopes], + ); + // Plaintext returned exactly once; only the hash was persisted. + res.status(201).json({ + id: rows[0].id, + name: rows[0].name, + scopes: rows[0].scopes, + token, + }); +}); + +router.get('/', async (req, res) => { + const { rows } = await query( + 'SELECT id, name, scopes, created_at, last_used_at FROM api_tokens WHERE user_id = $1 ORDER BY created_at DESC', + [req.session.userId], + ); + res.json({ tokens: rows }); +}); + +router.delete('/:id', async (req, res) => { + const { rowCount } = await query( + 'DELETE FROM api_tokens WHERE id = $1 AND user_id = $2', + [req.params.id, req.session.userId], + ); + if (!rowCount) return res.status(404).json({ error: 'not found' }); + res.status(204).end(); +}); + +export default router; diff --git a/backend/src/routes/apiTokens.test.js b/backend/src/routes/apiTokens.test.js new file mode 100644 index 00000000..92b08264 --- /dev/null +++ b/backend/src/routes/apiTokens.test.js @@ -0,0 +1,157 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import express from 'express'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +// Stub auth to inject a fixed session user. +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { req.session = { userId: 'user-1' }; next(); }, +})); +import { query } from '../services/db.js'; +import router from './apiTokens.js'; + +function appWith() { + const app = express(); + app.use(express.json()); + app.use('/api/tokens', router); + return app; +} +async function call(app, method, path, body) { + const { createServer } = await import('http'); + const server = createServer(app); + await new Promise((r) => server.listen(0, r)); + const base = `http://127.0.0.1:${server.address().port}`; + const res = await fetch(base + path, { + method, + headers: body ? { 'Content-Type': 'application/json' } : {}, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + server.close(); + return { status: res.status, body: text ? JSON.parse(text) : null }; +} + +function mockRes() { + return { + statusCode: 200, + body: null, + status(code) { this.statusCode = code; return this; }, + json(body) { this.body = body; return this; }, + end() { return this; }, + }; +} + +async function callRoute(method, { body = {}, params = {} } = {}) { + const layer = router.stack.find((entry) => entry.route?.path === '/' && entry.route.methods[method]); + const req = { body, params, session: { userId: 'user-1' } }; + const res = mockRes(); + await layer.route.stack[0].handle(req, res); + return { status: res.statusCode, body: res.body }; +} + +beforeEach(() => query.mockReset()); + +describe('POST /api/tokens', () => { + it('defaults a token with no requested scopes to read', async () => { + for (const scopes of [undefined, []]) { + query.mockResolvedValueOnce({ + rows: [{ id: 'tok-1', name: 'laptop', scopes: ['read'] }], + }); + const { status, body } = await callRoute('post', { body: { name: 'laptop', scopes } }); + expect(status).toBe(201); + expect(body.scopes).toEqual(['read']); + expect(query.mock.calls.at(-1)[0]).toMatch(/api_tokens \(user_id, token_hash, name, scopes\)/); + expect(query.mock.calls.at(-1)[1][3]).toEqual(['read']); + } + }); + + it('rejects unknown requested scopes', async () => { + query.mockResolvedValueOnce({ + rows: [{ id: 'tok-1', name: 'laptop', scopes: ['read', 'admin'] }], + }); + const { status, body } = await callRoute('post', { + body: { name: 'laptop', scopes: ['read', 'admin'] }, + }); + expect(status).toBe(400); + expect(body).toEqual({ error: 'unknown scope(s): admin' }); + expect(query).not.toHaveBeenCalled(); + }); + + it('mints a token with expanded requested scopes', async () => { + query.mockResolvedValueOnce({ + rows: [{ id: 'tok-2', name: 'sender', scopes: ['send', 'read'] }], + }); + const { status, body } = await callRoute('post', { + body: { name: 'sender', scopes: ['send'] }, + }); + expect(status).toBe(201); + expect(body).toMatchObject({ + id: 'tok-2', + name: 'sender', + scopes: ['send', 'read'], + }); + expect(body.token).toMatch(/^mcp_/); + expect(query.mock.calls[0][1][3]).toEqual(['send', 'read']); + expect(query.mock.calls[0][1]).not.toContain(body.token); + }); + + it('mints a token, returns the plaintext once, and stores only the hash', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop', scopes: ['read'] }] }); + const { status, body } = await call(appWith(), 'POST', '/api/tokens', { name: 'laptop' }); + expect(status).toBe(201); + expect(body.token).toMatch(/^mcp_/); + expect(body).toMatchObject({ id: 'tok-1', name: 'laptop' }); + // INSERT bound values: [user_id, token_hash, name, scopes] — never the plaintext. + const params = query.mock.calls[0][1]; + expect(params[0]).toBe('user-1'); + expect(params[1]).toMatch(/^[0-9a-f]{64}$/); + expect(params[2]).toBe('laptop'); + expect(params).not.toContain(body.token); + }); + it('rejects a missing name', async () => { + const { status } = await call(appWith(), 'POST', '/api/tokens', {}); + expect(status).toBe(400); + }); +}); + +describe('GET /api/tokens', () => { + it('selects and returns each token scopes', async () => { + query.mockResolvedValueOnce({ + rows: [{ + id: 'tok-1', + name: 'laptop', + scopes: ['read', 'write'], + created_at: 't', + last_used_at: null, + }], + }); + const { status, body } = await callRoute('get'); + expect(status).toBe(200); + expect(body.tokens[0].scopes).toEqual(['read', 'write']); + expect(query.mock.calls[0][0]).toMatch(/SELECT id, name, scopes, created_at, last_used_at/); + }); + + it('lists tokens without hashes or plaintext', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'tok-1', name: 'laptop', scopes: ['read'], created_at: 't', last_used_at: null }] }); + const { status, body } = await call(appWith(), 'GET', '/api/tokens'); + expect(status).toBe(200); + expect(body.tokens[0]).toEqual({ id: 'tok-1', name: 'laptop', scopes: ['read'], created_at: 't', last_used_at: null }); + expect(JSON.stringify(body)).not.toContain('token_hash'); + }); +}); + +describe('DELETE /api/tokens/:id', () => { + it('revokes only within the session user', async () => { + query.mockResolvedValueOnce({ rowCount: 1, rows: [] }); + const { status } = await call(appWith(), 'DELETE', '/api/tokens/tok-1'); + expect(status).toBe(204); + expect(query).toHaveBeenCalledWith( + expect.stringMatching(/DELETE FROM api_tokens WHERE id = \$1 AND user_id = \$2/), + ['tok-1', 'user-1'], + ); + }); + it('404s when the token is absent or not owned', async () => { + query.mockResolvedValueOnce({ rowCount: 0, rows: [] }); + const { status } = await call(appWith(), 'DELETE', '/api/tokens/nope'); + expect(status).toBe(404); + }); +}); diff --git a/backend/src/routes/auth.js b/backend/src/routes/auth.js index 4ae32060..7a91849b 100644 --- a/backend/src/routes/auth.js +++ b/backend/src/routes/auth.js @@ -19,6 +19,7 @@ import { sanitizeGtdPrefs } from '../utils/gtdPrefs.js'; import { sanitizeRightSidebarPrefs } from '../utils/rightSidebarPrefs.js'; import { redisClient } from '../services/redis.js'; import { consume as rlConsume, reset as rlReset } from '../services/rateLimiter.js'; +import { UNDO_CHOICES } from '../services/outboxService.js'; const router = Router(); @@ -769,7 +770,7 @@ export async function patchPreferences(req, res) { showAppBadge, showFaviconBadge, replyDefault, sidebarWidth, categorizationEnabled, markReadBehavior, markReadDelay, aiActions, autoLockMinutes, showMobileAvatars, gravatarAvatars, folderSyncInterval, - folderOrder, senderFavicons } = req.body; + folderOrder, senderFavicons, undoSendSeconds } = req.body; // GTD content and generic right-sidebar layout preferences are independent flat // top-level keys with separate allow-lists. gtdEnabled is intentionally NOT a user // preference — it lives per-account in email_accounts.gtd_enabled. @@ -794,6 +795,7 @@ export async function patchPreferences(req, res) { const autoLockMinutesVal = [0, 1, 5, 15, 30].includes(Number(autoLockMinutes)) ? String(Number(autoLockMinutes)) : null; // Folder-structure sync cadence in seconds; 0 = never. const folderSyncIntervalVal = folderSyncInterval != null && [0, 900, 1800, 3600].includes(Number(folderSyncInterval)) ? String(Number(folderSyncInterval)) : null; + const undoSendSecondsVal = UNDO_CHOICES.includes(Number(undoSendSeconds)) ? String(Number(undoSendSeconds)) : null; // User-defined AI actions: bound the array and each field so the JSONB can't grow unbounded. const aiActionsJson = (() => { if (!Array.isArray(aiActions)) return null; @@ -851,6 +853,7 @@ export async function patchPreferences(req, res) { || CASE WHEN $38::text IS NOT NULL THEN jsonb_build_object('folderSyncInterval', $38::text) ELSE '{}'::jsonb END || CASE WHEN $39::jsonb IS NOT NULL THEN jsonb_build_object('folderOrder', $39::jsonb) ELSE '{}'::jsonb END || CASE WHEN $40::boolean IS NOT NULL THEN jsonb_build_object('senderFavicons', $40::boolean) ELSE '{}'::jsonb END + || CASE WHEN $41::int IS NOT NULL THEN jsonb_build_object('undoSendSeconds', $41::int) ELSE '{}'::jsonb END WHERE id = $1 `, [req.session.userId, theme ?? null, font ?? null, layout ?? null, notificationSound ?? null, pageSize ?? null, scrollMode ?? null, syncInterval ?? null, @@ -860,7 +863,8 @@ export async function patchPreferences(req, res) { showAppBadge ?? null, showFaviconBadge ?? null, replyDefaultVal, sidebarWidthVal, categorizationEnabled ?? null, markReadBehaviorVal, markReadDelayVal, aiActionsJson, rightSidebarWidth, rightSidebarHidden, gtdCollapsedSectionsJson, gtdPetSlug, autoLockMinutesVal, - showMobileAvatars ?? null, gravatarAvatars ?? null, folderSyncIntervalVal, folderOrderJson, senderFaviconsVal]); + showMobileAvatars ?? null, gravatarAvatars ?? null, folderSyncIntervalVal, folderOrderJson, senderFaviconsVal, + undoSendSecondsVal]); if (syncInterval != null) { const ms = parseInt(syncInterval) * 1000; diff --git a/backend/src/routes/auth.preferences.test.js b/backend/src/routes/auth.preferences.test.js index 2a8ee730..4b449337 100644 --- a/backend/src/routes/auth.preferences.test.js +++ b/backend/src/routes/auth.preferences.test.js @@ -96,3 +96,29 @@ describe('PATCH /auth/preferences senderFavicons', () => { expect(query).not.toHaveBeenCalled(); }); }); + +describe('PATCH /auth/preferences undoSendSeconds', () => { + it('stores an allowed undo window after the existing preference binds', async () => { + const req = { session: { userId: 'user-1' }, body: { undoSendSeconds: 60 } }; + const res = { status: vi.fn().mockReturnThis(), json: vi.fn() }; + + await patchPreferences(req, res); + + const [sql, binds] = query.mock.calls[0]; + expect(sql).toContain( + "CASE WHEN $41::int IS NOT NULL THEN jsonb_build_object('undoSendSeconds', $41::int)", + ); + expect(binds).toHaveLength(41); + expect(binds[40]).toBe('60'); + }); + + it('does not store a value outside the supported undo choices', async () => { + const req = { session: { userId: 'user-1' }, body: { undoSendSeconds: 45 } }; + const res = { status: vi.fn().mockReturnThis(), json: vi.fn() }; + + await patchPreferences(req, res); + + const [, binds] = query.mock.calls[0]; + expect(binds[40]).toBeNull(); + }); +}); diff --git a/backend/src/routes/composeSessions.js b/backend/src/routes/composeSessions.js new file mode 100644 index 00000000..edf2db54 --- /dev/null +++ b/backend/src/routes/composeSessions.js @@ -0,0 +1,351 @@ +import { Router } from 'express'; +import { requireAuth } from '../middleware/auth.js'; +import { query, withTransaction } from '../services/db.js'; +import { redisClient as defaultRedisClient } from '../services/redis.js'; +import * as composeSessionService from '../services/composeSessionService.js'; +import * as composeSessionLifecycle from '../services/composeSessionLifecycle.js'; +import * as defaultOutboxService from '../services/outboxService.js'; +import * as defaultDraftService from '../services/draftService.js'; +import { + normalizeComposeChanges, + normalizeComposeClientId, + normalizeReplyAllRecipients, +} from '../services/composeSessionModel.js'; +import { UUID_RE } from '../utils/validation.js'; + +const EXPOSED_STATUSES = new Set([400, 404, 409, 413, 415, 422]); +const SERVICE_METHODS = [ + 'listComposeSessions', + 'createComposeSession', + 'claimDraftIntoComposeSession', + 'getComposeSession', + 'patchComposeSession', + 'setComposePresentation', + 'addComposeAttachment', + 'removeComposeAttachment', + 'closeComposeSession', + 'discardComposeSession', + 'sendComposeSession', + 'restoreQueuedComposeSession', +]; + +function requestError(code, message, status = 400) { + return Object.assign(new Error(message), { + code, + status, + details: {}, + expose: true, + }); +} + +function parseUuid(value, kind) { + if (typeof value !== 'string' || !UUID_RE.test(value)) { + const attachment = kind === 'attachment'; + throw requestError( + attachment ? 'invalid_compose_attachment_id' : 'invalid_compose_session_id', + attachment + ? 'Compose attachment id must be a UUID' + : 'Compose session id must be a UUID', + ); + } + return value; +} + +function parseClaimAccountId(value) { + if (typeof value !== 'string' || !UUID_RE.test(value)) { + throw requestError('invalid_compose_account_id', 'accountId must be a UUID'); + } + return value; +} + +function parseOutboxId(value) { + if (typeof value !== 'string' || !UUID_RE.test(value)) { + throw requestError('invalid_compose_outbox_id', 'Outbox id must be a UUID'); + } + return value; +} + +function parseClaimFolder(value) { + if (typeof value !== 'string' + || !value.trim() + || value.length > 500 + || /[\r\n\0]/.test(value) + || value.trim() !== value) { + throw requestError( + 'invalid_compose_draft_folder', + 'folder must be a non-empty folder path', + ); + } + return value; +} + +function parseClaimUid(value) { + if (!Number.isSafeInteger(value) || value < 1) { + throw requestError('invalid_compose_draft_uid', 'uid must be a positive integer'); + } + return value; +} + +function parseRequestedSlot(value) { + if (value === undefined || value === null) return undefined; + if (!Number.isInteger(value) || value < 1 || value > 9) { + throw requestError( + 'invalid_compose_slot', + 'requestedSlot must be an integer from 1 to 9', + ); + } + return value; +} + +function parseChanges(body) { + const value = body && typeof body === 'object' && !Array.isArray(body) + && Object.hasOwn(body, 'changes') + ? body.changes + : {}; + return normalizeComposeChanges(value); +} + +function parseExpectedRevision(value) { + const decimalString = typeof value === 'string' && /^[1-9][0-9]*$/.test(value); + const revision = decimalString ? Number(value) : value; + if (!Number.isSafeInteger(revision) || revision < 1) { + throw requestError( + 'invalid_compose_revision', + 'expectedRevision must be a positive integer', + ); + } + return revision; +} + +function parseUndoSendSeconds(value) { + if (value === undefined) return undefined; + if (!Number.isInteger(value) || value < 0 || value > 120) { + throw requestError( + 'invalid_compose_undo_seconds', + 'undoSendSeconds must be an integer from 0 to 120', + ); + } + return value; +} + +function lifecycleDependencies(deps, req) { + return deps.imapManager + ? deps + : { ...deps, imapManager: req.app.get('imapManager') }; +} + +async function sendLifecycleDependencies(deps, req) { + const refreshMicrosoftToken = deps.refreshMicrosoftToken + || (await import('./oauth.js')).refreshMicrosoftToken; + return { + ...lifecycleDependencies(deps, req), + redisClient: deps.redisClient || defaultRedisClient, + refreshMicrosoftToken, + outboxService: deps.outboxService || defaultOutboxService, + draftService: deps.draftService || defaultDraftService, + }; +} + +function decodeAttachmentFilename(value) { + try { + return decodeURIComponent(value || 'attachment'); + } catch { + throw requestError( + 'invalid_attachment_filename', + 'X-Mailflow-Filename must be valid percent encoding', + ); + } +} + +function route(handler) { + return async (req, res, next) => { + try { + await handler(req, res); + } catch (error) { + if ( + error?.expose === true + && EXPOSED_STATUSES.has(error.status) + && typeof error.code === 'string' + ) { + const details = error.details + && typeof error.details === 'object' + && !Array.isArray(error.details) + ? error.details + : {}; + return res.status(error.status).json({ + error: error.message, + code: error.code, + ...details, + }); + } + next(error); + } + }; +} + +export function createComposeSessionsRouter(deps = {}) { + const defaultServices = { ...composeSessionService, ...composeSessionLifecycle }; + const services = Object.fromEntries(SERVICE_METHODS.map(name => [ + name, + typeof deps[name] === 'function' ? deps[name] : defaultServices[name], + ])); + const router = Router(); + router.use(requireAuth); + + router.get('/', route(async (req, res) => { + const sessions = await services.listComposeSessions({ + userId: req.session.userId, + }, deps); + res.json(sessions); + })); + + router.post('/', route(async (req, res) => { + const body = req.body || {}; + const session = await services.createComposeSession({ + userId: req.session.userId, + requestedSlot: body.requestedSlot, + changes: parseChanges(body), + ...(body.replyAllRecipients === undefined ? {} : { + replyAllRecipients: normalizeReplyAllRecipients(body.replyAllRecipients), + }), + clientId: normalizeComposeClientId(body.clientId), + }, deps); + res.status(201).json(session); + })); + + router.post('/claim-draft', route(async (req, res) => { + const body = req.body || {}; + const lifecycleDeps = lifecycleDependencies(deps, req); + const session = await services.claimDraftIntoComposeSession({ + userId: req.session.userId, + accountId: parseClaimAccountId(body.accountId), + folder: parseClaimFolder(body.folder), + uid: parseClaimUid(body.uid), + requestedSlot: parseRequestedSlot(body.requestedSlot), + replyAllRecipients: normalizeReplyAllRecipients(body.replyAllRecipients ?? []), + }, lifecycleDeps); + res.status(201).json(session); + })); + + router.post('/:id/close', route(async (req, res) => { + const body = req.body || {}; + const lifecycleDeps = lifecycleDependencies(deps, req); + const result = await services.closeComposeSession({ + userId: req.session.userId, + id: parseUuid(req.params.id, 'session'), + expectedRevision: parseExpectedRevision(body.expectedRevision), + changes: parseChanges(body), + }, lifecycleDeps); + res.json(result); + })); + + router.post('/:id/discard', route(async (req, res) => { + const body = req.body || {}; + const lifecycleDeps = lifecycleDependencies(deps, req); + const result = await services.discardComposeSession({ + userId: req.session.userId, + id: parseUuid(req.params.id, 'session'), + expectedRevision: parseExpectedRevision(body.expectedRevision), + }, lifecycleDeps); + res.json(result); + })); + + router.post('/:id/send', route(async (req, res) => { + const body = req.body || {}; + const result = await services.sendComposeSession({ + userId: req.session.userId, + id: parseUuid(req.params.id, 'session'), + expectedRevision: parseExpectedRevision(body.expectedRevision), + undoSendSeconds: parseUndoSendSeconds(body.undoSendSeconds), + idempotencyKey: composeSessionLifecycle.normalizeComposeIdempotencyKey( + typeof req.headers['x-idempotency-key'] === 'string' + ? req.headers['x-idempotency-key'] + : null, + ), + }, await sendLifecycleDependencies(deps, req)); + res.status(result?.queued === true ? 202 : 200).json(result); + })); + + router.post('/outbox/:outboxId/restore', route(async (req, res) => { + const result = await services.restoreQueuedComposeSession({ + userId: req.session.userId, + outboxId: parseOutboxId(req.params.outboxId), + }, deps); + res.json(result); + })); + + router.get('/:id', route(async (req, res) => { + const session = await services.getComposeSession({ + userId: req.session.userId, + id: parseUuid(req.params.id, 'session'), + }, deps); + res.json(session); + })); + + router.patch('/:id', route(async (req, res) => { + const body = req.body || {}; + const session = await services.patchComposeSession({ + userId: req.session.userId, + id: parseUuid(req.params.id, 'session'), + expectedRevision: parseExpectedRevision(body.expectedRevision), + changes: parseChanges(body), + clientId: normalizeComposeClientId(body.clientId), + }, deps); + res.json(session); + })); + + router.put('/:id/presentation', route(async (req, res) => { + const body = req.body || {}; + const session = await services.setComposePresentation({ + userId: req.session.userId, + id: parseUuid(req.params.id, 'session'), + expectedRevision: parseExpectedRevision(body.expectedRevision), + state: body.state, + clientId: normalizeComposeClientId(body.clientId), + }, deps); + res.json(session); + })); + + router.post('/:id/attachments', route(async (req, res) => { + const id = parseUuid(req.params.id, 'session'); + const mediaType = req.get('Content-Type')?.split(';', 1)[0].trim().toLowerCase(); + if (mediaType !== 'application/octet-stream') { + throw requestError( + 'unsupported_attachment_media_type', + 'Content-Type must be application/octet-stream', + 415, + ); + } + if (!Buffer.isBuffer(req.body)) { + throw requestError('invalid_attachment_body', 'Attachment body must be raw bytes'); + } + const result = await services.addComposeAttachment({ + userId: req.session.userId, + id, + expectedRevision: parseExpectedRevision( + req.get('X-Mailflow-Expected-Revision') ?? req.query.expectedRevision, + ), + filename: decodeAttachmentFilename(req.get('X-Mailflow-Filename')), + contentType: req.get('X-Mailflow-Content-Type') || 'application/octet-stream', + content: req.body, + clientId: normalizeComposeClientId(req.get('X-Mailflow-Client-Id')), + }, deps); + res.status(201).json(result); + })); + + router.delete('/:id/attachments/:attachmentId', route(async (req, res) => { + const body = req.body || {}; + const result = await services.removeComposeAttachment({ + userId: req.session.userId, + id: parseUuid(req.params.id, 'session'), + attachmentId: parseUuid(req.params.attachmentId, 'attachment'), + expectedRevision: parseExpectedRevision(body.expectedRevision), + clientId: normalizeComposeClientId(body.clientId), + }, deps); + res.json(result); + })); + + return router; +} + +export default createComposeSessionsRouter({ query, withTransaction }); diff --git a/backend/src/routes/composeSessions.test.js b/backend/src/routes/composeSessions.test.js new file mode 100644 index 00000000..df58641c --- /dev/null +++ b/backend/src/routes/composeSessions.test.js @@ -0,0 +1,1142 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import express from 'express'; + +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, res, next) => { + const userId = req.get('X-Test-User-Id'); + if (!userId) return res.status(401).json({ error: 'Not authenticated' }); + req.session = { userId }; + next(); + }, +})); + +import { createComposeSessionsRouter } from './composeSessions.js'; + +const USER_ID = '11111111-1111-4111-8111-111111111111'; +const SESSION_ID = '22222222-2222-4222-8222-222222222222'; +const ATTACHMENT_ID = '33333333-3333-4333-8333-333333333333'; +const ACCOUNT_ID = '44444444-4444-4444-8444-444444444444'; +const APP_IMAP_MANAGER = { fetchMessageBody: vi.fn(), fetchAttachment: vi.fn() }; + +function exposedError(code, message, status, details = {}) { + return Object.assign(new Error(message), { + code, + status, + details, + expose: true, + }); +} + +const deps = { + query: vi.fn(), + withTransaction: vi.fn(), + broadcast: vi.fn(), + listComposeSessions: vi.fn(), + createComposeSession: vi.fn(), + claimDraftIntoComposeSession: vi.fn(), + getComposeSession: vi.fn(), + patchComposeSession: vi.fn(), + setComposePresentation: vi.fn(), + addComposeAttachment: vi.fn(), + removeComposeAttachment: vi.fn(), + closeComposeSession: vi.fn(), + discardComposeSession: vi.fn(), + sendComposeSession: vi.fn(), + restoreQueuedComposeSession: vi.fn(), + redisClient: { get: vi.fn(), set: vi.fn(), del: vi.fn() }, + refreshMicrosoftToken: vi.fn(), + outboxService: { enqueue: vi.fn(), normalizeUndoWindow: vi.fn() }, + draftService: { deleteDraft: vi.fn() }, +}; + +function buildApp({ useRawParser = true } = {}) { + const app = express(); + app.set('imapManager', APP_IMAP_MANAGER); + if (useRawParser) { + app.use( + '/api/compose-sessions/:id/attachments', + express.raw({ type: 'application/octet-stream', limit: '25mb' }), + ); + } + app.use('/api/compose-sessions', express.json({ limit: '35mb' })); + app.use('/api/compose-sessions', createComposeSessionsRouter(deps)); + // Mirrors index.js's non-leaking final error response for unknown failures. + // eslint-disable-next-line no-unused-vars + app.use((err, _req, res, _next) => { + res.status(500).json({ error: 'Internal server error' }); + }); + return app; +} + +async function request(base, method, path, { body, headers = {}, authenticated = true } = {}) { + const requestHeaders = { ...headers }; + if (authenticated) requestHeaders['X-Test-User-Id'] = USER_ID; + let requestBody = body; + if (body !== undefined && !Buffer.isBuffer(body)) { + requestHeaders['Content-Type'] ||= 'application/json'; + requestBody = JSON.stringify(body); + } + const response = await fetch(`${base}${path}`, { + method, + headers: requestHeaders, + body: requestBody, + }); + const text = await response.text(); + return { + status: response.status, + body: text ? JSON.parse(text) : null, + }; +} + +describe('compose session routes', () => { + let server; + let base; + + beforeAll(async () => { + server = buildApp().listen(0, '127.0.0.1'); + await new Promise(resolve => server.once('listening', resolve)); + base = `http://127.0.0.1:${server.address().port}`; + }); + + afterAll(async () => { + if (server) await new Promise(resolve => server.close(resolve)); + }); + + beforeEach(() => { + for (const value of Object.values(deps)) { + if (typeof value?.mockReset === 'function') value.mockReset(); + } + }); + + it('requires authentication for the compose-session router', async () => { + const response = await request(base, 'GET', '/api/compose-sessions', { + authenticated: false, + }); + + expect(response).toEqual({ status: 401, body: { error: 'Not authenticated' } }); + expect(deps.listComposeSessions).not.toHaveBeenCalled(); + }); + + it('restores a queued compose through the authenticated owner-scoped route', async () => { + const restored = { + restored: true, + replayed: false, + session: { id: SESSION_ID, slot: 2, attachments: [{ id: ATTACHMENT_ID }] }, + }; + deps.restoreQueuedComposeSession.mockResolvedValueOnce(restored); + + const response = await request( + base, + 'POST', + '/api/compose-sessions/outbox/55555555-5555-4555-8555-555555555555/restore', + { body: {} }, + ); + + expect(response).toEqual({ status: 200, body: restored }); + expect(deps.restoreQueuedComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + outboxId: '55555555-5555-4555-8555-555555555555', + }, expect.objectContaining({ withTransaction: deps.withTransaction })); + }); + + it('maps explicit queued-restore outcomes without exposing payload bytes', async () => { + deps.restoreQueuedComposeSession.mockRejectedValueOnce(exposedError( + 'compose_outbox_too_late', + 'Queued message can no longer be restored', + 409, + )); + const response = await request( + base, + 'POST', + '/api/compose-sessions/outbox/55555555-5555-4555-8555-555555555555/restore', + { body: {} }, + ); + expect(response).toEqual({ + status: 409, + body: { + error: 'Queued message can no longer be restored', + code: 'compose_outbox_too_late', + }, + }); + }); + + it.each([ + ['GET', '/api/compose-sessions/not-a-uuid', undefined, 'invalid_compose_session_id'], + [ + 'DELETE', + `/api/compose-sessions/${SESSION_ID}/attachments/not-a-uuid`, + { expectedRevision: 1 }, + 'invalid_compose_attachment_id', + ], + ])('rejects malformed UUID route locators for %s %s', async (method, path, body, code) => { + const response = await request(base, method, path, { body }); + + expect(response).toEqual({ + status: 400, + body: { + error: code === 'invalid_compose_session_id' + ? 'Compose session id must be a UUID' + : 'Compose attachment id must be a UUID', + code, + }, + }); + expect(deps.getComposeSession).not.toHaveBeenCalled(); + expect(deps.removeComposeAttachment).not.toHaveBeenCalled(); + }); + + it('lists only summaries returned for the authenticated user', async () => { + const summaries = [{ + id: SESSION_ID, + slot: 1, + subject: 'Synthetic subject', + revision: 2, + attachmentCount: 1, + }]; + deps.listComposeSessions.mockResolvedValueOnce(summaries); + + const response = await request(base, 'GET', '/api/compose-sessions'); + + expect(response).toEqual({ status: 200, body: summaries }); + expect(deps.listComposeSessions).toHaveBeenCalledWith({ userId: USER_ID }, deps); + }); + + it('creates a session and returns the service shape with 201', async () => { + const session = { + id: SESSION_ID, + slot: 4, + subject: 'Synthetic subject', + revision: 1, + }; + deps.createComposeSession.mockResolvedValueOnce(session); + + const response = await request(base, 'POST', '/api/compose-sessions', { + body: { + requestedSlot: 4, + changes: { subject: 'Synthetic subject' }, + clientId: 'browser-synthetic', + }, + }); + + expect(response).toEqual({ status: 201, body: session }); + expect(deps.createComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + requestedSlot: 4, + changes: { subject: 'Synthetic subject' }, + clientId: 'browser-synthetic', + }, deps); + }); + + it('accepts sanitized reply-all source recipients only at create', async () => { + deps.createComposeSession.mockResolvedValueOnce({ id: SESSION_ID, slot: 1 }); + await request(base, 'POST', '/api/compose-sessions', { + body: { + changes: { mode: 'reply' }, + replyAllRecipients: [' Synthetic Copied '], + }, + }); + expect(deps.createComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + requestedSlot: undefined, + changes: { mode: 'reply' }, + replyAllRecipients: ['Synthetic Copied '], + clientId: undefined, + }, deps); + }); + + it('claims an owned draft with strict transport values and production IMAP dependencies', async () => { + const session = { + id: SESSION_ID, + slot: 3, + accountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + revision: 1, + }; + deps.claimDraftIntoComposeSession.mockResolvedValueOnce(session); + + const response = await request(base, 'POST', '/api/compose-sessions/claim-draft', { + body: { + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + requestedSlot: 3, + replyAllRecipients: [' Synthetic Copied '], + }, + }); + + expect(response).toEqual({ status: 201, body: session }); + expect(deps.claimDraftIntoComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + requestedSlot: 3, + replyAllRecipients: ['Synthetic Copied '], + }, expect.objectContaining({ + query: deps.query, + withTransaction: deps.withTransaction, + imapManager: APP_IMAP_MANAGER, + })); + }); + + it.each([0, -1, 1.5, '1', '1e2', null])( + 'rejects a non-positive or non-integer draft uid %#', + async (uid) => { + const response = await request(base, 'POST', '/api/compose-sessions/claim-draft', { + body: { accountId: ACCOUNT_ID, folder: '[Synthetic]/Drafts', uid }, + }); + + expect(response).toEqual({ + status: 400, + body: { + error: 'uid must be a positive integer', + code: 'invalid_compose_draft_uid', + }, + }); + expect(deps.claimDraftIntoComposeSession).not.toHaveBeenCalled(); + }, + ); + + it.each([0, 10, 1.5, '1'])( + 'rejects an invalid requested claim slot %#', + async (requestedSlot) => { + const response = await request(base, 'POST', '/api/compose-sessions/claim-draft', { + body: { + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + requestedSlot, + }, + }); + + expect(response).toEqual({ + status: 400, + body: { + error: 'requestedSlot must be an integer from 1 to 9', + code: 'invalid_compose_slot', + }, + }); + expect(deps.claimDraftIntoComposeSession).not.toHaveBeenCalled(); + }, + ); + + it.each([ + [{ accountId: 'not-a-uuid', folder: '[Synthetic]/Drafts', uid: 41 }, + 'invalid_compose_account_id', 'accountId must be a UUID'], + [{ accountId: ACCOUNT_ID, folder: '', uid: 41 }, + 'invalid_compose_draft_folder', 'folder must be a non-empty folder path'], + ])('rejects malformed claim locators %#', async (body, code, error) => { + const response = await request(base, 'POST', '/api/compose-sessions/claim-draft', { body }); + + expect(response).toEqual({ status: 400, body: { error, code } }); + expect(deps.claimDraftIntoComposeSession).not.toHaveBeenCalled(); + }); + + it('maps duplicate draft ownership to the stable 409 response', async () => { + deps.claimDraftIntoComposeSession.mockRejectedValueOnce(exposedError( + 'compose_draft_claimed', + 'Source draft is already open in a compose session', + 409, + )); + + const response = await request(base, 'POST', '/api/compose-sessions/claim-draft', { + body: { accountId: ACCOUNT_ID, folder: '[Synthetic]/Drafts', uid: 41 }, + }); + + expect(response).toEqual({ + status: 409, + body: { + error: 'Source draft is already open in a compose session', + code: 'compose_draft_claimed', + }, + }); + }); + + it.each([ + [{ changes: null }, 'changes must be an object'], + [{ changes: { to: 'recipient@example.com' } }, 'to must be an array'], + [{ changes: { body: null } }, 'body must be a string'], + ])('rejects malformed compose changes at the route boundary %#', async (body, error) => { + const response = await request(base, 'POST', '/api/compose-sessions', { body }); + + expect(response).toEqual({ + status: 400, + body: { error, code: 'invalid_compose_changes' }, + }); + expect(deps.createComposeSession).not.toHaveBeenCalled(); + }); + + it.each([ + ['person@example.com'], + ['contains spaces'], + ['x'.repeat(65)], + ])('rejects non-opaque client identities %#', async (clientId) => { + const response = await request(base, 'POST', '/api/compose-sessions', { + body: { changes: {}, clientId }, + }); + + expect(response).toEqual({ + status: 400, + body: { + error: 'clientId must be 1-64 characters using letters, numbers, underscores, or hyphens', + code: 'invalid_client_id', + }, + }); + expect(deps.createComposeSession).not.toHaveBeenCalled(); + }); + + it('maps an exposed occupied-slot conflict without leaking internals', async () => { + deps.createComposeSession.mockRejectedValueOnce(exposedError( + 'compose_slot_occupied', + 'Compose slot 4 is already occupied', + 409, + )); + + const response = await request(base, 'POST', '/api/compose-sessions', { + body: { requestedSlot: 4, changes: {} }, + }); + + expect(response).toEqual({ + status: 409, + body: { + error: 'Compose slot 4 is already occupied', + code: 'compose_slot_occupied', + }, + }); + }); + + it('gets an owned session by route id', async () => { + const session = { + id: SESSION_ID, + slot: 1, + revision: 3, + attachments: [], + }; + deps.getComposeSession.mockResolvedValueOnce(session); + + const response = await request(base, 'GET', `/api/compose-sessions/${SESSION_ID}`); + + expect(response).toEqual({ status: 200, body: session }); + expect(deps.getComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + id: SESSION_ID, + }, deps); + }); + + it('maps an out-of-scope session to the service 404 shape', async () => { + deps.getComposeSession.mockRejectedValueOnce(exposedError( + 'compose_session_not_found', + 'Compose session not found', + 404, + )); + + const response = await request(base, 'GET', `/api/compose-sessions/${SESSION_ID}`); + + expect(response).toEqual({ + status: 404, + body: { + error: 'Compose session not found', + code: 'compose_session_not_found', + }, + }); + }); + + it('closes with a strict revision and normalized atomic final changes', async () => { + const result = { + closed: true, + slot: 3, + draft: { + accountId: ACCOUNT_ID, + uid: 72, + folder: '[Synthetic]/Drafts', + messageId: '', + }, + }; + deps.closeComposeSession.mockResolvedValueOnce(result); + + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/close`, + { body: { expectedRevision: '7', changes: { subject: 'Final synthetic subject' } } }, + ); + + expect(response).toEqual({ status: 200, body: result }); + expect(deps.closeComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + id: SESSION_ID, + expectedRevision: 7, + changes: { subject: 'Final synthetic subject' }, + }, expect.objectContaining({ + query: deps.query, + withTransaction: deps.withTransaction, + imapManager: APP_IMAP_MANAGER, + })); + }); + + it('discards with a strict revision and returns the terminal receipt', async () => { + const result = { discarded: true, slot: 3 }; + deps.discardComposeSession.mockResolvedValueOnce(result); + + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/discard`, + { body: { expectedRevision: 7 } }, + ); + + expect(response).toEqual({ status: 200, body: result }); + expect(deps.discardComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + id: SESSION_ID, + expectedRevision: 7, + }, expect.objectContaining({ + query: deps.query, + withTransaction: deps.withTransaction, + imapManager: APP_IMAP_MANAGER, + })); + }); + + it('sends immediately with strict inputs and preserves the complete shared receipt', async () => { + const result = { + ok: true, + messageId: '', + sentCopySaved: false, + receipt: { + subject: 'Synthetic send subject', + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + }, + }; + deps.sendComposeSession.mockResolvedValueOnce(result); + + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/send`, + { + body: { expectedRevision: '7', undoSendSeconds: 0 }, + headers: { 'X-Idempotency-Key': 'synthetic-send-key' }, + }, + ); + + expect(response).toEqual({ status: 200, body: result }); + expect(deps.sendComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + id: SESSION_ID, + expectedRevision: 7, + undoSendSeconds: 0, + idempotencyKey: 'synthetic-send-key', + }, expect.objectContaining({ + query: deps.query, + withTransaction: deps.withTransaction, + imapManager: APP_IMAP_MANAGER, + redisClient: deps.redisClient, + refreshMicrosoftToken: deps.refreshMicrosoftToken, + outboxService: deps.outboxService, + draftService: deps.draftService, + })); + }); + + it('returns 202 with the complete durable outbox result', async () => { + const result = { + queued: true, + outboxId: '55555555-5555-4555-8555-555555555555', + sendAt: '2026-08-01T12:00:30.000Z', + undoSeconds: 30, + }; + deps.sendComposeSession.mockResolvedValueOnce(result); + + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/send`, + { body: { expectedRevision: 7, undoSendSeconds: 30 } }, + ); + + expect(response).toEqual({ status: 202, body: result }); + expect(deps.sendComposeSession).toHaveBeenCalledWith( + expect.objectContaining({ + undoSendSeconds: 30, + idempotencyKey: null, + }), + expect.any(Object), + ); + }); + + it.each([ + ['same-prefix-a', `${'same-prefix-'.padEnd(128, 'x')}a`], + ['same-prefix-b', `${'same-prefix-'.padEnd(128, 'x')}b`], + ['blank', ''], + ['whitespace', 'contains spaces'], + ['content-like', 'recipient@example.com'], + ['slash', 'key/with/content'], + ])('rejects unsafe idempotency header %s before lifecycle send', async (_label, key) => { + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/send`, + { + body: { expectedRevision: 7 }, + headers: { 'X-Idempotency-Key': key }, + }, + ); + + expect(response).toEqual({ + status: 400, + body: { + error: 'X-Idempotency-Key must be 1-128 safe opaque characters', + code: 'invalid_compose_idempotency_key', + }, + }); + expect(deps.sendComposeSession).not.toHaveBeenCalled(); + }); + + it.each([-1, 121, 1.5, '30', null, true, [], {}])( + 'rejects invalid undoSendSeconds %# before send', + async (undoSendSeconds) => { + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/send`, + { body: { expectedRevision: 7, undoSendSeconds } }, + ); + + expect(response).toEqual({ + status: 400, + body: { + error: 'undoSendSeconds must be an integer from 0 to 120', + code: 'invalid_compose_undo_seconds', + }, + }); + expect(deps.sendComposeSession).not.toHaveBeenCalled(); + }, + ); + + it('maps an exposed pre-acceptance send conflict without a success response', async () => { + deps.sendComposeSession.mockRejectedValueOnce(exposedError( + 'compose_conflict', + 'Compose session changed', + 409, + { currentRevision: 8 }, + )); + + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/send`, + { body: { expectedRevision: 7 } }, + ); + + expect(response).toEqual({ + status: 409, + body: { + error: 'Compose session changed', + code: 'compose_conflict', + currentRevision: 8, + }, + }); + }); + + it.each([ + ['close', 0], + ['close', '1e2'], + ['close', '0x10'], + ['close', '+1'], + ['close', '1.0'], + ['discard', null], + ['discard', 1.5], + ['send', undefined], + ])('rejects invalid terminal revision %# for %s', async (operation, expectedRevision) => { + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/${operation}`, + { body: { expectedRevision, changes: {} } }, + ); + + expect(response).toEqual({ + status: 400, + body: { + error: 'expectedRevision must be a positive integer', + code: 'invalid_compose_revision', + }, + }); + expect(deps.closeComposeSession).not.toHaveBeenCalled(); + expect(deps.discardComposeSession).not.toHaveBeenCalled(); + expect(deps.sendComposeSession).not.toHaveBeenCalled(); + }); + + it.each(['close', 'discard', 'send'])('rejects malformed terminal UUID for %s', async (operation) => { + const response = await request( + base, + 'POST', + `/api/compose-sessions/not-a-uuid/${operation}`, + { body: { expectedRevision: 7, changes: {} } }, + ); + + expect(response).toEqual({ + status: 400, + body: { + error: 'Compose session id must be a UUID', + code: 'invalid_compose_session_id', + }, + }); + }); + + it('maps an exposed discard failure without claiming terminal success', async () => { + deps.discardComposeSession.mockRejectedValueOnce(exposedError( + 'compose_conflict', + 'Compose session changed', + 409, + { currentRevision: 8 }, + )); + + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/discard`, + { body: { expectedRevision: 7 } }, + ); + + expect(response).toEqual({ + status: 409, + body: { + error: 'Compose session changed', + code: 'compose_conflict', + currentRevision: 8, + }, + }); + }); + + it('returns a stable actionable 422 when the selected account has no Drafts folder', async () => { + deps.closeComposeSession.mockRejectedValueOnce(exposedError( + 'compose_drafts_folder_not_found', + 'No Drafts folder is available for this account', + 422, + )); + + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/close`, + { body: { expectedRevision: 7 } }, + ); + + expect(response).toEqual({ + status: 422, + body: { + error: 'No Drafts folder is available for this account', + code: 'compose_drafts_folder_not_found', + }, + }); + }); + + it.each([ + ['close', 'compose_close_accepted_cleanup_pending', 'Draft was saved'], + ['discard', 'compose_discard_accepted_cleanup_pending', 'Draft was deleted'], + ])('returns an actionable 409 for accepted %s cleanup pending', async ( + operation, + code, + prefix, + ) => { + deps[`${operation}ComposeSession`].mockRejectedValueOnce(exposedError( + code, + `${prefix} but compose cleanup is still pending; do not retry`, + 409, + )); + + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/${operation}`, + { body: { expectedRevision: 7, ...(operation === 'close' ? { changes: {} } : {}) } }, + ); + + expect(response).toEqual({ + status: 409, + body: { + error: `${prefix} but compose cleanup is still pending; do not retry`, + code, + }, + }); + }); + + it('patches fields with a parsed positive expected revision', async () => { + const session = { id: SESSION_ID, slot: 1, subject: 'New subject', revision: 4 }; + deps.patchComposeSession.mockResolvedValueOnce(session); + + const response = await request(base, 'PATCH', `/api/compose-sessions/${SESSION_ID}`, { + body: { + expectedRevision: '3', + changes: { subject: 'New subject' }, + clientId: 'browser-synthetic', + }, + }); + + expect(response).toEqual({ status: 200, body: session }); + expect(deps.patchComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + id: SESSION_ID, + expectedRevision: 3, + changes: { subject: 'New subject' }, + clientId: 'browser-synthetic', + }, deps); + }); + + it('returns structured revision-conflict details', async () => { + deps.patchComposeSession.mockRejectedValueOnce(exposedError( + 'compose_conflict', + 'Compose session changed in the requested fields', + 409, + { + conflictingFields: ['subject'], + currentRevision: 4, + remoteValues: { subject: 'Remote subject' }, + }, + )); + + const response = await request(base, 'PATCH', `/api/compose-sessions/${SESSION_ID}`, { + body: { expectedRevision: 3, changes: { subject: 'Local subject' } }, + }); + + expect(response).toEqual({ + status: 409, + body: { + error: 'Compose session changed in the requested fields', + code: 'compose_conflict', + conflictingFields: ['subject'], + currentRevision: 4, + remoteValues: { subject: 'Remote subject' }, + }, + }); + }); + + it('sets presentation intent with revision and client identity', async () => { + const session = { + id: SESSION_ID, + slot: 1, + presentationState: 'minimized', + revision: 5, + }; + deps.setComposePresentation.mockResolvedValueOnce(session); + + const response = await request( + base, + 'PUT', + `/api/compose-sessions/${SESSION_ID}/presentation`, + { + body: { + expectedRevision: 4, + state: 'minimized', + clientId: 'browser-synthetic', + }, + }, + ); + + expect(response).toEqual({ status: 200, body: session }); + expect(deps.setComposePresentation).toHaveBeenCalledWith({ + userId: USER_ID, + id: SESSION_ID, + expectedRevision: 4, + state: 'minimized', + clientId: 'browser-synthetic', + }, deps); + }); + + it.each([ + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { changes: {} }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: 0 }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: -1 }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: 1.5 }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: '1x' }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: '1e2' }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: '0x10' }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: '+1' }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: '1.0' }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: ' 1 ' }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: '0' }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: '-1' }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: '01' }, {}], + [ + 'PATCH', + `/api/compose-sessions/${SESSION_ID}`, + { expectedRevision: String(Number.MAX_SAFE_INTEGER + 1) }, + {}, + ], + [ + 'PATCH', + `/api/compose-sessions/${SESSION_ID}`, + { expectedRevision: Number.MAX_SAFE_INTEGER + 1 }, + {}, + ], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: true }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: null }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: [] }, {}], + ['PATCH', `/api/compose-sessions/${SESSION_ID}`, { expectedRevision: {} }, {}], + [ + 'PUT', + `/api/compose-sessions/${SESSION_ID}/presentation`, + { state: 'expanded' }, + {}, + ], + [ + 'POST', + `/api/compose-sessions/${SESSION_ID}/attachments`, + Buffer.from('synthetic bytes'), + { + 'Content-Type': 'application/octet-stream', + 'X-Mailflow-Filename': 'report.pdf', + 'X-Mailflow-Content-Type': 'application/pdf', + }, + ], + [ + 'POST', + `/api/compose-sessions/${SESSION_ID}/attachments`, + Buffer.from('synthetic bytes'), + { + 'Content-Type': 'application/octet-stream', + 'X-Mailflow-Expected-Revision': '1e2', + 'X-Mailflow-Filename': 'report.pdf', + 'X-Mailflow-Content-Type': 'application/pdf', + }, + ], + [ + 'POST', + `/api/compose-sessions/${SESSION_ID}/attachments?expectedRevision=0x10`, + Buffer.from('synthetic bytes'), + { + 'Content-Type': 'application/octet-stream', + 'X-Mailflow-Filename': 'report.pdf', + 'X-Mailflow-Content-Type': 'application/pdf', + }, + ], + [ + 'POST', + `/api/compose-sessions/${SESSION_ID}/attachments`, + Buffer.from('synthetic bytes'), + { + 'Content-Type': 'application/octet-stream', + 'X-Mailflow-Expected-Revision': 'not-an-integer', + 'X-Mailflow-Filename': 'report.pdf', + 'X-Mailflow-Content-Type': 'application/pdf', + }, + ], + [ + 'DELETE', + `/api/compose-sessions/${SESSION_ID}/attachments/${ATTACHMENT_ID}`, + {}, + {}, + ], + ])('rejects missing or invalid revisions for %s %s', async (method, path, body, headers) => { + const response = await request(base, method, path, { body, headers }); + + expect(response).toEqual({ + status: 400, + body: { + error: 'expectedRevision must be a positive integer', + code: 'invalid_compose_revision', + }, + }); + }); + + it('adds raw attachment bytes with decoded metadata and header revision', async () => { + const result = { + sessionId: SESSION_ID, + slot: 1, + revision: 6, + attachment: { + id: ATTACHMENT_ID, + filename: 'quarterly report.pdf', + contentType: 'application/pdf', + byteCount: 15, + }, + }; + deps.addComposeAttachment.mockResolvedValueOnce(result); + + const content = Buffer.from('synthetic bytes'); + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/attachments`, + { + body: content, + headers: { + 'Content-Type': 'application/octet-stream', + 'X-Mailflow-Expected-Revision': '5', + 'X-Mailflow-Filename': 'quarterly%20report.pdf', + 'X-Mailflow-Content-Type': 'application/pdf', + 'X-Mailflow-Client-Id': 'browser-synthetic', + }, + }, + ); + + expect(response).toEqual({ status: 201, body: result }); + expect(deps.addComposeAttachment).toHaveBeenCalledWith({ + userId: USER_ID, + id: SESSION_ID, + expectedRevision: 5, + filename: 'quarterly report.pdf', + contentType: 'application/pdf', + content, + clientId: 'browser-synthetic', + }, deps); + }); + + it('requires the binary attachment media type before calling the service', async () => { + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/attachments`, + { + body: 'synthetic text', + headers: { + 'Content-Type': 'text/plain', + 'X-Mailflow-Expected-Revision': '5', + 'X-Mailflow-Filename': 'report.txt', + 'X-Mailflow-Content-Type': 'text/plain', + }, + }, + ); + + expect(response).toEqual({ + status: 415, + body: { + error: 'Content-Type must be application/octet-stream', + code: 'unsupported_attachment_media_type', + }, + }); + expect(deps.addComposeAttachment).not.toHaveBeenCalled(); + }); + + it('rejects a non-Buffer attachment body before calling the service', async () => { + const bodyApp = buildApp({ useRawParser: false }); + const bodyServer = bodyApp.listen(0, '127.0.0.1'); + await new Promise(resolve => bodyServer.once('listening', resolve)); + const bodyBase = `http://127.0.0.1:${bodyServer.address().port}`; + try { + const response = await request( + bodyBase, + 'POST', + `/api/compose-sessions/${SESSION_ID}/attachments`, + { + body: Buffer.from('synthetic bytes'), + headers: { + 'Content-Type': 'application/octet-stream', + 'X-Mailflow-Expected-Revision': '5', + 'X-Mailflow-Filename': 'report.bin', + }, + }, + ); + + expect(response).toEqual({ + status: 400, + body: { + error: 'Attachment body must be raw bytes', + code: 'invalid_attachment_body', + }, + }); + expect(deps.addComposeAttachment).not.toHaveBeenCalled(); + } finally { + await new Promise(resolve => bodyServer.close(resolve)); + } + }); + + it('rejects malformed percent encoding in attachment filenames', async () => { + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/attachments`, + { + body: Buffer.from('synthetic bytes'), + headers: { + 'Content-Type': 'application/octet-stream', + 'X-Mailflow-Expected-Revision': '5', + 'X-Mailflow-Filename': '%E0%A4%A', + 'X-Mailflow-Content-Type': 'application/pdf', + }, + }, + ); + + expect(response).toEqual({ + status: 400, + body: { + error: 'X-Mailflow-Filename must be valid percent encoding', + code: 'invalid_attachment_filename', + }, + }); + expect(deps.addComposeAttachment).not.toHaveBeenCalled(); + }); + + it('maps the exposed attachment aggregate limit to 413', async () => { + deps.addComposeAttachment.mockRejectedValueOnce(exposedError( + 'attachment_limit', + 'Compose attachments exceed the 25 MiB limit', + 413, + )); + + const response = await request( + base, + 'POST', + `/api/compose-sessions/${SESSION_ID}/attachments`, + { + body: Buffer.from('synthetic bytes'), + headers: { + 'Content-Type': 'application/octet-stream', + 'X-Mailflow-Expected-Revision': '5', + 'X-Mailflow-Filename': 'report.pdf', + 'X-Mailflow-Content-Type': 'application/pdf', + }, + }, + ); + + expect(response).toEqual({ + status: 413, + body: { + error: 'Compose attachments exceed the 25 MiB limit', + code: 'attachment_limit', + }, + }); + }); + + it('removes an attachment with an expected revision', async () => { + const result = { + sessionId: SESSION_ID, + slot: 1, + revision: 7, + removedAttachmentId: ATTACHMENT_ID, + }; + deps.removeComposeAttachment.mockResolvedValueOnce(result); + + const response = await request( + base, + 'DELETE', + `/api/compose-sessions/${SESSION_ID}/attachments/${ATTACHMENT_ID}`, + { body: { expectedRevision: 6, clientId: 'browser-synthetic' } }, + ); + + expect(response).toEqual({ status: 200, body: result }); + expect(deps.removeComposeAttachment).toHaveBeenCalledWith({ + userId: USER_ID, + id: SESSION_ID, + attachmentId: ATTACHMENT_ID, + expectedRevision: 6, + clientId: 'browser-synthetic', + }, deps); + }); + + it('forwards unknown errors to the generic non-leaking handler', async () => { + deps.patchComposeSession.mockRejectedValueOnce( + new Error('database diagnostic containing private internals'), + ); + + const response = await request(base, 'PATCH', `/api/compose-sessions/${SESSION_ID}`, { + body: { expectedRevision: 1, changes: { subject: 'Synthetic subject' } }, + }); + + expect(response).toEqual({ + status: 500, + body: { error: 'Internal server error' }, + }); + expect(JSON.stringify(response.body)).not.toContain('database diagnostic'); + }); +}); diff --git a/backend/src/routes/draft.js b/backend/src/routes/draft.js index 630fbacd..8d507d7b 100644 --- a/backend/src/routes/draft.js +++ b/backend/src/routes/draft.js @@ -1,188 +1,63 @@ -import nodemailer from 'nodemailer'; -import { randomBytes } from 'crypto'; import { Router } from 'express'; -import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; -import sanitizeHtml from 'sanitize-html'; -import { sanitizeSignature, sanitizeComposeBody } from '../services/emailSanitizer.js'; -import { embedInlineDataImages } from '../utils/inlineImages.js'; import { imapManager } from '../index.js'; +import { query } from '../services/db.js'; +import { deleteDraft, saveDraft } from '../services/draftService.js'; const router = Router(); router.use(requireAuth); -function sanitizeHeaderValue(value) { - if (typeof value !== 'string') return ''; - return value.replace(/[\r\n\0]/g, '').trim(); -} - -// Extract { name, email } from an RFC 5322 address string ("Name ", -// "", or bare "email") for persisting to_addresses/cc_addresses. -function parseAddress(str) { - if (typeof str !== 'string') return { name: '', email: '' }; - const m = str.match(/^(.+?)\s*<([^>]+)>\s*$/); - if (m) return { name: m[1].trim().replace(/^"|"$/g, '').trim(), email: m[2].trim().toLowerCase() }; - const bare = str.match(/^\s*<([^>]+)>\s*$/); - if (bare) return { name: '', email: bare[1].trim().toLowerCase() }; - return { name: '', email: str.trim().toLowerCase() }; -} -function mapRecipientList(list) { - return (Array.isArray(list) ? list : []).filter(Boolean).map(addr => parseAddress(addr)); -} - -function textToHtml(text) { - return text.split('\n') - .map(l => `

${l.replace(/&/g, '&').replace(//g, '>') || ' '}

`) - .join(''); -} - -async function buildRawDraft({ accountId, aliasId, to, cc, bcc, subject, body, bodyIsHtml, quotedBody, quotedBodyHtml, editedSignature }) { - const acctResult = await query( - 'SELECT * FROM email_accounts WHERE id = $1', - [accountId] - ); - if (!acctResult.rows.length) throw Object.assign(new Error('Account not found'), { status: 404 }); - const account = acctResult.rows[0]; - - let fromName = account.sender_name || account.name; - let fromEmail = account.email_address; - let fromSignature = account.signature; - - if (aliasId) { - const aliasResult = await query( - 'SELECT * FROM account_aliases WHERE id = $1 AND account_id = $2', - [aliasId, accountId] - ); - if (aliasResult.rows.length) { - const alias = aliasResult.rows[0]; - fromName = alias.name; - fromEmail = alias.email; - if (alias.signature !== null) fromSignature = alias.signature; - } - } - - const rawSignature = editedSignature !== undefined ? (editedSignature || null) : fromSignature; - const effectiveSignature = rawSignature ? sanitizeSignature(rawSignature) : null; - - const sigText = effectiveSignature - ? sanitizeHtml(effectiveSignature, { allowedTags: [], allowedAttributes: {} }).trim() - : null; - - const bodyText = bodyIsHtml - ? sanitizeHtml(body || '', { allowedTags: [], allowedAttributes: {} }) - : (body || ''); - - const bodyHtml = bodyIsHtml - ? sanitizeComposeBody(body || '') - : textToHtml(body || ''); - - const rawHtml = bodyHtml + - (effectiveSignature ? `
${effectiveSignature}
` : '') + - (quotedBodyHtml || (quotedBody ? textToHtml(quotedBody) : '')); - const { html: draftHtml, attachments: inlineImageAttachments } = embedInlineDataImages(rawHtml); - - // Stable Message-ID so the appended MIME and the local DB row reference the same - // message (and a later sync reconciles cleanly). - const messageId = `<${randomBytes(16).toString('hex')}@${(fromEmail.split('@')[1] || 'mailflow.local')}>`; - const textBody = sigText ? `${bodyText}\n\n-- \n${sigText}${quotedBody || ''}` : `${bodyText}${quotedBody || ''}`; - - const mailOptions = { - messageId, - from: `${fromName} <${fromEmail}>`, - to: (Array.isArray(to) ? to : [to]).filter(Boolean).join(', ') || undefined, - cc: (Array.isArray(cc) ? cc : []).filter(Boolean).join(', ') || undefined, - bcc: (Array.isArray(bcc) ? bcc : []).filter(Boolean).join(', ') || undefined, - subject: sanitizeHeaderValue(subject || ''), - text: textBody, - html: draftHtml, - ...(inlineImageAttachments.length ? { attachments: inlineImageAttachments } : {}), - }; - - const streamTransport = nodemailer.createTransport({ streamTransport: true, newline: 'unix' }); - const streamInfo = await streamTransport.sendMail(mailOptions); - const chunks = []; - await new Promise((resolve, reject) => { - streamInfo.message.on('data', c => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c))); - streamInfo.message.on('end', resolve); - streamInfo.message.on('error', reject); - }); - // rawHtml (pre inline-image embedding) is what the composer should reopen with — - // inline data: URIs stay editable and getMessageBody serves body_html from the DB. - const snippet = textBody.replace(/\s+/g, ' ').trim().slice(0, 200); - return { - rawMessage: Buffer.concat(chunks), - account, - meta: { messageId, fromName, fromEmail, bodyHtml: rawHtml, bodyText: textBody, snippet }, - }; -} - -async function resolveDraftsFolder(account) { - const mapped = account.folder_mappings?.drafts; - if (mapped) return mapped; - const result = await query( - "SELECT path FROM folders WHERE account_id = $1 AND special_use = '\\Drafts' LIMIT 1", - [account.id] - ); - return result.rows[0]?.path || null; -} - router.post('/draft', async (req, res) => { - const { accountId, aliasId, to, cc, bcc, subject, body, bodyIsHtml = false, quotedBody, quotedBodyHtml, editedSignature, existingUid, existingFolder } = req.body; + const { + accountId, + aliasId, + to, + cc, + bcc, + subject, + body, + bodyIsHtml = false, + quotedBody, + quotedBodyHtml, + editedSignature, + inReplyTo, + references, + existingUid, + existingFolder, + } = req.body; if (!accountId) return res.status(400).json({ error: 'accountId required' }); - const ownerCheck = await query( - 'SELECT id FROM email_accounts WHERE id = $1 AND user_id = $2', - [accountId, req.session.userId] + const accountResult = await query( + 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', + [accountId, req.session.userId], ); - if (!ownerCheck.rows.length) return res.status(404).json({ error: 'Account not found' }); + if (!accountResult.rows.length) return res.status(404).json({ error: 'Account not found' }); try { - const { rawMessage, account, meta } = await buildRawDraft({ accountId, aliasId, to, cc, bcc, subject, body, bodyIsHtml, quotedBody, quotedBodyHtml, editedSignature }); - - const draftsFolder = await resolveDraftsFolder(account); - if (!draftsFolder) return res.status(422).json({ error: 'No Drafts folder found for this account' }); - - // APPEND the new draft first so we never lose the message - const { uid } = await imapManager.appendToFolder(account, draftsFolder, rawMessage, ['\\Draft', '\\Seen']); - - // Persist a local Drafts row immediately so the composer can reopen this draft - // (recipient/subject/body) even if the folder re-sync is delayed or fails on a - // flaky connection. Non-fatal — the append already stored the message on IMAP. - if (uid != null) { - try { - await imapManager.upsertDraftMessageRecord(account, draftsFolder, uid, { - messageId: meta.messageId, - subject, - fromName: meta.fromName, - fromEmail: meta.fromEmail, - to: mapRecipientList(to), - cc: mapRecipientList(cc), - snippet: meta.snippet, - bodyHtml: meta.bodyHtml, - bodyText: meta.bodyText, - }); - } catch (rowErr) { - console.error(`Draft: failed to persist local row uid=${uid}: ${rowErr.message}`); - } - } - - // Delete the old draft only after the new one is safely stored - if (existingUid && existingFolder) { - try { - await imapManager.permanentDeleteMessage(account, existingUid, existingFolder); - await query( - 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', - [account.id, existingUid, existingFolder] - ); - } catch (delErr) { - console.error(`Draft: failed to delete old uid=${existingUid}: ${delErr.message}`); - } - } - - res.json({ uid, folder: draftsFolder }); + const result = await saveDraft({ + userId: req.session.userId, + account: accountResult.rows[0], + aliasId, + to, + cc, + bcc, + subject, + body, + bodyIsHtml, + quotedBody, + quotedBodyHtml, + editedSignature, + inReplyTo, + references, + existingUid, + existingFolder, + }, { query, imapManager }); + return res.json({ uid: result.uid, folder: result.folder }); } catch (err) { console.error('Save draft failed:', err.message); - res.status(err.status || 500).json({ error: err.message || 'Failed to save draft' }); + const response = { error: err.message || 'Failed to save draft' }; + if (err.code === 'alias_not_found') response.code = err.code; + return res.status(err.status || 500).json(response); } }); @@ -191,25 +66,27 @@ router.delete('/draft/:uid', async (req, res) => { if (!uid || !Number.isFinite(uid)) return res.status(400).json({ error: 'Invalid uid' }); const { accountId, folder } = req.query; - if (!accountId || !folder) return res.status(400).json({ error: 'accountId and folder required' }); + if (!accountId || !folder) { + return res.status(400).json({ error: 'accountId and folder required' }); + } - const ownerCheck = await query( + const accountResult = await query( 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', - [accountId, req.session.userId] + [accountId, req.session.userId], ); - if (!ownerCheck.rows.length) return res.status(404).json({ error: 'Account not found' }); + if (!accountResult.rows.length) return res.status(404).json({ error: 'Account not found' }); try { - const account = ownerCheck.rows[0]; - await imapManager.permanentDeleteMessage(account, uid, folder); - await query( - 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', - [account.id, uid, folder] - ); - res.json({ ok: true }); + const result = await deleteDraft({ + userId: req.session.userId, + account: accountResult.rows[0], + uid, + folder, + }, { query, imapManager }); + return res.json(result); } catch (err) { console.error('Delete draft failed:', err.message); - res.status(500).json({ error: err.message || 'Failed to delete draft' }); + return res.status(500).json({ error: err.message || 'Failed to delete draft' }); } }); diff --git a/backend/src/routes/draft.test.js b/backend/src/routes/draft.test.js index 31b52296..fd1ab774 100644 --- a/backend/src/routes/draft.test.js +++ b/backend/src/routes/draft.test.js @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeAll, afterAll, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; vi.mock('../services/db.js', () => ({ query: vi.fn() })); vi.mock('../middleware/auth.js', () => ({ @@ -11,7 +11,6 @@ const imapManager = vi.hoisted(() => ({ })); vi.mock('../index.js', () => ({ imapManager })); -import express from 'express'; import draftRoutes from './draft.js'; import { query } from '../services/db.js'; @@ -21,27 +20,42 @@ const ACCOUNT_ROW = { sender_name: null, signature: null, folder_mappings: {}, }; -function buildApp() { - const app = express(); - app.use(express.json()); - app.use('/api/mail', draftRoutes); - return app; +function draftHandler() { + const layer = draftRoutes.stack.find(item => ( + item.route?.path === '/draft' && item.route.methods.post + )); + return layer.route.stack[0].handle; +} + +function responseRecorder() { + return { + statusCode: 200, + body: undefined, + status(code) { + this.statusCode = code; + return this; + }, + json(body) { + this.body = body; + return this; + }, + }; +} + +async function postDraft(body) { + const req = { body, session: { userId: 'user-1' } }; + const res = responseRecorder(); + await draftHandler()(req, res); + return res; } describe('POST /api/mail/draft — local row persistence', () => { - let server, base; - beforeAll(async () => { - await new Promise(r => { server = buildApp().listen(0, r); }); - base = `http://127.0.0.1:${server.address().port}`; - }); - afterAll(async () => { await new Promise(r => server.close(r)); }); beforeEach(() => { query.mockReset(); imapManager.appendToFolder.mockReset(); imapManager.upsertDraftMessageRecord.mockReset(); imapManager.permanentDeleteMessage.mockReset(); - // 1) owner check, 2) buildRawDraft account load, 3) resolveDraftsFolder lookup - query.mockResolvedValueOnce({ rows: [{ id: ACCOUNT_ID }] }); + // 1) scoped account row, 2) resolveDraftsFolder lookup query.mockResolvedValueOnce({ rows: [ACCOUNT_ROW] }); query.mockResolvedValueOnce({ rows: [{ path: 'Drafts' }] }); imapManager.appendToFolder.mockResolvedValue({ uid: 5, folder: 'Drafts' }); @@ -49,20 +63,18 @@ describe('POST /api/mail/draft — local row persistence', () => { }); it('persists a Drafts row with parsed recipient, subject and body after append', async () => { - const res = await fetch(`${base}/api/mail/draft`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - accountId: ACCOUNT_ID, - to: ['Mike Scanlan '], - cc: [], - subject: 'Re: MailFlow hero', - body: 'hello mike', - bodyIsHtml: false, - }), + const res = await postDraft({ + accountId: ACCOUNT_ID, + to: ['Mike Scanlan '], + cc: [], + subject: 'Re: MailFlow hero', + body: 'hello mike', + bodyIsHtml: false, + inReplyTo: '', + references: ' ', }); - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ uid: 5, folder: 'Drafts' }); + expect(res.statusCode).toBe(200); + expect(res.body).toEqual({ uid: 5, folder: 'Drafts' }); expect(imapManager.upsertDraftMessageRecord).toHaveBeenCalledTimes(1); const [acct, folder, uid, meta] = imapManager.upsertDraftMessageRecord.mock.calls[0]; @@ -74,28 +86,22 @@ describe('POST /api/mail/draft — local row persistence', () => { expect(meta.fromEmail).toBe('matthias@mailflow.sh'); expect(meta.bodyHtml).toContain('hello mike'); expect(meta.bodyText).toContain('hello mike'); + expect(meta.inReplyTo).toBe(''); + expect(meta.references).toBe(' '); expect(meta.messageId).toMatch(/^<[0-9a-f]+@mailflow\.sh>$/); }); it('still returns success if the local row persistence throws (append already stored it)', async () => { imapManager.upsertDraftMessageRecord.mockRejectedValueOnce(new Error('db down')); - const res = await fetch(`${base}/api/mail/draft`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ accountId: ACCOUNT_ID, to: ['a@b.com'], subject: 'x', body: 'y' }), - }); - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ uid: 5, folder: 'Drafts' }); + const res = await postDraft({ accountId: ACCOUNT_ID, to: ['a@b.com'], subject: 'x', body: 'y' }); + expect(res.statusCode).toBe(200); + expect(res.body).toEqual({ uid: 5, folder: 'Drafts' }); }); it('does not persist a row when the append returns no uid (no reliable key)', async () => { imapManager.appendToFolder.mockResolvedValueOnce({ uid: null, folder: 'Drafts' }); - const res = await fetch(`${base}/api/mail/draft`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ accountId: ACCOUNT_ID, to: ['a@b.com'], subject: 'x', body: 'y' }), - }); - expect(res.status).toBe(200); + const res = await postDraft({ accountId: ACCOUNT_ID, to: ['a@b.com'], subject: 'x', body: 'y' }); + expect(res.statusCode).toBe(200); expect(imapManager.upsertDraftMessageRecord).not.toHaveBeenCalled(); }); }); diff --git a/backend/src/routes/gtd.classify.test.js b/backend/src/routes/gtd.classify.test.js index 38a6ea86..0babeba8 100644 --- a/backend/src/routes/gtd.classify.test.js +++ b/backend/src/routes/gtd.classify.test.js @@ -22,11 +22,16 @@ vi.mock('../services/gtdConfig.js', async (importOriginal) => { const actual = await importOriginal(); return { ...actual, getGtdConfig: vi.fn() }; }); +vi.mock('../services/gtdDelegations.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, delegateMessages: vi.fn(), reconcileDelegatedRemovals: vi.fn() }; +}); import express from 'express'; import { query } from '../services/db.js'; import { imapManager } from '../index.js'; import { getGtdConfig, DEFAULT_GTD_FOLDERS } from '../services/gtdConfig.js'; +import { delegateMessages, GtdDelegationError, reconcileDelegatedRemovals } from '../services/gtdDelegations.js'; import gtdRoutes from './gtd.js'; const MSG_ID = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd'; @@ -63,6 +68,9 @@ const classify = (body) => fetch(`${base}/api/gtd/classify`, { const unclassify = (body) => fetch(`${base}/api/gtd/classify`, { method: 'DELETE', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), }); +const delegate = (body) => fetch(`${base}/api/gtd/delegations`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), +}); let server; let base; @@ -81,9 +89,44 @@ beforeEach(() => { Object.values(imapManager).forEach(fn => fn.mockReset()); getGtdConfig.mockReset(); getGtdConfig.mockResolvedValue({ enabled: true, folders: DEFAULT_GTD_FOLDERS }); + reconcileDelegatedRemovals.mockReset(); + delegateMessages.mockReset(); stubQueries(); }); +describe('POST /api/gtd/delegations', () => { + it('deduplicates valid IDs and returns the structured bulk result', async () => { + delegateMessages.mockResolvedValue({ + status: 'success', successCount: 1, failureCount: 0, + results: [{ messageId: MSG_ID, ok: true }], + }); + const res = await delegate({ messageIds: [MSG_ID, MSG_ID], contactId: null }); + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ status: 'success', successCount: 1 }); + expect(delegateMessages).toHaveBeenCalledWith(expect.objectContaining({ + userId: 'u1', messageIds: [MSG_ID], contactId: null, imapManager, + })); + }); + + it('rejects invalid shapes before calling the service', async () => { + for (const body of [ + {}, { messageIds: [], contactId: null }, { messageIds: ['bad'], contactId: null }, + { messageIds: [MSG_ID], contactId: 'bad' }, + ]) { + const res = await delegate(body); + expect(res.status).toBe(400); + } + expect(delegateMessages).not.toHaveBeenCalled(); + }); + + it('maps an unowned contact to the same 404 as an absent contact', async () => { + delegateMessages.mockRejectedValue(new GtdDelegationError('contact_not_found', 404)); + const res = await delegate({ messageIds: [MSG_ID], contactId: ACCT_ID }); + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ error: 'contact_not_found' }); + }); +}); + describe('POST /api/gtd/classify — request validation', () => { it('rejects a missing messageId/state with 400 before any lookup', async () => { const res = await classify({ state: 'todo' }); @@ -136,6 +179,17 @@ describe('POST /api/gtd/classify — apply a GTD label (COPY)', () => { }); describe('DELETE /api/gtd/classify — remove a GTD label', () => { + it('clears person metadata after removing the Delegated label', async () => { + const delegated = { ...inboxMsg, thread_key: 'thread-a' }; + stubQueries({ msg: delegated }); + const res = await unclassify({ messageId: MSG_ID, state: 'delegated' }); + expect(res.status).toBe(200); + expect(reconcileDelegatedRemovals).toHaveBeenCalledWith({ + userId: 'u1', accountId: ACCT_ID, + delegatedFolder: 'Delegated', threadKeys: ['thread-a'], + }); + }); + it('removes the sibling copy in the state folder and returns removed:true', async () => { const res = await unclassify({ messageId: MSG_ID, state: 'todo' }); expect(res.status).toBe(200); diff --git a/backend/src/routes/gtd.done.test.js b/backend/src/routes/gtd.done.test.js index aa3f47ee..33f45b3d 100644 --- a/backend/src/routes/gtd.done.test.js +++ b/backend/src/routes/gtd.done.test.js @@ -34,12 +34,16 @@ vi.mock('../services/gtdConfig.js', async (importOriginal) => { const actual = await importOriginal(); return { ...actual, getGtdConfig: vi.fn() }; }); +vi.mock('../services/gtdDelegations.js', async (importOriginal) => ({ + ...(await importOriginal()), reconcileDelegatedRemovals: vi.fn(), delegateMessages: vi.fn(), +})); import express from 'express'; import { query } from '../services/db.js'; import { imapManager } from '../index.js'; import { resolveArchiveFolder, isAllMailFolder, adjustFolderCounts, fanOutReadToSiblings } from '../utils/mailUtils.js'; import { getGtdConfig, DEFAULT_GTD_FOLDERS } from '../services/gtdConfig.js'; +import { reconcileDelegatedRemovals } from '../services/gtdDelegations.js'; import gtdRoutes from './gtd.js'; const MSG_ID = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd'; @@ -47,7 +51,7 @@ const ACCT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; // The rail acts on the Watch-folder copy; a distinct INBOX sibling is what the archive step // moves. is_read true on both keeps the mark-read path off the IMAP setFlag mock. -const msg = { id: MSG_ID, account_id: ACCT_ID, uid: 10, folder: 'Watch', message_id: '', is_read: true }; +const msg = { id: MSG_ID, account_id: ACCT_ID, uid: 10, folder: 'Watch', message_id: '', thread_key: 'thread-a', is_read: true }; const account = { id: ACCT_ID, user_id: 'u1', folder_mappings: {} }; const inboxCopy = { id: 'ib-1', uid: 77, is_read: true }; @@ -91,6 +95,7 @@ beforeEach(() => { query.mockReset(); Object.values(imapManager).forEach(fn => fn.mockReset()); [resolveArchiveFolder, isAllMailFolder, adjustFolderCounts, fanOutReadToSiblings, getGtdConfig].forEach(fn => fn.mockReset()); + reconcileDelegatedRemovals.mockReset(); getGtdConfig.mockResolvedValue({ enabled: true, folders: DEFAULT_GTD_FOLDERS }); resolveArchiveFolder.mockResolvedValue('Archive'); isAllMailFolder.mockResolvedValue(false); @@ -107,6 +112,16 @@ describe('POST /api/gtd/done — id validation', () => { }); describe('POST /api/gtd/done — archive count-adjust race', () => { + it('clears person metadata after successfully stripping Delegated', async () => { + stubQueries(); + const res = await done({ id: MSG_ID, states: ['delegated'] }); + expect(res.status).toBe(200); + expect(reconcileDelegatedRemovals).toHaveBeenCalledWith({ + userId: 'u1', accountId: ACCT_ID, + delegatedFolder: 'Delegated', threadKeys: ['thread-a'], + }); + }); + it('archives + adjusts both counts when the INBOX-scoped write applied (rowCount 1)', async () => { stubQueries({ archiveWrite: { rowCount: 1 } }); imapManager.moveMessage.mockResolvedValue(88); // UIDPLUS newUid diff --git a/backend/src/routes/gtd.js b/backend/src/routes/gtd.js index 39fdedd2..b50ab376 100644 --- a/backend/src/routes/gtd.js +++ b/backend/src/routes/gtd.js @@ -3,61 +3,40 @@ import { requireAuth } from '../middleware/auth.js'; import { getGtdSections } from '../services/gtdSections.js'; import { queueGistGeneration } from '../services/gtdGist.js'; import { importPet, decodeUploadedSheet, getPetMeta, getPetSheet, parsePetSlug, customPetSlug } from '../services/gtdPet.js'; -import { getGtdConfig, resolveGtdStateFolder, sanitizeGtdFolders, sanitizeGtdFoldersDetailed, DEFAULT_GTD_FOLDERS, planGtdFolderPersist, invalidateGtdConfigCache } from '../services/gtdConfig.js'; -import { fanOutReadToSiblings } from '../utils/mailUtils.js'; -import { archiveInboxCopy } from '../services/archiveInbox.js'; +import { sanitizeGtdFolders, sanitizeGtdFoldersDetailed, DEFAULT_GTD_FOLDERS, planGtdFolderPersist, invalidateGtdConfigCache } from '../services/gtdConfig.js'; import { query } from '../services/db.js'; import { imapManager } from '../index.js'; +import { + classifyTarget, + gtdClassify, + gtdDone, + gtdUnclassify, + resolveDoneFolders, +} from '../services/gtd/actions.js'; +import { delegateMessages, GtdDelegationError } from '../services/gtdDelegations.js'; const router = Router(); router.use(requireAuth); +export { classifyTarget, resolveDoneFolders }; + // Message/account ids are always UUIDs; pre-validate before any DB lookup so a malformed // id is a clean 400 rather than a parametrized query that just finds nothing (404) or a // driver cast error. Same idiom + regex as mail.js. const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -// Shared classify precondition: an account must have GTD enabled and the request's -// state must resolve to a designated folder. Returns { folder } to proceed, or -// { status, error } to reject. Pure — exported for unit tests. -export function classifyTarget({ enabled, folders, state }) { - if (!enabled) return { status: 400, error: 'GTD is not enabled for this account' }; - const folder = resolveGtdStateFolder(state, folders); - if (!folder) return { status: 400, error: `Unknown GTD state: ${state}` }; - return { folder }; -} - -// The "done" action's precondition, resolving the GTD label folders a done request must -// strip. Two contracts, per the caller: -// • Explicit `states` array (GTD sidebar entry): resolve each state to its designated folder, in -// order, deduped — a merged Waiting row carries both watch and delegated. Unknown state -// rejects. Mirrors classifyTarget but plural. -// • `states === 'all'` (inbox checkmark): strip every designated GTD label the thread -// actually carries. `existing` is the set of folder paths a live copy was found in; we -// intersect it with the account's designated GTD folders (map order, deduped). Absent -// labels are skipped, never an error; a thread with none resolves to { folders: [] } so -// the route degrades to mark-read + archive. -// Returns { folders } to proceed, or { status, error } to reject. Pure — exported for tests. -export function resolveDoneFolders({ enabled, folders, states, existing }) { - if (!enabled) return { status: 400, error: 'GTD is not enabled for this account' }; - if (states === 'all') { - const present = new Set(Array.isArray(existing) ? existing : []); - const resolved = []; - for (const folder of Object.values(folders || {})) { - if (present.has(folder) && !resolved.includes(folder)) resolved.push(folder); - } - return { folders: resolved }; +export function parseDelegationBody(body) { + if (!Array.isArray(body?.messageIds)) { + throw new GtdDelegationError('invalid_request', 400, 'messageIds must be an array'); } - if (!Array.isArray(states) || states.length === 0) { - return { status: 400, error: 'states must be a non-empty array' }; + const messageIds = [...new Set(body.messageIds)]; + if (messageIds.length < 1 || messageIds.length > 100 || messageIds.some(id => !UUID_RE.test(id))) { + throw new GtdDelegationError('invalid_request', 400, 'messageIds must contain 1 to 100 UUIDs'); } - const resolved = []; - for (const state of states) { - const folder = resolveGtdStateFolder(state, folders); - if (!folder) return { status: 400, error: `Unknown GTD state: ${state}` }; - if (!resolved.includes(folder)) resolved.push(folder); + if (body.contactId !== null && !UUID_RE.test(body.contactId || '')) { + throw new GtdDelegationError('invalid_request', 400, 'contactId must be a UUID or null'); } - return { folders: resolved }; + return { messageIds, contactId: body.contactId }; } // GET /api/gtd/sections — thread heads + counts per GTD state for GTD display surfaces. @@ -84,6 +63,24 @@ router.get('/sections', async (req, res) => { }).catch(err => console.warn('GTD gist generation error:', err.message)); }); +router.post('/delegations', async (req, res, next) => { + try { + const { messageIds, contactId } = parseDelegationBody(req.body); + const result = await delegateMessages({ + userId: req.session.userId, + messageIds, + contactId, + imapManager, + }); + res.json(result); + } catch (error) { + if (error instanceof GtdDelegationError) { + return res.status(error.status).json({ error: error.code }); + } + next(error); + } +}); + // ── GTD Inbox-Zero pet ──────────────────────────────────────────────────────── // POST /api/gtd/pet/import { petJson, sheet } — import a user's OWN pet by uploading the @@ -138,221 +135,53 @@ router.get('/pet/:slug/sheet', async (req, res) => { res.send(sheet.data); }); -// Load a message the caller owns, or send a 404. The email_accounts join is the -// ownership filter (a.user_id = $2); the message row itself carries everything the -// callers need (account_id, uid, folder, message_id), so no account column is selected. -async function loadOwnedMessage(userId, messageId) { - const result = await query( - `SELECT m.* - FROM messages m - JOIN email_accounts a ON a.id = m.account_id - WHERE m.id = $1 AND a.user_id = $2`, - [messageId, userId] - ); - return result.rows[0] || null; -} - -// POST /api/gtd/classify { messageId, state } — apply a GTD label by COPYing the -// message into the state's designated folder (the message stays in its current -// folder; classify never removes it from the inbox). Thin: resolve the folder, -// ensure it exists (callers own folder existence), then delegate to -// imapManager.copyMessage, which also emits gtd_sections_updated. +// Apply a GTD label by copying the message into the state folder. router.post('/classify', async (req, res) => { const { messageId, state } = req.body || {}; if (!messageId || !state) return res.status(400).json({ error: 'messageId and state are required' }); if (!UUID_RE.test(messageId)) return res.status(400).json({ error: 'Invalid message id' }); - const msg = await loadOwnedMessage(req.session.userId, messageId); - if (!msg) return res.status(404).json({ error: 'Message not found' }); - - const { enabled, folders } = await getGtdConfig(msg.account_id); - const target = classifyTarget({ enabled, folders, state }); - if (target.error) return res.status(target.status).json({ error: target.error }); - const toFolder = target.folder; - - // Already labelled with this state — nothing to copy. - if (msg.folder === toFolder) return res.json({ ok: true, folder: toFolder }); - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); - const account = accountResult.rows[0]; - - try { - await imapManager.ensureFolder(account, toFolder); - await imapManager.copyMessage(msg.account_id, msg.uid, msg.folder, toFolder); - } catch (err) { - console.error(`GTD classify failed for message ${messageId} -> ${toFolder}:`, err.message); - return res.status(500).json({ error: 'Failed to apply GTD label' }); - } - - res.json({ ok: true, folder: toFolder }); + const result = await gtdClassify(imapManager, { + userId: req.session.userId, + accountIds: null, + messageId, + state, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); -// Resolve the folder-copy uid a message has in `folder` for this account, or null. The -// acted row is used directly when it already lives there; otherwise the shared RFC -// Message-ID (COPY duplicates it verbatim) joins to the sibling copy. Shared by DELETE -// /classify (below) and POST /done's label strip. -async function resolveCopyUid(msg, folder) { - if (msg.folder === folder) return msg.uid; - const sib = await query( - 'SELECT uid FROM messages WHERE account_id = $1 AND folder = $2 AND message_id = $3 AND is_deleted = false LIMIT 1', - [msg.account_id, folder, msg.message_id] - ); - return sib.rows[0]?.uid ?? null; -} - -// DELETE /api/gtd/classify { messageId, state } — remove a GTD label by deleting -// the message's copy that lives in the state folder, leaving all other copies -// (INBOX, other labels) intact. The acted message id identifies the thread member -// by its RFC Message-ID; the copy in the state folder is resolved from that. +// Remove a GTD label copy while leaving all other message copies intact. router.delete('/classify', async (req, res) => { const { messageId, state } = req.body || {}; if (!messageId || !state) return res.status(400).json({ error: 'messageId and state are required' }); if (!UUID_RE.test(messageId)) return res.status(400).json({ error: 'Invalid message id' }); - const msg = await loadOwnedMessage(req.session.userId, messageId); - if (!msg) return res.status(404).json({ error: 'Message not found' }); - - const { enabled, folders } = await getGtdConfig(msg.account_id); - const target = classifyTarget({ enabled, folders, state }); - if (target.error) return res.status(target.status).json({ error: target.error }); - const stateFolder = target.folder; - - // Find the copy that lives in the state folder via resolveCopyUid: the acted row when it - // already lives there, else the shared RFC Message-ID (COPY duplicates it verbatim) joins to - // the sibling. A missing Message-ID only blocks that sibling lookup — the acted-row case - // needs no Message-ID — so guard it there and keep the explicit 400 the client relies on. - if (msg.folder !== stateFolder && !msg.message_id) { - return res.status(400).json({ error: 'Message has no Message-ID — cannot resolve GTD copy' }); - } - const siblingUid = await resolveCopyUid(msg, stateFolder); - if (siblingUid == null) return res.json({ ok: true, removed: false }); - - try { - await imapManager.removeMessageCopy(msg.account_id, siblingUid, stateFolder); - } catch (err) { - console.error(`GTD unclassify failed for message ${messageId} in ${stateFolder}:`, err.message); - return res.status(500).json({ error: 'Failed to remove GTD label' }); - } - - res.json({ ok: true, removed: true, folder: stateFolder }); + const result = await gtdUnclassify(imapManager, { + userId: req.session.userId, + accountIds: null, + messageId, + state, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); -// POST /api/gtd/done { id, states? } — the GTD "done" action. Two callers: the GTD sidebar passes -// an explicit `states` array (strip that section's labels); the inbox checkmark omits states -// (or sends 'all') for "done from anywhere" — strip every GTD label the thread carries. The -// id is its GTD-label-folder copy for the sidebar, or the INBOX copy from the inbox; either way -// the archive step resolves the INBOX copy from the shared Message-ID rather than acting on -// `id` directly. In one round trip this: (a) marks the whole thread read (DB fan-out across -// sibling copies, \Seen on the INBOX copy so it rides the archive move); (b) strips the -// row's own GTD label copies named in `states` — a merged Waiting row passes both -// watch+delegated — leaving any GTD labels in OTHER sections intact; (c) archives the INBOX -// copy if one exists (reusing resolveArchiveFolder + moveMessage, snooze's in-place UPDATE). -// One terminal gtd_sections_updated broadcast makes the row disappear cleanly on refetch. +// Mark a GTD message done, strip labels, and archive its INBOX copy when available. router.post('/done', async (req, res) => { const { id, states } = req.body || {}; if (!id) return res.status(400).json({ error: 'id is required' }); if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); - const msg = await loadOwnedMessage(req.session.userId, id); - if (!msg) return res.status(404).json({ error: 'Message not found' }); - if (!msg.message_id) return res.status(400).json({ error: 'Message has no Message-ID — cannot mark done' }); - - const { enabled, folders } = await getGtdConfig(msg.account_id); - - // All-states mode (inbox checkmark) resolves against the label copies that actually exist - // for this thread; the GTD sidebar's explicit-states path is untouched. Only look copies up when - // we'll use them (enabled + all-states); resolveDoneFolders still owns the gtd_enabled gate. - const allStates = states == null || states === 'all'; - let existing; - if (allStates && enabled) { - const copies = await query( - 'SELECT DISTINCT folder FROM messages WHERE account_id = $1 AND message_id = $2 AND is_deleted = false', - [msg.account_id, msg.message_id] - ); - existing = copies.rows.map(r => r.folder); - } - const target = allStates - ? resolveDoneFolders({ enabled, folders, states: 'all', existing }) - : resolveDoneFolders({ enabled, folders, states }); - if (target.error) return res.status(target.status).json({ error: target.error }); - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); - const account = accountResult.rows[0]; - - // (a) Mark the whole thread read. The DB fan-out (by Message-ID) covers every sibling - // copy and adjusts each folder's unread count; \Seen is set on the durable INBOX copy - // only (it rides the archive move; Gmail propagates message-wide) — the same per-copy - // asymmetry the ordinary read route accepts. A best-effort flag push is never fatal. - const inbox = await query( - 'SELECT id, uid, is_read FROM messages WHERE account_id = $1 AND folder = $2 AND message_id = $3 AND is_deleted = false LIMIT 1', - [msg.account_id, 'INBOX', msg.message_id] - ); - const inboxCopy = inbox.rows[0] || null; - try { - await fanOutReadToSiblings(msg.account_id, msg.message_id, true); - if (inboxCopy && !inboxCopy.is_read) { - await imapManager.setFlag(account, inboxCopy.uid, 'INBOX', '\\Seen', true); - } - } catch (err) { - console.warn(`GTD done: mark-read for ${id} degraded:`, err.message); - } - - // (b) Strip this row's GTD label copies. Each is a distinct folder copy resolved from - // the shared Message-ID; removeMessageCopy deletes the IMAP + DB copy and adjusts that - // label folder's counts, leaving INBOX and other-section labels untouched. - // - // Strip the acted row's OWN folder (msg.folder) LAST. A merged Waiting done passes both - // watch+delegated, and if an earlier copy's removal throws we 500 — but the same-id retry - // must still resolve the acted message via loadOwnedMessage. Deleting the acted row first - // would 404 that retry and orphan the copies not yet stripped, so keep it alive until every - // other copy is gone. (msg.folder is absent from target.folders in the inbox 'all' case — - // the acted INBOX row is never stripped anyway — so the ordering is a no-op there.) - const stripOrder = [ - ...target.folders.filter(f => f !== msg.folder), - ...target.folders.filter(f => f === msg.folder), - ]; - const removed = []; - try { - for (const folder of stripOrder) { - const uid = await resolveCopyUid(msg, folder); - if (uid == null) continue; // already gone - await imapManager.removeMessageCopy(msg.account_id, uid, folder); - removed.push(folder); - } - } catch (err) { - console.error(`GTD done: label strip for ${id} failed:`, err.message); - return res.status(500).json({ error: 'Failed to mark done' }); - } - - // (c) Archive the INBOX copy if one is present, via the shared per-copy archive primitive - // (resolveArchiveFolder + guarded moveMessage + race-safe DB repoint + count adjust, with - // the Gmail All-Mail DELETE branch). No archive folder configured is a soft outcome, not a - // failure — the GTD labels are already gone, so the thread has left all GTD sections regardless. - let archived = false; - let noArchiveFolder = false; - let archiveFailed = false; - if (inboxCopy) { - try { - const result = await archiveInboxCopy(imapManager, account, inboxCopy); - archived = result.archived; - noArchiveFolder = result.noArchiveFolder; - } catch (err) { - // Step (b) already stripped the labels (or had nothing to strip). A failed archive must - // not 500: that misreports a mostly-successful action, and — with the label row now - // gone — a retry by the same id would 404. Report a partial success (200, archiveFailed - // true, archived false) so the client can surface it and the id stays retryable. - console.error(`GTD done: archive of INBOX copy for ${id} failed:`, err.message); - archiveFailed = true; - } - } - - // One terminal refresh so GTD section data converges to the post-done state (removeMessageCopy - // also emits mid-op, but this covers the archive that follows it). - imapManager.broadcast({ type: 'gtd_sections_updated', accountId: msg.account_id }, account.user_id); - - res.json({ ok: true, removed, archived, noArchiveFolder, archiveFailed }); + const result = await gtdDone(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + states, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // POST /api/gtd/folders/ensure { accountId, folders } — create any of the account's // designated GTD label folders that are missing on the IMAP server, reporting per // folder whether it was created now or already existed. `folders` is the (possibly diff --git a/backend/src/routes/indexing.js b/backend/src/routes/indexing.js new file mode 100644 index 00000000..d8966d2d --- /dev/null +++ b/backend/src/routes/indexing.js @@ -0,0 +1,13 @@ +import { Router } from 'express'; +import { requireAdmin } from '../middleware/auth.js'; +import { listJobs } from '../services/backgroundJobs.js'; + +const router = Router(); + +// GET /api/admin/indexing/status — admin-gated (same guard as routes/ai.js). +// Surfaces every background drainer's progress, e.g. "FTS backfill: N/M". +router.get('/status', requireAdmin, async (_req, res) => { + res.json({ jobs: await listJobs() }); +}); + +export default router; diff --git a/backend/src/routes/mail.composeClaims.test.js b/backend/src/routes/mail.composeClaims.test.js new file mode 100644 index 00000000..c29cd5d4 --- /dev/null +++ b/backend/src/routes/mail.composeClaims.test.js @@ -0,0 +1,125 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'user-1' }; + next(); + }, +})); +vi.mock('../index.js', () => ({ + imapManager: { + prefetchFolderBodies: vi.fn().mockResolvedValue(undefined), + }, +})); +vi.mock('../services/gtdSections.js', () => ({ + emitGtdIfRelevant: vi.fn().mockResolvedValue(undefined), +})); + +import express from 'express'; +import { query } from '../services/db.js'; +import { imapManager } from '../index.js'; +import mailRoutes from './mail.js'; + +const ACCOUNT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const CLAIMED_ID = '11111111-1111-4111-8111-111111111111'; +const UNCLAIMED_ID = '22222222-2222-4222-8222-222222222222'; + +function row(id, folder) { + return { + id, + account_id: ACCOUNT_ID, + uid: id === CLAIMED_ID ? 41 : 42, + folder, + subject: id === CLAIMED_ID ? 'Claimed draft' : 'Visible message', + delegation: null, + }; +} + +let server; +let base; + +beforeAll(async () => { + const app = express(); + app.use('/api/mail', mailRoutes); + await new Promise(resolve => { + server = app.listen(0, resolve); + }); + base = `http://127.0.0.1:${server.address().port}`; +}); + +afterAll(async () => { + await new Promise(resolve => server.close(resolve)); +}); + +beforeEach(() => { + query.mockReset(); + imapManager.prefetchFolderBodies.mockClear(); +}); + +function installListQuery({ folder, folderSpecialUse }) { + query.mockImplementation(async (sql, params) => { + if (sql.includes('SELECT id, include_in_unified_inbox')) { + return { + rows: [{ + id: ACCOUNT_ID, + include_in_unified_inbox: true, + folder_mappings: { drafts: 'Drafts' }, + }], + }; + } + if (sql.includes('FROM folders WHERE account_id')) { + return { + rows: [{ total_count: 2, unread_count: 0, special_use: folderSpecialUse }], + }; + } + if (sql.includes('COUNT(*)::int AS total')) { + return { rows: [{ total: sql.includes('compose_sessions') ? 1 : 2 }] }; + } + if (sql.includes('FROM messages m')) { + return { + rows: sql.includes('compose_sessions') + ? [row(UNCLAIMED_ID, folder)] + : [row(CLAIMED_ID, folder), row(UNCLAIMED_ID, folder)], + }; + } + throw new Error(`Unexpected query: ${sql} ${JSON.stringify(params)}`); + }); +} + +describe('GET /api/mail/messages compose source claims', () => { + it('filters the owner\'s claimed source tuple from a Drafts-folder response', async () => { + installListQuery({ folder: 'Drafts', folderSpecialUse: '\\Drafts' }); + + const res = await fetch( + `${base}/api/mail/messages?accountId=${ACCOUNT_ID}&folder=Drafts&limit=25&offset=5`, + ); + + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ + messages: [{ id: UNCLAIMED_ID, uid: 42, folder: 'Drafts' }], + total: 1, + }); + const claimedQueries = query.mock.calls.filter(([sql]) => sql.includes('compose_sessions')); + expect(claimedQueries).toHaveLength(2); + for (const [sql, params] of claimedQueries) { + expect(sql).toContain('cs.user_id = $3'); + expect(sql).toContain('cs.source_draft_account_id = m.account_id'); + expect(sql).toContain('cs.source_draft_folder = m.folder'); + expect(sql).toContain('cs.source_draft_uid = m.uid'); + expect(params.slice(0, 3)).toEqual([ACCOUNT_ID, 'Drafts', 'user-1']); + } + }); + + it('does not add compose-claim filtering to a non-Drafts mail response', async () => { + installListQuery({ folder: 'INBOX', folderSpecialUse: '\\Inbox' }); + + const res = await fetch( + `${base}/api/mail/messages?accountId=${ACCOUNT_ID}&folder=INBOX`, + ); + + expect(res.status).toBe(200); + expect((await res.json()).messages).toHaveLength(2); + expect(query.mock.calls.some(([sql]) => sql.includes('compose_sessions'))).toBe(false); + }); +}); diff --git a/backend/src/routes/mail.js b/backend/src/routes/mail.js index 5aa52b4e..de042ac7 100644 --- a/backend/src/routes/mail.js +++ b/backend/src/routes/mail.js @@ -7,16 +7,32 @@ import { requireAuth } from '../middleware/auth.js'; import { imapManager } from '../index.js'; import { sanitizeEmail, stripEmailHead, hasRemoteImages, blockRemoteImages, rewriteEbayImageserUrls, rewriteAnchorHrefs } from '../services/emailSanitizer.js'; import { snippetFromBody, decodeMimeWords, parseRawHeaders, buildHeadersFromMessage } from '../services/messageParser.js'; -import { resolveTrashFolder, resolveAllTrashPaths, resolveAllDraftsPaths, resolveArchiveFolder, isAllMailFolder, resolveSpamFolder, resolveAllSpamPaths, getDeleteStrategy, adjustFolderCounts, fanOutReadToSiblings, fanOutStarToSiblings, fanOutBulkReadToSiblings } from '../utils/mailUtils.js'; +import { resolveTrashFolder, resolveAllTrashPaths, resolveAllDraftsPaths, getDeleteStrategy, adjustFolderCounts } from '../utils/mailUtils.js'; import { emitGtdIfRelevant } from '../services/gtdSections.js'; import { listMessages } from '../services/messageService.js'; import { resolveAccountScope } from '../services/unifiedInbox.js'; import { validateHost } from '../services/hostValidation.js'; import { safeFetch } from '../services/safeFetch.js'; +import { DELEGATION_SELECT_SQL, delegationJoinSql, mapDelegationRow } from '../services/gtdDelegations.js'; +import { bulkSetRead, setRead, setStarred } from '../services/mailbox/flags.js'; +import { bulkMoveToFolder } from '../services/mailbox/move.js'; +import { bulkArchive } from '../services/mailbox/archive.js'; +import { bulkTrash } from '../services/mailbox/trash.js'; +import { createFolder, deleteFolder, renameFolder } from '../services/mailbox/folders.js'; +import { setCategory } from '../services/mailbox/category.js'; +import { + gatherSnoozeConversation, + snoozeConversation, + unsnoozeConversation, +} from '../services/mailbox/snooze.js'; +import { markNotSpam, markSpam } from '../services/mailbox/spamLabel.js'; +import { UUID_RE, areValidUUIDs, isValidFolderName } from '../utils/validation.js'; const router = Router(); router.use(requireAuth); +export { gatherSnoozeConversation }; + // Sanitize an attachment filename for use in Content-Disposition. // Strips path separators and control characters; falls back to 'attachment'. function safeFilename(name) { @@ -33,17 +49,6 @@ function safeFilename(name) { return cleaned || 'attachment'; } -// Validate a folder name / path component: no control chars, max 255 chars. -function isValidFolderName(name) { - // eslint-disable-next-line no-control-regex -- intentionally rejecting control characters - return typeof name === 'string' && name.length > 0 && name.length <= 255 && !/[\x00-\x1f\x7f]/.test(name); -} - -const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -function areValidUUIDs(ids) { - return ids.every(id => typeof id === 'string' && UUID_RE.test(id)); -} - // Strip NUL bytes from strings before DB writes. PostgreSQL UTF-8 text columns // reject 0x00, and malformed MIME bodies can contain embedded NUL characters. function sanitizeDbText(value) { @@ -51,19 +56,6 @@ function sanitizeDbText(value) { return value.replace(/\0/g, ''); } -// Process IMAP operations in bounded batches so a 500-message bulk action -// does not spawn hundreds of parallel temporary IMAP connections. -async function runInBatches(items, concurrency, fn) { - const results = []; - for (let i = 0; i < items.length; i += concurrency) { - const batch = items.slice(i, i + concurrency); - const batchResults = await Promise.allSettled(batch.map(fn)); - results.push(...batchResults); - } - return results; -} - - // Returns true if a snippet contains content that should never appear in plain-text // preview, indicating it was generated from unclean HTML and needs regeneration: // - &entity; — undecoded HTML entities from before the entity-stripping fix @@ -125,6 +117,7 @@ router.get('/messages', async (req, res) => { unreadOnly, threaded, category: safeCategory, + excludeClaimedSourceDrafts: true, }); if (resolvedAccountId && messages.length) { @@ -147,15 +140,17 @@ router.get('/messages/:id', async (req, res) => { m.has_attachments, m.account_id, m.category, m.list_unsubscribe, m.list_unsubscribe_post, m.unsubscribed_at, m.delivery_addresses, a.name AS account_name, a.email_address AS account_email, - a.color AS account_color + a.color AS account_color, + ${DELEGATION_SELECT_SQL} FROM messages m JOIN email_accounts a ON m.account_id = a.id + ${delegationJoinSql('m', 'a')} WHERE m.id = $1 AND a.user_id = $2 AND m.is_deleted = false `, [id, req.session.userId]); if (!result.rows.length) return res.status(404).json({ error: 'Message not found' }); - res.json(result.rows[0]); + res.json({ ...result.rows[0], delegation: mapDelegationRow(result.rows[0]) }); } catch (err) { console.error('GET /messages/:id error:', err.message); res.status(500).json({ error: 'Failed to load message' }); @@ -184,7 +179,8 @@ router.get('/resolve-message', async (req, res) => { m.has_attachments, m.account_id, m.category, m.list_unsubscribe, m.list_unsubscribe_post, m.unsubscribed_at, m.delivery_addresses, a.name AS account_name, a.email_address AS account_email, - a.color AS account_color`; + a.color AS account_color, + ${DELEGATION_SELECT_SQL}`; try { // Durable match on the stable Message-ID header. When the same email exists in more // than one folder (e.g. INBOX + Archive), prefer the INBOX copy, then the most recent. @@ -192,6 +188,7 @@ router.get('/resolve-message', async (req, res) => { SELECT ${COLS} FROM messages m JOIN email_accounts a ON m.account_id = a.id + ${delegationJoinSql('m', 'a')} WHERE m.message_id = $1 AND a.user_id = $2 AND m.is_deleted = false @@ -205,6 +202,7 @@ router.get('/resolve-message', async (req, res) => { SELECT ${COLS} FROM messages m JOIN email_accounts a ON m.account_id = a.id + ${delegationJoinSql('m', 'a')} WHERE m.id = $1 AND a.user_id = $2 AND m.is_deleted = false @@ -212,7 +210,7 @@ router.get('/resolve-message', async (req, res) => { `, [ref, req.session.userId, accountId]); } if (result.rows.length === 0) return res.status(404).json({ error: 'Message not found' }); - res.json(result.rows[0]); + res.json({ ...result.rows[0], delegation: mapDelegationRow(result.rows[0]) }); } catch (err) { console.error('GET /resolve-message error:', err.message); res.status(500).json({ error: 'Failed to resolve message' }); @@ -262,9 +260,11 @@ router.get('/thread/:threadId', async (req, res) => { m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, m.category, m.list_unsubscribe, m.list_unsubscribe_post, m.unsubscribed_at, m.delivery_addresses, - a.name AS account_name, a.email_address AS account_email, a.color AS account_color + a.name AS account_name, a.email_address AS account_email, a.color AS account_color, + ${DELEGATION_SELECT_SQL} FROM messages m JOIN email_accounts a ON m.account_id = a.id + ${delegationJoinSql('m', 'a')} WHERE m.is_deleted = false AND m.account_id = ANY($1) AND m.thread_key = $2 @@ -275,7 +275,7 @@ router.get('/thread/:threadId', async (req, res) => { SELECT * FROM deduped ORDER BY date ASC `, [accountIds, threadId]); - res.json({ messages: result.rows }); + res.json({ messages: result.rows.map(row => ({ ...row, delegation: mapDelegationRow(row) })) }); } catch (err) { console.error('Thread fetch error:', err); res.status(500).json({ error: 'Failed to load thread' }); @@ -655,62 +655,14 @@ router.patch('/messages/:id/read', async (req, res) => { const { id } = req.params; if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); const { read } = req.body; - - const result = await query(` - SELECT m.*, a.user_id, - CASE WHEN m.message_id IS NULL THEN 1 - ELSE (SELECT COUNT(*) FROM messages s - WHERE s.account_id = m.account_id AND s.message_id = m.message_id) - END AS sibling_count - FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [id, req.session.userId]); - - if (!result.rows.length) return res.status(404).json({ error: 'Message not found' }); - const message = result.rows[0]; - - // Run DB update and account fetch concurrently — no dependency between them. - // read_changed_at tells the IMAP sync not to overwrite this change for 30 s, - // preventing a race where a concurrent sync fetch sees the old IMAP flag. - const [, accountResult] = await Promise.all([ - query('UPDATE messages SET is_read = $1, read_changed_at = NOW() WHERE id = $2', [read, id]), - query('SELECT * FROM email_accounts WHERE id = $1', [message.account_id]), - ]); - - // Keep the cached folder unread_count in sync so pagination totals stay accurate. - if (!!message.is_read !== !!read) { - adjustFolderCounts(message.account_id, message.folder, 0, read ? -1 : 1); - // Notify the user's OTHER sessions so a read/unread on one device reflects on the rest - // in place, without a full folder refetch (the originating device already applied it). - imapManager.broadcast({ type: 'message_flags', accountId: message.account_id, changes: [{ id, is_read: read }] }, req.session.userId); - } - - // GTD: a labeled message owns a sibling row per folder. Fan the read change out to - // those rows (and their folder unread counts) so label views don't go stale. Gated on - // gtd_enabled (so a non-GTD account is byte-identical to pre-GTD behaviour) AND on the - // message actually having siblings — a plain single-folder message keeps the PK-only - // fast path. The IMAP \Seen flag is written to the acted folder only (below): Gmail - // propagates \Seen message-wide server-side, and per-copy writes to N folders would - // multiply round-trips — an asymmetry accepted in the GTD design. - if (accountResult.rows[0]?.gtd_enabled && Number(message.sibling_count) > 1) { - await fanOutReadToSiblings(message.account_id, message.message_id, read); - } - - try { - await imapManager.setFlag(accountResult.rows[0], message.uid, message.folder, '\\Seen', read); - imapManager._resolveFlagPush(message.account_id, id, '\\Seen'); // confirmed — drop any stale queued op - } catch (err) { - console.error('IMAP flag update failed:', err.message); - // Push failed — queue a durable retry so a later flag-sync pull can't silently revert - // the user's change once the 30s local-wins window lapses. - imapManager._enqueueFlagPush(message.account_id, id, '\\Seen', read); - } - - // Refresh GTD section data if this message's thread carries a GTD label (its head shows read state). - emitGtdSectionsRefresh([message], req.session.userId); - - res.json({ ok: true, is_read: read }); + const result = await setRead(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + read, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // Star/unstar @@ -718,53 +670,14 @@ router.patch('/messages/:id/star', async (req, res) => { const { id } = req.params; if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); const { starred } = req.body; - - const result = await query(` - SELECT m.*, a.user_id, - CASE WHEN m.message_id IS NULL THEN 1 - ELSE (SELECT COUNT(*) FROM messages s - WHERE s.account_id = m.account_id AND s.message_id = m.message_id) - END AS sibling_count - FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [id, req.session.userId]); - - if (!result.rows.length) return res.status(404).json({ error: 'Message not found' }); - const message = result.rows[0]; - - // Run DB update and account fetch concurrently — no dependency between them. - // star_changed_at tells the IMAP sync not to overwrite this change for 30 s. - const [, accountResult] = await Promise.all([ - query('UPDATE messages SET is_starred = $1, star_changed_at = NOW() WHERE id = $2', [starred, id]), - query('SELECT * FROM email_accounts WHERE id = $1', [message.account_id]), - ]); - - // GTD: fan the star change out to the message's sibling label rows (see the read - // handler). Gated on gtd_enabled to keep a non-GTD account byte-identical to pre-GTD. - // Stars don't affect folder unread counts, so no count adjustment. The IMAP \Flagged - // write below stays on the acted folder only. - if (accountResult.rows[0]?.gtd_enabled && Number(message.sibling_count) > 1) { - await fanOutStarToSiblings(message.account_id, message.message_id, starred); - } - - try { - await imapManager.setFlag(accountResult.rows[0], message.uid, message.folder, '\\Flagged', starred); - imapManager._resolveFlagPush(message.account_id, id, '\\Flagged'); // confirmed — drop any stale queued op - } catch (err) { - console.error('IMAP star update failed:', err.message); - // Push failed — queue a durable retry so a later flag-sync pull can't silently revert it. - imapManager._enqueueFlagPush(message.account_id, id, '\\Flagged', starred); - } - - // Refresh GTD section data if this message's thread carries a GTD label (its head shows star state). - emitGtdSectionsRefresh([message], req.session.userId); - // Reflect the star change on the user's other sessions in place (no full refetch). - if (!!message.is_starred !== !!starred) { - imapManager.broadcast({ type: 'message_flags', accountId: message.account_id, changes: [{ id, is_starred: starred }] }, req.session.userId); - } - - res.json({ ok: true, is_starred: starred }); + const result = await setStarred(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + starred, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // Manual sync (INBOX) @@ -851,80 +764,49 @@ router.post('/folders', async (req, res) => { if (!accountId || !name?.trim()) return res.status(400).json({ error: 'accountId and name required' }); if (!isValidFolderName(name.trim())) return res.status(400).json({ error: 'Invalid folder name' }); if (parentPath && !isValidFolderName(parentPath)) return res.status(400).json({ error: 'Invalid parent path' }); - const check = await query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [accountId, req.session.userId]); - if (!check.rows.length) return res.status(404).json({ error: 'Account not found' }); - - // Build path: if parentPath given, look up the delimiter used by this account's folders - let path = name.trim(); - if (parentPath) { - const delimResult = await query('SELECT delimiter FROM folders WHERE account_id = $1 LIMIT 1', [accountId]); - const delim = delimResult.rows[0]?.delimiter || '/'; - path = `${parentPath}${delim}${name.trim()}`; - } - try { - await imapManager.createFolder(check.rows[0], path); - await query( - `INSERT INTO folders (account_id, path, name) VALUES ($1, $2, $3) - ON CONFLICT (account_id, path) DO NOTHING`, - [accountId, path, name.trim()] - ); - res.json({ ok: true, path }); - } catch (err) { - console.error('Create folder error:', err); - res.status(500).json({ error: 'Failed to create folder' }); - } + const result = await createFolder(imapManager, { + userId: req.session.userId, + accountIds: null, + accountId, + name, + parentPath, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // Delete folder router.post('/folders/delete', async (req, res) => { const { accountId, path } = req.body; if (!accountId || !path) return res.status(400).json({ error: 'accountId and path required' }); if (!isValidFolderName(path)) return res.status(400).json({ error: 'Invalid folder path' }); - const check = await query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [accountId, req.session.userId]); - if (!check.rows.length) return res.status(404).json({ error: 'Account not found' }); - try { - await imapManager.deleteFolder(check.rows[0], path); - } catch (err) { - console.error(`IMAP deleteFolder failed for ${path}:`, err.message); - return res.status(500).json({ error: 'Failed to delete folder on server' }); - } - await query('DELETE FROM folders WHERE account_id = $1 AND path = $2', [accountId, path]); - await query('DELETE FROM messages WHERE account_id = $1 AND folder = $2', [accountId, path]); - res.json({ ok: true }); + const result = await deleteFolder(imapManager, { + userId: req.session.userId, + accountIds: null, + accountId, + path, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // Rename folder router.post('/folders/rename', async (req, res) => { const { accountId, oldPath, newName } = req.body; if (!accountId || !oldPath || !newName?.trim()) return res.status(400).json({ error: 'Missing required fields' }); if (!isValidFolderName(newName.trim())) return res.status(400).json({ error: 'Invalid folder name' }); if (!isValidFolderName(oldPath)) return res.status(400).json({ error: 'Invalid folder path' }); - const check = await query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [accountId, req.session.userId]); - if (!check.rows.length) return res.status(404).json({ error: 'Account not found' }); - - // Build the new path by replacing only the last path component - const delimResult = await query('SELECT delimiter FROM folders WHERE account_id = $1 AND path = $2', [accountId, oldPath]); - const delim = delimResult.rows[0]?.delimiter || '/'; - const parts = oldPath.split(delim); - parts[parts.length - 1] = newName.trim(); - const newPath = parts.join(delim); - try { - await imapManager.renameFolder(check.rows[0], oldPath, newPath); - await query( - 'UPDATE folders SET path = $1, name = $2, updated_at = NOW() WHERE account_id = $3 AND path = $4', - [newPath, newName.trim(), accountId, oldPath] - ); - await query('UPDATE messages SET folder = $1 WHERE account_id = $2 AND folder = $3', [newPath, accountId, oldPath]); - res.json({ ok: true, newPath }); - } catch (err) { - console.error('Rename folder error:', err); - res.status(500).json({ error: 'Failed to rename folder' }); - } + const result = await renameFolder(imapManager, { + userId: req.session.userId, + accountIds: null, + accountId, + oldPath, + newName, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // Empty folder (delete all messages) router.post('/folders/empty', async (req, res) => { const { accountId, path } = req.body; @@ -964,85 +846,14 @@ router.post('/messages/bulk-read', async (req, res) => { return res.status(400).json({ error: 'read must be a boolean' }); } - try { - const result = await query( - `SELECT m.id, m.uid, m.folder, m.is_read, m.account_id, m.message_id, a.gtd_enabled FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1`, - [req.session.userId, ids] - ); - - const owned = result.rows; - if (!owned.length) return res.json({ ok: true, updated: [] }); - - // Skip messages whose state already matches — avoid spurious DB writes and IMAP round-trips. - const toUpdate = owned.filter(m => !!m.is_read !== !!read); - if (!toUpdate.length) return res.json({ ok: true, updated: [] }); - - await query( - 'UPDATE messages SET is_read = $1, read_changed_at = NOW() WHERE id = ANY($2::uuid[])', - [read, toUpdate.map(m => m.id)] - ); - - // Adjust cached unread counts per account+folder. - const folderDeltas = {}; - for (const msg of toUpdate) { - const key = `${msg.account_id}:${msg.folder}`; - if (!folderDeltas[key]) folderDeltas[key] = { accountId: msg.account_id, folder: msg.folder, delta: 0 }; - folderDeltas[key].delta += read ? -1 : 1; - } - for (const { accountId, folder, delta } of Object.values(folderDeltas)) { - adjustFolderCounts(accountId, folder, 0, delta); - } - - // GTD: fan the read change out to sibling label rows of every updated message that - // belongs to a gtd_enabled account, adjusting each sibling folder's unread count. - // Gating on gtd_enabled keeps a non-GTD account byte-identical to pre-GTD (no extra - // fan-out query); the fan-out itself is also self-limiting for messages without - // siblings. IMAP \Seen is still written per acted row only (below); Gmail propagates - // it message-wide server-side. - // gtdUpdatedIds is scoped to toUpdate (rows whose read-state actually changed), so a - // message already at the target state never triggers sibling fan-out here — unlike the - // single-message handler above, which fans out unconditionally regardless of whether the - // acted message's own state changed. That asymmetry is acceptable: nothing else in this - // path can push a sibling out of sync with its head, and the label-folder tick already - // self-heals any divergence on the next read. - const gtdUpdatedIds = toUpdate.filter(m => m.gtd_enabled).map(m => m.id); - if (gtdUpdatedIds.length) await fanOutBulkReadToSiblings(gtdUpdatedIds, read); - // Reflect the bulk read/unread change on the user's other sessions in place (no full refetch). - imapManager.broadcast({ type: 'message_flags', changes: toUpdate.map(m => ({ id: m.id, is_read: read })) }, req.session.userId); - - // IMAP flag updates — group by account to fetch each account row once. - const byAccount = {}; - for (const msg of toUpdate) { - (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); - } - for (const [accountId, msgs] of Object.entries(byAccount)) { - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); - const account = accountResult.rows[0]; - const results = await runInBatches( - msgs, 3, - msg => imapManager.setFlag(account, msg.uid, msg.folder, '\\Seen', read) - ); - results.forEach((r, i) => { - if (r.status === 'rejected') { - console.error(`bulk-read IMAP ${msgs[i].id}:`, r.reason.message); - // Durable retry so a later flag-sync pull can't revert this message to unread. - imapManager._enqueueFlagPush(accountId, msgs[i].id, '\\Seen', read); - } else { - imapManager._resolveFlagPush(accountId, msgs[i].id, '\\Seen'); // confirmed - } - }); - } - - // Refresh GTD section data for any updated thread that carries a GTD label. - emitGtdSectionsRefresh(toUpdate, req.session.userId); - - res.json({ ok: true, updated: toUpdate.map(m => m.id) }); - } catch (err) { - console.error('bulk-read error:', err); - res.status(500).json({ error: 'Failed to update messages' }); - } + const result = await bulkSetRead(imapManager, { + userId: req.session.userId, + accountIds: null, + ids, + read, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // Bulk delete (move to trash) @@ -1058,202 +869,15 @@ router.post('/messages/bulk-delete', async (req, res) => { return res.status(400).json({ error: 'Invalid message id format' }); } - const moveGuards = []; - try { - const result = await query( - `SELECT m.*, a.user_id, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1`, - [req.session.userId, ids] - ); - - const owned = result.rows; - if (!owned.length) return res.json({ ok: true, deleted: [] }); - - // Guard source UIDs for the whole operation so reconcileDeletes can't delete a - // trash-move source row between the IMAP move and the re-INSERT CTE (message vanishing - // from both folders). Harmless for the expunge path (those rows are deleted anyway). - // Released in the finally below. - for (const m of owned) { - moveGuards.push({ accountId: m.account_id, folder: m.folder, uid: m.uid }); - imapManager._guardMoveUid(m.account_id, m.folder, m.uid); - } - - const byAccount = {}; - for (const msg of owned) { - (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); - } - - // expungeSucceeded: permanently deleted (already in Trash, or no Trash folder on account). - // trashMoveSucceeded: moved from a non-Trash folder into Trash. - const expungeSucceeded = []; - const trashMoveSucceeded = []; // { msg, trashPath, newUid } - const accountsById = {}; - - for (const [accountId, msgs] of Object.entries(byAccount)) { - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); - const account = accountResult.rows[0]; - accountsById[accountId] = account; - const trashPath = await resolveTrashFolder(accountId, msgs[0].folder_mappings); - const allTrashPaths = await resolveAllTrashPaths(accountId, msgs[0].folder_mappings); - const allDraftsPaths = await resolveAllDraftsPaths(accountId, msgs[0].folder_mappings); - - if (!trashPath) { - console.error(`bulk-delete: no Trash folder found for account ${accountId} — skipping ${msgs.length} messages`); - continue; - } - - // Drafts and messages already in Trash are permanently deleted; others move to Trash. - const toExpunge = msgs.filter(m => allTrashPaths.has(m.folder) || allDraftsPaths.has(m.folder)); - const toMove = msgs.filter(m => !allTrashPaths.has(m.folder) && !allDraftsPaths.has(m.folder)); - - // Permanently delete messages already in a trash-like folder (grouped by actual folder). - if (toExpunge.length) { - const byExpungeFolder = {}; - for (const msg of toExpunge) { - (byExpungeFolder[msg.folder] = byExpungeFolder[msg.folder] || []).push(msg); - } - for (const [expungeFolder, folderMsgs] of Object.entries(byExpungeFolder)) { - const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); - const { succeeded, failed } = await imapManager.bulkPermanentDelete(account, folderMsgs.map(m => m.uid), expungeFolder); - for (const uid of succeeded) expungeSucceeded.push(uidToMsg.get(String(uid))); - for (const uid of failed) console.error(`bulk-delete IMAP expunge uid ${uid} from ${expungeFolder}: IMAP delete failed`); - } - } - - // Move messages from non-Trash folders into Trash. - if (toMove.length) { - const byFolder = {}; - for (const msg of toMove) { - (byFolder[msg.folder] = byFolder[msg.folder] || []).push(msg); - } - for (const [srcFolder, folderMsgs] of Object.entries(byFolder)) { - const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); - const { uidMap, succeeded, failed } = await imapManager.bulkMoveMessages(account, folderMsgs.map(m => m.uid), srcFolder, trashPath); - for (const uid of succeeded) { - trashMoveSucceeded.push({ msg: uidToMsg.get(String(uid)), trashPath, newUid: uidMap.get(Number(uid)) || null }); - } - for (const uid of failed) console.error(`bulk-delete IMAP move uid ${uid}: IMAP move failed`); - } - } - } - - // Permanently deleted: remove DB rows immediately. - if (expungeSucceeded.length) { - await query('DELETE FROM messages WHERE id = ANY($1::uuid[])', [expungeSucceeded.map(m => m.id)]); - } - - // Trash moves: same CTE approach as bulk-move — DELETE source rows and - // immediately re-INSERT at the destination when new UIDs are known. - // Group by trashPath since different accounts may have different Trash folders. - if (trashMoveSucceeded.length) { - const byTrashPath = {}; - for (const u of trashMoveSucceeded) { - (byTrashPath[u.trashPath] = byTrashPath[u.trashPath] || []).push(u); - } - for (const [trashPath, entries] of Object.entries(byTrashPath)) { - const allIds = entries.map(u => u.msg.id); - const withUid = entries.filter(u => u.newUid); - await query(` - WITH deleted AS ( - DELETE FROM messages WHERE id = ANY($1::uuid[]) RETURNING * - ), - uid_map(src_id, new_uid) AS ( - SELECT * FROM unnest($2::uuid[], $3::bigint[]) - ) - INSERT INTO messages ( - account_id, uid, folder, message_id, subject, - from_name, from_email, to_addresses, cc_addresses, - reply_to, in_reply_to, date, snippet, is_read, is_starred, - has_attachments, flags, body_html, body_text, attachments, - thread_references, thread_id, is_bulk, - read_changed_at, star_changed_at, spam_score_sa, spam_score_ml, - spam_verdict, spam_analyzed_at, spam_details, spam_user_override, - category, list_unsubscribe, list_unsubscribe_post, unsubscribed_at - ) - SELECT - d.account_id, u.new_uid, $4, d.message_id, d.subject, - d.from_name, d.from_email, d.to_addresses, d.cc_addresses, - d.reply_to, d.in_reply_to, d.date, d.snippet, d.is_read, d.is_starred, - d.has_attachments, d.flags, d.body_html, d.body_text, d.attachments, - d.thread_references, d.thread_id, d.is_bulk, - d.read_changed_at, d.star_changed_at, d.spam_score_sa, d.spam_score_ml, - d.spam_verdict, d.spam_analyzed_at, d.spam_details, d.spam_user_override, - d.category, d.list_unsubscribe, d.list_unsubscribe_post, d.unsubscribed_at - FROM deleted d - JOIN uid_map u ON d.id = u.src_id - ON CONFLICT (account_id, uid, folder) DO NOTHING - `, [allIds, withUid.map(u => u.msg.id), withUid.map(u => u.newUid), trashPath]); - } - // Non-UIDPLUS trash moves were deleted with no reinsert; pull each affected - // (account, trash folder) now so they reappear promptly instead of via IDLE. - const needResync = new Map(); // accountId -> Set - for (const u of trashMoveSucceeded) { - if (u.newUid) continue; - if (!needResync.has(u.msg.account_id)) needResync.set(u.msg.account_id, new Set()); - needResync.get(u.msg.account_id).add(u.trashPath); - } - for (const [acctId, paths] of needResync) { - const acct = accountsById[acctId]; - if (!acct) continue; - for (const tp of paths) { - imapManager.syncFolderOnDemand(acct, tp) - .catch(err => console.warn('post-trash destination sync failed:', err.message)); - } - } - } - - // Adjust cached folder counts. - // Source folders always lose the message; Trash gains only for non-Trash moves. - const allSucceeded = [ - ...expungeSucceeded.map(m => m.id), - ...trashMoveSucceeded.map(u => u.msg.id), - ]; - if (allSucceeded.length) { - const srcDeltas = {}; - for (const msg of expungeSucceeded) { - const key = `${msg.account_id}:${msg.folder}`; - if (!srcDeltas[key]) srcDeltas[key] = { accountId: msg.account_id, path: msg.folder, total: 0, unread: 0 }; - srcDeltas[key].total++; - if (!msg.is_read) srcDeltas[key].unread++; - } - for (const { msg } of trashMoveSucceeded) { - const key = `${msg.account_id}:${msg.folder}`; - if (!srcDeltas[key]) srcDeltas[key] = { accountId: msg.account_id, path: msg.folder, total: 0, unread: 0 }; - srcDeltas[key].total++; - if (!msg.is_read) srcDeltas[key].unread++; - } - for (const { accountId, path, total, unread } of Object.values(srcDeltas)) { - adjustFolderCounts(accountId, path, -total, -unread); - } - const dstDeltas = {}; - for (const { msg, trashPath } of trashMoveSucceeded) { - const key = `${msg.account_id}:${trashPath}`; - if (!dstDeltas[key]) dstDeltas[key] = { accountId: msg.account_id, path: trashPath, total: 0, unread: 0 }; - dstDeltas[key].total++; - if (!msg.is_read) dstDeltas[key].unread++; - } - for (const { accountId, path, total, unread } of Object.values(dstDeltas)) { - adjustFolderCounts(accountId, path, total, unread); - } - // Notify clients viewing each Trash folder to refresh silently. - for (const { accountId, path } of Object.values(dstDeltas)) { - imapManager.broadcast({ type: 'folder_updated', folder: path, accountId }, req.session.userId); - } - } - - // Refresh GTD section data for any deleted thread that still carries a GTD label sibling. - emitGtdSectionsRefresh(owned, req.session.userId); - - res.json({ ok: true, deleted: allSucceeded }); - } catch (err) { - console.error('bulk-delete error:', err); - res.status(500).json({ error: 'Failed to delete messages' }); - } finally { - for (const g of moveGuards) imapManager._unguardMoveUid(g.accountId, g.folder, g.uid); - } + const result = await bulkTrash(imapManager, { + userId: req.session.userId, + accountIds: null, + ids, + allowPermanent: true, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); - // Bulk move to folder router.post('/messages/bulk-move', async (req, res) => { const { ids, folder } = req.body; @@ -1270,149 +894,17 @@ router.post('/messages/bulk-move', async (req, res) => { return res.status(400).json({ error: 'Invalid message id format' }); } - const moveGuards = []; - try { - const result = await query( - `SELECT m.*, a.user_id FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1`, - [req.session.userId, ids] - ); - - const owned = result.rows; - if (!owned.length) return res.json({ ok: true, moved: [] }); - - // Guard every source (account, folder, uid) for the whole bulk move. bulkMoveMessages - // removes the UIDs from the server (seconds of wall-clock), and a concurrent - // reconcileDeletes tick would otherwise see the source rows as orphans and delete them - // before the DELETE...RETURNING CTE re-inserts them at the destination — dropping the - // message from BOTH folders. Unguarded in the finally once the CTE has committed. - // Mirrors the single-message move paths. - for (const m of owned) { - moveGuards.push({ accountId: m.account_id, folder: m.folder, uid: m.uid }); - imapManager._guardMoveUid(m.account_id, m.folder, m.uid); - } - - const byAccount = {}; - for (const msg of owned) { - (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); - } - - const movedIds = []; - const uidUpdates = []; - const resyncAccounts = []; // accounts whose moved msgs lacked new UIDs (non-UIDPLUS) - for (const [accountId, msgs] of Object.entries(byAccount)) { - // Verify the destination folder exists for this account - const folderCheck = await query( - 'SELECT 1 FROM folders WHERE account_id = $1 AND path = $2', - [accountId, folder] - ); - if (!folderCheck.rows.length) { - console.warn(`bulk-move: folder "${folder}" not found for account ${accountId}, skipping`); - continue; - } - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); - const account = accountResult.rows[0]; - const byFolder = {}; - for (const msg of msgs) { - (byFolder[msg.folder] = byFolder[msg.folder] || []).push(msg); - } - let accountMissingUid = false; - for (const [srcFolder, folderMsgs] of Object.entries(byFolder)) { - const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); - const { uidMap, succeeded, failed } = await imapManager.bulkMoveMessages(account, folderMsgs.map(m => m.uid), srcFolder, folder); - for (const uid of succeeded) { - const msg = uidToMsg.get(String(uid)); - movedIds.push(msg.id); - const newUid = uidMap.get(Number(uid)) || null; - if (newUid) uidUpdates.push({ id: msg.id, newUid }); - else accountMissingUid = true; - } - for (const uid of failed) console.error(`bulk-move IMAP uid ${uid}: IMAP move failed`); - } - if (accountMissingUid) resyncAccounts.push(account); - } - - if (movedIds.length > 0) { - // DELETE source rows and, when we have UIDPLUS-provided new UIDs, immediately - // re-INSERT at the destination in one atomic CTE statement. This avoids any - // transient folder/uid state that could collide with existing rows (UIDs are - // per-folder, so the same UID number is valid in two different folders). - // If IMAP IDLE already inserted the destination row, ON CONFLICT DO NOTHING - // keeps it intact. For messages without new UIDs the DELETE-only path relies - // on IMAP IDLE + the message_id pre-check in processMsg to re-insert them. - const uidUpdateMap = new Map(uidUpdates.map(u => [u.id, u.newUid])); - const withNewUid = movedIds.filter(id => uidUpdateMap.has(id)); - await query(` - WITH deleted AS ( - DELETE FROM messages WHERE id = ANY($1::uuid[]) RETURNING * - ), - uid_map(src_id, new_uid) AS ( - SELECT * FROM unnest($2::uuid[], $3::bigint[]) - ) - INSERT INTO messages ( - account_id, uid, folder, message_id, subject, - from_name, from_email, to_addresses, cc_addresses, - reply_to, in_reply_to, date, snippet, is_read, is_starred, - has_attachments, flags, body_html, body_text, attachments, - thread_references, thread_id, is_bulk, - read_changed_at, star_changed_at, spam_score_sa, spam_score_ml, - spam_verdict, spam_analyzed_at, spam_details, spam_user_override, - category, list_unsubscribe, list_unsubscribe_post, unsubscribed_at - ) - SELECT - d.account_id, u.new_uid, $4, d.message_id, d.subject, - d.from_name, d.from_email, d.to_addresses, d.cc_addresses, - d.reply_to, d.in_reply_to, d.date, d.snippet, d.is_read, d.is_starred, - d.has_attachments, d.flags, d.body_html, d.body_text, d.attachments, - d.thread_references, d.thread_id, d.is_bulk, - d.read_changed_at, d.star_changed_at, d.spam_score_sa, d.spam_score_ml, - d.spam_verdict, d.spam_analyzed_at, d.spam_details, d.spam_user_override, - d.category, d.list_unsubscribe, d.list_unsubscribe_post, d.unsubscribed_at - FROM deleted d - JOIN uid_map u ON d.id = u.src_id - ON CONFLICT (account_id, uid, folder) DO NOTHING - `, [movedIds, withNewUid, withNewUid.map(id => uidUpdateMap.get(id)), folder]); - // Messages moved on a non-UIDPLUS server were deleted with no reinsert; pull the - // destination folder now so they reappear promptly instead of waiting for IDLE. - for (const acct of resyncAccounts) { - imapManager.syncFolderOnDemand(acct, folder) - .catch(err => console.warn('post-move destination sync failed:', err.message)); - } - // Adjust cached counts: decrement source folders, increment the destination. - const movedSet = new Set(movedIds); - const srcTotals = {}; - for (const msg of owned) { - if (!movedSet.has(msg.id)) continue; - const key = `${msg.account_id}:${msg.folder}`; - if (!srcTotals[key]) srcTotals[key] = { accountId: msg.account_id, path: msg.folder, total: 0, unread: 0 }; - srcTotals[key].total++; - if (!msg.is_read) srcTotals[key].unread++; - } - for (const { accountId, path, total, unread } of Object.values(srcTotals)) { - adjustFolderCounts(accountId, path, -total, -unread); - adjustFolderCounts(accountId, folder, total, unread); - } - - // Notify clients that the destination folder has new content so they - // refresh without sounds or alerts (unlike new_messages). - for (const accountId of Object.keys(srcTotals).map(k => k.split(':')[0])) { - imapManager.broadcast({ type: 'folder_updated', folder, accountId }, req.session.userId); - } - } - - // Refresh GTD section data for any moved thread that still carries a GTD label sibling. - emitGtdSectionsRefresh(owned, req.session.userId); - - res.json({ ok: true, moved: movedIds }); - } catch (err) { - console.error('bulk-move error:', err); - res.status(500).json({ error: 'Failed to move messages' }); - } finally { - for (const g of moveGuards) imapManager._unguardMoveUid(g.accountId, g.folder, g.uid); - } + const result = await bulkMoveToFolder(imapManager, { + userId: req.session.userId, + accountIds: null, + ids, + folder, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + // MCP consumes the richer receipt (movedDetails/skippedAccounts/failed) via the + // service return value directly; REST keeps its original {ok, moved} wire shape. + res.json({ ok: result.ok, moved: result.moved }); }); - // Bulk archive — moves messages to the archive folder for each account router.post('/messages/bulk-archive', async (req, res) => { const { ids } = req.body; @@ -1426,261 +918,17 @@ router.post('/messages/bulk-archive', async (req, res) => { return res.status(400).json({ error: 'Invalid message IDs' }); } - const moveGuards = []; - try { - const result = await query( - `SELECT m.*, a.user_id, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1`, - [req.session.userId, ids] - ); - - const owned = result.rows; - if (!owned.length) return res.json({ ok: true, archived: [], noArchiveFolder: [] }); - - // Guard source UIDs for the whole operation so reconcileDeletes can't delete a source - // row between the IMAP move and the re-INSERT CTE (message vanishing from both folders). - // Released in the finally below. - for (const m of owned) { - moveGuards.push({ accountId: m.account_id, folder: m.folder, uid: m.uid }); - imapManager._guardMoveUid(m.account_id, m.folder, m.uid); - } - - const byAccount = {}; - for (const msg of owned) { - (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); - } - - const archivedIds = []; - const noArchiveFolder = []; - const accountsById = {}; - // Archive-folder paths that resolved to Gmail's All Mail (special_use '\All'). - // All Mail is excluded from sync/backfill and the relocate guard (imapManager.js), - // so messages archived there get their DB row deleted below instead of re-homed. - const allMailDestFolders = new Set(); - - for (const [accountId, msgs] of Object.entries(byAccount)) { - const archiveFolder = await resolveArchiveFolder(accountId, msgs[0].folder_mappings); - if (!archiveFolder) { - noArchiveFolder.push(accountId); - continue; - } - if (await isAllMailFolder(accountId, archiveFolder)) { - allMailDestFolders.add(archiveFolder); - } - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); - const account = accountResult.rows[0]; - accountsById[accountId] = account; - const byFolder = {}; - for (const msg of msgs) { - (byFolder[msg.folder] = byFolder[msg.folder] || []).push(msg); - } - for (const [srcFolder, folderMsgs] of Object.entries(byFolder)) { - const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); - const { uidMap, succeeded, failed } = await imapManager.bulkMoveMessages(account, folderMsgs.map(m => m.uid), srcFolder, archiveFolder); - for (const uid of succeeded) { - const msg = uidToMsg.get(String(uid)); - archivedIds.push({ id: msg.id, accountId, folder: archiveFolder, newUid: uidMap.get(Number(uid)) || null }); - } - for (const uid of failed) console.error(`bulk-archive IMAP uid ${uid}: IMAP move failed`); - } - } - - // Update DB: same CTE DELETE+INSERT pattern as bulk-move — except when the - // destination is Gmail's All Mail, where the message just vanishes from our view - // (see allMailDestFolders above), so a plain DELETE with no reinsert is correct. - const byFolder = {}; - for (const { id, folder, newUid } of archivedIds) { - (byFolder[folder] = byFolder[folder] || []).push({ id, newUid }); - } - for (const [archiveFolder, entries] of Object.entries(byFolder)) { - const allIds = entries.map(e => e.id); - if (allMailDestFolders.has(archiveFolder)) { - await query('DELETE FROM messages WHERE id = ANY($1::uuid[])', [allIds]); - continue; - } - const withUid = entries.filter(e => e.newUid != null); - await query(` - WITH deleted AS ( - DELETE FROM messages WHERE id = ANY($1::uuid[]) RETURNING * - ), - uid_map(src_id, new_uid) AS ( - SELECT * FROM unnest($2::uuid[], $3::bigint[]) - ) - INSERT INTO messages ( - account_id, uid, folder, message_id, subject, - from_name, from_email, to_addresses, cc_addresses, - reply_to, in_reply_to, date, snippet, is_read, is_starred, - has_attachments, flags, body_html, body_text, attachments, - thread_references, thread_id, is_bulk, - read_changed_at, star_changed_at, spam_score_sa, spam_score_ml, - spam_verdict, spam_analyzed_at, spam_details, spam_user_override, - category, list_unsubscribe, list_unsubscribe_post, unsubscribed_at - ) - SELECT - d.account_id, u.new_uid, $4, d.message_id, d.subject, - d.from_name, d.from_email, d.to_addresses, d.cc_addresses, - d.reply_to, d.in_reply_to, d.date, d.snippet, d.is_read, d.is_starred, - d.has_attachments, d.flags, d.body_html, d.body_text, d.attachments, - d.thread_references, d.thread_id, d.is_bulk, - d.read_changed_at, d.star_changed_at, d.spam_score_sa, d.spam_score_ml, - d.spam_verdict, d.spam_analyzed_at, d.spam_details, d.spam_user_override, - d.category, d.list_unsubscribe, d.list_unsubscribe_post, d.unsubscribed_at - FROM deleted d - JOIN uid_map u ON d.id = u.src_id - ON CONFLICT (account_id, uid, folder) DO NOTHING - `, [allIds, withUid.map(e => e.id), withUid.map(e => e.newUid), archiveFolder]); - } - - // Non-UIDPLUS archive moves were deleted with no reinsert; pull each affected - // (account, archive folder) now so they reappear promptly instead of via IDLE. - const needResync = new Map(); // accountId -> Set - for (const e of archivedIds) { - if (e.newUid) continue; - if (allMailDestFolders.has(e.folder)) continue; // no DB row there to keep fresh - if (!needResync.has(e.accountId)) needResync.set(e.accountId, new Set()); - needResync.get(e.accountId).add(e.folder); - } - for (const [acctId, paths] of needResync) { - const acct = accountsById[acctId]; - if (!acct) continue; - for (const fp of paths) { - imapManager.syncFolderOnDemand(acct, fp) - .catch(err => console.warn('post-archive destination sync failed:', err.message)); - } - } - - // Adjust cached folder counts: use signed deltas so source and dest share one pass. - if (archivedIds.length > 0) { - const idToArchiveDest = new Map(archivedIds.map(({ id, folder: dest }) => [id, dest])); - const folderDeltas = {}; // key: `${accountId}:${path}` -> { accountId, path, totalDelta, unreadDelta } - for (const msg of owned) { - const dest = idToArchiveDest.get(msg.id); - if (!dest) continue; - const wasUnread = !msg.is_read ? 1 : 0; - const srcKey = `${msg.account_id}:${msg.folder}`; - if (!folderDeltas[srcKey]) folderDeltas[srcKey] = { accountId: msg.account_id, path: msg.folder, totalDelta: 0, unreadDelta: 0 }; - folderDeltas[srcKey].totalDelta--; - folderDeltas[srcKey].unreadDelta -= wasUnread; - if (allMailDestFolders.has(dest)) continue; // All Mail counts aren't tracked - const dstKey = `${msg.account_id}:${dest}`; - if (!folderDeltas[dstKey]) folderDeltas[dstKey] = { accountId: msg.account_id, path: dest, totalDelta: 0, unreadDelta: 0 }; - folderDeltas[dstKey].totalDelta++; - folderDeltas[dstKey].unreadDelta += wasUnread; - } - for (const { accountId, path, totalDelta, unreadDelta } of Object.values(folderDeltas)) { - adjustFolderCounts(accountId, path, totalDelta, unreadDelta); - } - // Notify clients viewing each destination folder to refresh silently. - const destFolders = [...new Set(archivedIds.map(a => a.folder))].filter(f => !allMailDestFolders.has(f)); - for (const dest of destFolders) { - const accountIds = [...new Set(archivedIds.filter(a => a.folder === dest).map(a => { - const msg = owned.find(m => m.id === a.id); - return msg?.account_id; - }).filter(Boolean))]; - for (const accountId of accountIds) { - imapManager.broadcast({ type: 'folder_updated', folder: dest, accountId }, req.session.userId); - } - } - } - - // Refresh GTD section data for any archived thread that still carries a GTD label sibling. - emitGtdSectionsRefresh(owned, req.session.userId); - - res.json({ ok: true, archived: archivedIds.map(a => a.id), noArchiveFolder }); - } catch (err) { - console.error('bulk-archive error:', err); - res.status(500).json({ error: 'Failed to archive messages' }); - } finally { - for (const g of moveGuards) imapManager._unguardMoveUid(g.accountId, g.folder, g.uid); - } + const result = await bulkArchive(imapManager, { + userId: req.session.userId, + accountIds: null, + ids, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + // MCP consumes the richer receipt (archivedDetails/failed) via the service return + // value directly; REST keeps its original {ok, archived, noArchiveFolder} wire shape. + res.json({ ok: result.ok, archived: result.archived, noArchiveFolder: result.noArchiveFolder }); }); -// Gather the reply-chain conversation that should be snoozed alongside `msg`. -// -// Snoozing a single message doesn't work on Gmail: Gmail groups the inbox by -// conversation, so moving one message to Snoozed only strips \Inbox from that -// message — its thread siblings keep \Inbox and the whole conversation stays in -// the inbox (#271). MailFlow's own inbox is thread-grouped too. So we snooze the -// entire conversation, but bounded to the RFC 5322 reply chain (Message-ID / -// In-Reply-To / References links) rather than thread_id: thread_id falls back to -// subject grouping and can lump hundreds of unrelated messages together (e.g. -// identical automated-notification emails), which must never be swept into Snoozed. -// -// Returns the messages in `msg`'s source folder reachable from `msg` through -// header links (always including `msg` itself); excludes already-snoozed messages. -export async function gatherSnoozeConversation(msg) { - if (!msg.thread_id) return [msg]; - - // Load the whole thread across ALL folders. thread_id is a superset of the true - // conversation, and the messages that hold a real conversation together — the - // other party's replies, your own Sent messages, the thread root — frequently - // live in Sent / All Mail rather than the inbox. They must be present as graph - // connectors or a genuine thread fragments and only part of it snoozes. The - // reply-chain walk below filters out the subject-only collisions that thread_id - // also collects (e.g. identical automated-notification emails). - const pool = (await query( - `SELECT id, uid, account_id, folder, message_id, in_reply_to, thread_references, is_read - FROM messages - WHERE account_id = $1 AND thread_id = $2 AND message_id IS NOT NULL`, - [msg.account_id, msg.thread_id] - )).rows; - - // Ensure the triggering message is present (the query above could miss it on a - // transient read skew). - if (!pool.some(r => r.message_id === msg.message_id)) pool.push(msg); - - const refsOf = (r) => { - const ids = (r.thread_references || '').match(/<[^>]+>/g) || []; - if (r.in_reply_to) ids.push(r.in_reply_to); - return ids; - }; - - // Undirected reply-chain graph over the whole thread; take the connected - // component containing `msg`. Messages with no header link into that component - // (subject-only collisions) are left out. - const adj = new Map(); - const node = (m) => { let s = adj.get(m); if (!s) { s = new Set(); adj.set(m, s); } return s; }; - for (const r of pool) node(r.message_id); - for (const r of pool) { - for (const ref of refsOf(r)) { - if (adj.has(ref)) { node(r.message_id).add(ref); node(ref).add(r.message_id); } - } - } - const seen = new Set([msg.message_id]); - const queue = [msg.message_id]; - while (queue.length) { - const cur = queue.shift(); - for (const nb of (adj.get(cur) || [])) if (!seen.has(nb)) { seen.add(nb); queue.push(nb); } - } - - // Snooze only the conversation members in the acted-on message's source folder - // (the inbox copies — Sent copies carry no \Inbox and shouldn't move), skipping - // any already snoozed. Already-snoozed messages stay valid graph connectors above. - const already = new Set( - (await query( - 'SELECT message_id_header FROM snoozed_messages WHERE account_id = $1 AND message_id_header = ANY($2)', - [msg.account_id, [...seen]] - )).rows.map(r => r.message_id_header) - ); - // Dedupe by Message-ID so a message that somehow has two rows in the source - // folder isn't moved (and recorded) twice. - const picked = new Map(); - for (const r of pool) { - if (seen.has(r.message_id) && r.folder === msg.folder && !already.has(r.message_id) && !picked.has(r.message_id)) { - picked.set(r.message_id, r); - } - } - // Return the acted-on message first so the caller can treat a failure moving it - // as fatal before any sibling has been touched (no partial snooze on error). - const rest = [...picked.values()].filter(r => r.message_id !== msg.message_id); - // msg always qualifies (it's the acted-on, not-yet-snoozed message in its own - // folder); fall back to it directly if the pool row for it was missed. - const self = picked.get(msg.message_id) || msg; - return [self, ...rest]; -} - // Snooze a message: move it to a Snoozed IMAP folder and record when to restore it router.post('/messages/:id/snooze', async (req, res) => { const { id } = req.params; @@ -1696,84 +944,27 @@ router.post('/messages/:id/snooze', async (req, res) => { maxDate.setDate(maxDate.getDate() + 30); if (untilDate > maxDate) return res.status(400).json({ error: 'until must be within 30 days' }); - // Ownership check - const msgResult = await query( - `SELECT m.*, a.user_id FROM messages m - JOIN email_accounts a ON a.id = m.account_id - WHERE m.id = $1 AND a.user_id = $2`, - [id, req.session.userId] - ); - if (!msgResult.rows.length) return res.status(404).json({ error: 'Message not found' }); - const msg = msgResult.rows[0]; - - if (!msg.message_id) return res.status(400).json({ error: 'Message has no Message-ID header — cannot snooze' }); - - const snoozedFolder = 'Snoozed'; - - if (msg.folder === snoozedFolder) { - return res.status(400).json({ error: 'Message is already in Snoozed folder' }); - } - - // Check if already snoozed - const existing = await query( - 'SELECT id FROM snoozed_messages WHERE account_id = $1 AND message_id_header = $2', - [msg.account_id, msg.message_id] - ); - if (existing.rows.length) return res.status(400).json({ error: 'Message is already snoozed' }); - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); - const account = accountResult.rows[0]; - - // Snooze the whole reply-chain conversation, not just this message (see - // gatherSnoozeConversation for why Gmail requires this and why it's bounded - // to the header reply chain rather than thread_id). - const convo = await gatherSnoozeConversation(msg); - - try { - await imapManager.ensureFolder(account, snoozedFolder); - } catch (err) { - console.error(`Snooze ensureFolder failed for message ${id}:`, err.message); - return res.status(500).json({ error: 'Failed to move message to Snoozed folder' }); - } - - for (const tm of convo) { - imapManager._guardMoveUid(tm.account_id, tm.folder, tm.uid); - try { - let snoozedUid; - try { - snoozedUid = await imapManager.moveMessage(account, tm.uid, tm.folder, snoozedFolder); - } catch (err) { - console.error(`Snooze IMAP move failed for message ${tm.id}:`, err.message); - // The message the user acted on must succeed; a failed sibling is logged - // and skipped so the rest of the conversation still snoozes. - if (tm.id === msg.id) return res.status(500).json({ error: 'Failed to move message to Snoozed folder' }); - continue; - } - if (snoozedUid != null) { - await query('UPDATE messages SET folder = $1, uid = $2 WHERE id = $3', [snoozedFolder, snoozedUid, tm.id]); - } else { - imapManager._guardMoveUid(tm.account_id, snoozedFolder, tm.uid); - await query('UPDATE messages SET folder = $1 WHERE id = $2', [snoozedFolder, tm.id]); - setTimeout(() => imapManager._unguardMoveUid(tm.account_id, snoozedFolder, tm.uid), 10_000); - } - - await query( - `INSERT INTO snoozed_messages (user_id, account_id, message_id_header, original_folder, snooze_until, snoozed_folder) - VALUES ($1, $2, $3, $4, $5, $6)`, - [req.session.userId, tm.account_id, tm.message_id, tm.folder, untilDate.toISOString(), snoozedFolder] - ); - - adjustFolderCounts(tm.account_id, tm.folder, -1, tm.is_read ? 0 : -1); - adjustFolderCounts(tm.account_id, snoozedFolder, 1, tm.is_read ? 0 : 1); - } finally { - imapManager._unguardMoveUid(tm.account_id, tm.folder, tm.uid); - } - } + const result = await snoozeConversation(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + until: untilDate, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); +}); - // Refresh GTD section data if the snoozed conversation carries a GTD label (its in_inbox flips). - emitGtdSectionsRefresh(convo, req.session.userId); +router.delete('/messages/:id/snooze', async (req, res) => { + const { id } = req.params; + if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); - res.json({ ok: true }); + const result = await unsnoozeConversation(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // Delete (move to trash; drafts are permanently deleted) @@ -1874,108 +1065,6 @@ router.delete('/messages/:id', async (req, res) => { // // No automatic classification runs here — that ships in v0.2 (ML) and v0.3 (SA). -// Helper: move a single message to a destination folder, update DB, log to -// training_log, and broadcast folder_updated. Shared between /spam and /ham. -async function moveForSpamLabel(messageId, userId, destinationFolder, label) { - const result = await query(` - SELECT m.*, a.user_id, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [messageId, userId]); - - if (!result.rows.length) return { ok: false, status: 404, error: 'Message not found' }; - const message = result.rows[0]; - - // No-op: message already in the destination folder. - if (message.folder === destinationFolder) { - // Still record the training label so the user's intent is captured - // (e.g. re-confirming a verdict), but skip the IMAP move. - await query( - `INSERT INTO spam_training_log - (user_id, account_id, message_id_header, message_uid, folder, label) - VALUES ($1, $2, $3, $4, $5, $6)`, - [userId, message.account_id, message.message_id, message.uid, message.folder, label] - ); - await query( - `UPDATE messages SET spam_user_override = $1, spam_verdict = $1, spam_analyzed_at = NOW() WHERE id = $2`, - [label, messageId] - ); - return { ok: true, status: 200, body: { ok: true, alreadyInFolder: true, folder: destinationFolder } }; - } - - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [message.account_id]); - const account = accountResult.rows[0]; - - // Guard the source UID before the IMAP move so reconcileDeletes cannot - // delete the DB row if an EXPUNGE arrives while the move is in flight. - imapManager._guardMoveUid(account.id, message.folder, message.uid); - let newUid; - try { - try { - newUid = await imapManager.moveMessage(account, message.uid, message.folder, destinationFolder); - } catch (err) { - console.error(`IMAP move for /${label} failed:`, err.message); - return { ok: false, status: 502, error: `IMAP move failed: ${err.message}` }; - } - if (newUid != null) { - await query('DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3 AND id != $4', - [account.id, newUid, destinationFolder, messageId]); - await query( - `UPDATE messages SET folder = $1, uid = $2, - spam_user_override = $3, spam_verdict = $3, spam_analyzed_at = NOW() - WHERE id = $4`, - [destinationFolder, newUid, label, messageId] - ); - } else { - // Non-UIDPLUS server: DB holds the stale source UID at the destination. - imapManager._guardMoveUid(account.id, destinationFolder, message.uid); - await query( - `UPDATE messages SET folder = $1, - spam_user_override = $2, spam_verdict = $2, spam_analyzed_at = NOW() - WHERE id = $3`, - [destinationFolder, label, messageId] - ); - setTimeout(() => imapManager._unguardMoveUid(account.id, destinationFolder, message.uid), 10_000); - } - } finally { - imapManager._unguardMoveUid(account.id, message.folder, message.uid); - } - - // Adjust cached folder counts. - const wasUnread = !message.is_read ? 1 : 0; - adjustFolderCounts(account.id, message.folder, -1, -wasUnread); - adjustFolderCounts(account.id, destinationFolder, 1, wasUnread); - - // Training log: capture the decision for future model training. - await query( - `INSERT INTO spam_training_log - (user_id, account_id, message_id_header, message_uid, folder, label, source) - VALUES ($1, $2, $3, $4, $5, $6, 'manual')`, - [userId, account.id, message.message_id, message.uid, destinationFolder, label] - ); - - // If folder_mappings.spam is not yet configured, learn from the discovered folder. - if (label === 'spam' && !account.folder_mappings?.spam) { - await query( - `UPDATE email_accounts SET folder_mappings = folder_mappings || jsonb_build_object('spam', $1::text) - WHERE id = $2 AND NOT (folder_mappings ? 'spam')`, - [destinationFolder, account.id] - ).catch(err => console.warn('Failed to auto-persist folder_mappings.spam:', err.message)); - } - - imapManager.broadcast( - { type: 'folder_updated', folder: destinationFolder, accountId: account.id }, - userId - ); - - // Refresh GTD section data if the (un)spammed message's thread carries a GTD label. Covers both - // /spam and /ham, which share this mover. The already-in-folder no-op path above returns - // early without a move, so GTD section data is untouched there. - emitGtdSectionsRefresh([message], userId); - - return { ok: true, status: 200, body: { ok: true, folder: destinationFolder, newUid: newUid || null } }; -} - // POST /api/mail/messages/:id/spam // Moves the message to the account's spam/junk folder and records the user // override as spam. Coexists with the future ML/SA auto-classification: @@ -1984,17 +1073,11 @@ router.post('/messages/:id/spam', async (req, res) => { const { id } = req.params; if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); - const lookup = await query(` - SELECT m.account_id, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [id, req.session.userId]); - - if (!lookup.rows.length) return res.status(404).json({ error: 'Message not found' }); - const spamFolder = await resolveSpamFolder(lookup.rows[0].account_id, lookup.rows[0].folder_mappings); - if (!spamFolder) return res.status(422).json({ error: 'No spam folder configured for this account' }); - - const result = await moveForSpamLabel(id, req.session.userId, spamFolder, 'spam'); + const result = await markSpam(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + }); if (!result.ok) return res.status(result.status).json({ error: result.error }); res.json(result.body); }); @@ -2046,17 +1129,14 @@ router.patch('/messages/:id/category', async (req, res) => { return res.status(400).json({ error: 'Invalid category' }); } - const result = await query( - `UPDATE messages SET category = $1 - FROM email_accounts a - WHERE messages.id = $2 - AND messages.account_id = a.id - AND a.user_id = $3 - RETURNING messages.id`, - [category === 'primary' ? null : category, id, req.session.userId] - ); - if (!result.rows.length) return res.status(404).json({ error: 'Message not found' }); - res.json({ ok: true, category }); + const result = await setCategory(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + category, + }); + if (!result.ok) return res.status(result.status).json({ error: result.error }); + res.json(result); }); // POST /api/mail/messages/:id/unsubscribe @@ -2138,23 +1218,11 @@ router.post('/messages/:id/ham', async (req, res) => { const { id } = req.params; if (!UUID_RE.test(id)) return res.status(400).json({ error: 'Invalid message id' }); - const lookup = await query(` - SELECT m.account_id, m.folder, a.folder_mappings FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2 - `, [id, req.session.userId]); - - if (!lookup.rows.length) return res.status(404).json({ error: 'Message not found' }); - const allSpam = await resolveAllSpamPaths(lookup.rows[0].account_id, lookup.rows[0].folder_mappings); - if (!allSpam.has(lookup.rows[0].folder)) { - return res.status(400).json({ error: 'Message is not in the spam folder' }); - } - - // Resolve inbox folder per account — Gmail, Exchange and others may not use - // the literal 'INBOX' (e.g. 'Inbox' on Dovecot, 'Posteingang', etc.). - // Same pattern as folder_mappings.sent / .drafts in send.js and draft.js. - const inboxFolder = lookup.rows[0].folder_mappings?.inbox || 'INBOX'; - const result = await moveForSpamLabel(id, req.session.userId, inboxFolder, 'ham'); + const result = await markNotSpam(imapManager, { + userId: req.session.userId, + accountIds: null, + id, + }); if (!result.ok) return res.status(result.status).json({ error: result.error }); res.json(result.body); }); diff --git a/backend/src/routes/mail.resolve.test.js b/backend/src/routes/mail.resolve.test.js index 774d4e5e..70bd18f2 100644 --- a/backend/src/routes/mail.resolve.test.js +++ b/backend/src/routes/mail.resolve.test.js @@ -56,13 +56,18 @@ describe('GET /api/mail/resolve-message account scope', () => { }); it('keeps unscoped deep-link resolution backward compatible', async () => { - query.mockResolvedValueOnce({ rows: [{ id: 'current-row', account_id: ACCOUNT_ID }] }); + query.mockResolvedValueOnce({ rows: [{ + id: 'current-row', account_id: ACCOUNT_ID, + delegation: '{"contact_id":null,"display_name":"Casey"}', + }] }); const response = await fetch(`${base}/api/mail/resolve-message?ref=${encodeURIComponent(MESSAGE_ID)}`); expect(response.status).toBe(200); const [, params] = query.mock.calls[0]; expect(params).toEqual([MESSAGE_ID, 'user-1', null]); + expect((await response.json()).delegation).toEqual({ contact_id: null, display_name: 'Casey' }); + expect(query.mock.calls[0][0]).toContain('gtd_delegations'); }); it('rejects a malformed account scope before querying', async () => { diff --git a/backend/src/routes/mail.test.js b/backend/src/routes/mail.test.js new file mode 100644 index 00000000..4b44a899 --- /dev/null +++ b/backend/src/routes/mail.test.js @@ -0,0 +1,311 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../services/db.js', () => ({ query: vi.fn() })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'u1' }; + next(); + }, +})); +vi.mock('../index.js', () => ({ + imapManager: { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + bulkMoveMessages: vi.fn(), + bulkPermanentDelete: vi.fn(), + syncFolderOnDemand: vi.fn(), + moveMessage: vi.fn(), + moveMessageGetNewUid: vi.fn(), + setFlag: vi.fn(), + broadcast: vi.fn(), + }, +})); +vi.mock('../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + adjustFolderCounts: vi.fn(), + resolveTrashFolder: vi.fn(), + resolveAllTrashPaths: vi.fn(), + resolveAllDraftsPaths: vi.fn(), + resolveArchiveFolder: vi.fn(), + isAllMailFolder: vi.fn(), + resolveSpamFolder: vi.fn(), + resolveAllSpamPaths: vi.fn(), + }; +}); +vi.mock('../services/gtdSections.js', () => ({ emitGtdIfRelevant: vi.fn().mockResolvedValue(undefined) })); + +import express from 'express'; +import { query } from '../services/db.js'; +import { imapManager } from '../index.js'; +import { + adjustFolderCounts, + isAllMailFolder, + resolveAllDraftsPaths, + resolveAllSpamPaths, + resolveAllTrashPaths, + resolveArchiveFolder, + resolveSpamFolder, + resolveTrashFolder, +} from '../utils/mailUtils.js'; +import mailRoutes from './mail.js'; + +const MSG_ID = '11111111-1111-4111-8111-111111111111'; +const ACCOUNT_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const account = { id: ACCOUNT_ID, user_id: 'u1', folder_mappings: {} }; +const message = { + id: MSG_ID, + account_id: ACCOUNT_ID, + uid: 10, + folder: 'INBOX', + message_id: '', + is_read: false, + folder_mappings: {}, +}; + +function buildApp() { + const app = express(); + app.use(express.json()); + app.use('/api/mail', mailRoutes); + return app; +} + +const post = (path, body) => fetch(`${base}/api/mail${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), +}); +const del = (path) => fetch(`${base}/api/mail${path}`, { method: 'DELETE' }); + +let server; +let base; + +beforeAll(async () => { + await new Promise((resolve) => { + server = buildApp().listen(0, resolve); + }); + base = `http://127.0.0.1:${server.address().port}`; +}); + +afterAll(async () => { + await new Promise((resolve) => server.close(resolve)); +}); + +beforeEach(() => { + query.mockReset(); + Object.values(imapManager).forEach(fn => fn.mockReset()); + [ + adjustFolderCounts, + isAllMailFolder, + resolveAllDraftsPaths, + resolveAllSpamPaths, + resolveAllTrashPaths, + resolveArchiveFolder, + resolveSpamFolder, + resolveTrashFolder, + ].forEach(fn => fn.mockReset()); + imapManager.syncFolderOnDemand.mockResolvedValue(undefined); +}); + +describe('mail mutation route characterization', () => { + it('bulk-move preserves the guarded UIDPLUS move and response contract', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) return { rows: [message] }; + if (sql.includes('SELECT 1 FROM folders')) return { rows: [{ '?column?': 1 }] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + if (sql.includes('WITH deleted AS')) return { rows: [] }; + return { rows: [] }; + }); + imapManager.bulkMoveMessages.mockResolvedValue({ + uidMap: new Map([[10, 110]]), + succeeded: [10], + failed: [], + }); + + const res = await post('/messages/bulk-move', { ids: [MSG_ID], folder: 'Archive' }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, moved: [MSG_ID] }); + expect(imapManager._guardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + expect(imapManager.bulkMoveMessages).toHaveBeenCalledWith(account, [10], 'INBOX', 'Archive'); + expect(imapManager._unguardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + const cte = query.mock.calls.find(([sql]) => sql.includes('WITH deleted AS')); + expect(cte[1]).toEqual([[MSG_ID], [MSG_ID], [110], 'Archive']); + }); + + it('bulk-archive preserves the guarded archive move and receipt', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + if (sql.includes('WITH deleted AS')) return { rows: [] }; + return { rows: [] }; + }); + resolveArchiveFolder.mockResolvedValue('Archive'); + isAllMailFolder.mockResolvedValue(false); + imapManager.bulkMoveMessages.mockResolvedValue({ + uidMap: new Map([[10, 110]]), + succeeded: [10], + failed: [], + }); + + const res = await post('/messages/bulk-archive', { ids: [MSG_ID] }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, archived: [MSG_ID], noArchiveFolder: [] }); + expect(imapManager.bulkMoveMessages).toHaveBeenCalledWith(account, [10], 'INBOX', 'Archive'); + expect(imapManager._guardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + expect(imapManager._unguardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + }); + + it('bulk-delete preserves the move-to-trash path and deleted-id receipt', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + if (sql.includes('WITH deleted AS')) return { rows: [] }; + return { rows: [] }; + }); + resolveTrashFolder.mockResolvedValue('Trash'); + resolveAllTrashPaths.mockResolvedValue(new Set(['Trash'])); + resolveAllDraftsPaths.mockResolvedValue(new Set(['Drafts'])); + imapManager.bulkMoveMessages.mockResolvedValue({ + uidMap: new Map([[10, 210]]), + succeeded: [10], + failed: [], + }); + + const res = await post('/messages/bulk-delete', { ids: [MSG_ID] }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, deleted: [MSG_ID] }); + expect(imapManager.bulkPermanentDelete).not.toHaveBeenCalled(); + expect(imapManager.bulkMoveMessages).toHaveBeenCalledWith(account, [10], 'INBOX', 'Trash'); + expect(imapManager._guardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + expect(imapManager._unguardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + }); + + it('/spam resolves the spam folder and preserves the move receipt', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.account_id, a.folder_mappings')) { + return { rows: [{ account_id: ACCOUNT_ID, folder_mappings: {} }] }; + } + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + resolveSpamFolder.mockResolvedValue('Junk'); + imapManager.moveMessage.mockResolvedValue(310); + + const res = await post(`/messages/${MSG_ID}/spam`, {}); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, folder: 'Junk', newUid: 310 }); + expect(imapManager.moveMessage).toHaveBeenCalledWith(account, 10, 'INBOX', 'Junk'); + expect(imapManager._guardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + expect(imapManager._unguardMoveUid).toHaveBeenCalledWith(ACCOUNT_ID, 'INBOX', 10); + }); + + it('/ham requires a spam-like source and preserves the inbox move receipt', async () => { + const spamMessage = { ...message, folder: 'Junk' }; + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.account_id, m.folder, a.folder_mappings')) { + return { rows: [{ account_id: ACCOUNT_ID, folder: 'Junk', folder_mappings: { inbox: 'INBOX' } }] }; + } + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [spamMessage] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [{ ...account, folder_mappings: { inbox: 'INBOX' } }] }; + return { rows: [] }; + }); + resolveAllSpamPaths.mockResolvedValue(new Set(['Junk'])); + imapManager.moveMessage.mockResolvedValue(410); + + const res = await post(`/messages/${MSG_ID}/ham`, {}); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, folder: 'INBOX', newUid: 410 }); + expect(imapManager.moveMessage).toHaveBeenCalledWith( + { ...account, folder_mappings: { inbox: 'INBOX' } }, + 10, + 'Junk', + 'INBOX', + ); + }); +}); + +describe('DELETE /api/mail/messages/:id/snooze', () => { + it('returns 404 when the message is not owned', async () => { + query.mockResolvedValue({ rows: [] }); + + const res = await del(`/messages/${MSG_ID}/snooze`); + + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ error: 'Message not found' }); + expect(imapManager.moveMessageGetNewUid).not.toHaveBeenCalled(); + }); + + it('returns 400 when the message is not currently snoozed', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) { + return { rows: [{ ...message, folder: 'Snoozed', thread_id: null }] }; + } + if (sql.includes('FROM snoozed_messages sm')) return { rows: [] }; + return { rows: [] }; + }); + + const res = await del(`/messages/${MSG_ID}/snooze`); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: 'Message is not currently snoozed' }); + }); + + it('restores the whole snoozed reply chain and reports its count', async () => { + const root = { ...message, folder: 'Snoozed', thread_id: 'thread', is_read: true }; + const reply = { + ...root, + id: '22222222-2222-4222-8222-222222222222', + uid: 11, + message_id: '', + in_reply_to: root.message_id, + thread_references: root.message_id, + is_read: false, + }; + const snoozedRows = [ + { + snooze_id: 's1', + user_id: 'u1', + account_id: ACCOUNT_ID, + message_id_header: root.message_id, + original_folder: 'INBOX', + snoozed_folder: 'Snoozed', + uid: 10, + is_read: true, + }, + { + snooze_id: 's2', + user_id: 'u1', + account_id: ACCOUNT_ID, + message_id_header: reply.message_id, + original_folder: 'INBOX', + snoozed_folder: 'Snoozed', + uid: 11, + is_read: false, + }, + ]; + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) return { rows: [root] }; + if (sql.includes('WHERE account_id = $1 AND thread_id = $2')) return { rows: [root, reply] }; + if (sql.includes('FROM snoozed_messages sm')) return { rows: snoozedRows }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + imapManager.moveMessageGetNewUid + .mockResolvedValueOnce(110) + .mockResolvedValueOnce(111); + + const res = await del(`/messages/${MSG_ID}/snooze`); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true, restored: 2, folder: 'INBOX' }); + expect(imapManager.moveMessageGetNewUid).toHaveBeenCalledTimes(2); + expect(imapManager.setFlag).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/routes/mcpAccountStages.js b/backend/src/routes/mcpAccountStages.js new file mode 100644 index 00000000..005d6250 --- /dev/null +++ b/backend/src/routes/mcpAccountStages.js @@ -0,0 +1,43 @@ +import { Router } from 'express'; +import { requireAuth } from '../middleware/auth.js'; +import { + completeAccountStage, + discardAccountStage, + listStages, +} from '../services/accountService.js'; + +const router = Router(); +router.use(requireAuth); + +router.get('/', async (req, res) => { + res.json(await listStages(req.session.userId)); +}); + +router.post('/:id/execute', async (req, res) => { + try { + const account = await completeAccountStage({ + stageId: req.params.id, + userId: req.session.userId, + credentials: req.body, + }); + if (!account) return res.status(404).json({ error: 'not found' }); + res.json(account); + } catch (err) { + // completeAccountStage throws when defense-in-depth revalidation (host/port) + // fails on the merged credentials — surface that as a real client error + // rather than falling through to the generic 500 handler. + if (!err.expose) throw err; + res.status(err.status || 400).json({ error: err.message }); + } +}); + +router.delete('/:id', async (req, res) => { + const ok = await discardAccountStage({ + stageId: req.params.id, + userId: req.session.userId, + }); + if (!ok) return res.status(404).json({ error: 'not found' }); + res.status(204).end(); +}); + +export default router; diff --git a/backend/src/routes/mcpAccountStages.test.js b/backend/src/routes/mcpAccountStages.test.js new file mode 100644 index 00000000..7b054b00 --- /dev/null +++ b/backend/src/routes/mcpAccountStages.test.js @@ -0,0 +1,165 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import express from 'express'; + +const { + listStages, + completeAccountStage, + discardAccountStage, +} = vi.hoisted(() => ({ + listStages: vi.fn(), + completeAccountStage: vi.fn(), + discardAccountStage: vi.fn(), +})); + +vi.mock('../services/accountService.js', () => ({ + listStages, + completeAccountStage, + discardAccountStage, +})); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'user-1' }; + next(); + }, +})); + +import router from './mcpAccountStages.js'; + +function appWith() { + const app = express(); + app.use(express.json()); + app.use('/api/mcp-account-stages', router); + return app; +} + +async function call(app, method, path, body) { + const { createServer } = await import('node:http'); + const server = createServer(app); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const base = `http://127.0.0.1:${server.address().port}`; + try { + const response = await fetch(base + path, { + method, + headers: body ? { 'Content-Type': 'application/json' } : undefined, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await response.text(); + return { + status: response.status, + body: text ? JSON.parse(text) : null, + }; + } finally { + await new Promise(resolve => server.close(resolve)); + } +} + +beforeEach(() => { + listStages.mockReset(); + completeAccountStage.mockReset(); + discardAccountStage.mockReset(); +}); + +describe('GET /api/mcp-account-stages', () => { + it('lists stages scoped to the session user', async () => { + const rows = [{ id: 'stage-1', status: 'staged', payload: { name: 'Work' } }]; + listStages.mockResolvedValueOnce(rows); + + const response = await call(appWith(), 'GET', '/api/mcp-account-stages'); + + expect(response).toEqual({ status: 200, body: rows }); + expect(listStages).toHaveBeenCalledWith('user-1'); + }); +}); + +describe('POST /api/mcp-account-stages/:id/execute', () => { + it.each(['foreign', 'absent', 'completed'])( + '404s a %s stage', + async suffix => { + completeAccountStage.mockResolvedValueOnce(null); + + const response = await call( + appWith(), + 'POST', + `/api/mcp-account-stages/${suffix}/execute`, + { auth_pass: 'fresh-password' } + ); + + expect(response.status).toBe(404); + } + ); + + it('surfaces a defense-in-depth revalidation failure as a shaped error, not a generic 500', async () => { + completeAccountStage.mockRejectedValueOnce( + Object.assign(new Error('IMAP: Host cannot be a local address'), { status: 400, expose: true }) + ); + + const response = await call( + appWith(), + 'POST', + '/api/mcp-account-stages/stage-1/execute', + { auth_pass: 'fresh-password' } + ); + + expect(response).toEqual({ + status: 400, + body: { error: 'IMAP: Host cannot be a local address' }, + }); + }); + + it('passes fresh credentials to completion and returns the created safe account', async () => { + const credentials = { + auth_pass: 'fresh-password', + oauth_access_token: 'fresh-token', + }; + const account = { + id: 'account-1', + name: 'Work', + email_address: 'work@example.com', + }; + completeAccountStage.mockResolvedValueOnce(account); + + const response = await call( + appWith(), + 'POST', + '/api/mcp-account-stages/stage-1/execute', + credentials + ); + + expect(response).toEqual({ status: 200, body: account }); + expect(completeAccountStage).toHaveBeenCalledWith({ + stageId: 'stage-1', + userId: 'user-1', + credentials, + }); + }); +}); + +describe('DELETE /api/mcp-account-stages/:id', () => { + it('discards a scoped staged account with 204', async () => { + discardAccountStage.mockResolvedValueOnce(true); + + const response = await call( + appWith(), + 'DELETE', + '/api/mcp-account-stages/stage-1' + ); + + expect(response).toEqual({ status: 204, body: null }); + expect(discardAccountStage).toHaveBeenCalledWith({ + stageId: 'stage-1', + userId: 'user-1', + }); + }); + + it('404s an absent, foreign, completed, or discarded stage', async () => { + discardAccountStage.mockResolvedValueOnce(false); + + const response = await call( + appWith(), + 'DELETE', + '/api/mcp-account-stages/not-staged' + ); + + expect(response.status).toBe(404); + }); +}); diff --git a/backend/src/routes/mcpDeletions.js b/backend/src/routes/mcpDeletions.js new file mode 100644 index 00000000..c7cf924c --- /dev/null +++ b/backend/src/routes/mcpDeletions.js @@ -0,0 +1,25 @@ +import { Router } from 'express'; +import { requireAuth } from '../middleware/auth.js'; +import { executeDeletionBatch, unstageDeletionBatch } from '../mcp/engineAdapter.js'; + +// Session-authenticated execute/unstage endpoints for MCP deletion batches. The +// stage_deletion tool only RECORDS a batch; flipping messages.is_deleted (Mailflow's +// soft delete) happens here, behind a browser session, never from the token. Both +// operations are scoped to req.session.userId, so a user cannot execute/cancel +// another user's batch. +const router = Router(); +router.use(requireAuth); + +router.post('/:id/execute', async (req, res) => { + const n = await executeDeletionBatch(req.params.id, req.session.userId); + if (n === null) return res.status(404).json({ error: 'not found' }); + res.json({ batch_id: req.params.id, status: 'executed', deleted: n }); +}); + +router.delete('/:id', async (req, res) => { + const ok = await unstageDeletionBatch(req.params.id, req.session.userId); + if (!ok) return res.status(404).json({ error: 'not found' }); + res.status(204).end(); +}); + +export default router; diff --git a/backend/src/routes/mcpDeletions.test.js b/backend/src/routes/mcpDeletions.test.js new file mode 100644 index 00000000..1a4eff25 --- /dev/null +++ b/backend/src/routes/mcpDeletions.test.js @@ -0,0 +1,59 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import express from 'express'; + +vi.mock('../mcp/engineAdapter.js', () => ({ executeDeletionBatch: vi.fn(), unstageDeletionBatch: vi.fn() })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { req.session = { userId: 'user-1' }; next(); }, +})); +import { executeDeletionBatch, unstageDeletionBatch } from '../mcp/engineAdapter.js'; +import router from './mcpDeletions.js'; + +function appWith() { + const app = express(); + app.use(express.json()); + app.use('/api/mcp-deletions', router); + return app; +} +async function call(app, method, path) { + const { createServer } = await import('http'); + const server = createServer(app); + await new Promise((r) => server.listen(0, r)); + const base = `http://127.0.0.1:${server.address().port}`; + const res = await fetch(base + path, { method }); + const text = await res.text(); + server.close(); + return { status: res.status, body: text ? JSON.parse(text) : null }; +} + +beforeEach(() => { executeDeletionBatch.mockReset(); unstageDeletionBatch.mockReset(); }); + +describe('POST /api/mcp-deletions/:id/execute', () => { + it('soft-deletes the owner batch and reports the count, scoped to the session user', async () => { + executeDeletionBatch.mockResolvedValue(3); + const { status, body } = await call(appWith(), 'POST', '/api/mcp-deletions/b1/execute'); + expect(status).toBe(200); + expect(body).toEqual({ batch_id: 'b1', status: 'executed', deleted: 3 }); + expect(executeDeletionBatch).toHaveBeenCalledWith('b1', 'user-1'); + }); + + it('404s a batch not owned / not found (cross-user isolation, no update)', async () => { + executeDeletionBatch.mockResolvedValue(null); + const { status } = await call(appWith(), 'POST', '/api/mcp-deletions/other/execute'); + expect(status).toBe(404); + }); +}); + +describe('DELETE /api/mcp-deletions/:id', () => { + it('unstages only the owner batch', async () => { + unstageDeletionBatch.mockResolvedValue(true); + const { status } = await call(appWith(), 'DELETE', '/api/mcp-deletions/b1'); + expect(status).toBe(204); + expect(unstageDeletionBatch).toHaveBeenCalledWith('b1', 'user-1'); + }); + + it('404s when absent or not owned', async () => { + unstageDeletionBatch.mockResolvedValue(false); + const { status } = await call(appWith(), 'DELETE', '/api/mcp-deletions/nope'); + expect(status).toBe(404); + }); +}); diff --git a/backend/src/routes/rules.js b/backend/src/routes/rules.js index cbbfb8ee..66dc5d7f 100644 --- a/backend/src/routes/rules.js +++ b/backend/src/routes/rules.js @@ -1,7 +1,7 @@ import { Router } from 'express'; import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; -import { applyInboxRules, isDangerousRegex } from '../services/inboxRules.js'; +import { applyInboxRules, isDangerousRegex, toRuleMessage } from '../services/inboxRules.js'; const router = Router(); router.use(requireAuth); @@ -145,30 +145,7 @@ router.post('/run', async (req, res) => { lastId = msgResult.rows[msgResult.rows.length - 1].id; - const messages = msgResult.rows.map(row => { - let toArr = []; - try { - const raw = typeof row.to_addresses === 'string' - ? JSON.parse(row.to_addresses) - : row.to_addresses; - if (Array.isArray(raw)) { - toArr = raw.map(a => ({ email: a.address || a.email || '', name: a.name || '' })); - } - } catch { /* malformed to_addresses — leave toArr empty */ } - return { - id: row.id, - uid: row.uid, - folder: row.folder, - fromEmail: row.from_email || '', - fromName: row.from_name || '', - to: toArr, - subject: row.subject || '', - hasAttachments: !!row.has_attachments, - isRead: !!row.is_read, - is_read: !!row.is_read, - parsedHeaders: {}, - }; - }); + const messages = msgResult.rows.map(toRuleMessage); const before = messages.length; const { remaining } = await applyInboxRules(messages, account, imapMgr); diff --git a/backend/src/routes/search.js b/backend/src/routes/search.js index 9572f57d..63980648 100644 --- a/backend/src/routes/search.js +++ b/backend/src/routes/search.js @@ -1,11 +1,14 @@ import { Router } from 'express'; import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; -import { resolveAccountScope } from '../services/unifiedInbox.js'; +import { parseQuery } from '../services/search/queryParser.js'; +import { search } from '../services/search/searchService.js'; const router = Router(); router.use(requireAuth); +const ALLOWED_MODES = new Set(['lexical', 'vector', 'hybrid']); + // Simple in-memory rate limiter: 20 searches per minute per user. const searchBuckets = new Map(); setInterval(() => { @@ -31,227 +34,27 @@ function searchLimiter(req, res, next) { next(); } -// Parses a raw search string into structured operator filters and free-text -// terms. Supports from: to: subject: has: is: after: before: in:, quoted values -// (from:"John Smith"), and a leading '-' that negates either an operator -// (-from:smith) or a bare word (-invoice). Filters are a list (not a map) so -// repeated/negated operators like `from:a -from:b` are all preserved. -export function parseSearchQuery(raw) { - const filters = []; - const terms = []; - - // The leading (-?) captures optional negation. \b sits between an optional '-' - // and the operator name, so both `from:` and `-from:` match. - const opPattern = /(-?)\b(from|to|subject|has|is|after|before|in):("([^"]*)"|([\S]+))/gi; - const remaining = raw.replace(opPattern, (_, neg, key, _v, quoted, unquoted) => { - const k = key.toLowerCase(); - const v = (quoted !== undefined ? quoted : (unquoted || '')).toLowerCase().trim(); - if (v) filters.push({ key: k, value: v, negate: neg === '-' }); - return ' '; - }).trim(); - - for (const word of remaining.split(/\s+/)) { - let w = word.trim(); - if (!w || w === '-') continue; // skip blanks and a lone '-' (nothing to negate) - let negate = false; - if (w[0] === '-' && w.length > 1) { negate = true; w = w.slice(1); } - terms.push({ value: w, negate }); - } - - return { filters, terms }; -} - -// Wraps a positive condition so that when negated it also matches rows where the -// underlying columns are NULL (COALESCE(..., false) treats NULL as "not a match", -// which NOT then flips to a match — the intuitive meaning of exclusion). -function negateCond(sql) { - return `NOT COALESCE((${sql}), false)`; -} - -export function resolveSearchFolderScope(filters, folderParam = '') { - let folderScope; - let folderFuzzy = false; // in: matches loosely; the folder param is exact - - for (const f of filters) { - if (f.key !== 'in') continue; - if (f.value === 'all') { folderScope = null; } - else { folderScope = f.value; folderFuzzy = true; } - } - - if (folderScope === undefined) { - folderScope = (folderParam || '').trim() || null; - folderFuzzy = false; - } - - return { folderScope, folderFuzzy }; -} - -export function shouldExcludeTrashFromSearch(folderScope) { - return folderScope === null; -} - -export function trashFolderExclusionCondition() { - return `NOT EXISTS ( - SELECT 1 - FROM folders f - WHERE f.account_id = m.account_id - AND f.path = m.folder - AND (f.special_use = '\\Trash' - OR lower(f.name) LIKE '%trash%' - OR lower(f.name) LIKE '%deleted%') - )`; -} - -// Postgres refuses to build a tsvector larger than ~1MB of packed lexemes -// (SQLSTATE 54000), so an oversized body would 500 the whole search. Cap the -// text fed to to_tsvector at 600k chars — matching msgvault's maxFTSBodyChars -// (internal/store/dialect_pg.go) — so one huge email can't crash the query. -// Exported because slice 02's search_fts trigger caps the same way. -export const FTS_BODY_CHAR_CAP = 600000; - -// Builds the per-term free-text OR-condition: a term matches if it appears in -// the sender, the subject, the stored search_vector, or the length-capped body. -// Extracted so the body cap is a single, testable source of truth. -export function freeTextTermCondition(likeIdx, ftsIdx) { - return `( - m.from_name ILIKE $${likeIdx} - OR m.from_email ILIKE $${likeIdx} - OR m.subject ILIKE $${likeIdx} - OR m.search_vector @@ plainto_tsquery('english', $${ftsIdx}) - OR to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP})) @@ plainto_tsquery('english', $${ftsIdx}) - )`; -} - router.get('/', searchLimiter, async (req, res) => { const { q, accountId, limit = 50, offset = 0 } = req.query; const trimmed = (q || '').trim(); if (!trimmed) return res.json({ messages: [] }); if (trimmed.length > 500) return res.status(400).json({ error: 'Search query too long' }); - const accountsResult = await query( - 'SELECT id, include_in_unified_inbox FROM email_accounts WHERE user_id = $1 AND enabled = true', - [req.session.userId] - ); - const { accountIds: targetIds } = resolveAccountScope(accountsResult.rows, accountId); - if (!targetIds.length) return res.json({ messages: [] }); - - const cap = Math.max(1, Math.min(parseInt(limit) || 50, 200)); - const { filters, terms } = parseSearchQuery(trimmed); - - const conditions = []; - const params = [targetIds]; - let p = 2; - - // Folder scope. `in:` in the query wins; otherwise the client-supplied `folder` - // param (the folder the user is currently viewing) applies. `undefined` means - // no in: operator was given, so we fall back to the param below. - // folderScope === null → search all folders - // folderScope === string → restrict to that folder - - // ── Operator filters ────────────────────────────────────────────────────── - - for (const f of filters) { - // in: controls scope rather than adding a row condition; negation is - // meaningless here so it's ignored. - if (f.key === 'in') { - continue; - } - - let cond = null; - - if (f.key === 'from') { - params.push(`%${f.value}%`); - cond = `(m.from_email ILIKE $${p} OR m.from_name ILIKE $${p})`; - p++; - } else if (f.key === 'subject') { - params.push(`%${f.value}%`); - cond = `m.subject ILIKE $${p++}`; - } else if (f.key === 'to') { - // to: searches the to/cc address JSON — cast to text covers name and email - params.push(`%${f.value}%`); - cond = `(m.to_addresses::text ILIKE $${p} OR m.cc_addresses::text ILIKE $${p})`; - p++; - } else if (f.key === 'has') { - if (f.value === 'attachment' || f.value === 'attachments') cond = `m.has_attachments = true`; - } else if (f.key === 'is') { - if (f.value === 'unread') cond = `m.is_read = false`; - else if (f.value === 'read') cond = `m.is_read = true`; - else if (f.value === 'starred') cond = `m.is_starred = true`; - } else if (f.key === 'after') { - const d = new Date(f.value); - if (!isNaN(d)) { params.push(d.toISOString()); cond = `m.date >= $${p++}`; } - } else if (f.key === 'before') { - const d = new Date(f.value); - if (!isNaN(d)) { params.push(d.toISOString()); cond = `m.date < $${p++}`; } - } - - if (cond) conditions.push(f.negate ? negateCond(cond) : cond); - } - - // ── Free-text terms ─────────────────────────────────────────────────────── - // Each term must match at least one of: from, subject (ILIKE — good for names - // and partial words), or body content (FTS — good for large text with stemming). - // AND between all terms: every word must appear somewhere in the email. - // A negated term (-word) must appear nowhere. - - for (const term of terms.slice(0, 10)) { - if (term.value.length < 2) continue; // single-char terms are too broad and expensive - params.push(`%${term.value}%`); // ILIKE pattern - const likeIdx = p++; - - params.push(term.value); // raw term for plainto_tsquery - const ftsIdx = p++; - - const cond = freeTextTermCondition(likeIdx, ftsIdx); - conditions.push(term.negate ? negateCond(cond) : cond); - } - - // Require at least one real search condition before applying folder scope, so a - // bare `in:inbox` (or a lone folder param) never dumps an entire folder. - if (!conditions.length) return res.json({ messages: [], query: q }); - - // Resolve folder scope: in: operator wins; otherwise use the param. - const { folderScope, folderFuzzy } = resolveSearchFolderScope(filters, req.query.folder || ''); - if (folderScope) { - if (folderFuzzy) { - // in: — case-insensitive match on a folder named exactly that, or a - // nested folder whose path ends in it (in:sent → "Sent" or "Personal/Sent"). - // A multi-word leaf like "[Gmail]/Sent Mail" needs the quoted form in:"sent mail". - params.push(folderScope); - params.push(`%/${folderScope}`); - conditions.push(`(m.folder ILIKE $${p} OR m.folder ILIKE $${p + 1})`); - p += 2; - } else { - params.push(folderScope); - conditions.push(`m.folder = $${p++}`); - } - } else if (shouldExcludeTrashFromSearch(folderScope)) { - // Deleting moves mail into Trash, where it remains searchable by explicit - // folder queries like in:trash. Keep ordinary all-folder searches from - // resurfacing freshly-deleted messages after the optimistic UI guard expires. - conditions.push(trashFolderExclusionCondition()); - } - - const off = Math.max(0, parseInt(offset) || 0); - params.push(cap); - params.push(off); + const parsed = parseQuery(trimmed); + const mode = ALLOWED_MODES.has(req.query.mode) ? req.query.mode : 'lexical'; try { - const result = await query(` - SELECT - m.id, m.uid, m.folder, m.subject, m.from_name, m.from_email, - m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, - a.name as account_name, a.email_address as account_email, a.color as account_color - FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.account_id = ANY($1) - AND m.is_deleted = false - AND ${conditions.join('\n AND ')} - ORDER BY m.date DESC - LIMIT $${p} OFFSET $${p + 1} - `, params); - - res.json({ messages: result.rows, query: q }); + const result = await search({ + userId: req.session.userId, + accountId, + parsed, + folderParam: req.query.folder || '', + limit, + offset, + mode, + }); + // Response is a strict superset of the historical { messages, query }. + res.json({ ...result, query: q, unsupported: parsed.unsupported, errors: parsed.errors }); } catch (err) { console.error('Search error:', err); res.status(500).json({ error: 'Search failed' }); diff --git a/backend/src/routes/search.test.js b/backend/src/routes/search.test.js index 3ef54c76..bed322d2 100644 --- a/backend/src/routes/search.test.js +++ b/backend/src/routes/search.test.js @@ -1,157 +1,103 @@ -import { describe, it, expect, vi } from 'vitest'; - -// search.js opens a DB handle and registers auth middleware at import time; -// neither is exercised by the pure parser under test, so stub them out. +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { readFileSync } from 'node:fs'; +import express from 'express'; + +// The route must not hold SQL: it parses, calls the service, and JSON-frames. +// Mock inline + import the mocked binding (avoids the vi.mock hoisting TDZ trap). +vi.mock('../services/search/searchService.js', () => ({ search: vi.fn() })); +vi.mock('../middleware/auth.js', () => ({ requireAuth: (req, _res, next) => { req.session = { userId: 'u1' }; next(); } })); vi.mock('../services/db.js', () => ({ query: vi.fn() })); -vi.mock('../middleware/auth.js', () => ({ requireAuth: vi.fn() })); - -import { - parseSearchQuery, - resolveSearchFolderScope, - shouldExcludeTrashFromSearch, - trashFolderExclusionCondition, - freeTextTermCondition, - FTS_BODY_CHAR_CAP, -} from './search.js'; - -describe('parseSearchQuery', () => { - it('treats bare words as free-text terms', () => { - const { filters, terms } = parseSearchQuery('hello world'); - expect(filters).toEqual([]); - expect(terms).toEqual([ - { value: 'hello', negate: false }, - { value: 'world', negate: false }, - ]); - }); - - it('extracts positive operators and lowercases their values', () => { - const { filters, terms } = parseSearchQuery('from:Amazon subject:Invoice hello'); - expect(filters).toEqual([ - { key: 'from', value: 'amazon', negate: false }, - { key: 'subject', value: 'invoice', negate: false }, - ]); - expect(terms).toEqual([{ value: 'hello', negate: false }]); - }); - - it('supports quoted operator values with spaces', () => { - const { filters, terms } = parseSearchQuery('from:"John Smith" report'); - expect(filters).toEqual([{ key: 'from', value: 'john smith', negate: false }]); - expect(terms).toEqual([{ value: 'report', negate: false }]); - }); - - it('negates an operator when prefixed with -', () => { - const { filters, terms } = parseSearchQuery('-from:Smith report'); - expect(filters).toEqual([{ key: 'from', value: 'smith', negate: true }]); - expect(terms).toEqual([{ value: 'report', negate: false }]); - }); - - it('negates a free-text term when prefixed with -', () => { - const { filters, terms } = parseSearchQuery('report -invoice'); - expect(filters).toEqual([]); - expect(terms).toEqual([ - { value: 'report', negate: false }, - { value: 'invoice', negate: true }, - ]); - }); - it('parses the in: scope operator (in:all and named folders)', () => { - expect(parseSearchQuery('in:all invoice').filters).toEqual([ - { key: 'in', value: 'all', negate: false }, - ]); - expect(parseSearchQuery('in:Sent proposal').filters).toEqual([ - { key: 'in', value: 'sent', negate: false }, - ]); - }); - - it('preserves repeated and mixed positive/negative operators', () => { - const { filters } = parseSearchQuery('from:alice -from:bob is:unread'); - expect(filters).toEqual([ - { key: 'from', value: 'alice', negate: false }, - { key: 'from', value: 'bob', negate: true }, - { key: 'is', value: 'unread', negate: false }, - ]); +import { search } from '../services/search/searchService.js'; +import searchRouter from './search.js'; + +function makeApp() { + const app = express(); + app.use((req, _res, next) => { req.session = { userId: 'u1' }; next(); }); + app.use('/api/search', searchRouter); + return app; +} + +async function get(app, path) { + const { default: request } = await import('node:http'); + return new Promise((resolve) => { + const server = app.listen(0, () => { + const port = server.address().port; + request.get(`http://127.0.0.1:${port}${path}`, (res) => { + let body = ''; + res.on('data', c => (body += c)); + res.on('end', () => { server.close(); resolve({ status: res.statusCode, json: JSON.parse(body || '{}') }); }); + }); + }); }); +} - it('ignores a lone - so it is not treated as a negated empty term', () => { - const { filters, terms } = parseSearchQuery('report - draft'); - expect(filters).toEqual([]); - expect(terms).toEqual([ - { value: 'report', negate: false }, - { value: 'draft', negate: false }, - ]); - }); +beforeEach(() => search.mockReset()); - it('returns empty structures for blank input', () => { - expect(parseSearchQuery('')).toEqual({ filters: [], terms: [] }); - expect(parseSearchQuery(' ')).toEqual({ filters: [], terms: [] }); +describe('GET /api/search', () => { + it('short-circuits a blank query with { messages: [] } and never calls the service', async () => { + const res = await get(makeApp(), '/api/search?q='); + expect(res.status).toBe(200); + expect(res.json).toEqual({ messages: [] }); + expect(search).not.toHaveBeenCalled(); }); - it('does not treat unknown prefixes as operators', () => { - const { filters, terms } = parseSearchQuery('label:work'); - expect(filters).toEqual([]); - expect(terms).toEqual([{ value: 'label:work', negate: false }]); + it('rejects an over-500-char query with 400', async () => { + const res = await get(makeApp(), `/api/search?q=${'a'.repeat(501)}`); + expect(res.status).toBe(400); }); - it('handles all supported operator keys', () => { - const raw = 'from:a to:b subject:c has:attachment is:starred after:2024-01-01 before:2024-12-31 in:archive'; - const keys = parseSearchQuery(raw).filters.map(f => f.key); - expect(keys).toEqual(['from', 'to', 'subject', 'has', 'is', 'after', 'before', 'in']); + it('returns a superset response: messages + query + mode + page + unsupported + errors', async () => { + search.mockResolvedValue({ messages: [{ id: 'm1' }], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + const res = await get(makeApp(), '/api/search?q=larger:5M%20invoice'); + expect(res.status).toBe(200); + expect(res.json.messages).toEqual([{ id: 'm1' }]); + expect(res.json.query).toBe('larger:5M invoice'); + expect(res.json.mode).toBe('lexical'); + expect(res.json.page).toEqual({ offset: 0, limit: 50, hasMore: false }); + // larger: is recognized but unserviceable — surfaced, not silently dropped. + expect(res.json.unsupported).toEqual([{ key: 'larger', token: 'larger:5m' }]); + expect(res.json.errors).toEqual([]); }); +}); - it('scopes search to the client folder param when no in: operator is present', () => { - const { filters } = parseSearchQuery('subject:newsletter'); - expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ - folderScope: 'INBOX', - folderFuzzy: false, - }); - }); +it('projects delegation metadata in message search without an N+1 lookup', () => { + const source = readFileSync(new URL('../services/search/lexicalRepo.js', import.meta.url), 'utf8'); + expect(source).toContain("delegationJoinSql('m', 'a')"); + expect(source).toContain('mapDelegationRow(row)'); +}); - it('lets in: override the client folder param', () => { - const { filters } = parseSearchQuery('in:trash subject:newsletter'); - expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ - folderScope: 'trash', - folderFuzzy: true, - }); - }); +describe('GET /api/search mode passthrough (Phase 4 Task 7)', () => { + beforeEach(() => search.mockReset()); - it('excludes trash-like folders from ordinary all-folder searches', () => { - const { filters } = parseSearchQuery('subject:newsletter'); - const { folderScope } = resolveSearchFolderScope(filters); - expect(folderScope).toBeNull(); - expect(shouldExcludeTrashFromSearch(folderScope)).toBe(true); - expect(trashFolderExclusionCondition()).toContain('NOT EXISTS'); - expect(trashFolderExclusionCondition()).toContain('%trash%'); - expect(trashFolderExclusionCondition()).toContain('%deleted%'); + it('defaults mode to lexical when absent', async () => { + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello'); + expect(search.mock.calls[0][0].mode).toBe('lexical'); }); - it('keeps explicit folder searches eligible to find trash messages', () => { - const { filters } = parseSearchQuery('in:trash subject:newsletter'); - const { folderScope } = resolveSearchFolderScope(filters); - expect(shouldExcludeTrashFromSearch(folderScope)).toBe(false); + it('passes mode=hybrid straight through', async () => { + search.mockResolvedValue({ messages: [], mode: 'hybrid', pool_saturated: false, generation: null, page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello&mode=hybrid'); + expect(search.mock.calls[0][0].mode).toBe('hybrid'); }); -}); -describe('freeTextTermCondition (oversized-body crash hotfix)', () => { - it('caps the body tsvector so a multi-megabyte email cannot exceed the 1MB tsvector limit', () => { - const cond = freeTextTermCondition(3, 4); - // The body text is fed to to_tsvector through LEFT(..., FTS_BODY_CHAR_CAP). - // Without this cap a single oversized body raises SQLSTATE 54000 and 500s search. - expect(cond).toContain( - `to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}))` - ); - // Guard against a regression back to the uncapped expression. - expect(cond).not.toContain("to_tsvector('english', coalesce(m.body_text,'')) @@"); + it('passes mode=vector straight through', async () => { + search.mockResolvedValue({ messages: [], mode: 'vector', pool_saturated: false, generation: null, page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello&mode=vector'); + expect(search.mock.calls[0][0].mode).toBe('vector'); }); - it('pins the cap at 600000 chars (msgvault maxFTSBodyChars parity)', () => { - expect(FTS_BODY_CHAR_CAP).toBe(600000); + it('coerces an unknown mode to lexical', async () => { + search.mockResolvedValue({ messages: [], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + await get(makeApp(), '/api/search?q=hello&mode=bogus'); + expect(search.mock.calls[0][0].mode).toBe('lexical'); }); - it('still matches sender, subject, and the stored search_vector at the given param positions', () => { - const cond = freeTextTermCondition(3, 4); - expect(cond).toContain('m.from_name ILIKE $3'); - expect(cond).toContain('m.from_email ILIKE $3'); - expect(cond).toContain('m.subject ILIKE $3'); - expect(cond).toContain("m.search_vector @@ plainto_tsquery('english', $4)"); + it('returns the service response including fellBack (superset)', async () => { + search.mockResolvedValue({ messages: [], mode: 'lexical', fellBack: true, page: { offset: 0, limit: 50, hasMore: false } }); + const res = await get(makeApp(), '/api/search?q=hello&mode=hybrid'); + expect(res.json.mode).toBe('lexical'); + expect(res.json.fellBack).toBe(true); }); }); diff --git a/backend/src/routes/search.unifiedInbox.test.js b/backend/src/routes/search.unifiedInbox.test.js index 256a81c6..7dc62531 100644 --- a/backend/src/routes/search.unifiedInbox.test.js +++ b/backend/src/routes/search.unifiedInbox.test.js @@ -45,7 +45,7 @@ describe('GET /api/search unified account scope', () => { { id: 'excluded', include_in_unified_inbox: false }, ], }) - .mockResolvedValueOnce({ rows: [] }); + .mockResolvedValue({ rows: [] }); const response = await fetch(`${base}/api/search?q=invoice`); @@ -58,7 +58,7 @@ describe('GET /api/search unified account scope', () => { .mockResolvedValueOnce({ rows: [{ id: 'excluded', include_in_unified_inbox: false }], }) - .mockResolvedValueOnce({ rows: [] }); + .mockResolvedValue({ rows: [] }); const response = await fetch(`${base}/api/search?q=invoice&accountId=excluded`); diff --git a/backend/src/routes/send.js b/backend/src/routes/send.js index 3aec21da..6fcd1f8e 100644 --- a/backend/src/routes/send.js +++ b/backend/src/routes/send.js @@ -1,524 +1,111 @@ -import nodemailer from 'nodemailer'; -import { randomBytes, createHash, randomUUID } from 'crypto'; import { Router } from 'express'; -import { query } from '../services/db.js'; import { requireAuth } from '../middleware/auth.js'; -import sanitizeHtml from 'sanitize-html'; -import { sanitizeSignature, sanitizeComposeBody } from '../services/emailSanitizer.js'; -import { embedInlineDataImages } from '../utils/inlineImages.js'; -import { redisClient } from '../services/redis.js'; -import { redactEmail } from '../utils/redact.js'; -import { generateVCard } from '../utils/vcard.js'; -import { createAccountSmtpTransport } from '../services/smtpTransport.js'; import { imapManager } from '../index.js'; -import { runTransitionsForSentMessage } from '../services/gtdTransitions.js'; - -const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; - -function escapeHtml(str) { - return str.replace(/&/g, '&').replace(//g, '>'); -} - -// Map SMTP/connection errors to user-friendly messages that don't expose server internals. -function sanitizeSmtpError(err) { - const msg = err.message || ''; - if (/ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ECONNRESET|EHOSTUNREACH/i.test(msg)) { - return 'Could not connect to the mail server. Check your SMTP settings.'; - } - if (/535|534|530|invalid.?login|authentication.?fail|bad.*credentials|username.*password|password.*username/i.test(msg)) { - return 'Authentication failed. Check your email account credentials.'; - } - if (/throttl|rate.?limit|too many|4\.2\.|4\.7\.94/i.test(msg)) { - return 'The mail server is rate limiting sends. Please try again shortly.'; - } - if (/550|5\.[13]\.|reject|blacklist|spam|not.?accept/i.test(msg)) { - return 'Message was rejected by the mail server.'; - } - if (/TLS|SSL|certificate|handshake/i.test(msg)) { - return 'Secure connection to the mail server failed. Check your TLS settings.'; - } - return 'Failed to send message. Please try again.'; -} - -// Extract name and email from an RFC 5322 address string. -// Handles "Name ", "Name", bare "", and bare "email" forms. -function parseAddress(str) { - const m = str.match(/^(.+?)\s*<([^>]+)>\s*$/); - if (m) return { name: m[1].trim().replace(/^"|"$/g, '').trim(), email: m[2].trim().toLowerCase() }; - const bare = str.match(/^\s*<([^>]+)>\s*$/); - if (bare) return { name: '', email: bare[1].trim().toLowerCase() }; - return { name: '', email: str.trim().toLowerCase() }; -} - -function mapRecipientList(list) { - return (list || []).map(addr => parseAddress(addr)); -} - -function buildSentSnippet(body, bodyIsHtml) { - return bodyToPlain(body, bodyIsHtml).replace(/\s+/g, ' ').trim().substring(0, 200); -} - -function scheduleSentMetadataUpsert(account, sentFolder, mailOptions, meta) { - if (!sentFolder || !mailOptions.messageId) return; - setImmediate(async () => { - for (const delay of [3000, 10000, 20000]) { - await new Promise(r => setTimeout(r, delay)); - try { - const uid = await imapManager.findUidByMessageId(account, sentFolder, mailOptions.messageId); - if (uid) { - await imapManager.upsertSentMessageRecord(account, sentFolder, uid, meta); - return; - } - } catch (err) { - console.warn('Post-send sent metadata upsert failed:', err.message); - } - } - }); -} - -// Reject any recipient address that contains newlines, null bytes, or looks -// malformed — these are the classic email header-injection vectors. -function normalizeRecipients(list, fieldName) { - if (!Array.isArray(list)) throw Object.assign(new Error(`${fieldName} must be an array`), { status: 400 }); - return list.map((addr, i) => { - if (typeof addr !== 'string' || !addr.trim()) { - throw Object.assign(new Error(`${fieldName}[${i}] is empty or not a string`), { status: 400 }); - } - const trimmed = addr.trim(); - if (/[\r\n\0]/.test(trimmed)) { - throw Object.assign(new Error(`${fieldName}[${i}] contains invalid characters`), { status: 400 }); - } - const at = trimmed.lastIndexOf('@'); - if (at < 1 || at === trimmed.length - 1) { - throw Object.assign(new Error(`${fieldName}[${i}] is not a valid email address`), { status: 400 }); - } - return trimmed; - }); -} - -// Strip header-injection characters from single-line header values. -function sanitizeHeaderValue(value) { - if (typeof value !== 'string') return ''; - return value.replace(/[\r\n\0]/g, '').trim(); -} - -function textToHtml(text) { - return '
' + - text.split('\n').map(l => `

${escapeHtml(l) || ' '}

`).join('') + - '
'; -} - -function sigToPlainText(html) { - return sanitizeHtml(html, { allowedTags: [], allowedAttributes: {} }).trim(); -} - -function bodyToPlain(body, isHtml) { - if (!isHtml) return body; - return sanitizeHtml(body, { allowedTags: [], allowedAttributes: {} }); -} - -function bodyToHtml(body, isHtml) { - if (!isHtml) return textToHtml(body); - return sanitizeComposeBody(body); -} +import { query } from '../services/db.js'; +import { redisClient } from '../services/redis.js'; +import { sendOrEnqueue } from '../services/sendService.js'; +import * as outboxService from '../services/outboxService.js'; +import { normalizeUndoWindow } from '../services/outboxService.js'; +import { sanitizeSmtpError } from '../services/mail/smtp.js'; +import { refreshMicrosoftToken } from './oauth.js'; const router = Router(); router.use(requireAuth); - router.post('/send', async (req, res) => { - const { accountId, aliasId, to, cc = [], bcc = [], subject, body, bodyIsHtml = false, quotedBody, quotedBodyHtml, inReplyTo, references, attachments, editedSignature, forwardedAttachments, priority } = req.body; - const VALID_PRIORITIES = new Set(['high', 'normal', 'low']); - const emailPriority = VALID_PRIORITIES.has(priority) ? priority : 'normal'; - if (!accountId || !to?.length) return res.status(400).json({ error: 'accountId and to required' }); - - // Idempotency guard. The client sends a stable X-Idempotency-Key per logical send: a - // sequential retry after a lost success response returns the cached result, and a - // concurrent same-key submit is blocked by the reservation set just before delivery - // (below). Neither can produce a duplicate email. - const idempotencyKey = typeof req.headers['x-idempotency-key'] === 'string' - ? req.headers['x-idempotency-key'].slice(0, 128) - : null; - const idemKeyRedis = idempotencyKey ? `send_idem:${req.session.userId}:${idempotencyKey}` : null; - if (idemKeyRedis) { - const cached = await redisClient.get(idemKeyRedis).catch(() => null); - if (cached === '__inflight__') return res.status(409).json({ error: 'This message is already being sent.' }); - if (cached) return res.json(JSON.parse(cached)); + const { accountId, to } = req.body; + if (!accountId || !to?.length) { + return res.status(400).json({ error: 'accountId and to required' }); } - - if (attachments !== undefined) { - if (!Array.isArray(attachments)) return res.status(400).json({ error: 'attachments must be an array' }); - if (attachments.length > 100) return res.status(400).json({ error: 'Too many attachments (max 100)' }); - const totalBytes = attachments.reduce((sum, a) => sum + (typeof a.content === 'string' ? Math.ceil(a.content.length * 0.75) : 0), 0); - if (totalBytes > 26_214_400) return res.status(400).json({ error: 'Total attachment size exceeds 25 MB' }); - for (const [i, a] of attachments.entries()) { - if (typeof a.filename !== 'string' || !a.filename.trim()) return res.status(400).json({ error: `attachments[${i}].filename is required` }); - if (typeof a.content !== 'string') return res.status(400).json({ error: `attachments[${i}].content must be a base64 string` }); - } + const requestedUndo = req.body.undoSendSeconds; + if ( + requestedUndo !== undefined && + ( + typeof requestedUndo !== 'number' || + !Number.isInteger(requestedUndo) || + requestedUndo < 0 || + requestedUndo > 120 + ) + ) { + return res.status(400).json({ error: 'undoSendSeconds must be an integer from 0 to 120' }); } - if (forwardedAttachments !== undefined) { - if (!Array.isArray(forwardedAttachments)) return res.status(400).json({ error: 'forwardedAttachments must be an array' }); - for (const [i, fa] of forwardedAttachments.entries()) { - if (typeof fa.messageId !== 'string' || !UUID_RE.test(fa.messageId)) return res.status(400).json({ error: `forwardedAttachments[${i}].messageId is invalid` }); - if (typeof fa.part !== 'string' || !fa.part.trim()) return res.status(400).json({ error: `forwardedAttachments[${i}].part is required` }); - } - } - - let normalizedTo, normalizedCc, normalizedBcc; try { - normalizedTo = normalizeRecipients(to, 'to'); - normalizedCc = normalizeRecipients(cc, 'cc'); - normalizedBcc = normalizeRecipients(bcc, 'bcc'); + // Ownership remains at the HTTP boundary. Services only receive this already-scoped row. + const [accountResult, prefResult] = await Promise.all([ + query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [ + accountId, + req.session.userId, + ]), + query('SELECT preferences FROM users WHERE id = $1', [req.session.userId]), + ]); + if (!accountResult.rows.length) { + return res.status(404).json({ error: 'Account not found' }); + } + + const preferences = prefResult.rows[0]?.preferences || {}; + const undoSeconds = normalizeUndoWindow( + requestedUndo, + preferences.undoSendSeconds, + ); + const result = await sendOrEnqueue({ + ...req.body, + userId: req.session.userId, + account: accountResult.rows[0], + plaintextEmail: preferences.plaintextEmail === true, + undoSeconds, + idempotencyKey: typeof req.headers['x-idempotency-key'] === 'string' + ? req.headers['x-idempotency-key'] + : null, + }, { + query, + imapManager, + redisClient, + refreshMicrosoftToken, + outboxService, + }); + + if (result.queued) return res.status(202).json(result); + + // Preserve the REST response contract while the service exposes a richer receipt to MCP. + const response = { ok: true }; + if (result.sentCopySaved === false) response.sentCopySaved = false; + return res.json(response); } catch (err) { - return res.status(err.status || 400).json({ error: err.message }); + console.error('Send failed:', err.message); + const body = { error: err.expose ? err.message : sanitizeSmtpError(err) }; + if (err.code === 'alias_not_found') body.code = err.code; + return res.status(err.status || 500).json(body); } - const normalizedSubject = sanitizeHeaderValue(subject || ''); - - const [result, prefResult] = await Promise.all([ - query('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', [accountId, req.session.userId]), - query('SELECT preferences FROM users WHERE id = $1', [req.session.userId]), - ]); - if (!result.rows.length) return res.status(404).json({ error: 'Account not found' }); - const plaintextEmail = prefResult.rows[0]?.preferences?.plaintextEmail === true; - let account = result.rows[0]; - - // Resolve the From identity — account by default, alias if requested - let fromName = account.sender_name || account.name; - let fromEmail = account.email_address; - let fromSignature = account.signature; - let fromReplyTo = null; +}); - if (aliasId) { - const aliasResult = await query( - 'SELECT * FROM account_aliases WHERE id = $1 AND account_id = $2', - [aliasId, accountId] +router.post('/outbox/:id/cancel', async (req, res) => { + try { + const result = await outboxService.cancel( + { id: req.params.id, userId: req.session.userId }, + { query }, ); - if (aliasResult.rows.length) { - const alias = aliasResult.rows[0]; - fromName = alias.name; - fromEmail = alias.email; - fromReplyTo = alias.reply_to || null; - // null (DB default) means inherit from account; only override when alias has an explicit signature set - if (alias.signature !== null) fromSignature = alias.signature; - } - } - - // Allow the client to override the signature per-send (editedSignature === undefined means use DB value). - // Sanitize client-supplied HTML to prevent injecting scripts or tracking pixels into sent mail. - const effectiveSignature = editedSignature !== undefined - ? (editedSignature ? sanitizeSignature(editedSignature) : null) - : fromSignature; // fromSignature from DB is already sanitized on write - - // Fetch forwarded attachment content from IMAP before entering the SMTP try-block so that - // attachment errors return descriptive messages rather than being sanitized as SMTP errors. - let resolvedFwdAttachments = []; - if (forwardedAttachments?.length) { - try { - resolvedFwdAttachments = await Promise.all(forwardedAttachments.map(async (fa) => { - const msgResult = await query( - `SELECT m.uid, m.folder, m.attachments, m.account_id FROM messages m - JOIN email_accounts a ON m.account_id = a.id - WHERE m.id = $1 AND a.user_id = $2`, - [fa.messageId, req.session.userId] - ); - if (!msgResult.rows.length) throw Object.assign(new Error('Forwarded message not found'), { status: 404 }); - const msg = msgResult.rows[0]; - - const storedAtts = typeof msg.attachments === 'string' - ? JSON.parse(msg.attachments || '[]') - : (msg.attachments || []); - const att = storedAtts.find(a => a.part === fa.part); - if (!att) throw Object.assign(new Error('Attachment not found in message'), { status: 404 }); - - const accResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); - if (!accResult.rows.length) throw Object.assign(new Error('Account not found'), { status: 404 }); - - const buffer = await imapManager.fetchAttachment(accResult.rows[0], msg.uid, msg.folder, fa.part); - if (!buffer) throw Object.assign(new Error(`Could not fetch attachment: ${att.filename}`), { status: 502 }); - - return { - filename: sanitizeHeaderValue(att.filename || 'attachment'), - content: buffer, - contentType: att.type || 'application/octet-stream', - }; - })); - - // Combined size check: user uploads + forwarded content - const uploadedBytes = (attachments || []).reduce( - (sum, a) => sum + (typeof a.content === 'string' ? Math.ceil(a.content.length * 0.75) : 0), 0 - ); - const fwdBytes = resolvedFwdAttachments.reduce((sum, a) => sum + (a.content?.length || 0), 0); - if (uploadedBytes + fwdBytes > 26_214_400) { - return res.status(400).json({ error: 'Total attachment size exceeds 25 MB' }); - } - } catch (err) { - return res.status(err.status || 500).json({ error: err.message || 'Failed to fetch forwarded attachments' }); + if (result.cancelled || result.reason === 'cancelled') return res.json({ ok: true }); + if (result.reason === 'already_sent') { + return res.status(409).json({ error: 'already_sent' }); } + return res.status(404).json({ error: 'not_found' }); + } catch (err) { + console.error('Outbox cancel failed:', err.message); + return res.status(500).json({ error: 'Internal server error' }); } +}); - let delivered = false; // true once transport.sendMail has actually handed off the message +router.get('/outbox', async (req, res) => { try { - const smtp = await createAccountSmtpTransport(account); - if (smtp.error) return res.status(smtp.status).json({ error: smtp.error }); - account = smtp.account; - const transport = smtp.transport; - - // Use a stable Message-ID so the SMTP copy and any IMAP APPEND reference the same message. - const domain = fromEmail.split('@')[1] || 'mailflow.local'; - const mailOptions = { - messageId: `<${randomBytes(16).toString('hex')}@${domain}>`, - from: `${fromName} <${fromEmail}>`, - ...(fromReplyTo ? { replyTo: fromReplyTo } : {}), - to: normalizedTo.join(', '), - cc: normalizedCc.join(', ') || undefined, - bcc: normalizedBcc.join(', ') || undefined, - subject: normalizedSubject, - ...(emailPriority !== 'normal' ? { priority: emailPriority } : {}), - text: effectiveSignature - ? bodyToPlain(body, bodyIsHtml) + '\n\n-- \n' + sigToPlainText(effectiveSignature) + (quotedBody || '') - : bodyToPlain(body, bodyIsHtml) + (quotedBody || ''), - }; - - let inlineImageAttachments = []; - if (!plaintextEmail) { - const rawHtml = bodyToHtml(body, bodyIsHtml) + - (effectiveSignature - ? '
' + effectiveSignature + '
' - : '') + - (quotedBodyHtml || (quotedBody ? textToHtml(quotedBody) : '')); - const embedded = embedInlineDataImages(rawHtml); - mailOptions.html = embedded.html; - inlineImageAttachments = embedded.attachments; - } - - if (inReplyTo) { - mailOptions.inReplyTo = sanitizeHeaderValue(inReplyTo); - // Use the full prior references chain if available; fall back to just inReplyTo. - mailOptions.references = sanitizeHeaderValue(references || inReplyTo); - } - const allAttachments = [ - ...inlineImageAttachments, - ...(attachments?.length ? attachments.map(a => ({ - filename: sanitizeHeaderValue(a.filename), - content: Buffer.from(a.content, 'base64'), - contentType: typeof a.contentType === 'string' ? a.contentType : 'application/octet-stream', - })) : []), - ...resolvedFwdAttachments, - ]; - if (allAttachments.length) { - mailOptions.attachments = allAttachments; - } - - // OAuth providers (Gmail, Microsoft) save sent mail to IMAP automatically via their - // servers — skip APPEND and sync after a delay. All other accounts use direct IMAP - // APPEND so sent mail reliably appears regardless of what the SMTP server does. - const serverAutoSaves = !!account.oauth_provider; - - // For servers that don't auto-save, generate the raw MIME now so we can APPEND it. - let rawMessage = null; - if (!serverAutoSaves) { - const streamTransport = nodemailer.createTransport({ streamTransport: true, newline: 'unix' }); - const streamInfo = await streamTransport.sendMail(mailOptions); - const chunks = []; - await new Promise((resolve, reject) => { - streamInfo.message.on('data', c => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c))); - streamInfo.message.on('end', resolve); - streamInfo.message.on('error', reject); - }); - rawMessage = Buffer.concat(chunks); - } - - // Reserve the idempotency key atomically right before delivery so a concurrent - // same-key submit cannot also send (the post-send cache alone can't stop concurrent - // duplicates). Overwritten with the result on success; released in the catch only if - // delivery never happened, so a genuine retry after a pre-send failure can proceed. - if (idemKeyRedis) { - // TTL comfortably above the worst-case send (large attachment over a slow SMTP - // server) so the in-flight guard cannot lapse while this request is still running. - const reserved = await redisClient.set(idemKeyRedis, '__inflight__', { NX: true, EX: 300 }).catch(() => 'OK'); - if (reserved === null) return res.status(409).json({ error: 'This message is already being sent.' }); - } - - await transport.sendMail(mailOptions); - delivered = true; - - // Auto-learn sent recipients so they rank above inbound-only senders in autocomplete. - // Fire-and-forget — a DB error here must never affect the send response. - const allRecipients = [...normalizedTo, ...normalizedCc, ...normalizedBcc]; - if (allRecipients.length) { - const userId = req.session.userId; - const now = new Date(); - setImmediate(async () => { - try { - // Ensure the user's default address book exists - const abResult = await query( - `INSERT INTO address_books (user_id, name) VALUES ($1, 'Personal') - ON CONFLICT (user_id, name) DO UPDATE SET updated_at = NOW() - RETURNING id`, - [userId] - ); - const addressBookId = abResult.rows[0].id; - - const results = await Promise.allSettled(allRecipients.map(addr => { - const { name, email } = parseAddress(addr); - if (!email) return Promise.resolve(); - const primaryEmail = email.toLowerCase(); - const displayName = name || primaryEmail; - const uid = randomUUID(); - const emails = [{ value: primaryEmail, type: 'other', primary: true }]; - const vcard = generateVCard({ uid, displayName, emails }); - const etag = createHash('md5').update(vcard).digest('hex'); - // Upsert by (user_id, primary_email) — bump send_count and promote from is_auto. - // On conflict, preserve an existing vcard; only fill it in if the row had none. - return query(` - INSERT INTO contacts ( - address_book_id, user_id, uid, vcard, etag, - display_name, primary_email, emails, is_auto, send_count, last_sent - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, false, 1, $9) - ON CONFLICT (address_book_id, primary_email) WHERE primary_email IS NOT NULL DO UPDATE - SET send_count = contacts.send_count + 1, - last_sent = $9, - is_auto = false, - display_name = CASE WHEN contacts.is_auto THEN $6 ELSE contacts.display_name END, - vcard = COALESCE(contacts.vcard, EXCLUDED.vcard), - etag = COALESCE(contacts.etag, EXCLUDED.etag), - updated_at = NOW() - RETURNING address_book_id - `, [addressBookId, userId, uid, vcard, etag, displayName, primaryEmail, JSON.stringify(emails), now]); - })); - - const failed = results.filter(r => r.status === 'rejected'); - if (failed.length) console.warn('Contact upsert errors:', failed.map(r => r.reason?.message)); - - // Collect distinct address books actually modified (contacts may live in non-default books). - const booksToSync = new Set(); - for (const r of results) { - if (r.status === 'fulfilled' && r.value?.rows?.[0]?.address_book_id) { - booksToSync.add(r.value.rows[0].address_book_id); - } - } - if (!booksToSync.size) booksToSync.add(addressBookId); - - await Promise.all([...booksToSync].map(bookId => - query('UPDATE address_books SET sync_token = gen_random_uuid()::text, updated_at = NOW() WHERE id = $1', [bookId]) - )); - } catch (err) { - console.warn('Contact upsert setup error:', err.message); - } - }); - } - - // Get the Sent folder path (manual mapping takes priority over special_use auto-detect) - let sentFolder = account.folder_mappings?.sent || null; - if (!sentFolder) { - const folderResult = await query( - "SELECT path FROM folders WHERE account_id = $1 AND special_use = '\\Sent' LIMIT 1", - [accountId] - ); - sentFolder = folderResult.rows[0]?.path || null; - } - console.log(`Post-send: ${redactEmail(account.email_address)} sentFolder=${sentFolder} autoSaves=${serverAutoSaves}`); - - // sentCopySaved: null = not applicable (server auto-saves, or no Sent folder resolved); - // true/false = whether OUR IMAP APPEND landed the Sent copy. Surfaced to the client so - // it can warn when a delivered message could not be saved to Sent. - let sentCopySaved = null; - const sentMeta = sentFolder ? { - messageId: mailOptions.messageId, - subject: normalizedSubject, - fromName, - fromEmail, - to: mapRecipientList(normalizedTo), - cc: mapRecipientList(normalizedCc), - snippet: buildSentSnippet(body, bodyIsHtml), - date: new Date(), - } : null; - - if (sentFolder) { - if (rawMessage) { - // Non-auto-saving account: APPEND the Sent copy ourselves — exactly ONCE. IMAP - // APPEND is NOT idempotent (unlike a \Seen flag), so we must not retry: a retry - // whose first attempt merely timed out (but still lands on the server) would store - // a SECOND copy. Bound the wait so a stalled connection can't hang the response; - // the abandoned append can at worst still save the single copy. On failure, warn - // the user and schedule a fallback sync in case the append landed late. Audit [2]. - sentCopySaved = false; - try { - const { uid } = await Promise.race([ - imapManager.appendToSent(account, sentFolder, rawMessage), - new Promise((_, rej) => setTimeout(() => rej(new Error('Sent APPEND timed out')), 20000)), - ]); - sentCopySaved = true; - if (uid && sentMeta) { - await imapManager.upsertSentMessageRecord(account, sentFolder, uid, sentMeta) - .catch(err => console.warn('Sent metadata upsert failed:', err.message)); - } - setTimeout(() => { - imapManager.syncFolderOnDemand(account, sentFolder) - // Once the Sent copy is in the DB, re-run GTD transitions for its thread: a reply - // to a Todo/Someday thread means the owner acted, so that label should drop. The - // sent message reaches no other GTD hook (Sent isn't INBOX, and the tick watches - // only the state folders), so this is the only trigger. Swallow on failure — the - // next inbound sync / GTD tick self-heals. - .then(() => runTransitionsForSentMessage(imapManager, account, mailOptions.messageId) - .catch(e => console.warn(`Post-append GTD transition failed: ${e.message}`))) - .catch(e => console.error(`Post-append sync failed: ${e.message}`)); - }, 1000); - } catch (appendErr) { - console.error(`IMAP append to Sent failed for ${redactEmail(account.email_address)}/${sentFolder}: ${appendErr.message}`); - // The append may still have landed (or land shortly) — pull the folder so a - // late-completing append self-corrects the DB rather than staying invisible. - setTimeout(() => { - imapManager.syncFolderOnDemand(account, sentFolder) - .catch(e => console.error(`Post-append fallback sync failed: ${e.message}`)); - }, 8000); - } - } else { - // Server auto-saves via SMTP; seed metadata once the Sent copy is searchable. - if (sentMeta) scheduleSentMetadataUpsert(account, sentFolder, mailOptions, sentMeta); - // Server auto-saves via SMTP; just sync after a delay. Two attempts because the - // provider (e.g. Gmail) can be slow to expose the sent message; the 3s pass usually - // catches it, the 15s pass is the safety net. GTD transitions run after each: the 3s - // attempt may miss (Sent copy not yet visible → empty thread set → no-op) and the 15s - // attempt then catches it; if 3s already stripped, 15s is an idempotent no-op. - const syncAttempt = (label) => imapManager.syncFolderOnDemand(account, sentFolder) - .then(() => { - console.log(`Post-send ${label} sync done: ${redactEmail(account.email_address)}/${sentFolder}`); - return runTransitionsForSentMessage(imapManager, account, mailOptions.messageId) - .catch(e => console.warn(`Post-send ${label} GTD transition failed: ${e.message}`)); - }) - .catch(e => console.error(`Post-send ${label} sync failed: ${e.message}`)); - setTimeout(() => syncAttempt('3s'), 3000); - setTimeout(() => syncAttempt('15s'), 15000); - } - } - - const sendResult = { ok: true }; - // Surface only the problem case so existing success handling is unchanged; the UI warns - // when a delivered message could not be saved to the account's Sent folder. - if (sentCopySaved === false) sendResult.sentCopySaved = false; - // Overwrite the in-flight reservation with the final result so a retry after a lost - // response returns this instead of re-sending. - if (idemKeyRedis) redisClient.set(idemKeyRedis, JSON.stringify(sendResult), { EX: 86400 }).catch(() => {}); - res.json(sendResult); + const pending = await outboxService.listPending( + { userId: req.session.userId }, + { query }, + ); + return res.json({ pending }); } catch (err) { - console.error('Send failed:', err.message); - if (idemKeyRedis) { - if (delivered) { - // The message WAS delivered but a later step threw. Persist a DURABLE success - // result (not just the short-lived reservation) so a retry at ANY time returns it - // instead of re-running transport.sendMail — otherwise the reservation would lapse - // and the same key could deliver a second copy. - redisClient.set(idemKeyRedis, JSON.stringify({ ok: true }), { EX: 86400 }).catch(() => {}); - } else { - // Delivery never happened — release so a genuine retry after a pre-send failure - // can proceed immediately. - redisClient.del(idemKeyRedis).catch(() => {}); - } - } - res.status(500).json({ error: sanitizeSmtpError(err) }); + console.error('Outbox list failed:', err.message); + return res.status(500).json({ error: 'Internal server error' }); } }); diff --git a/backend/src/routes/send.outbox.test.js b/backend/src/routes/send.outbox.test.js new file mode 100644 index 00000000..4738a175 --- /dev/null +++ b/backend/src/routes/send.outbox.test.js @@ -0,0 +1,250 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const smtpSendMail = vi.hoisted(() => vi.fn()); +const query = vi.hoisted(() => vi.fn()); +const redisClient = vi.hoisted(() => ({ + get: vi.fn(), + set: vi.fn(), + del: vi.fn(), +})); +const outbox = vi.hoisted(() => ({ + enqueue: vi.fn(), + cancel: vi.fn(), + listPending: vi.fn(), +})); +const imapManager = vi.hoisted(() => ({ + appendToSent: vi.fn(), + upsertSentMessageRecord: vi.fn(), + syncFolderOnDemand: vi.fn(), + findUidByMessageId: vi.fn(), +})); + +vi.mock('nodemailer', async (importOriginal) => { + const actual = await importOriginal(); + return { + default: { + ...actual.default, + createTransport: vi.fn(options => ( + options?.streamTransport + ? actual.default.createTransport(options) + : { sendMail: smtpSendMail } + )), + }, + }; +}); +vi.mock('../services/db.js', () => ({ query })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'user-1' }; + next(); + }, +})); +vi.mock('./oauth.js', () => ({ refreshMicrosoftToken: vi.fn() })); +vi.mock('../services/encryption.js', () => ({ decrypt: vi.fn(() => 'smtp-password') })); +vi.mock('../services/redis.js', () => ({ redisClient })); +vi.mock('../services/hostValidation.js', () => ({ + resolveForConnection: vi.fn().mockResolvedValue({ + host: '203.0.113.25', + servername: 'smtp.example.com', + }), +})); +vi.mock('../services/connectionPolicy.js', () => ({ + getConnectionPolicy: vi.fn().mockResolvedValue({ + allowPrivateHosts: false, + allowInsecureTls: false, + }), +})); +vi.mock('../index.js', () => ({ imapManager })); +vi.mock('../services/gtdTransitions.js', () => ({ + runTransitionsForSentMessage: vi.fn().mockResolvedValue(undefined), +})); +vi.mock('../services/outboxService.js', async (importOriginal) => ({ + ...(await importOriginal()), + ...outbox, +})); + +import express from 'express'; +import sendRoutes from './send.js'; + +const ACCOUNT_ID = '11111111-1111-4111-8111-111111111111'; +const SEND_AT = new Date('2026-07-28T12:00:30.000Z'); +const ACCOUNT_ROW = { + id: ACCOUNT_ID, + user_id: 'user-1', + email_address: 'sender@example.com', + name: 'Sender', + sender_name: null, + signature: null, + auth_user: 'sender@example.com', + auth_pass: 'encrypted', + oauth_provider: null, + smtp_host: 'smtp.example.com', + smtp_port: 587, + smtp_tls: 'STARTTLS', + imap_skip_tls_verify: false, + folder_mappings: { sent: 'Sent' }, +}; + +function buildApp() { + const app = express(); + app.use(express.json()); + app.use('/api/mail', sendRoutes); + return app; +} + +function post(base, path, body, headers = {}) { + return fetch(`${base}/api/mail${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(body), + }); +} + +function compose(overrides = {}) { + return { + accountId: ACCOUNT_ID, + to: ['Recipient '], + subject: 'Undo send', + body: 'hello', + ...overrides, + }; +} + +describe('undo-send REST routes', () => { + let server; + let base; + + beforeAll(async () => { + await new Promise(resolve => { + server = buildApp().listen(0, resolve); + }); + base = `http://127.0.0.1:${server.address().port}`; + }); + + afterAll(async () => { + if (server) await new Promise(resolve => server.close(resolve)); + }); + + beforeEach(() => { + query.mockReset(); + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2')) { + return { rows: [ACCOUNT_ROW] }; + } + if (sql.includes('SELECT preferences FROM users')) { + return { rows: [{ preferences: { plaintextEmail: false } }] }; + } + if (sql.includes('INSERT INTO address_books')) return { rows: [{ id: 'book-1' }] }; + if (sql.includes('INSERT INTO contacts')) return { rows: [{ address_book_id: 'book-1' }] }; + return { rows: [] }; + }); + redisClient.get.mockReset().mockResolvedValue(null); + redisClient.set.mockReset().mockResolvedValue('OK'); + redisClient.del.mockReset().mockResolvedValue(1); + smtpSendMail.mockReset().mockResolvedValue({ messageId: '' }); + imapManager.appendToSent.mockReset().mockResolvedValue({ uid: null }); + imapManager.upsertSentMessageRecord.mockReset().mockResolvedValue(undefined); + imapManager.syncFolderOnDemand.mockReset().mockResolvedValue(undefined); + imapManager.findUidByMessageId.mockReset().mockResolvedValue(null); + outbox.enqueue.mockReset().mockResolvedValue({ + outbox_id: 'outbox-1', + send_at: SEND_AT, + undo_seconds: 30, + }); + outbox.cancel.mockReset(); + outbox.listPending.mockReset(); + }); + + it('sends immediately with the unchanged 200 response when undoSendSeconds is 0', async () => { + const response = await post(base, '/send', compose({ undoSendSeconds: 0 })); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + expect(smtpSendMail).toHaveBeenCalledTimes(1); + expect(outbox.enqueue).not.toHaveBeenCalled(); + }); + + it('queues for 30 seconds without calling sendMail', async () => { + const response = await post( + base, + '/send', + compose({ undoSendSeconds: 30 }), + { 'X-Idempotency-Key': 'queued-1' }, + ); + + expect(response.status).toBe(202); + expect(await response.json()).toEqual({ + queued: true, + outboxId: 'outbox-1', + sendAt: SEND_AT.toISOString(), + undoSeconds: 30, + }); + expect(smtpSendMail).not.toHaveBeenCalled(); + expect(outbox.enqueue).toHaveBeenCalledWith( + expect.objectContaining({ idempotencyKey: 'queued-1', undoSeconds: 30 }), + expect.any(Object), + ); + }); + + it('sends immediately when the field and user preference are both absent', async () => { + const response = await post(base, '/send', compose()); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + expect(smtpSendMail).toHaveBeenCalledTimes(1); + expect(outbox.enqueue).not.toHaveBeenCalled(); + }); + + it.each([-1, 121, 1.5, '30'])('rejects invalid undoSendSeconds value %j', async (value) => { + const response = await post(base, '/send', compose({ undoSendSeconds: value })); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ error: 'undoSendSeconds must be an integer from 0 to 120' }); + expect(smtpSendMail).not.toHaveBeenCalled(); + expect(outbox.enqueue).not.toHaveBeenCalled(); + }); + + it('maps cancel results to 200, 409, and 404 without changing user scope', async () => { + outbox.cancel + .mockResolvedValueOnce({ cancelled: true }) + .mockResolvedValueOnce({ cancelled: false, reason: 'already_sent' }) + .mockResolvedValueOnce({ cancelled: false, reason: 'not_found' }); + + const cancelled = await post(base, '/outbox/outbox-1/cancel', {}); + expect(cancelled.status).toBe(200); + expect(await cancelled.json()).toEqual({ ok: true }); + + const sent = await post(base, '/outbox/outbox-2/cancel', {}); + expect(sent.status).toBe(409); + expect(await sent.json()).toEqual({ error: 'already_sent' }); + + const missing = await post(base, '/outbox/outbox-3/cancel', {}); + expect(missing.status).toBe(404); + expect(await missing.json()).toEqual({ error: 'not_found' }); + + expect(outbox.cancel.mock.calls.map(([input]) => input)).toEqual([ + { id: 'outbox-1', userId: 'user-1' }, + { id: 'outbox-2', userId: 'user-1' }, + { id: 'outbox-3', userId: 'user-1' }, + ]); + }); + + it('lists pending rows through the session user scope', async () => { + const pending = [{ + id: 'outbox-1', + subject: 'Undo send', + to_preview: ['recipient@example.com'], + send_at: SEND_AT.toISOString(), + }]; + outbox.listPending.mockResolvedValue(pending); + + const response = await fetch(`${base}/api/mail/outbox`); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ pending }); + expect(outbox.listPending).toHaveBeenCalledWith( + { userId: 'user-1' }, + expect.any(Object), + ); + }); +}); diff --git a/backend/src/routes/send.test.js b/backend/src/routes/send.test.js new file mode 100644 index 00000000..40b1a49a --- /dev/null +++ b/backend/src/routes/send.test.js @@ -0,0 +1,185 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const smtpSendMail = vi.hoisted(() => vi.fn()); +const query = vi.hoisted(() => vi.fn()); +const redisClient = vi.hoisted(() => ({ + get: vi.fn(), + set: vi.fn(), + del: vi.fn(), +})); +const imapManager = vi.hoisted(() => ({ + appendToSent: vi.fn(), + upsertSentMessageRecord: vi.fn(), + syncFolderOnDemand: vi.fn(), + findUidByMessageId: vi.fn(), +})); + +vi.mock('nodemailer', async (importOriginal) => { + const actual = await importOriginal(); + return { + default: { + ...actual.default, + createTransport: vi.fn(options => ( + options?.streamTransport + ? actual.default.createTransport(options) + : { sendMail: smtpSendMail } + )), + }, + }; +}); +vi.mock('../services/db.js', () => ({ query })); +vi.mock('../middleware/auth.js', () => ({ + requireAuth: (req, _res, next) => { + req.session = { userId: 'user-1' }; + next(); + }, +})); +vi.mock('./oauth.js', () => ({ refreshMicrosoftToken: vi.fn() })); +vi.mock('../services/encryption.js', () => ({ decrypt: vi.fn(() => 'smtp-password') })); +vi.mock('../services/redis.js', () => ({ redisClient })); +vi.mock('../services/hostValidation.js', () => ({ + resolveForConnection: vi.fn().mockResolvedValue({ + host: '203.0.113.25', + servername: 'smtp.example.com', + }), +})); +vi.mock('../services/connectionPolicy.js', () => ({ + getConnectionPolicy: vi.fn().mockResolvedValue({ + allowPrivateHosts: false, + allowInsecureTls: false, + }), +})); +vi.mock('../index.js', () => ({ imapManager })); +vi.mock('../services/gtdTransitions.js', () => ({ + runTransitionsForSentMessage: vi.fn().mockResolvedValue(undefined), +})); + +import express from 'express'; +import sendRoutes from './send.js'; + +const ACCOUNT_ID = '11111111-1111-4111-8111-111111111111'; +const ACCOUNT_ROW = { + id: ACCOUNT_ID, + user_id: 'user-1', + email_address: 'sender@example.com', + name: 'Sender', + sender_name: null, + signature: null, + auth_user: 'sender@example.com', + auth_pass: 'encrypted', + oauth_provider: null, + smtp_host: 'smtp.example.com', + smtp_port: 587, + smtp_tls: 'STARTTLS', + imap_skip_tls_verify: false, + folder_mappings: { sent: 'Sent' }, +}; + +function buildApp() { + const app = express(); + app.use(express.json()); + app.use('/api/mail', sendRoutes); + return app; +} + +function postSend(base, { headers = {}, body = {} } = {}) { + return fetch(`${base}/api/mail/send`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify({ + accountId: ACCOUNT_ID, + to: ['Recipient '], + subject: 'Golden send', + body: 'hello', + ...body, + }), + }); +} + +describe('POST /api/mail/send — golden parity', () => { + let server; + let base; + + beforeAll(async () => { + await new Promise(resolve => { + server = buildApp().listen(0, resolve); + }); + base = `http://127.0.0.1:${server.address().port}`; + }); + + afterAll(async () => { + if (server) await new Promise(resolve => server.close(resolve)); + }); + + beforeEach(() => { + query.mockReset(); + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2')) { + return { rows: [ACCOUNT_ROW] }; + } + if (sql.includes('SELECT preferences FROM users')) { + return { rows: [{ preferences: { plaintextEmail: false } }] }; + } + if (sql.includes('INSERT INTO address_books')) return { rows: [{ id: 'book-1' }] }; + if (sql.includes('INSERT INTO contacts')) return { rows: [{ address_book_id: 'book-1' }] }; + return { rows: [] }; + }); + redisClient.get.mockReset().mockResolvedValue(null); + redisClient.set.mockReset().mockResolvedValue('OK'); + redisClient.del.mockReset().mockResolvedValue(1); + smtpSendMail.mockReset().mockResolvedValue({ messageId: '' }); + imapManager.appendToSent.mockReset().mockResolvedValue({ uid: null }); + imapManager.upsertSentMessageRecord.mockReset().mockResolvedValue(undefined); + imapManager.syncFolderOnDemand.mockReset().mockResolvedValue(undefined); + imapManager.findUidByMessageId.mockReset().mockResolvedValue(null); + }); + + it('returns the exact current success response shape', async () => { + const response = await postSend(base); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + }); + + it('passes through sentCopySaved:false when the single Sent APPEND fails', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + imapManager.appendToSent.mockRejectedValueOnce(new Error('append unavailable')); + + const response = await postSend(base); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true, sentCopySaved: false }); + expect(imapManager.appendToSent).toHaveBeenCalledTimes(1); + }); + + it('returns 409 for an in-flight idempotency key without sending', async () => { + redisClient.get.mockResolvedValueOnce('__inflight__'); + + const response = await postSend(base, { + headers: { 'X-Idempotency-Key': 'same-send' }, + }); + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ error: 'This message is already being sent.' }); + expect(smtpSendMail).not.toHaveBeenCalled(); + }); + + it('replays a completed cached result without sending', async () => { + redisClient.get.mockResolvedValueOnce(JSON.stringify({ ok: true, sentCopySaved: false })); + + const response = await postSend(base, { + headers: { 'X-Idempotency-Key': 'completed-send' }, + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true, sentCopySaved: false }); + expect(smtpSendMail).not.toHaveBeenCalled(); + }); + + it('sanitizes SMTP failures instead of exposing raw server details', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + smtpSendMail.mockRejectedValueOnce(new Error('ECONNREFUSED smtp.secret.internal:2525')); + + const response = await postSend(base); + expect(response.status).toBe(500); + expect(await response.json()).toEqual({ + error: 'Could not connect to the mail server. Check your SMTP settings.', + }); + }); +}); diff --git a/backend/src/services/accountFields.js b/backend/src/services/accountFields.js new file mode 100644 index 00000000..bec8b09d --- /dev/null +++ b/backend/src/services/accountFields.js @@ -0,0 +1,20 @@ +import { sanitizeSignature } from './emailSanitizer.js'; + +// Fields safe to return to the client — matches the GET list, excludes credentials and tokens +export const SAFE_FIELDS = [ + 'id', 'name', 'sender_name', 'email_address', 'color', 'protocol', + 'imap_host', 'imap_port', 'imap_skip_tls_verify', + 'smtp_host', 'smtp_port', 'smtp_tls', + 'auth_user', 'oauth_provider', 'enabled', + 'include_in_unified_inbox', + 'last_sync', 'sync_error', 'sort_order', 'folder_mappings', + 'signature', 'created_at', 'categorization_enabled', + 'gtd_enabled', 'gtd_folders', +]; + +export function safeAccount(row) { + const obj = Object.fromEntries(SAFE_FIELDS.map(k => [k, row[k]])); + // Sanitize on read so legacy values stored before the write-time sanitizer are safe + if (obj.signature) obj.signature = sanitizeSignature(obj.signature); + return obj; +} diff --git a/backend/src/services/accountFields.test.js b/backend/src/services/accountFields.test.js new file mode 100644 index 00000000..925a5af3 --- /dev/null +++ b/backend/src/services/accountFields.test.js @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { SAFE_FIELDS, safeAccount } from './accountFields.js'; + +describe('safeAccount', () => { + it('returns only SAFE_FIELDS and sanitizes the signature on read', () => { + const row = Object.fromEntries(SAFE_FIELDS.map(field => [field, `${field}-value`])); + row.signature = 'Safe'; + row.auth_pass = 'secret'; + row.oauth_access_token = 'access'; + row.oauth_refresh_token = 'refresh'; + row.user_id = 'user-1'; + + const account = safeAccount(row); + + expect(Object.keys(account)).toEqual(SAFE_FIELDS); + expect(account.signature).toBe('Safe'); + expect(account).not.toHaveProperty('auth_pass'); + expect(account).not.toHaveProperty('oauth_access_token'); + expect(account).not.toHaveProperty('oauth_refresh_token'); + expect(account).not.toHaveProperty('user_id'); + }); + + it('keeps every safe field in parity with the REST GET account column list', () => { + const accountsRoute = readFileSync( + fileURLToPath(new URL('../routes/accounts.js', import.meta.url)), + 'utf8' + ); + const select = accountsRoute.match( + /router\.get\('\/'[\s\S]*?`SELECT\s+([\s\S]*?)\s+FROM email_accounts/ + ); + expect(select).not.toBeNull(); + + const selectedFields = select[1] + .split(',') + .map(field => field.trim()) + .filter(Boolean); + + expect(SAFE_FIELDS.every(field => selectedFields.includes(field))).toBe(true); + }); +}); diff --git a/backend/src/services/accountService.js b/backend/src/services/accountService.js new file mode 100644 index 00000000..6647f763 --- /dev/null +++ b/backend/src/services/accountService.js @@ -0,0 +1,225 @@ +import { query } from './db.js'; +import { imapManager } from '../index.js'; +import { encrypt } from './encryption.js'; +import { hasHeaderInjectionChars, sanitizeSignature } from './emailSanitizer.js'; +import { validateHost } from './hostValidation.js'; +import { getConnectionPolicy } from './connectionPolicy.js'; +import { safeAccount } from './accountFields.js'; +import { createKeyedSerializer } from '../utils/keyedSerializer.js'; + +// Serialize an account's reconnect triggers so a rapid settings change (e.g. a +// gtd_enabled double-toggle) can't fire two overlapping disconnect→connect chains — +// connectAccount's in-progress guard would drop the second and leave the GTD sync +// tick armed inconsistently with the final DB value. Queued per account id. +const reconnectQueue = createKeyedSerializer(); + +export const ALLOWED_IMAP_PORTS = new Set([143, 993]); +export const ALLOWED_SMTP_PORTS = new Set([465, 587]); + +export function validatePort(port, allowed) { + const n = Number(port); + if (!Number.isInteger(n) || n < 1 || n > 65535) { + return `Port ${port} is not a valid port number`; + } + // When private/local hosts are explicitly allowed (e.g. Proton Mail Bridge on 1143/1025), + // skip the whitelist — the operator has already opted into unrestricted host access. + if (process.env.ALLOW_PRIVATE_IMAP_HOSTS === 'true') return null; + if (!allowed.has(n)) { + return `Port ${port} is not allowed. Allowed: ${[...allowed].join(', ')}`; + } + return null; +} + +async function validateAccountFields(fields) { + const { + name, sender_name = null, email_address, + imap_host, imap_port = 993, + smtp_host, smtp_port = 587, + } = fields; + + if (!name || !email_address) return { error: 'Name and email required', status: 400 }; + if (hasHeaderInjectionChars(name) || hasHeaderInjectionChars(email_address)) { + return { error: 'Name and email address cannot contain control characters', status: 400 }; + } + if (sender_name && hasHeaderInjectionChars(sender_name)) { + return { error: 'Sender name cannot contain control characters', status: 400 }; + } + + const policy = await getConnectionPolicy(); + + if (imap_host) { + const err = (await validateHost(imap_host, { allowPrivate: policy.allowPrivateHosts })) + || (!policy.allowNonstandardPorts && validatePort(imap_port, ALLOWED_IMAP_PORTS)); + if (err) return { error: `IMAP: ${err}`, status: 400 }; + } + if (smtp_host) { + const err = (await validateHost(smtp_host, { allowPrivate: policy.allowPrivateHosts })) + || (!policy.allowNonstandardPorts && validatePort(smtp_port, ALLOWED_SMTP_PORTS)); + if (err) return { error: `SMTP: ${err}`, status: 400 }; + } + return null; +} + +export async function createAccount({ userId, fields }) { + const validation = await validateAccountFields(fields); + if (validation) return validation; + + const { + name, sender_name = null, email_address, color = '#6366f1', protocol = 'imap', + imap_host, imap_port = 993, imap_skip_tls_verify = false, + smtp_host, smtp_port = 587, smtp_tls = 'STARTTLS', + auth_user, auth_pass, + oauth_provider, oauth_access_token, oauth_refresh_token, + signature = null + } = fields; + + try { + const result = await query(` + INSERT INTO email_accounts ( + user_id, name, sender_name, email_address, color, protocol, + imap_host, imap_port, imap_tls, imap_skip_tls_verify, smtp_host, smtp_port, smtp_tls, + auth_user, auth_pass, oauth_provider, oauth_access_token, oauth_refresh_token, + signature + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) + RETURNING * + `, [ + userId, name, sender_name || null, email_address, color, protocol, + imap_host, imap_port, Number(imap_port) % 1000 === 993, !!imap_skip_tls_verify, smtp_host, smtp_port, smtp_tls, + auth_user, encrypt(auth_pass), oauth_provider, encrypt(oauth_access_token), encrypt(oauth_refresh_token), + sanitizeSignature(signature) || null + ]); + + const account = result.rows[0]; + + // Immediately try to connect — needs full credentials from DB row + if (protocol === 'imap') { + imapManager.connectAccount(account).catch(console.error); + } + + return { account: safeAccount(account) }; + } catch (err) { + console.error(err); + return { error: 'Failed to add account', status: 500 }; + } +} + +export function reconcileConnectionState({ id, updates, before, updated }) { + const gtdFoldersChanged = !!before?.gtdFoldersChanged; + const isDisabling = 'enabled' in updates && !updates.enabled; + const needsReconnect = !isDisabling && ( + 'enabled' in updates || + 'auth_user' in updates || + 'auth_pass' in updates || + 'imap_host' in updates || + 'imap_port' in updates || + 'imap_tls' in updates || + 'imap_skip_tls_verify' in updates || + 'gtd_enabled' in updates || + gtdFoldersChanged + ); + + // Both branches queue through the per-account serializer so overlapping settings + // changes (e.g. a rapid gtd_enabled double-toggle) apply their connection-state + // effects in order, never as two overlapping chains. + if (isDisabling) { + reconnectQueue(id, () => imapManager.disconnectAccount(id)) + .catch(err => console.error(`Failed to disconnect account ${id} after disable:`, err.message)); + } else if (needsReconnect && updated.protocol === 'imap' && updated.enabled) { + reconnectQueue(id, () => + imapManager.disconnectAccount(id) + .then(() => query('SELECT * FROM email_accounts WHERE id = $1', [id])) + .then(r => { if (r.rows.length) return imapManager.connectAccount(r.rows[0]); }) + ).catch(err => console.error(`Failed to reconnect account ${id} after update:`, err.message)); + } +} + +const STAGED_SECRET_FIELDS = [ + 'auth_pass', + 'oauth_access_token', + 'oauth_refresh_token', +]; + +export async function stageAccount({ userId, payload }) { + for (const key of STAGED_SECRET_FIELDS) { + if (Object.prototype.hasOwnProperty.call(payload, key)) { + throw new Error(`Staged account payload cannot contain ${key}`); + } + } + + const validation = await validateAccountFields(payload); + if (validation) return validation; + + const stagedPayload = { + ...payload, + signature: sanitizeSignature(payload.signature) || null, + }; + const result = await query( + `INSERT INTO mcp_account_stages (user_id, payload) + VALUES ($1, $2) + RETURNING *`, + [userId, stagedPayload] + ); + return result.rows[0]; +} + +export async function listStages(userId) { + const result = await query( + `SELECT id, status, payload, created_at, completed_account_id + FROM mcp_account_stages + WHERE user_id = $1 AND status = 'staged' + ORDER BY created_at`, + [userId] + ); + return result.rows; +} + +export async function completeAccountStage({ stageId, userId, credentials }) { + const stageResult = await query( + `SELECT id, payload + FROM mcp_account_stages + WHERE id = $1 AND user_id = $2 AND status = 'staged'`, + [stageId, userId] + ); + if (!stageResult.rows.length) return null; + + const freshCredentials = {}; + for (const key of STAGED_SECRET_FIELDS) { + if (Object.prototype.hasOwnProperty.call(credentials || {}, key)) { + freshCredentials[key] = credentials[key]; + } + } + const fields = { + ...stageResult.rows[0].payload, + ...freshCredentials, + }; + + // createAccount performs the same host/port/header validation again before + // writing, providing defense in depth if a staged payload was tampered with. + const created = await createAccount({ userId, fields }); + if (created.error) { + throw Object.assign(new Error(created.error), { + status: created.status, + expose: true, + }); + } + + await query( + `UPDATE mcp_account_stages + SET status = 'completed', completed_account_id = $1 + WHERE id = $2 AND user_id = $3 AND status = 'staged' + RETURNING id`, + [created.account.id, stageId, userId] + ); + return created.account; +} + +export async function discardAccountStage({ stageId, userId }) { + const result = await query( + `UPDATE mcp_account_stages + SET status = 'discarded' + WHERE id = $1 AND user_id = $2 AND status = 'staged' + RETURNING id`, + [stageId, userId] + ); + return result.rows.length > 0; +} diff --git a/backend/src/services/accountService.test.js b/backend/src/services/accountService.test.js new file mode 100644 index 00000000..904eaaa4 --- /dev/null +++ b/backend/src/services/accountService.test.js @@ -0,0 +1,302 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { + query, + encrypt, + validateHost, + getConnectionPolicy, + connectAccount, + disconnectAccount, +} = vi.hoisted(() => ({ + query: vi.fn(), + encrypt: vi.fn(value => value ? `encrypted:${value}` : value), + validateHost: vi.fn(), + getConnectionPolicy: vi.fn(), + connectAccount: vi.fn(), + disconnectAccount: vi.fn(), +})); + +vi.mock('./db.js', () => ({ query })); +vi.mock('./encryption.js', () => ({ encrypt })); +vi.mock('./hostValidation.js', () => ({ validateHost })); +vi.mock('./connectionPolicy.js', () => ({ getConnectionPolicy })); +vi.mock('../index.js', () => ({ + imapManager: { connectAccount, disconnectAccount }, +})); + +import { SAFE_FIELDS } from './accountFields.js'; +import { + completeAccountStage, + createAccount, + discardAccountStage, + listStages, + reconcileConnectionState, + stageAccount, +} from './accountService.js'; + +const validFields = { + name: 'Primary', + sender_name: 'Sender', + email_address: 'sender@example.com', + protocol: 'imap', + imap_host: 'imap.example.com', + imap_port: 993, + smtp_host: 'smtp.example.com', + smtp_port: 587, + auth_user: 'sender@example.com', + auth_pass: 'password', +}; + +beforeEach(() => { + query.mockReset(); + encrypt.mockClear(); + validateHost.mockReset().mockResolvedValue(null); + getConnectionPolicy.mockReset().mockResolvedValue({ + allowPrivateHosts: false, + allowNonstandardPorts: false, + }); + connectAccount.mockReset().mockResolvedValue(undefined); + disconnectAccount.mockReset().mockResolvedValue(undefined); +}); + +describe('createAccount', () => { + it('rejects an invalid host', async () => { + validateHost.mockResolvedValueOnce('Host cannot be a local address'); + + await expect(createAccount({ userId: 'user-1', fields: validFields })).resolves.toEqual({ + error: 'IMAP: Host cannot be a local address', + status: 400, + }); + expect(query).not.toHaveBeenCalled(); + }); + + it('rejects an invalid port', async () => { + const result = await createAccount({ + userId: 'user-1', + fields: { ...validFields, imap_port: 25 }, + }); + + expect(result).toEqual({ + error: 'IMAP: Port 25 is not allowed. Allowed: 143, 993', + status: 400, + }); + expect(query).not.toHaveBeenCalled(); + }); + + it('encrypts auth_pass, starts a fire-and-forget connection, and returns only safe fields', async () => { + const inserted = { + id: 'account-1', + user_id: 'user-1', + ...validFields, + auth_pass: 'encrypted:password', + oauth_access_token: 'secret-access', + oauth_refresh_token: 'secret-refresh', + signature: 'Sender', + enabled: true, + }; + query.mockResolvedValueOnce({ rows: [inserted] }); + connectAccount.mockReturnValueOnce(new Promise(() => {})); + + const result = await createAccount({ userId: 'user-1', fields: validFields }); + + expect(encrypt).toHaveBeenCalledWith('password'); + expect(query.mock.calls[0][1]).toContain('encrypted:password'); + expect(connectAccount).toHaveBeenCalledWith(inserted); + expect(Object.keys(result.account)).toEqual(SAFE_FIELDS); + expect(result.account.signature).toBe('Sender'); + expect(result.account).not.toHaveProperty('auth_pass'); + }); +}); + +describe('reconcileConnectionState', () => { + it('disconnects only when disabling', async () => { + reconcileConnectionState({ + id: 'disable-1', + updates: { enabled: false }, + before: { gtdFoldersChanged: false }, + updated: { protocol: 'imap', enabled: false }, + }); + + await vi.waitFor(() => expect(disconnectAccount).toHaveBeenCalledWith('disable-1')); + expect(query).not.toHaveBeenCalled(); + expect(connectAccount).not.toHaveBeenCalled(); + }); + + it.each([ + ['enabled', { enabled: true }, false], + ['auth_user', { auth_user: 'new-user' }, false], + ['auth_pass', { auth_pass: 'new-pass' }, false], + ['imap_host', { imap_host: 'new.example.com' }, false], + ['imap_port', { imap_port: 143 }, false], + ['imap_tls', { imap_tls: false }, false], + ['imap_skip_tls_verify', { imap_skip_tls_verify: true }, false], + ['gtd_enabled', { gtd_enabled: true }, false], + ['gtdFoldersChanged', {}, true], + ])('reconnects for the %s trigger', async (name, updates, gtdFoldersChanged) => { + const id = `trigger-${name}`; + const fresh = { id, protocol: 'imap', enabled: true }; + query.mockResolvedValueOnce({ rows: [fresh] }); + + reconcileConnectionState({ + id, + updates, + before: { gtdFoldersChanged }, + updated: fresh, + }); + + await vi.waitFor(() => expect(connectAccount).toHaveBeenCalledWith(fresh)); + expect(disconnectAccount).toHaveBeenCalledWith(id); + }); + + it('does nothing when no reconnect field changed', async () => { + reconcileConnectionState({ + id: 'noop-1', + updates: { color: '#ffffff' }, + before: { gtdFoldersChanged: false }, + updated: { protocol: 'imap', enabled: true }, + }); + await Promise.resolve(); + + expect(disconnectAccount).not.toHaveBeenCalled(); + expect(connectAccount).not.toHaveBeenCalled(); + expect(query).not.toHaveBeenCalled(); + }); + + it('gives disabling precedence over another reconnect trigger', async () => { + reconcileConnectionState({ + id: 'disable-precedence', + updates: { enabled: false, auth_user: 'new-user' }, + before: { gtdFoldersChanged: false }, + updated: { protocol: 'imap', enabled: false }, + }); + + await vi.waitFor(() => expect(disconnectAccount).toHaveBeenCalledWith('disable-precedence')); + expect(query).not.toHaveBeenCalled(); + expect(connectAccount).not.toHaveBeenCalled(); + }); +}); + +describe('staged accounts', () => { + it.each([ + ['auth_pass', ''], + ['oauth_access_token', null], + ['oauth_refresh_token', false], + ])('hard-rejects payloads containing the secret key %s', async (key, value) => { + const payload = { ...validFields }; + delete payload.auth_pass; + payload[key] = value; + await expect(stageAccount({ + userId: 'user-1', + payload, + })).rejects.toThrow(`Staged account payload cannot contain ${key}`); + expect(query).not.toHaveBeenCalled(); + }); + + it('validates staged hosts before inserting', async () => { + validateHost.mockResolvedValueOnce('Host cannot be a local address'); + const payload = { ...validFields }; + delete payload.auth_pass; + + await expect(stageAccount({ + userId: 'user-1', + payload, + })).resolves.toEqual({ + error: 'IMAP: Host cannot be a local address', + status: 400, + }); + expect(query).not.toHaveBeenCalled(); + }); + + it('validates staged ports before inserting', async () => { + const payload = { ...validFields, smtp_port: 25 }; + delete payload.auth_pass; + const result = await stageAccount({ + userId: 'user-1', + payload, + }); + + expect(result).toEqual({ + error: 'SMTP: Port 25 is not allowed. Allowed: 465, 587', + status: 400, + }); + expect(query).not.toHaveBeenCalled(); + }); + + it('lists only staged rows scoped to the user', async () => { + const rows = [{ id: 'stage-1', status: 'staged' }]; + query.mockResolvedValueOnce({ rows }); + + await expect(listStages('user-1')).resolves.toEqual(rows); + expect(query).toHaveBeenCalledWith( + expect.stringContaining("user_id = $1 AND status = 'staged'"), + ['user-1'] + ); + }); + + it('completes a scoped stage by merging fresh credentials and marking it completed', async () => { + const payload = { ...validFields }; + delete payload.auth_pass; + const inserted = { + id: 'account-from-stage', + user_id: 'user-1', + ...payload, + auth_pass: 'encrypted:fresh-password', + enabled: true, + }; + query + .mockResolvedValueOnce({ rows: [{ id: 'stage-1', payload }] }) + .mockResolvedValueOnce({ rows: [inserted] }) + .mockResolvedValueOnce({ rows: [{ id: 'stage-1', status: 'completed' }] }); + + const result = await completeAccountStage({ + stageId: 'stage-1', + userId: 'user-1', + credentials: { auth_pass: 'fresh-password' }, + }); + + expect(result.id).toBe('account-from-stage'); + expect(encrypt).toHaveBeenCalledWith('fresh-password'); + expect(query.mock.calls[0]).toEqual([ + expect.stringContaining("user_id = $2 AND status = 'staged'"), + ['stage-1', 'user-1'], + ]); + expect(query.mock.calls[1][0]).toContain('INSERT INTO email_accounts'); + expect(query.mock.calls[2]).toEqual([ + expect.stringContaining("SET status = 'completed', completed_account_id = $1"), + ['account-from-stage', 'stage-1', 'user-1'], + ]); + }); + + it('returns null when completing a missing, foreign, or non-staged row', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(completeAccountStage({ + stageId: 'foreign-stage', + userId: 'user-1', + credentials: { auth_pass: 'fresh-password' }, + })).resolves.toBeNull(); + expect(query).toHaveBeenCalledTimes(1); + }); + + it('discards only a scoped stage that is still staged', async () => { + query.mockResolvedValueOnce({ rows: [{ id: 'stage-1' }] }); + + await expect(discardAccountStage({ + stageId: 'stage-1', + userId: 'user-1', + })).resolves.toBe(true); + expect(query).toHaveBeenCalledWith( + expect.stringContaining("status = 'staged'"), + ['stage-1', 'user-1'] + ); + }); + + it('cannot discard a missing, foreign, completed, or discarded stage', async () => { + query.mockResolvedValueOnce({ rows: [] }); + + await expect(discardAccountStage({ + stageId: 'not-staged', + userId: 'user-1', + })).resolves.toBe(false); + }); +}); diff --git a/backend/src/services/aiProvider.js b/backend/src/services/aiProvider.js index 5ae2714b..cffcd3a7 100644 --- a/backend/src/services/aiProvider.js +++ b/backend/src/services/aiProvider.js @@ -5,6 +5,7 @@ import { validateHost } from './hostValidation.js'; import { createRequestSignal, parseJson, readLimited, readSseData, sanitizeText } from './aiHttp.js'; import { completeCodexText, streamCodexResponses } from './openaiCodexResponses.js'; import { getCodexAccess, getCodexStatus } from './openaiCodexAuth.js'; +import { applyEmbedDefaults } from './embeddings/config.js'; export const AI_PROVIDER_API_KEY = 'api-key'; export const AI_PROVIDER_CHATGPT = 'chatgpt'; @@ -26,6 +27,14 @@ export class AiProviderError extends Error { } } +export function buildEmbeddingsConfig(body = {}, existing = null, encryptFn = encrypt) { + const resolved = applyEmbedDefaults(body); + resolved.apiKey = body.apiKey && body.apiKey !== MASKED_API_KEY + ? encryptFn(body.apiKey) + : (existing?.apiKey || null); + return resolved; +} + function cleanString(value) { return typeof value === 'string' ? value.trim() : ''; } @@ -41,6 +50,9 @@ export function normalizeAiConfig(raw = {}) { ? raw.chatgptConfig : {}; const provider = PROVIDERS.has(raw.provider) ? raw.provider : AI_PROVIDER_API_KEY; + const embeddings = raw.embeddings && typeof raw.embeddings === 'object' + ? applyEmbedDefaults(raw.embeddings) + : null; return { enabled: raw.enabled !== false, provider, @@ -56,6 +68,7 @@ export function normalizeAiConfig(raw = {}) { compose: raw.features?.compose !== false, summarize: raw.features?.summarize !== false, }, + ...(embeddings ? { embeddings } : {}), }; } @@ -67,6 +80,12 @@ function publicConfig(config) { ...config.apiKeyConfig, apiKey: config.apiKeyConfig.apiKey ? MASKED_API_KEY : '', }, + ...(config.embeddings ? { + embeddings: { + ...config.embeddings, + apiKey: config.embeddings.apiKey ? MASKED_API_KEY : '', + }, + } : {}), }; } @@ -166,9 +185,13 @@ export function createAiProvider({ const storedKey = typeof incomingKey === 'string' && incomingKey && incomingKey !== MASKED_API_KEY ? encryptFn(incomingKey) : (existing?.apiKeyConfig.apiKey || null); + const embeddings = input.embeddings + ? buildEmbeddingsConfig(input.embeddings, existing?.embeddings, encryptFn) + : existing?.embeddings; const config = normalizeAiConfig({ ...input, apiKeyConfig: { ...incomingApi, apiKey: storedKey }, + ...(embeddings ? { embeddings } : {}), }); if (config.enabled && config.provider === AI_PROVIDER_API_KEY @@ -186,6 +209,17 @@ export function createAiProvider({ const hostError = await validateHostFn(hostname, { allowPrivate: policy.allowPrivateHosts }); if (hostError) throw new AiProviderError(`API base URL: ${hostError}`, { status: 400 }); } + if (config.embeddings?.endpoint) { + let hostname; + try { + hostname = new URL(config.embeddings.endpoint).hostname; + } catch { + throw new AiProviderError('Invalid embeddings endpoint URL', { status: 400 }); + } + const policy = await getConnectionPolicyFn(); + const hostError = await validateHostFn(hostname, { allowPrivate: policy.allowPrivateHosts }); + if (hostError) throw new AiProviderError(`Embeddings endpoint: ${hostError}`, { status: 400 }); + } await queryFn( `INSERT INTO system_settings (key, value, updated_at) VALUES ('ai_config', $1, NOW()) diff --git a/backend/src/services/backgroundJobs.js b/backend/src/services/backgroundJobs.js new file mode 100644 index 00000000..799b05e0 --- /dev/null +++ b/backend/src/services/backgroundJobs.js @@ -0,0 +1,26 @@ +import { query } from './db.js'; + +// Upsert a drainer's progress row. One row per (kind, account); global jobs +// pass accountId = null. started_at is set once and preserved across updates. +export async function upsertJob({ kind, accountId = null, state, processed = 0, total = 0, lastError = null }) { + await query(` + INSERT INTO background_jobs (kind, account_id, state, processed, total, last_error, started_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, NOW(), NOW()) + ON CONFLICT (kind, COALESCE(account_id::text, '')) DO UPDATE SET + state = EXCLUDED.state, + processed = EXCLUDED.processed, + total = EXCLUDED.total, + last_error = EXCLUDED.last_error, + started_at = COALESCE(background_jobs.started_at, EXCLUDED.started_at), + updated_at = NOW() + `, [kind, accountId, state, processed, total, lastError]); +} + +export async function listJobs() { + const { rows } = await query( + `SELECT kind, account_id, state, processed, total, last_error, started_at, updated_at + FROM background_jobs + ORDER BY updated_at DESC` + ); + return rows; +} diff --git a/backend/src/services/backgroundJobs.test.js b/backend/src/services/backgroundJobs.test.js new file mode 100644 index 00000000..f3ecfd24 --- /dev/null +++ b/backend/src/services/backgroundJobs.test.js @@ -0,0 +1,25 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('./db.js', () => ({ query: vi.fn() })); +import { query } from './db.js'; +import { upsertJob, listJobs } from './backgroundJobs.js'; + +beforeEach(() => query.mockReset()); + +describe('backgroundJobs', () => { + it('upserts one row per (kind, account) via the COALESCE unique index', async () => { + query.mockResolvedValue({ rows: [] }); + await upsertJob({ kind: 'fts_backfill', state: 'running', processed: 10, total: 100 }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('INSERT INTO background_jobs'); + expect(sql).toContain("ON CONFLICT (kind, COALESCE(account_id::text, ''))"); + expect(sql).toContain('started_at = COALESCE(background_jobs.started_at, EXCLUDED.started_at)'); + expect(params).toEqual(['fts_backfill', null, 'running', 10, 100, null]); + }); + + it('lists jobs newest-first', async () => { + query.mockResolvedValue({ rows: [{ kind: 'fts_backfill' }] }); + const jobs = await listJobs(); + expect(jobs).toEqual([{ kind: 'fts_backfill' }]); + expect(query.mock.calls[0][0]).toContain('ORDER BY updated_at DESC'); + }); +}); diff --git a/backend/src/services/bodyBackfill.js b/backend/src/services/bodyBackfill.js new file mode 100644 index 00000000..11e6cbea --- /dev/null +++ b/backend/src/services/bodyBackfill.js @@ -0,0 +1,182 @@ +import { query } from './db.js'; +import { providerProfile } from './imapManager.js'; + +// Body-materialization drainer. Fills messages.body_text/body_html over IMAP so weight-D +// lexical search and embeddings have material to work with, WITHOUT growing the imapManager +// singleton (README invariant) and WITHOUT touching throttle-hostile providers. ALL policy — +// the scan predicate, batching, pacing, provider gating, session cap, quiet-window +// backpressure, and the per-account circuit breaker — lives here. The singleton exposes only +// the narrow fetchBodiesForMessages() entry point this module calls. +// +// Shape is copied from imapManager.startSnippetIndexer, but this module is standalone: its +// per-account run guard and circuit-breaker state are module-level (not instance fields), and +// its IMAP/quiet-window/progress/clock dependencies are injected so it unit-tests against a +// fake fetchBodies with no real IMAP. + +const BATCH_SIZE = 50; // messages fetched per batch +const MIN_BATCH_DELAY_MS = 2000; // floor between batches (provider batchDelay may raise it) +const MAX_BATCHES_PER_RUN = 200; // session cap: 10,000 messages per run, then resume next kick +const QUIET_WINDOW_MS = 8000; // pause extra when the user opened a message this recently +const MAX_CONSECUTIVE_ERRORS = 3; // abort the run after this many failing batches in a row +const BACKOFF_BASE_MS = 10 * 60 * 1000; // first circuit-breaker back-off +const BACKOFF_MAX_MS = 2 * 60 * 60 * 1000; // cap + +// Per-account run guard and circuit breaker. Module-level so a single process runs at most one +// body drainer per account (the snippet indexer uses instance fields for the same purpose). +const running = new Set(); // accountId +const backoff = new Map(); // accountId -> { failures, until } + +// Test seam: reset module state between unit tests. +export function resetBodyBackfillState() { + running.clear(); + backoff.clear(); +} + +const realSleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Core drainer. Dependencies are injected so tests supply a fake fetchBodies: +// fetchBodies(accountId, ids) -> Promise<{ fetched: number }> (writes bodies; throws on a +// connection-level failure so this loop can back off) +// getLastActivityMs(accountId) -> number (ms timestamp of the user's last live body open) +// isSnippetIndexerRunning(accountId) -> boolean (the snippet indexer does its own sustained +// BODY[]-ish fetches over an independent connection; running +// both loops for the same account at once double-books IMAP +// connections against the same provider limit) +// upsertJobProgress({ accountId, kind, processed, total, state }) -> Promise (this is +// backgroundJobs.upsertJob's own shape — callers inject it +// directly; the drainer speaks its vocabulary, no adapter) +// sleep(ms) -> Promise (injectable for fast, deterministic tests) +// now() -> number (injectable clock) +export async function startBodyBackfill(account, deps) { + const { + fetchBodies, + getLastActivityMs = () => 0, + isSnippetIndexerRunning = () => false, + upsertJobProgress = async () => {}, + sleep = realSleep, + now = Date.now, + } = deps; + + const cfg = providerProfile(account); + if (!cfg.bodyBackfill) return; // provider gate (Gmail/PurelyMail/etc.) + if (running.has(account.id)) return; // one drainer per account + const bo = backoff.get(account.id); + if (bo && now() < bo.until) return; // circuit breaker open + running.add(account.id); + + const batchDelay = Math.max(cfg.batchDelay || 0, MIN_BATCH_DELAY_MS); + const kind = 'body_backfill'; + let batchCount = 0; + let failed = false; + + try { + // Denominator + starting coverage for progress. total is the whole eligible mailbox so the + // progress bar reflects real coverage, not just this run. + const countRes = await query( + `SELECT count(*)::int AS total, + count(*) FILTER (WHERE body_text IS NOT NULL OR body_html IS NOT NULL)::int AS have_body + FROM messages WHERE account_id = $1 AND is_deleted = false`, + [account.id] + ); + const total = countRes.rows[0].total; + let haveBody = countRes.rows[0].have_body; + if (total - haveBody <= 0) { + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'done' }); + return; + } + + // Keyset by id ascending: id is a non-null, unique UUID, so the cursor advances past every + // row exactly once per run — including rows whose body cannot be fetched (they stay NULL but + // are not re-selected this run, so the run always terminates). Recency ordering is + // intentionally not used: body coverage is a background quality lever, and the embedding + // backstop picks up late arrivals regardless (README D4). + let cursorId = null; + let consecutiveErrors = 0; + + while (true) { + // Stop if the account was deleted mid-run. + const alive = await query('SELECT id FROM email_accounts WHERE id = $1', [account.id]); + if (!alive.rows.length) return; + + // Defer to the snippet indexer rather than double-book BODY[] IMAP connections for the + // same account — checked every iteration (not just at kick time) so a session already in + // progress backs off cleanly the moment the indexer starts, instead of racing it. + if (isSnippetIndexerRunning(account.id)) { + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'deferred' }); + return; + } + + if (batchCount >= MAX_BATCHES_PER_RUN) { + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'paused' }); + return; + } + + const batchRes = await query( + `SELECT id FROM messages + WHERE account_id = $1 AND body_text IS NULL AND body_html IS NULL AND is_deleted = false + AND ($2::uuid IS NULL OR id > $2) + ORDER BY id ASC + LIMIT $3`, + [account.id, cursorId, BATCH_SIZE] + ); + if (!batchRes.rows.length) break; // drained + const ids = batchRes.rows.map((r) => r.id); + + try { + const { fetched } = await fetchBodies(account.id, ids); + haveBody += fetched; + cursorId = ids[ids.length - 1]; // advance only on success + batchCount++; + consecutiveErrors = 0; + } catch { + consecutiveErrors++; + await sleep(cfg.errorDelay || batchDelay); + if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { + failed = true; + // Write a terminal 'error' state (ftsBackfill.js convention) so background_jobs + // doesn't keep showing a stale 'running' row for an account that's actually stalled. + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'error' }); + return; // finally trips the circuit breaker + } + continue; // retry the same batch (cursor not advanced); fetchBodies reopens IMAP + } + + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'running' }); + + // Quiet-window backpressure: pause longer when the user is actively opening messages so + // background BODY[] traffic doesn't compete with click-time fetches. + const quietFor = now() - getLastActivityMs(account.id); + const extraDelay = quietFor < QUIET_WINDOW_MS ? QUIET_WINDOW_MS - quietFor : 0; + await sleep(batchDelay + extraDelay); + } + + await upsertJobProgress({ accountId: account.id, kind, processed: haveBody, total, state: 'done' }); + } catch { + failed = true; + } finally { + running.delete(account.id); + // Circuit breaker: a run that failed without draining a single batch (e.g. the provider + // refuses the extra connection) backs off exponentially so we stop reopening competing IMAP + // connections. Any progress — or a clean finish — clears the backoff. + if (failed && batchCount === 0) { + const failures = (backoff.get(account.id)?.failures || 0) + 1; + const delay = Math.min(BACKOFF_BASE_MS * 2 ** (failures - 1), BACKOFF_MAX_MS); + backoff.set(account.id, { failures, until: now() + delay }); + } else { + backoff.delete(account.id); + } + } +} + +// Convenience wrapper for the reindex route: builds the injected deps from the imapManager +// singleton and the injected progress sink (backgroundJobs.upsertJob — the drainer already +// emits its shape), then runs the core drainer. Kept here (not in the route) so the wiring is +// unit-testable and bodyBackfill.js never imports backgroundJobs.js directly. +export function startAccountBodyBackfill(account, imapManager, upsertJobProgress) { + return startBodyBackfill(account, { + fetchBodies: (accountId, ids) => imapManager.fetchBodiesForMessages(accountId, ids), + getLastActivityMs: (accountId) => imapManager.lastUserActivity.get(accountId) || 0, + isSnippetIndexerRunning: (accountId) => imapManager.snippetIndexerRunning.has(accountId), + upsertJobProgress, + }); +} diff --git a/backend/src/services/bodyBackfill.test.js b/backend/src/services/bodyBackfill.test.js new file mode 100644 index 00000000..3b3da49f --- /dev/null +++ b/backend/src/services/bodyBackfill.test.js @@ -0,0 +1,290 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('./db.js', () => ({ query: vi.fn() })); +vi.mock('./imapManager.js', () => ({ providerProfile: vi.fn() })); + +import { query } from './db.js'; +import { providerProfile } from './imapManager.js'; +import { startBodyBackfill, startAccountBodyBackfill, resetBodyBackfillState } from './bodyBackfill.js'; + +const account = { id: 'acct-1', imap_host: 'imap.icloud.com' }; +const enabledProfile = { bodyBackfill: true, batchDelay: 2000, errorDelay: 100 }; + +// Instant, assertable deps so the drainer runs with no wall-clock waits. +function baseDeps(overrides = {}) { + return { + fetchBodies: vi.fn().mockResolvedValue({ fetched: 0 }), + getLastActivityMs: () => 0, + isSnippetIndexerRunning: () => false, + upsertJobProgress: vi.fn().mockResolvedValue(), + sleep: vi.fn().mockResolvedValue(), + now: () => 1_000_000, + ...overrides, + }; +} + +// Route the mocked query by SQL shape. `batches` is an array of id-arrays returned in order; +// once exhausted it returns an empty batch (drained). +function mockQuery({ total = 0, haveBody = 0, batches = [], alive = true } = {}) { + let i = 0; + query.mockImplementation((sql) => { + if (/AS total/.test(sql)) return Promise.resolve({ rows: [{ total, have_body: haveBody }] }); + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: alive ? [{ id: 'acct-1' }] : [] }); + if (/body_text IS NULL/.test(sql)) { + const rows = (batches[i++] || []).map((id) => ({ id })); + return Promise.resolve({ rows }); + } + return Promise.resolve({ rows: [] }); + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + resetBodyBackfillState(); +}); + +describe('startBodyBackfill — provider gate', () => { + it('no-ops for a provider with bodyBackfill:false and never queries', async () => { + providerProfile.mockReturnValue({ bodyBackfill: false }); + const deps = baseDeps(); + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).not.toHaveBeenCalled(); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('startBodyBackfill — nothing to do', () => { + it('reports complete and fetches nothing when coverage is already full', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 100, haveBody: 100 }); + const deps = baseDeps(); + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).not.toHaveBeenCalled(); + expect(deps.upsertJobProgress).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'body_backfill', state: 'done', processed: 100, total: 100 }) + ); + }); +}); + +describe('startBodyBackfill — batching + keyset + progress', () => { + it('drains 120 messages in batches of 50/50/20 and advances the cursor', async () => { + providerProfile.mockReturnValue(enabledProfile); + const b1 = Array.from({ length: 50 }, (_, k) => `a${k}`); + const b2 = Array.from({ length: 50 }, (_, k) => `b${k}`); + const b3 = Array.from({ length: 20 }, (_, k) => `c${k}`); + mockQuery({ total: 120, haveBody: 0, batches: [b1, b2, b3] }); + const deps = baseDeps({ fetchBodies: vi.fn().mockImplementation((_id, ids) => Promise.resolve({ fetched: ids.length })) }); + + await startBodyBackfill(account, deps); + + expect(deps.fetchBodies).toHaveBeenCalledTimes(3); + expect(deps.fetchBodies).toHaveBeenNthCalledWith(1, 'acct-1', b1); + expect(deps.fetchBodies).toHaveBeenNthCalledWith(2, 'acct-1', b2); + expect(deps.fetchBodies).toHaveBeenNthCalledWith(3, 'acct-1', b3); + // Cursor advances to the last id of the previous batch (keyset param $2). + const scanCalls = query.mock.calls.filter(([sql]) => /body_text IS NULL/.test(sql)); + expect(scanCalls[0][1][1]).toBeNull(); // first scan: cursor null + expect(scanCalls[1][1][1]).toBe('a49'); // second scan: last id of batch 1 + expect(scanCalls[2][1][1]).toBe('b49'); // third scan: last id of batch 2 + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', processed: 120, total: 120 }) + ); + }); +}); + +describe('startBodyBackfill — session cap + resume', () => { + it('pauses after MAX_BATCHES_PER_RUN batches, then resumes on the next call', async () => { + providerProfile.mockReturnValue(enabledProfile); + // Never drains: every scan returns 50 fresh ids so the run hits the batch cap. + let n = 0; + query.mockImplementation((sql) => { + if (/AS total/.test(sql)) return Promise.resolve({ rows: [{ total: 100000, have_body: 0 }] }); + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/body_text IS NULL/.test(sql)) return Promise.resolve({ rows: Array.from({ length: 50 }, () => ({ id: `id-${n++}` })) }); + return Promise.resolve({ rows: [] }); + }); + const deps = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 50 }) }); + + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).toHaveBeenCalledTimes(200); // MAX_BATCHES_PER_RUN + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'paused' }) + ); + + // Resume: the run guard was released in finally, so a second call proceeds (not gated). + await startBodyBackfill(account, deps); + expect(deps.fetchBodies).toHaveBeenCalledTimes(400); + }); +}); + +describe('startBodyBackfill — single drainer per account', () => { + it('a second concurrent call for the same account is a no-op', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 50, haveBody: 0, batches: [['x1']] }); + const deps = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }) }); + + // Both are invoked synchronously in the same tick: the first adds the account to the + // run guard before its first await, so the second sees it and returns immediately. + const p1 = startBodyBackfill(account, deps); + const p2 = startBodyBackfill(account, deps); + await Promise.all([p1, p2]); + + expect(deps.fetchBodies).toHaveBeenCalledTimes(1); // only p1 did work + }); +}); + +describe('startBodyBackfill — circuit breaker trip + recovery', () => { + it('trips after 3 consecutive batch errors, blocks retries in the window, recovers after it', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 100, haveBody: 0, batches: [['x1'], ['x1'], ['x1']] }); + let clock = 1_000_000; + const failing = baseDeps({ + fetchBodies: vi.fn().mockRejectedValue(new Error('Command failed')), + now: () => clock, + }); + + // Trip: 3 consecutive rejections abort the run and open the breaker. + await startBodyBackfill(account, failing); + expect(failing.fetchBodies).toHaveBeenCalledTimes(3); + // A terminal 'error' progress row is written so background_jobs doesn't keep showing a + // stale 'running' state for an account that's actually stalled. + expect(failing.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ kind: 'body_backfill', state: 'error' }) + ); + + // Blocked: a call inside the back-off window no-ops (breaker open). + const blocked = baseDeps({ fetchBodies: vi.fn(), now: () => clock }); + await startBodyBackfill(account, blocked); + expect(blocked.fetchBodies).not.toHaveBeenCalled(); + + // Recover: advance past the back-off window; a fresh call runs again. + clock += 3 * 60 * 60 * 1000; // beyond BACKOFF_MAX_MS (2h) + mockQuery({ total: 100, haveBody: 0, batches: [['y1']] }); + const recovered = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }), now: () => clock }); + await startBodyBackfill(account, recovered); + expect(recovered.fetchBodies).toHaveBeenCalledTimes(1); + }); +}); + +describe('startBodyBackfill — quiet-window backpressure', () => { + it('adds the remaining quiet window to the post-batch delay when the user is active', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 1, haveBody: 0, batches: [['m1']] }); + const NOW = 5_000_000; + const deps = baseDeps({ + fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }), + now: () => NOW, + getLastActivityMs: () => NOW - 2000, // user opened a message 2s ago + }); + + await startBodyBackfill(account, deps); + + // batchDelay = max(2000, 2000) = 2000; quiet window 8000 - 2000 elapsed = 6000 extra. + expect(deps.sleep).toHaveBeenCalledWith(8000); + }); +}); + +describe('startBodyBackfill — defers to the snippet indexer', () => { + it('gates the kick: never fetches when the indexer is already running', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 100, haveBody: 0, batches: [['x1']] }); + const deps = baseDeps({ isSnippetIndexerRunning: () => true }); + + await startBodyBackfill(account, deps); + + expect(deps.fetchBodies).not.toHaveBeenCalled(); + expect(deps.upsertJobProgress).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'body_backfill', state: 'deferred' }) + ); + }); + + it('re-checks every iteration: defers mid-session once the indexer starts, without tripping the breaker', async () => { + providerProfile.mockReturnValue(enabledProfile); + const b1 = ['a1']; + const b2 = ['b1']; + mockQuery({ total: 100, haveBody: 0, batches: [b1, b2] }); + let indexerRunning = false; + const deps = baseDeps({ + fetchBodies: vi.fn().mockImplementation(() => { + indexerRunning = true; // simulate the snippet indexer kicking in right after batch 1 + return Promise.resolve({ fetched: 1 }); + }), + isSnippetIndexerRunning: () => indexerRunning, + }); + + await startBodyBackfill(account, deps); + + // Batch 1 runs (indexer not yet running); batch 2's scan never happens because the + // re-check at the top of the next iteration sees the indexer is now running and bails. + expect(deps.fetchBodies).toHaveBeenCalledTimes(1); + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'deferred' }) + ); + + // Deferring is not a failure: the circuit breaker must not be armed by it. + const resumed = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 1 }) }); + mockQuery({ total: 100, haveBody: 1, batches: [['c1']] }); + await startBodyBackfill(account, resumed); + expect(resumed.fetchBodies).toHaveBeenCalledTimes(1); + }); +}); + +describe('startBodyBackfill — forward progress', () => { + it('drains to completion even when every batch reports {fetched:0} (resolved, not rejected)', async () => { + providerProfile.mockReturnValue(enabledProfile); + const b1 = Array.from({ length: 50 }, (_, k) => `a${k}`); + const b2 = Array.from({ length: 10 }, (_, k) => `b${k}`); + mockQuery({ total: 60, haveBody: 0, batches: [b1, b2] }); + const deps = baseDeps({ fetchBodies: vi.fn().mockResolvedValue({ fetched: 0 }) }); + + await startBodyBackfill(account, deps); + + // The cursor still advances on a resolved (non-throwing) call even when it wrote nothing — + // e.g. every message in the batch legitimately has no fetchable body — so the run still + // reaches completion instead of looping forever or being mistaken for a failure. + expect(deps.fetchBodies).toHaveBeenCalledTimes(2); + expect(deps.upsertJobProgress).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', processed: 0, total: 60 }) + ); + }); +}); + +describe('startAccountBodyBackfill — adapter wiring', () => { + it('wires imapManager.fetchBodiesForMessages and lastUserActivity into the core drainer', async () => { + vi.useFakeTimers(); + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 1, haveBody: 0, batches: [['m1']] }); + const fakeImap = { + fetchBodiesForMessages: vi.fn().mockResolvedValue({ fetched: 1 }), + lastUserActivity: new Map([['acct-1', Date.now()]]), + snippetIndexerRunning: new Set(), + }; + const upsert = vi.fn().mockResolvedValue(); + + const p = startAccountBodyBackfill(account, fakeImap, upsert); + await vi.runAllTimersAsync(); // flush the real setTimeout used by the default sleep + await p; + + expect(fakeImap.fetchBodiesForMessages).toHaveBeenCalledWith('acct-1', ['m1']); + expect(upsert).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'body_backfill', accountId: 'acct-1' }) + ); + vi.useRealTimers(); + }); + + it('wires imapManager.snippetIndexerRunning so an already-running indexer defers the kick', async () => { + providerProfile.mockReturnValue(enabledProfile); + mockQuery({ total: 1, haveBody: 0, batches: [['m1']] }); + const fakeImap = { + fetchBodiesForMessages: vi.fn().mockResolvedValue({ fetched: 1 }), + lastUserActivity: new Map(), + snippetIndexerRunning: new Set(['acct-1']), + }; + const upsert = vi.fn().mockResolvedValue(); + + await startAccountBodyBackfill(account, fakeImap, upsert); + + expect(fakeImap.fetchBodiesForMessages).not.toHaveBeenCalled(); + expect(upsert).toHaveBeenCalledWith(expect.objectContaining({ state: 'deferred' })); + }); +}); diff --git a/backend/src/services/composeSessionLifecycle.js b/backend/src/services/composeSessionLifecycle.js new file mode 100644 index 00000000..a83afa20 --- /dev/null +++ b/backend/src/services/composeSessionLifecycle.js @@ -0,0 +1,1163 @@ +import { isDeepStrictEqual } from 'node:util'; +import { createHash, randomUUID } from 'node:crypto'; +import { + MAX_COMPOSE_ATTACHMENT_BYTES, + MAX_COMPOSE_ATTACHMENTS, + composeSessionError, + meaningfulComposeSession, + normalizeReplyAllRecipients, +} from './composeSessionModel.js'; +import * as composeSessionService from './composeSessionService.js'; +import * as draftService from './draftService.js'; +import * as sendService from './sendService.js'; +import * as outboxService from './outboxService.js'; +import { sanitizeHeaderValue } from './mail/addresses.js'; +import { sanitizeEmail } from './emailSanitizer.js'; +import { UUID_RE } from '../utils/validation.js'; + +const VALID_MODES = new Set(['new', 'reply', 'reply_all', 'forward']); +const VALID_PRIORITIES = new Set(['low', 'normal', 'high']); + +function normalizedList(value) { + if (!Array.isArray(value)) return []; + return value + .filter(item => typeof item === 'string') + .map(item => item.trim()) + .filter(Boolean); +} + +function nullableString(value) { + return typeof value === 'string' ? value : null; +} + +function normalizedForwardedAttachments(value) { + if (!Array.isArray(value)) return []; + return value.map(attachment => ({ + messageId: attachment.messageId, + part: attachment.part, + })); +} + +function attachmentFingerprint(attachment) { + return { + id: attachment.id, + filename: attachment.filename, + contentType: attachment.contentType, + byteCount: Number(attachment.byteCount), + }; +} + +function invalidAttachmentContent() { + return Object.assign(new Error('Compose attachment content is missing or unsupported'), { + code: 'invalid_compose_attachment_content', + status: 500, + expose: false, + }); +} + +function isCanonicalBase64(value) { + if (value.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(value)) return false; + return Buffer.from(value, 'base64').toString('base64') === value; +} + +function attachmentInput(attachment) { + let content = attachment.content; + if (Buffer.isBuffer(content)) { + content = content.toString('base64'); + } else if (content instanceof Uint8Array) { + content = Buffer.from(content).toString('base64'); + } else if (typeof content !== 'string' || !isCanonicalBase64(content)) { + throw invalidAttachmentContent(); + } + return { + filename: attachment.filename, + content, + contentType: typeof attachment.contentType === 'string' + ? attachment.contentType + : 'application/octet-stream', + }; +} + +function attachmentContentSha256(attachment) { + const encoded = attachmentInput(attachment).content; + return createHash('sha256').update(Buffer.from(encoded, 'base64')).digest('hex'); +} + +function jsonValue(value, fallback) { + if (value == null) return fallback; + if (typeof value !== 'string') return value; + try { + return JSON.parse(value); + } catch { + return fallback; + } +} + +function recipientStrings(value) { + const recipients = jsonValue(value, []); + if (!Array.isArray(recipients)) return []; + return recipients.map((recipient) => { + if (typeof recipient === 'string') return sanitizeHeaderValue(recipient); + if (!recipient || typeof recipient !== 'object') return ''; + const email = sanitizeHeaderValue(recipient.email || recipient.address || ''); + const name = sanitizeHeaderValue(recipient.name || ''); + if (!email) return ''; + return name ? `${name} <${email}>` : email; + }).filter(Boolean); +} + +function referenceStrings(value) { + if (Array.isArray(value)) return value.map(sanitizeHeaderValue).filter(Boolean); + if (typeof value !== 'string' || !value.trim()) return []; + const parsed = jsonValue(value, null); + if (Array.isArray(parsed)) return parsed.map(sanitizeHeaderValue).filter(Boolean); + const messageIds = value.match(/<[^>]+>/g); + return messageIds?.length ? messageIds.map(sanitizeHeaderValue) : [sanitizeHeaderValue(value)]; +} + +function requireClaimInput(input) { + if (typeof input.accountId !== 'string' || !UUID_RE.test(input.accountId)) { + throw composeSessionError( + 'invalid_compose_account_id', + 'accountId must be a UUID', + 400, + ); + } + if (typeof input.folder !== 'string' + || !input.folder.trim() + || input.folder.length > 500 + || sanitizeHeaderValue(input.folder) !== input.folder) { + throw composeSessionError( + 'invalid_compose_draft_folder', + 'folder must be a non-empty folder path', + 400, + ); + } + if (!Number.isSafeInteger(input.uid) || input.uid < 1) { + throw composeSessionError( + 'invalid_compose_draft_uid', + 'uid must be a positive integer', + 400, + ); + } + const requestedSlot = input.requestedSlot ?? null; + if (requestedSlot !== null + && (!Number.isInteger(requestedSlot) || requestedSlot < 1 || requestedSlot > 9)) { + throw composeSessionError( + 'invalid_compose_slot', + 'requestedSlot must be an integer from 1 to 9', + 400, + ); + } + return requestedSlot; +} + +function accountFromJoinedRow(row, accountId) { + return { ...row, id: accountId }; +} + +async function requireDraftsFolder(row, input, deps) { + const mappings = jsonValue(row.folder_mappings, {}); + if (mappings?.drafts === input.folder) return; + const specialUse = await deps.query( + `SELECT path FROM folders + WHERE account_id=$1 AND path=$2 AND special_use='\\Drafts' + LIMIT 1`, + [input.accountId, input.folder], + ); + if (!specialUse.rows.length) { + throw composeSessionError( + 'compose_source_not_draft', + 'The selected message is not in this account\'s Drafts folder', + 400, + ); + } +} + +async function resolveClaimAlias(row, account, deps) { + const fromEmail = typeof row.from_email === 'string' ? row.from_email.trim() : ''; + const accountEmail = typeof account.email_address === 'string' + ? account.email_address.trim() + : ''; + if (!fromEmail || fromEmail.toLowerCase() === accountEmail.toLowerCase()) return null; + const alias = await deps.query( + `SELECT id FROM account_aliases + WHERE account_id=$1 AND LOWER(email)=LOWER($2) + LIMIT 1`, + [account.id, fromEmail], + ); + return alias.rows[0]?.id || null; +} + +function attachmentDescriptors(value) { + const descriptors = jsonValue(value, []); + return Array.isArray(descriptors) ? descriptors : []; +} + +function nonemptyString(value) { + return typeof value === 'string' && value.trim().length > 0; +} + +function normalizedClaimAttachment(descriptor, content, id) { + if (!Buffer.isBuffer(content)) { + throw composeSessionError( + 'compose_source_attachment_unavailable', + 'A source draft attachment could not be fetched', + 409, + ); + } + const filename = sanitizeHeaderValue(descriptor?.filename || 'attachment') || 'attachment'; + const candidateType = sanitizeHeaderValue(descriptor?.type || descriptor?.contentType || ''); + const contentType = /^[A-Za-z0-9!#$&^_.+-]+\/[A-Za-z0-9!#$&^_.+-]+$/.test(candidateType) + ? candidateType + : 'application/octet-stream'; + return { + id, + part: String(descriptor?.part || ''), + filename, + contentType, + byteCount: content.length, + content: Buffer.from(content), + }; +} + +function claimedSession(values, row, attachments, sourceInitialRevision) { + return { + id: row.id, + slot: Number(row.slot), + accountId: values.accountId, + aliasId: values.aliasId, + mode: 'new', + to: values.to, + cc: values.cc, + bcc: [], + subject: values.subject, + body: values.body, + bodyIsHtml: values.bodyIsHtml, + quotedBody: null, + quotedBodyHtml: null, + editedSignature: null, + forwardedAttachments: [], + priority: 'normal', + inReplyTo: values.inReplyTo, + references: values.references, + fromChanged: false, + replyAllRecipients: values.replyAllRecipients, + sourceDraftAccountId: values.accountId, + sourceDraftFolder: values.folder, + sourceDraftUid: values.uid, + sourceDraftMessageId: values.messageId, + sourceInitialRevision, + presentationState: row.presentation_state, + operationState: row.operation_state, + operationToken: row.operation_token, + revision: Number(row.revision), + fieldRevisions: jsonValue(row.field_revisions, {}), + lastFocusedAt: row.last_focused_at, + createdAt: row.created_at, + updatedAt: row.updated_at, + attachments: attachments.map(attachment => ({ + id: attachment.id, + filename: attachment.filename, + contentType: attachment.contentType, + byteCount: attachment.byteCount, + createdAt: attachment.createdAt, + })), + }; +} + +export function canonicalCompose(session = {}) { + const attachments = Array.isArray(session.attachments) + ? session.attachments.map(attachmentFingerprint) + .sort((left, right) => String(left.id).localeCompare(String(right.id))) + : []; + + return { + accountId: session.accountId ?? null, + aliasId: session.aliasId ?? null, + mode: VALID_MODES.has(session.mode) ? session.mode : 'new', + to: normalizedList(session.to), + cc: normalizedList(session.cc), + bcc: normalizedList(session.bcc), + subject: typeof session.subject === 'string' ? session.subject : '', + body: typeof session.body === 'string' ? session.body : '', + bodyIsHtml: typeof session.bodyIsHtml === 'boolean' ? session.bodyIsHtml : true, + quotedBody: nullableString(session.quotedBody), + quotedBodyHtml: nullableString(session.quotedBodyHtml), + editedSignature: nullableString(session.editedSignature), + forwardedAttachments: normalizedForwardedAttachments(session.forwardedAttachments), + priority: VALID_PRIORITIES.has(session.priority) ? session.priority : 'normal', + inReplyTo: nullableString(session.inReplyTo), + references: normalizedList(session.references), + fromChanged: session.fromChanged === true, + attachments, + }; +} + +export async function claimDraftIntoComposeSession(input, deps) { + const requestedSlot = requireClaimInput(input); + const replyAllRecipients = normalizeReplyAllRecipients(input.replyAllRecipients ?? []); + const source = await deps.query( + `SELECT m.*, a.* + FROM messages m + JOIN email_accounts a ON a.id=m.account_id + WHERE m.account_id=$1 AND m.folder=$2 AND m.uid=$3 + AND a.user_id=$4 AND m.is_deleted=false + LIMIT 1`, + [input.accountId, input.folder, input.uid, input.userId], + ); + if (!source.rows.length) { + throw composeSessionError( + 'compose_source_draft_not_found', + 'Source draft not found', + 404, + ); + } + + const draft = source.rows[0]; + const account = accountFromJoinedRow(draft, input.accountId); + await requireDraftsFolder(draft, input, deps); + const aliasId = await resolveClaimAlias(draft, account, deps); + + let bodyHtml = draft.body_html; + let bodyText = draft.body_text; + let descriptors = attachmentDescriptors(draft.attachments); + const hasCachedBody = nonemptyString(bodyHtml) || nonemptyString(bodyText); + const missingDeclaredDescriptors = draft.has_attachments === true && descriptors.length === 0; + if (!hasCachedBody || missingDeclaredDescriptors) { + const fetched = await deps.imapManager.fetchMessageBody( + account, + input.uid, + input.folder, + ); + if (!hasCachedBody) { + const fetchedHtml = nonemptyString(fetched?.html) + ? sanitizeEmail(fetched.html) + : null; + if (nonemptyString(fetchedHtml)) { + bodyHtml = fetchedHtml; + bodyText = fetched?.text ?? null; + } else if (nonemptyString(fetched?.text)) { + bodyHtml = null; + bodyText = fetched.text; + } + } + if (Array.isArray(fetched?.attachments) && fetched.attachments.length) { + descriptors = fetched.attachments; + } + } + if (missingDeclaredDescriptors && descriptors.length === 0) { + throw composeSessionError( + 'compose_source_attachments_incomplete', + 'Source draft attachment metadata could not be recovered', + 409, + ); + } + if (descriptors.length > MAX_COMPOSE_ATTACHMENTS) { + throw composeSessionError( + 'attachment_count_limit', + 'Compose sessions support at most 100 attachments', + 413, + ); + } + + const makeUuid = typeof deps.randomUUID === 'function' ? deps.randomUUID : randomUUID; + const attachments = await Promise.all(descriptors.map(async (descriptor) => { + const part = typeof descriptor?.part === 'string' ? descriptor.part.trim() : ''; + if (!part) { + throw composeSessionError( + 'compose_source_attachment_unavailable', + 'A source draft attachment has no fetchable IMAP part', + 409, + ); + } + const content = await deps.imapManager.fetchAttachment( + account, + input.uid, + input.folder, + part, + ); + return normalizedClaimAttachment({ ...descriptor, part }, content, makeUuid()); + })); + const totalAttachmentBytes = attachments.reduce( + (total, attachment) => total + attachment.byteCount, + 0, + ); + if (totalAttachmentBytes > MAX_COMPOSE_ATTACHMENT_BYTES) { + throw composeSessionError( + 'attachment_limit', + 'Compose attachments exceed the 25 MiB limit', + 413, + ); + } + + const values = { + accountId: input.accountId, + aliasId, + mode: 'new', + to: recipientStrings(draft.to_addresses), + cc: recipientStrings(draft.cc_addresses), + bcc: [], + subject: sanitizeHeaderValue(draft.subject || ''), + body: nonemptyString(bodyHtml) ? bodyHtml : (bodyText ?? ''), + bodyIsHtml: nonemptyString(bodyHtml), + quotedBody: null, + quotedBodyHtml: null, + editedSignature: null, + forwardedAttachments: [], + priority: 'normal', + inReplyTo: draft.in_reply_to ? sanitizeHeaderValue(draft.in_reply_to) : null, + references: referenceStrings(draft.thread_references), + fromChanged: false, + replyAllRecipients, + attachments, + folder: input.folder, + uid: input.uid, + messageId: draft.message_id ? sanitizeHeaderValue(draft.message_id) : null, + }; + // Attachment ids are allocated before this snapshot and used verbatim by the + // subsequent inserts, so equality checks refer to the persisted child rows. + const sourceInitialRevision = canonicalCompose(values); + + const result = await deps.withTransaction(async (client) => { + await client.query( + 'SELECT pg_advisory_xact_lock(hashtext($1))', + [`compose-slots:${input.userId}`], + ); + const alreadyClaimed = await client.query( + `SELECT id FROM compose_sessions + WHERE user_id=$1 + AND source_draft_account_id=$2 + AND source_draft_folder=$3 + AND source_draft_uid=$4 + LIMIT 1`, + [input.userId, input.accountId, input.folder, input.uid], + ); + if (alreadyClaimed.rows.length) { + throw composeSessionError( + 'compose_draft_claimed', + 'Source draft is already open in a compose session', + 409, + ); + } + + const allocated = await client.query( + `SELECT candidate.slot + FROM generate_series(1, 9) AS candidate(slot) + WHERE ($2::smallint IS NULL OR candidate.slot=$2) + AND NOT EXISTS ( + SELECT 1 FROM compose_sessions existing + WHERE existing.user_id=$1 AND existing.slot=candidate.slot + ) + ORDER BY candidate.slot + LIMIT 1`, + [input.userId, requestedSlot], + ); + const slot = allocated.rows[0]?.slot; + if (!slot) { + if (requestedSlot !== null) { + throw composeSessionError( + 'compose_slot_occupied', + `Compose slot ${requestedSlot} is already occupied`, + 409, + ); + } + throw composeSessionError( + 'compose_session_limit', + 'Nine compose sessions are already open', + 409, + ); + } + + const inserted = await client.query( + `INSERT INTO compose_sessions ( + user_id, slot, account_id, alias_id, + to_recipients, cc_recipients, bcc_recipients, + subject, body, body_is_html, in_reply_to, thread_references, + reply_all_recipients, + source_draft_account_id, source_draft_folder, source_draft_uid, + source_draft_message_id, source_initial_revision + ) VALUES ( + $1, $2, $3, $4, + $5::jsonb, $6::jsonb, '[]'::jsonb, + $7, $8, $9, $10, $11::jsonb, + $12::jsonb, + $13, $14, $15, $16, $17::jsonb + ) + RETURNING *`, + [ + input.userId, + Number(slot), + values.accountId, + values.aliasId, + JSON.stringify(values.to), + JSON.stringify(values.cc), + values.subject, + values.body, + values.bodyIsHtml, + values.inReplyTo, + JSON.stringify(values.references), + JSON.stringify(values.replyAllRecipients), + values.accountId, + values.folder, + values.uid, + values.messageId, + JSON.stringify(sourceInitialRevision), + ], + ); + + const persistedAttachments = []; + for (const attachment of attachments) { + const persisted = await client.query( + `INSERT INTO compose_session_attachments ( + id, session_id, filename, content_type, byte_count, content + ) VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id, filename, content_type, byte_count, created_at`, + [ + attachment.id, + inserted.rows[0].id, + attachment.filename, + attachment.contentType, + attachment.byteCount, + attachment.content, + ], + ); + persistedAttachments.push({ + id: persisted.rows[0].id, + filename: persisted.rows[0].filename, + contentType: persisted.rows[0].content_type, + byteCount: Number(persisted.rows[0].byte_count), + createdAt: persisted.rows[0].created_at, + }); + } + return claimedSession(values, inserted.rows[0], persistedAttachments, sourceInitialRevision); + }); + + if (typeof deps.broadcast === 'function') { + deps.broadcast({ + type: 'compose_sessions_updated', + action: 'claimed', + sessionId: result.id, + slot: result.slot, + revision: result.revision, + }, input.userId); + } + return result; +} + +export function sourceDraftChanged(session = {}) { + return !isDeepStrictEqual(canonicalCompose(session), session.sourceInitialRevision); +} + +export function sessionToComposeInput(session = {}, account, options = {}) { + const canonical = canonicalCompose(session); + const uploadedAttachments = Array.isArray(session.attachments) + ? session.attachments.map(attachmentInput) + : []; + const materializedForwarded = options.materializedForwardedAttachments; + const attachments = materializedForwarded === undefined + ? uploadedAttachments + : [ + ...uploadedAttachments, + ...(Array.isArray(materializedForwarded) + ? materializedForwarded.map(attachmentInput) + : []), + ]; + + return { + userId: options.userId, + account, + aliasId: canonical.aliasId, + to: canonical.to, + cc: canonical.cc, + bcc: canonical.bcc, + subject: canonical.subject, + body: canonical.body, + bodyIsHtml: canonical.bodyIsHtml, + quotedBody: canonical.quotedBody, + quotedBodyHtml: canonical.quotedBodyHtml, + editedSignature: canonical.editedSignature, + priority: canonical.priority, + inReplyTo: canonical.inReplyTo, + references: canonical.references, + attachments, + forwardedAttachments: materializedForwarded === undefined + ? canonical.forwardedAttachments + : [], + }; +} + +function lifecycleMethod(deps, name, fallback) { + return typeof deps[name] === 'function' ? deps[name] : fallback; +} + +function draftMethod(deps, name, fallback) { + return typeof deps.draftService?.[name] === 'function' + ? deps.draftService[name] + : fallback; +} + +function sendMethod(deps, name, fallback) { + return typeof deps.sendService?.[name] === 'function' + ? deps.sendService[name] + : fallback; +} + +function outboxMethod(deps, name, fallback) { + return typeof deps.outboxService?.[name] === 'function' + ? deps.outboxService[name] + : fallback; +} + +function broadcastLifecycleInvalidation(deps, userId, action, session) { + if (typeof deps.broadcast !== 'function') return; + try { + deps.broadcast({ + type: 'compose_sessions_updated', + action, + sessionId: session.id, + slot: session.slot, + revision: session.revision, + }, userId); + } catch (error) { + console.error('Compose session invalidation failed', { code: error?.code || 'unknown' }); + } +} + +async function resolveOwnedAccount(accountId, userId, deps) { + const result = await deps.query( + 'SELECT * FROM email_accounts WHERE id=$1 AND user_id=$2 LIMIT 1', + [accountId, userId], + ); + if (!result.rows.length) { + throw composeSessionError( + 'compose_account_not_found', + 'Compose account not found', + 404, + ); + } + return result.rows[0]; +} + +function forwardedDescriptors(value) { + if (Array.isArray(value)) return value; + if (typeof value !== 'string') return []; + try { + const parsed = JSON.parse(value || '[]'); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } +} + +async function materializeForwardedAttachments(session, deps) { + const refs = Array.isArray(session.forwardedAttachments) + ? session.forwardedAttachments + : []; + const materialized = []; + for (const forwarded of refs) { + const result = await deps.query( + `SELECT m.uid, m.folder, m.attachments, to_jsonb(a) AS account + FROM messages m + JOIN email_accounts a ON a.id=m.account_id + WHERE m.id=$1 AND a.user_id=$2 AND m.is_deleted=false + LIMIT 1`, + [forwarded.messageId, session.userId], + ); + if (!result.rows.length) { + throw composeSessionError( + 'compose_forwarded_message_not_found', + 'Forwarded message not found', + 404, + ); + } + const message = result.rows[0]; + const descriptor = forwardedDescriptors(message.attachments) + .find(candidate => String(candidate?.part) === forwarded.part); + if (!descriptor) { + throw composeSessionError( + 'compose_forwarded_attachment_not_found', + 'Forwarded attachment not found', + 404, + ); + } + const content = await deps.imapManager.fetchAttachment( + message.account, + message.uid, + message.folder, + forwarded.part, + ); + if (!Buffer.isBuffer(content)) { + throw composeSessionError( + 'compose_forwarded_attachment_unavailable', + 'Forwarded attachment content is unavailable', + 409, + ); + } + const filename = sanitizeHeaderValue(descriptor.filename || 'attachment') || 'attachment'; + const candidateType = sanitizeHeaderValue(descriptor.type || descriptor.contentType || ''); + const contentType = /^[A-Za-z0-9!#$&^_.+-]+\/[A-Za-z0-9!#$&^_.+-]+$/.test(candidateType) + ? candidateType + : 'application/octet-stream'; + materialized.push({ filename, contentType, content: Buffer.from(content) }); + } + + const uploadedBytes = Array.isArray(session.attachments) + ? session.attachments.reduce((total, attachment) => total + Number(attachment.byteCount || 0), 0) + : 0; + const forwardedBytes = materialized.reduce( + (total, attachment) => total + attachment.content.length, + 0, + ); + if (uploadedBytes + forwardedBytes > MAX_COMPOSE_ATTACHMENT_BYTES) { + throw composeSessionError( + 'attachment_limit', + 'Compose attachments exceed the 25 MiB limit', + 413, + ); + } + return materialized; +} + +async function releaseClaim(input, session, deps, originalError) { + const release = lifecycleMethod( + deps, + 'releaseComposeOperation', + composeSessionService.releaseComposeOperation, + ); + try { + const released = await release({ + userId: input.userId, + id: session.id, + token: session.operationToken, + }, deps); + if (released) { + broadcastLifecycleInvalidation(deps, input.userId, 'operation_released', session); + } + } catch (releaseError) { + console.error('Compose session operation release failed', { + originalCode: originalError?.code || 'unknown', + releaseCode: releaseError?.code || 'unknown', + }); + } +} + +function normalizeExternalLifecycleError(error) { + if (error?.expose !== true || typeof error.code === 'string') return error; + if (error.status === 422) { + return composeSessionError( + 'compose_drafts_folder_not_found', + 'No Drafts folder is available for this account', + 422, + ); + } + const status = Number.isInteger(error.status) && error.status >= 400 && error.status < 500 + ? error.status + : 500; + return composeSessionError( + 'compose_draft_operation_failed', + 'Draft operation failed', + status, + ); +} + +function lifecycleAcceptedCleanupPending(operation) { + const close = operation === 'close'; + return composeSessionError( + close + ? 'compose_close_accepted_cleanup_pending' + : 'compose_discard_accepted_cleanup_pending', + close + ? 'Draft was saved but compose cleanup is still pending; do not retry' + : 'Draft was deleted but compose cleanup is still pending; do not retry', + 409, + ); +} + +async function deleteTerminalComposeSession(input, session, deps, allowAlreadyAbsent) { + const deleteClaimed = lifecycleMethod( + deps, + 'deleteClaimedComposeSession', + composeSessionService.deleteClaimedComposeSession, + ); + const deleted = await deleteClaimed({ + userId: input.userId, + id: session.id, + token: session.operationToken, + }, deps); + if (deleted) return; + + if (allowAlreadyAbsent) { + const remaining = await deps.query( + `SELECT operation_state, operation_token + FROM compose_sessions + WHERE id=$1 AND user_id=$2 + LIMIT 1`, + [session.id, input.userId], + ); + if (!remaining.rows.length) return; + } + + throw Object.assign(new Error('Claimed compose session could not be deleted'), { + code: 'compose_claim_lost', + status: 500, + expose: false, + }); +} + +export async function closeComposeSession(input, deps) { + const claim = lifecycleMethod( + deps, + 'claimComposeOperation', + composeSessionService.claimComposeOperation, + ); + const claimed = await claim({ + userId: input.userId, + id: input.id, + slot: input.slot, + expectedRevision: input.expectedRevision, + operation: 'closing', + changes: input.changes === undefined ? {} : input.changes, + }, deps); + const session = { ...claimed, userId: input.userId }; + let saved = null; + let saveAccount = null; + let accepted = false; + try { + const hasSource = session.sourceDraftUid != null + && session.sourceDraftFolder + && session.sourceDraftAccountId; + const shouldSave = hasSource + ? sourceDraftChanged(session) + : meaningfulComposeSession({ + ...session, + attachmentCount: Array.isArray(session.attachments) ? session.attachments.length : 0, + }); + if (shouldSave) { + saveAccount = await resolveOwnedAccount(session.accountId, input.userId, deps); + const crossesAccounts = hasSource && session.sourceDraftAccountId !== saveAccount.id; + const sourceAccount = crossesAccounts + ? await resolveOwnedAccount(session.sourceDraftAccountId, input.userId, deps) + : saveAccount; + const materializedForwardedAttachments = await materializeForwardedAttachments(session, deps); + const save = draftMethod(deps, 'saveDraft', draftService.saveDraft); + saved = await save({ + ...sessionToComposeInput(session, saveAccount, { + userId: input.userId, + materializedForwardedAttachments, + }), + ...(hasSource && !crossesAccounts ? { + existingUid: session.sourceDraftUid, + existingFolder: session.sourceDraftFolder, + reportSourceDraftDeletion: true, + } : {}), + }, deps); + accepted = true; + if (!crossesAccounts && saved?.sourceDraftDeleted === false) { + throw lifecycleAcceptedCleanupPending('close'); + } + if (crossesAccounts) { + const deleteDraft = draftMethod(deps, 'deleteDraft', draftService.deleteDraft); + await deleteDraft({ + account: sourceAccount, + uid: session.sourceDraftUid, + folder: session.sourceDraftFolder, + }, deps); + } + } + + await deleteTerminalComposeSession(input, session, deps, accepted); + } catch (error) { + if (accepted) { + if (error?.code === 'compose_close_accepted_cleanup_pending') throw error; + throw lifecycleAcceptedCleanupPending('close'); + } + const exposedError = normalizeExternalLifecycleError(error); + await releaseClaim(input, session, deps, exposedError); + throw exposedError; + } + + broadcastLifecycleInvalidation(deps, input.userId, 'closed', session); + return { + closed: true, + slot: session.slot, + draft: saved ? { + accountId: saveAccount.id, + account: saveAccount.email_address, + uid: saved.uid, + folder: saved.folder, + messageId: saved.messageId, + } : null, + }; +} + +function sourceDraftDeleteContract(session) { + if (session.sourceDraftUid == null + || !session.sourceDraftFolder + || !session.sourceDraftAccountId) return null; + return { + accountId: session.sourceDraftAccountId, + uid: session.sourceDraftUid, + folder: session.sourceDraftFolder, + }; +} + +function composeSessionRestoreContract(session) { + const canonical = canonicalCompose(session); + const changes = { ...canonical }; + delete changes.attachments; + return { + version: 1, + originalSessionId: session.id, + preferredSlot: session.slot, + // Uploaded bytes are restored as child rows, never as editable fields. + changes, + replyAllRecipients: normalizedList(session.replyAllRecipients), + sourceDraft: session.sourceDraftAccountId + && session.sourceDraftFolder + && session.sourceDraftUid != null + ? { + accountId: session.sourceDraftAccountId, + folder: session.sourceDraftFolder, + uid: session.sourceDraftUid, + messageId: session.sourceDraftMessageId ?? null, + initialRevision: session.sourceInitialRevision ?? null, + } + : null, + attachments: (Array.isArray(session.attachments) ? session.attachments : []).map(attachment => ({ + id: attachment.id, + filename: attachment.filename, + contentType: attachment.contentType, + byteCount: attachment.byteCount, + contentSha256: attachmentContentSha256(attachment), + })), + }; +} + +const COMPOSE_IDEMPOTENCY_KEY_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; + +export function normalizeComposeIdempotencyKey(value) { + if (value === undefined || value === null) return null; + if (typeof value !== 'string' || !COMPOSE_IDEMPOTENCY_KEY_RE.test(value)) { + throw composeSessionError( + 'invalid_compose_idempotency_key', + 'X-Idempotency-Key must be 1-128 safe opaque characters', + 400, + ); + } + return value; +} + +function composeSendKey(input, session) { + const callerKey = normalizeComposeIdempotencyKey(input.idempotencyKey); + if (!callerKey) return `compose-session:${session.id}`; + const digest = createHash('sha256').update(callerKey, 'utf8').digest('hex'); + return `compose-session:${session.id}:${digest}`; +} + +function invalidSendAcceptance() { + return Object.assign(new Error('Send service did not accept the compose session'), { + code: 'compose_send_unaccepted', + status: 500, + expose: false, + }); +} + +function acceptedCleanupPending() { + return Object.assign(new Error('The message was accepted but compose cleanup is still pending'), { + code: 'compose_send_accepted_cleanup_pending', + status: 500, + expose: false, + }); +} + +function safeErrorCode(error) { + const code = typeof error?.code === 'string' ? error.code : ''; + return /^[A-Za-z0-9_-]{1,64}$/.test(code) ? code : 'unknown'; +} + +export async function sendComposeSession(input, deps) { + const claim = lifecycleMethod( + deps, + 'claimComposeOperation', + composeSessionService.claimComposeOperation, + ); + const claimed = await claim({ + userId: input.userId, + id: input.id, + slot: input.slot, + expectedRevision: input.expectedRevision, + operation: 'sending', + changes: {}, + }, deps); + const session = { ...claimed, userId: input.userId }; + let result; + let immediate; + let queued; + let deleteDraftOnSend; + let sourceAccount; + + try { + const idempotencyKey = composeSendKey(input, session); + if (!session.accountId) { + throw composeSessionError( + 'compose_account_required', + 'A sending account is required', + 400, + ); + } + if (![session.to, session.cc, session.bcc].some(list => ( + Array.isArray(list) && list.some(recipient => typeof recipient === 'string' && recipient.trim()) + ))) { + throw composeSessionError( + 'compose_recipients_required', + 'At least one recipient is required', + 400, + ); + } + + const [account, preferenceResult] = await Promise.all([ + resolveOwnedAccount(session.accountId, input.userId, deps), + deps.query('SELECT preferences FROM users WHERE id=$1 LIMIT 1', [input.userId]), + ]); + const preferences = jsonValue(preferenceResult.rows[0]?.preferences, {}); + const normalizeUndoWindow = outboxMethod( + deps, + 'normalizeUndoWindow', + outboxService.normalizeUndoWindow, + ); + const undoSeconds = normalizeUndoWindow( + input.undoSendSeconds, + preferences.undoSendSeconds, + ); + deleteDraftOnSend = sourceDraftDeleteContract(session); + sourceAccount = deleteDraftOnSend + ? (deleteDraftOnSend.accountId === account.id + ? account + : await resolveOwnedAccount(deleteDraftOnSend.accountId, input.userId, deps)) + : null; + const sendOrEnqueue = sendMethod(deps, 'sendOrEnqueue', sendService.sendOrEnqueue); + result = await sendOrEnqueue({ + ...sessionToComposeInput(session, account, { userId: input.userId }), + ...(undoSeconds > 0 ? { + composeSessionRestore: composeSessionRestoreContract(session), + } : {}), + plaintextEmail: preferences.plaintextEmail === true, + undoSeconds, + idempotencyKey, + ...(deleteDraftOnSend ? { deleteDraftOnSend } : {}), + }, deps); + + immediate = result?.ok === true; + queued = result?.queued === true + && typeof result.outboxId === 'string' + && result.outboxId.length > 0; + if (immediate === queued) throw invalidSendAcceptance(); + } catch (error) { + await releaseClaim(input, session, deps, error); + throw error; + } + + if (immediate && deleteDraftOnSend) { + const deleteDraft = draftMethod(deps, 'deleteDraft', draftService.deleteDraft); + try { + await deleteDraft({ + account: sourceAccount, + uid: deleteDraftOnSend.uid, + folder: deleteDraftOnSend.folder, + }, deps); + } catch (error) { + console.error('Compose source cleanup failed after accepted send', { + code: safeErrorCode(error), + }); + } + } + + const deleteClaimed = lifecycleMethod( + deps, + 'deleteClaimedComposeSession', + composeSessionService.deleteClaimedComposeSession, + ); + try { + const deleted = await deleteClaimed({ + userId: input.userId, + id: session.id, + token: session.operationToken, + }, deps); + if (!deleted) { + const remaining = await deps.query( + `SELECT operation_state, operation_token + FROM compose_sessions + WHERE id=$1 AND user_id=$2 + LIMIT 1`, + [session.id, input.userId], + ); + if (remaining.rows.length) throw acceptedCleanupPending(); + } + } catch (error) { + if (error?.code === 'compose_send_accepted_cleanup_pending') throw error; + throw acceptedCleanupPending(); + } + + broadcastLifecycleInvalidation( + deps, + input.userId, + queued ? 'queued' : 'sent', + session, + ); + return result; +} + +export async function discardComposeSession(input, deps) { + const claim = lifecycleMethod( + deps, + 'claimComposeOperation', + composeSessionService.claimComposeOperation, + ); + const claimed = await claim({ + userId: input.userId, + id: input.id, + slot: input.slot, + expectedRevision: input.expectedRevision, + operation: 'discarding', + changes: {}, + }, deps); + const session = { ...claimed, userId: input.userId }; + let accepted = false; + try { + const hasSource = session.sourceDraftUid != null + && session.sourceDraftFolder + && session.sourceDraftAccountId; + if (hasSource) { + const account = await resolveOwnedAccount( + session.sourceDraftAccountId, + input.userId, + deps, + ); + const deleteDraft = draftMethod(deps, 'deleteDraft', draftService.deleteDraft); + const deletion = await deleteDraft({ + account, + uid: session.sourceDraftUid, + folder: session.sourceDraftFolder, + reportDeletionAcceptance: true, + }, deps); + accepted = true; + if (deletion?.localCleanupPending === true) { + throw lifecycleAcceptedCleanupPending('discard'); + } + } + + await deleteTerminalComposeSession(input, session, deps, accepted); + } catch (error) { + if (accepted) { + if (error?.code === 'compose_discard_accepted_cleanup_pending') throw error; + throw lifecycleAcceptedCleanupPending('discard'); + } + const exposedError = normalizeExternalLifecycleError(error); + await releaseClaim(input, session, deps, exposedError); + throw exposedError; + } + + broadcastLifecycleInvalidation(deps, input.userId, 'discarded', session); + return { discarded: true, slot: session.slot }; +} diff --git a/backend/src/services/composeSessionLifecycle.test.js b/backend/src/services/composeSessionLifecycle.test.js new file mode 100644 index 00000000..ceda0140 --- /dev/null +++ b/backend/src/services/composeSessionLifecycle.test.js @@ -0,0 +1,2165 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + canonicalCompose, + claimDraftIntoComposeSession, + closeComposeSession, + discardComposeSession, + sendComposeSession, + sessionToComposeInput, + sourceDraftChanged, +} from './composeSessionLifecycle.js'; + +const ACCOUNT_ID = '00000000-0000-4000-8000-000000000001'; +const DESTINATION_ACCOUNT_ID = '00000000-0000-4000-8000-000000000040'; +const ALIAS_ID = '00000000-0000-4000-8000-000000000002'; +const MESSAGE_ID = '00000000-0000-4000-8000-000000000003'; +const ATTACHMENT_ID = '00000000-0000-4000-8000-000000000004'; +const USER_ID = '00000000-0000-4000-8000-000000000010'; +const CLAIMED_SESSION_ID = '00000000-0000-4000-8000-000000000011'; +const CLAIMED_ATTACHMENT_ID = '00000000-0000-4000-8000-000000000012'; +const SECOND_SESSION_ID = '00000000-0000-4000-8000-000000000013'; + +function completeSession(overrides = {}) { + return { + id: '00000000-0000-4000-8000-000000000005', + slot: 4, + accountId: ACCOUNT_ID, + aliasId: ALIAS_ID, + mode: 'reply', + to: ['A '], + cc: [], + bcc: [], + subject: 'Hello', + body: '

Body

', + bodyIsHtml: true, + quotedBody: null, + quotedBodyHtml: null, + editedSignature: '

Sig

', + forwardedAttachments: [{ messageId: MESSAGE_ID, part: '2' }], + priority: 'high', + inReplyTo: '', + references: ['', ''], + fromChanged: true, + attachments: [{ + id: ATTACHMENT_ID, + filename: 'report.pdf', + contentType: 'application/pdf', + byteCount: 3, + content: Buffer.from('pdf'), + createdAt: '2026-08-01T00:00:00.000Z', + }], + sourceInitialRevision: null, + presentationState: 'minimized', + operationState: 'closing', + operationToken: '00000000-0000-4000-8000-000000000006', + revision: 9, + fieldRevisions: { subject: 8 }, + lastFocusedAt: '2026-08-01T01:00:00.000Z', + createdAt: '2026-08-01T00:00:00.000Z', + updatedAt: '2026-08-01T01:00:00.000Z', + ...overrides, + }; +} + +describe('sessionToComposeInput', () => { + it('maps every shared draft/send input and base64-encodes uploaded bytes', () => { + const account = { id: ACCOUNT_ID, email_address: 'sender@example.com' }; + + expect(sessionToComposeInput(completeSession(), account, { userId: 'user-1' })) + .toEqual({ + userId: 'user-1', + account, + aliasId: ALIAS_ID, + to: ['A '], + cc: [], + bcc: [], + subject: 'Hello', + body: '

Body

', + bodyIsHtml: true, + quotedBody: null, + quotedBodyHtml: null, + editedSignature: '

Sig

', + priority: 'high', + inReplyTo: '', + references: ['', ''], + attachments: [{ + filename: 'report.pdf', + content: Buffer.from('pdf').toString('base64'), + contentType: 'application/pdf', + }], + forwardedAttachments: [{ messageId: MESSAGE_ID, part: '2' }], + }); + }); + + it('preserves plaintext representation and normalized empty defaults', () => { + const input = sessionToComposeInput(completeSession({ + aliasId: null, + to: undefined, + cc: undefined, + bcc: undefined, + subject: undefined, + body: 'Plain body', + bodyIsHtml: false, + quotedBody: undefined, + quotedBodyHtml: undefined, + editedSignature: undefined, + forwardedAttachments: undefined, + priority: undefined, + inReplyTo: undefined, + references: undefined, + attachments: undefined, + }), { id: ACCOUNT_ID }, { userId: 'user-1' }); + + expect(input).toMatchObject({ + aliasId: null, + to: [], + cc: [], + bcc: [], + subject: '', + body: 'Plain body', + bodyIsHtml: false, + quotedBody: null, + quotedBodyHtml: null, + editedSignature: null, + priority: 'normal', + inReplyTo: null, + references: [], + attachments: [], + forwardedAttachments: [], + }); + }); + + it('provides a close-time seam for materialized forwarded attachment bytes', () => { + const input = sessionToComposeInput(completeSession(), { id: ACCOUNT_ID }, { + userId: 'user-1', + materializedForwardedAttachments: [{ + filename: 'forwarded.txt', + contentType: 'text/plain', + content: Buffer.from('forwarded'), + }], + }); + + expect(input.forwardedAttachments).toEqual([]); + expect(input.attachments).toEqual([ + { + filename: 'report.pdf', + content: Buffer.from('pdf').toString('base64'), + contentType: 'application/pdf', + }, + { + filename: 'forwarded.txt', + content: Buffer.from('forwarded').toString('base64'), + contentType: 'text/plain', + }, + ]); + }); + + it.each([ + ['uploaded', completeSession({ + attachments: [{ ...completeSession().attachments[0], content: undefined }], + }), {}], + ['materialized forwarded', completeSession(), { + materializedForwardedAttachments: [{ + filename: 'forwarded.txt', + contentType: 'text/plain', + content: { unsupported: true }, + }], + }], + ])('rejects missing or unsupported %s attachment content', (_kind, session, options) => { + let thrown; + try { + sessionToComposeInput(session, { id: ACCOUNT_ID }, { userId: 'user-1', ...options }); + } catch (error) { + thrown = error; + } + + expect(thrown).toMatchObject({ + code: 'invalid_compose_attachment_content', + status: 500, + expose: false, + }); + }); + + it('preserves canonical base64 and converts Uint8Array attachment content', () => { + const input = sessionToComposeInput(completeSession({ + attachments: [ + { ...completeSession().attachments[0], content: 'cGRm' }, + { + id: '00000000-0000-4000-8000-000000000009', + filename: 'typed.bin', + contentType: 'application/octet-stream', + byteCount: 3, + content: new Uint8Array([1, 2, 3]), + }, + ], + }), { id: ACCOUNT_ID }, { userId: 'user-1' }); + + expect(input.attachments.map(attachment => attachment.content)).toEqual([ + 'cGRm', + Buffer.from([1, 2, 3]).toString('base64'), + ]); + }); +}); + +describe('canonicalCompose and sourceDraftChanged', () => { + it('captures deterministic meaningful state and attachment fingerprints only', () => { + const session = completeSession(); + const canonical = canonicalCompose(session); + + expect(canonical).toEqual({ + accountId: ACCOUNT_ID, + aliasId: ALIAS_ID, + mode: 'reply', + to: ['A '], + cc: [], + bcc: [], + subject: 'Hello', + body: '

Body

', + bodyIsHtml: true, + quotedBody: null, + quotedBodyHtml: null, + editedSignature: '

Sig

', + forwardedAttachments: [{ messageId: MESSAGE_ID, part: '2' }], + priority: 'high', + inReplyTo: '', + references: ['', ''], + fromChanged: true, + attachments: [{ + id: ATTACHMENT_ID, + filename: 'report.pdf', + contentType: 'application/pdf', + byteCount: 3, + }], + }); + expect(canonical).not.toHaveProperty('slot'); + expect(canonical).not.toHaveProperty('revision'); + expect(canonical).not.toHaveProperty('presentationState'); + expect(canonical).not.toHaveProperty('operationState'); + expect(canonical.attachments[0]).not.toHaveProperty('content'); + expect(canonical.attachments[0]).not.toHaveProperty('createdAt'); + }); + + it('treats revision, timestamps, presentation, operation, slot, and bytes as irrelevant', () => { + const initial = completeSession(); + const sourceInitialRevision = canonicalCompose(initial); + const current = completeSession({ + sourceInitialRevision, + slot: 9, + revision: 99, + fieldRevisions: { body: 99 }, + presentationState: 'expanded', + operationState: 'sending', + operationToken: '00000000-0000-4000-8000-000000000007', + updatedAt: '2026-08-02T00:00:00.000Z', + attachments: [{ ...initial.attachments[0], content: Buffer.from('new') }], + }); + + expect(sourceDraftChanged(current)).toBe(false); + }); + + it.each([ + ['recipient', { to: ['B '] }], + ['alias', { aliasId: null }], + ['plaintext mode', { bodyIsHtml: false }], + ['priority', { priority: 'low' }], + ['quoted content', { quotedBody: 'Quoted' }], + ['signature', { editedSignature: '

Different

' }], + ['forwarded reference', { forwardedAttachments: [] }], + ['attachment fingerprint', { + attachments: [{ ...completeSession().attachments[0], byteCount: 4 }], + }], + ])('detects a changed %s', (_label, change) => { + const initial = completeSession(); + const current = completeSession({ + ...change, + sourceInitialRevision: canonicalCompose(initial), + }); + + expect(sourceDraftChanged(current)).toBe(true); + }); + + it('sorts attachment fingerprints by stable id', () => { + const second = { + id: '00000000-0000-4000-8000-000000000008', + filename: 'second.txt', + contentType: 'text/plain', + byteCount: 2, + content: Buffer.from('ok'), + }; + const initial = completeSession({ + attachments: [second, completeSession().attachments[0]], + }); + const current = completeSession({ + attachments: [completeSession().attachments[0], second], + sourceInitialRevision: canonicalCompose(initial), + }); + + expect(sourceDraftChanged(current)).toBe(false); + }); +}); + +function sourceDraft(overrides = {}) { + return { + id: ACCOUNT_ID, + account_id: ACCOUNT_ID, + uid: 41, + folder: '[Synthetic]/Drafts', + message_id: '', + subject: 'Synthetic draft', + to_addresses: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc_addresses: [], + from_email: 'sender@example.com', + body_html: '

Synthetic body

', + body_text: 'Synthetic body', + in_reply_to: '', + thread_references: ' ', + attachments: [{ + part: '2', + filename: 'synthetic.txt', + type: 'text/plain', + size: 15, + }], + folder_mappings: { drafts: '[Synthetic]/Drafts' }, + email_address: 'sender@example.com', + ...overrides, + }; +} + +function sessionRowFromInsert(params) { + return { + id: CLAIMED_SESSION_ID, + user_id: params[0], + slot: params[1], + account_id: params[2], + alias_id: params[3], + mode: 'new', + to_recipients: params[4], + cc_recipients: params[5], + bcc_recipients: '[]', + subject: params[6], + body: params[7], + body_is_html: params[8], + quoted_body: null, + quoted_body_html: null, + edited_signature: null, + forwarded_attachments: '[]', + priority: 'normal', + in_reply_to: params[9], + thread_references: params[10], + reply_all_recipients: params[11], + from_changed: false, + source_draft_account_id: params[12], + source_draft_folder: params[13], + source_draft_uid: params[14], + source_draft_message_id: params[15], + source_initial_revision: params[16], + presentation_state: 'expanded', + operation_state: 'idle', + operation_token: null, + revision: 1, + field_revisions: '{}', + last_focused_at: '2026-08-01T00:00:00.000Z', + created_at: '2026-08-01T00:00:00.000Z', + updated_at: '2026-08-01T00:00:00.000Z', + }; +} + +function claimDeps({ draft = sourceDraft(), occupiedSlots = [], duplicate = false } = {}) { + const attachmentRows = []; + const clientQuery = async (sql, params = []) => { + const normalized = sql.replace(/\s+/g, ' ').trim(); + if (normalized.includes('pg_advisory_xact_lock')) return { rows: [{}] }; + if (normalized.includes('FROM compose_sessions') + && normalized.includes('source_draft_account_id')) { + return { rows: duplicate ? [{ id: CLAIMED_SESSION_ID }] : [] }; + } + if (normalized.includes('FROM generate_series(1, 9)')) { + const requested = params[1]; + const slot = requested == null + ? Array.from({ length: 9 }, (_, index) => index + 1) + .find(candidate => !occupiedSlots.includes(candidate)) + : (!occupiedSlots.includes(requested) ? requested : undefined); + return { rows: slot ? [{ slot }] : [] }; + } + if (normalized.startsWith('INSERT INTO compose_sessions')) { + const sessionRow = sessionRowFromInsert(params); + return { rows: [sessionRow] }; + } + if (normalized.startsWith('INSERT INTO compose_session_attachments')) { + const [id, sessionId, filename, contentType, byteCount, content] = params; + const row = { + id, + session_id: sessionId, + filename, + content_type: contentType, + byte_count: byteCount, + content, + created_at: '2026-08-01T00:00:00.000Z', + }; + attachmentRows.push(row); + return { rows: [row] }; + } + throw new Error(`Unexpected transaction SQL: ${normalized}`); + }; + const query = vi.fn(async (sql, params = []) => { + const normalized = sql.replace(/\s+/g, ' ').trim(); + if (normalized.startsWith('SELECT m.*, a.*')) return { rows: [draft] }; + if (normalized.includes("special_use='\\Drafts'")) return { rows: [] }; + if (normalized.startsWith('SELECT id FROM account_aliases')) return { rows: [] }; + throw new Error(`Unexpected SQL: ${normalized} ${JSON.stringify(params)}`); + }); + const imapManager = { + fetchMessageBody: vi.fn(), + fetchAttachment: vi.fn(async () => Buffer.from('synthetic bytes')), + permanentDeleteMessage: vi.fn(), + }; + const withTransaction = vi.fn(async callback => callback({ query: clientQuery })); + return { + deps: { + query, + withTransaction, + imapManager, + randomUUID: vi.fn(() => CLAIMED_ATTACHMENT_ID), + broadcast: vi.fn(), + }, + attachmentRows, + }; +} + +describe('claimDraftIntoComposeSession', () => { + it('owner-scopes the source, fetches every attachment before allocation, and snapshots persisted ids', async () => { + const fake = claimDeps({ occupiedSlots: [1] }); + + const claimed = await claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + replyAllRecipients: [' Synthetic Copied '], + }, fake.deps); + + const sourceCall = fake.deps.query.mock.calls[0]; + expect(sourceCall[0].replace(/\s+/g, ' ')).toContain( + 'SELECT m.*, a.* FROM messages m JOIN email_accounts a ON a.id=m.account_id', + ); + expect(sourceCall[0].replace(/\s+/g, ' ')).toContain( + 'm.account_id=$1 AND m.folder=$2 AND m.uid=$3 AND a.user_id=$4 AND m.is_deleted=false', + ); + expect(sourceCall[1]).toEqual([ACCOUNT_ID, '[Synthetic]/Drafts', 41, USER_ID]); + expect(fake.deps.imapManager.fetchAttachment).toHaveBeenCalledWith( + expect.objectContaining({ id: ACCOUNT_ID }), + 41, + '[Synthetic]/Drafts', + '2', + ); + expect(fake.deps.imapManager.fetchAttachment.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.withTransaction.mock.invocationCallOrder[0]); + expect(claimed).toMatchObject({ + id: CLAIMED_SESSION_ID, + slot: 2, + accountId: ACCOUNT_ID, + subject: 'Synthetic draft', + body: '

Synthetic body

', + bodyIsHtml: true, + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + replyAllRecipients: ['Synthetic Copied '], + attachments: [{ + id: CLAIMED_ATTACHMENT_ID, + filename: 'synthetic.txt', + contentType: 'text/plain', + byteCount: 15, + }], + }); + expect(claimed.to).toEqual(['Recipient ']); + expect(claimed.references).toEqual([ + '', + '', + ]); + expect(claimed.sourceInitialRevision).toEqual(canonicalCompose(claimed)); + expect(fake.attachmentRows[0].id).toBe(CLAIMED_ATTACHMENT_ID); + expect(claimed.sourceInitialRevision.attachments[0].id).toBe(fake.attachmentRows[0].id); + expect(fake.attachmentRows[0].content).toEqual(Buffer.from('synthetic bytes')); + expect(fake.deps.imapManager.permanentDeleteMessage).not.toHaveBeenCalled(); + expect(fake.deps.query.mock.calls.flatMap(call => call[0])).not.toContain('DELETE FROM messages'); + }); + + it('fetches a missing body and uses plaintext plus fetched attachment descriptors truthfully', async () => { + const fake = claimDeps({ + draft: sourceDraft({ body_html: null, body_text: null, attachments: [] }), + }); + fake.deps.imapManager.fetchMessageBody.mockResolvedValueOnce({ + html: null, + text: 'Fetched synthetic body', + attachments: [{ + part: '3', filename: 'fetched.bin', type: 'application/octet-stream', size: 4, + }], + }); + fake.deps.imapManager.fetchAttachment.mockResolvedValueOnce(Buffer.from('data')); + + const claimed = await claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + }, fake.deps); + + expect(fake.deps.imapManager.fetchMessageBody).toHaveBeenCalledWith( + expect.objectContaining({ id: ACCOUNT_ID }), 41, '[Synthetic]/Drafts', + ); + expect(fake.deps.imapManager.fetchAttachment).toHaveBeenCalledWith( + expect.objectContaining({ id: ACCOUNT_ID }), 41, '[Synthetic]/Drafts', '3', + ); + expect(claimed).toMatchObject({ body: 'Fetched synthetic body', bodyIsHtml: false }); + expect(claimed.attachments[0]).toMatchObject({ filename: 'fetched.bin', byteCount: 4 }); + }); + + it('fetches missing descriptors for a cached attachment-bearing draft without replacing its body', async () => { + const fake = claimDeps({ + draft: sourceDraft({ + body_html: '

Cached synthetic body

', + body_text: 'Cached synthetic body', + has_attachments: true, + attachments: [], + }), + }); + fake.deps.imapManager.fetchMessageBody.mockResolvedValueOnce({ + html: '

Freshly fetched body

', + text: 'Freshly fetched body', + attachments: [{ + part: '4', filename: 'recovered.bin', type: 'application/octet-stream', size: 5, + }], + }); + fake.deps.imapManager.fetchAttachment.mockResolvedValueOnce(Buffer.from('bytes')); + + const claimed = await claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + }, fake.deps); + + expect(fake.deps.imapManager.fetchMessageBody).toHaveBeenCalledOnce(); + expect(claimed).toMatchObject({ + body: '

Cached synthetic body

', + bodyIsHtml: true, + attachments: [{ filename: 'recovered.bin', byteCount: 5 }], + }); + expect(fake.deps.imapManager.fetchAttachment.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.withTransaction.mock.invocationCallOrder[0]); + }); + + it('fails closed when an attachment-bearing draft still has no descriptors after live fetch', async () => { + const fake = claimDeps({ + draft: sourceDraft({ has_attachments: true, attachments: [] }), + }); + fake.deps.imapManager.fetchMessageBody.mockResolvedValueOnce({ + html: '

Fetched synthetic body

', + text: 'Fetched synthetic body', + attachments: [], + }); + + await expect(claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_source_attachments_incomplete', + status: 409, + expose: true, + }); + expect(fake.deps.withTransaction).not.toHaveBeenCalled(); + }); + + it('sanitizes live-fetched HTML before placing it in the authoritative session', async () => { + const fake = claimDeps({ + draft: sourceDraft({ body_html: null, body_text: null, attachments: [] }), + }); + fake.deps.imapManager.fetchMessageBody.mockResolvedValueOnce({ + html: '

Safe text

', + text: 'Safe text', + attachments: [], + }); + + const claimed = await claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + }, fake.deps); + + expect(claimed.body).toContain('Safe text'); + expect(claimed.body).not.toMatch(/script|onclick|syntheticHandler|syntheticScript/i); + expect(claimed.bodyIsHtml).toBe(true); + }); + + it('treats whitespace-only live HTML as absent so nonempty plaintext wins', async () => { + const fake = claimDeps({ + draft: sourceDraft({ body_html: null, body_text: null, attachments: [] }), + }); + fake.deps.imapManager.fetchMessageBody.mockResolvedValueOnce({ + html: ' \n\t ', + text: 'Fetched plaintext body', + attachments: [], + }); + + await expect(claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + }, fake.deps)).resolves.toMatchObject({ + body: 'Fetched plaintext body', + bodyIsHtml: false, + }); + }); + + it('accepts an actual special-use Drafts path when no account mapping matches', async () => { + const draft = sourceDraft({ folder_mappings: {}, folder: 'Localized/Entwürfe', attachments: [] }); + const fake = claimDeps({ draft }); + fake.deps.query.mockImplementation(async (sql) => { + const normalized = sql.replace(/\s+/g, ' ').trim(); + if (normalized.startsWith('SELECT m.*, a.*')) return { rows: [draft] }; + if (normalized.includes("special_use='\\Drafts'")) { + return { rows: [{ path: 'Localized/Entwürfe' }] }; + } + if (normalized.startsWith('SELECT id FROM account_aliases')) return { rows: [] }; + throw new Error(`Unexpected SQL: ${normalized}`); + }); + + await expect(claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: 'Localized/Entwürfe', + uid: 41, + }, fake.deps)).resolves.toMatchObject({ sourceDraftFolder: 'Localized/Entwürfe' }); + expect(fake.deps.query).toHaveBeenCalledWith( + expect.stringContaining("special_use='\\Drafts'"), + [ACCOUNT_ID, 'Localized/Entwürfe'], + ); + }); + + it('rejects a folder that is neither mapped nor marked as Drafts before IMAP or allocation', async () => { + const fake = claimDeps({ + draft: sourceDraft({ folder_mappings: {}, folder: 'Synthetic/Archive' }), + }); + + await expect(claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: 'Synthetic/Archive', + uid: 41, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_source_not_draft', + status: 400, + }); + expect(fake.deps.imapManager.fetchMessageBody).not.toHaveBeenCalled(); + expect(fake.deps.imapManager.fetchAttachment).not.toHaveBeenCalled(); + expect(fake.deps.withTransaction).not.toHaveBeenCalled(); + }); + + it('returns an owner-scoped not-found without opening an allocation transaction', async () => { + const fake = claimDeps(); + fake.deps.query.mockResolvedValueOnce({ rows: [] }); + + await expect(claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_source_draft_not_found', + status: 404, + }); + expect(fake.deps.withTransaction).not.toHaveBeenCalled(); + }); + + it('rejects a duplicate source tuple with a stable conflict and leaves the source intact', async () => { + const fake = claimDeps({ duplicate: true }); + + await expect(claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + }, fake.deps)).rejects.toMatchObject({ code: 'compose_draft_claimed', status: 409 }); + expect(fake.deps.imapManager.permanentDeleteMessage).not.toHaveBeenCalled(); + }); + + it('honors a requested free slot and reports occupied and exhausted allocation distinctly', async () => { + const requested = claimDeps({ occupiedSlots: [1, 2] }); + await expect(claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + requestedSlot: 7, + }, requested.deps)).resolves.toMatchObject({ slot: 7 }); + + const occupied = claimDeps({ occupiedSlots: [7] }); + await expect(claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + requestedSlot: 7, + }, occupied.deps)).rejects.toMatchObject({ code: 'compose_slot_occupied', status: 409 }); + + const exhausted = claimDeps({ occupiedSlots: [1, 2, 3, 4, 5, 6, 7, 8, 9] }); + await expect(claimDraftIntoComposeSession({ + userId: USER_ID, + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + }, exhausted.deps)).rejects.toMatchObject({ code: 'compose_session_limit', status: 409 }); + }); +}); + +function deferred() { + let resolve; + let reject; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +function closeSession(overrides = {}) { + return completeSession({ + id: CLAIMED_SESSION_ID, + slot: 3, + accountId: ACCOUNT_ID, + aliasId: null, + mode: 'new', + to: [], + cc: [], + bcc: [], + subject: '', + body: '', + bodyIsHtml: true, + quotedBody: null, + quotedBodyHtml: null, + editedSignature: null, + forwardedAttachments: [], + priority: 'normal', + inReplyTo: null, + references: [], + fromChanged: false, + attachments: [], + sourceDraftAccountId: null, + sourceDraftFolder: null, + sourceDraftUid: null, + sourceDraftMessageId: null, + sourceInitialRevision: null, + operationState: 'closing', + operationToken: '00000000-0000-4000-8000-000000000020', + revision: 7, + ...overrides, + }); +} + +function closeDeps(session = closeSession()) { + const account = { + id: ACCOUNT_ID, + user_id: USER_ID, + email_address: 'sender@example.com', + }; + const claimComposeOperation = vi.fn().mockResolvedValue(session); + const releaseComposeOperation = vi.fn().mockResolvedValue(true); + const deleteClaimedComposeSession = vi.fn().mockResolvedValue(true); + const saveDraft = vi.fn().mockResolvedValue({ + uid: 72, + folder: '[Synthetic]/Drafts', + messageId: '', + }); + const deleteDraft = vi.fn().mockResolvedValue({ ok: true }); + const query = vi.fn(async (sql) => { + if (sql.includes('FROM email_accounts')) return { rows: [account] }; + throw new Error(`Unexpected close SQL: ${sql.replace(/\s+/g, ' ').trim()}`); + }); + return { + account, + deps: { + query, + imapManager: { fetchAttachment: vi.fn() }, + draftService: { saveDraft, deleteDraft }, + claimComposeOperation, + releaseComposeOperation, + deleteClaimedComposeSession, + broadcast: vi.fn(), + }, + }; +} + +function sendSession(overrides = {}) { + return closeSession({ + operationState: 'sending', + to: ['Recipient '], + subject: 'Synthetic send subject', + body: 'Synthetic send body', + bodyIsHtml: false, + ...overrides, + }); +} + +function sendDeps(session = sendSession(), { preference = 0 } = {}) { + const destinationAccount = { + id: session.accountId || ACCOUNT_ID, + user_id: USER_ID, + email_address: 'sender@example.com', + }; + const sourceAccount = { + id: session.sourceDraftAccountId || ACCOUNT_ID, + user_id: USER_ID, + email_address: 'source@example.com', + }; + const claimComposeOperation = vi.fn().mockResolvedValue(session); + const releaseComposeOperation = vi.fn().mockResolvedValue(true); + const deleteClaimedComposeSession = vi.fn().mockResolvedValue(true); + const sendOrEnqueue = vi.fn().mockResolvedValue({ + ok: true, + messageId: '', + sentCopySaved: true, + receipt: { subject: 'Synthetic send subject' }, + }); + const deleteDraft = vi.fn().mockResolvedValue({ ok: true }); + const normalizeUndoWindow = vi.fn((requested, fallback) => requested ?? fallback ?? 0); + const query = vi.fn(async (sql, params) => { + if (sql.includes('FROM email_accounts')) { + if (params[0] === destinationAccount.id) return { rows: [destinationAccount] }; + if (params[0] === sourceAccount.id) return { rows: [sourceAccount] }; + return { rows: [] }; + } + if (sql.includes('FROM users')) { + return { rows: [{ preferences: { undoSendSeconds: preference, plaintextEmail: true } }] }; + } + throw new Error(`Unexpected send SQL: ${sql.replace(/\s+/g, ' ').trim()}`); + }); + return { + destinationAccount, + sourceAccount, + deps: { + query, + imapManager: { fetchAttachment: vi.fn() }, + redisClient: { get: vi.fn(), set: vi.fn(), del: vi.fn() }, + refreshMicrosoftToken: vi.fn(), + sendService: { sendOrEnqueue }, + outboxService: { normalizeUndoWindow, enqueue: vi.fn() }, + draftService: { deleteDraft }, + claimComposeOperation, + releaseComposeOperation, + deleteClaimedComposeSession, + broadcast: vi.fn(), + }, + }; +} + +describe('closeComposeSession', () => { + it('claims an atomic final patch and deletes an empty new session by token without saving', async () => { + const fake = closeDeps(); + const changes = { subject: '' }; + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 6, + changes, + }, fake.deps)).resolves.toEqual({ closed: true, slot: 3, draft: null }); + + expect(fake.deps.claimComposeOperation).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 6, + operation: 'closing', + changes, + }, fake.deps); + expect(fake.deps.draftService.saveDraft).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + token: '00000000-0000-4000-8000-000000000020', + }, fake.deps); + expect(fake.deps.broadcast).toHaveBeenCalledWith({ + type: 'compose_sessions_updated', + action: 'closed', + sessionId: CLAIMED_SESSION_ID, + slot: 3, + revision: 7, + }, USER_ID); + }); + + it('awaits saving a meaningful new draft before deleting the claimed session', async () => { + const fake = closeDeps(closeSession({ subject: 'Synthetic subject' })); + const save = deferred(); + fake.deps.draftService.saveDraft.mockReturnValueOnce(save.promise); + + const closing = closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps); + await vi.waitFor(() => expect(fake.deps.draftService.saveDraft).toHaveBeenCalledOnce()); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + + save.resolve({ + uid: 72, + folder: '[Synthetic]/Drafts', + messageId: '', + }); + await expect(closing).resolves.toEqual({ + closed: true, + slot: 3, + draft: { + accountId: ACCOUNT_ID, + account: 'sender@example.com', + uid: 72, + folder: '[Synthetic]/Drafts', + messageId: '', + }, + }); + expect(fake.deps.draftService.saveDraft.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.deleteClaimedComposeSession.mock.invocationCallOrder[0]); + }); + + it('deletes the token but leaves an unchanged claimed source draft intact', async () => { + const initial = closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + sourceDraftMessageId: '', + }); + const fake = closeDeps({ ...initial, sourceInitialRevision: canonicalCompose(initial) }); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).resolves.toEqual({ closed: true, slot: 3, draft: null }); + + expect(fake.deps.draftService.saveDraft).not.toHaveBeenCalled(); + expect(fake.deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).toHaveBeenCalledOnce(); + }); + + it('saves a changed claimed source with existing locators before token deletion', async () => { + const initial = closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + subject: 'Initial synthetic subject', + }); + const claimed = { + ...initial, + subject: 'Changed synthetic subject', + sourceInitialRevision: canonicalCompose(initial), + }; + const fake = closeDeps(claimed); + const save = deferred(); + fake.deps.draftService.saveDraft.mockReturnValueOnce(save.promise); + + const closing = closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps); + await vi.waitFor(() => expect(fake.deps.draftService.saveDraft).toHaveBeenCalledOnce()); + expect(fake.deps.draftService.saveDraft.mock.calls[0][0]).toMatchObject({ + existingUid: 41, + existingFolder: '[Synthetic]/Drafts', + reportSourceDraftDeletion: true, + subject: 'Changed synthetic subject', + }); + expect(fake.deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + + save.resolve({ uid: 73, folder: '[Synthetic]/Drafts', messageId: null }); + await closing; + expect(fake.deps.draftService.saveDraft.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.deleteClaimedComposeSession.mock.invocationCallOrder[0]); + }); + + it('moves a claimed source across owned accounts without passing source locators to destination save', async () => { + const initial = closeSession({ + accountId: ACCOUNT_ID, + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + subject: 'Initial synthetic subject', + }); + const claimed = { + ...initial, + accountId: DESTINATION_ACCOUNT_ID, + subject: 'Changed synthetic subject', + sourceInitialRevision: canonicalCompose(initial), + }; + const fake = closeDeps(claimed); + const sourceAccount = fake.account; + const destinationAccount = { + id: DESTINATION_ACCOUNT_ID, + user_id: USER_ID, + email_address: 'destination@example.com', + }; + fake.deps.query.mockImplementation(async (sql, params) => { + if (!sql.includes('FROM email_accounts')) throw new Error('Unexpected cross-account SQL'); + if (params[0] === DESTINATION_ACCOUNT_ID) return { rows: [destinationAccount] }; + if (params[0] === ACCOUNT_ID) return { rows: [sourceAccount] }; + return { rows: [] }; + }); + const save = deferred(); + const sourceDelete = deferred(); + fake.deps.draftService.saveDraft.mockReturnValueOnce(save.promise); + fake.deps.draftService.deleteDraft.mockReturnValueOnce(sourceDelete.promise); + + const closing = closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps); + await vi.waitFor(() => expect(fake.deps.draftService.saveDraft).toHaveBeenCalledOnce()); + + expect(fake.deps.query).toHaveBeenNthCalledWith( + 1, + expect.stringContaining('FROM email_accounts'), + [DESTINATION_ACCOUNT_ID, USER_ID], + ); + expect(fake.deps.query).toHaveBeenNthCalledWith( + 2, + expect.stringContaining('FROM email_accounts'), + [ACCOUNT_ID, USER_ID], + ); + expect(fake.deps.draftService.saveDraft.mock.calls[0][0]).toMatchObject({ + account: destinationAccount, + subject: 'Changed synthetic subject', + }); + expect(fake.deps.draftService.saveDraft.mock.calls[0][0]).not.toHaveProperty('existingUid'); + expect(fake.deps.draftService.saveDraft.mock.calls[0][0]).not.toHaveProperty('existingFolder'); + expect(fake.deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + + save.resolve({ uid: 73, folder: '[Destination]/Drafts', messageId: null }); + await vi.waitFor(() => expect(fake.deps.draftService.deleteDraft).toHaveBeenCalledOnce()); + expect(fake.deps.draftService.deleteDraft).toHaveBeenCalledWith({ + account: sourceAccount, + uid: 41, + folder: '[Synthetic]/Drafts', + }, fake.deps); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + + sourceDelete.resolve({ ok: true }); + await expect(closing).resolves.toMatchObject({ + closed: true, + draft: { accountId: DESTINATION_ACCOUNT_ID, uid: 73 }, + }); + expect(fake.deps.draftService.saveDraft.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.draftService.deleteDraft.mock.invocationCallOrder[0]); + expect(fake.deps.draftService.deleteDraft.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.deleteClaimedComposeSession.mock.invocationCallOrder[0]); + expect(fake.deps.deleteClaimedComposeSession.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.broadcast.mock.invocationCallOrder[0]); + }); + + it('keeps a cross-account close claimed when source deletion fails after destination save', async () => { + const initial = closeSession({ + accountId: ACCOUNT_ID, + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + subject: 'Initial synthetic subject', + }); + const fake = closeDeps({ + ...initial, + accountId: DESTINATION_ACCOUNT_ID, + subject: 'Changed synthetic subject', + sourceInitialRevision: canonicalCompose(initial), + }); + const destinationAccount = { + id: DESTINATION_ACCOUNT_ID, + user_id: USER_ID, + email_address: 'destination@example.com', + }; + fake.deps.query.mockImplementation(async (_sql, params) => ({ + rows: [params[0] === DESTINATION_ACCOUNT_ID ? destinationAccount : fake.account], + })); + const failure = Object.assign(new Error('Synthetic source deletion failure'), { + code: 'synthetic_source_delete', + }); + fake.deps.draftService.deleteDraft.mockRejectedValueOnce(failure); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_close_accepted_cleanup_pending', + status: 409, + expose: true, + }); + + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + expect(failure.code).toBe('synthetic_source_delete'); + }); + + it('keeps a same-account replacement claimed when APPEND succeeds but source cleanup is incomplete', async () => { + const initial = closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + subject: 'Initial synthetic subject', + }); + const fake = closeDeps({ + ...initial, + subject: 'Changed synthetic subject', + sourceInitialRevision: canonicalCompose(initial), + }); + fake.deps.draftService.saveDraft.mockResolvedValueOnce({ + uid: 73, + folder: '[Synthetic]/Drafts', + messageId: null, + sourceDraftDeleted: false, + }); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_close_accepted_cleanup_pending', + status: 409, + expose: true, + }); + + expect(fake.deps.draftService.saveDraft).toHaveBeenCalledWith( + expect.objectContaining({ reportSourceDraftDeletion: true }), + fake.deps, + ); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + }); + + it('blocks close retry when token deletion fails after a meaningful draft was accepted', async () => { + const session = closeSession({ subject: 'Synthetic subject' }); + const fake = closeDeps(session); + const operationInProgress = Object.assign(new Error('Operation in progress'), { + code: 'compose_operation_in_progress', status: 409, expose: true, + }); + fake.deps.claimComposeOperation + .mockResolvedValueOnce(session) + .mockRejectedValueOnce(operationInProgress); + fake.deps.deleteClaimedComposeSession.mockRejectedValueOnce( + new Error('Synthetic database disconnect'), + ); + const input = { + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }; + + await expect(closeComposeSession(input, fake.deps)).rejects.toMatchObject({ + code: 'compose_close_accepted_cleanup_pending', + status: 409, + expose: true, + }); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + + await expect(closeComposeSession(input, fake.deps)).rejects.toBe(operationInProgress); + expect(fake.deps.draftService.saveDraft).toHaveBeenCalledOnce(); + }); + + it('treats an owner-scoped absent row as completed close cleanup after acceptance', async () => { + const fake = closeDeps(closeSession({ subject: 'Synthetic subject' })); + const query = fake.deps.query.getMockImplementation(); + fake.deps.query.mockImplementation((sql, params) => ( + sql.includes('FROM compose_sessions') ? { rows: [] } : query(sql, params) + )); + fake.deps.deleteClaimedComposeSession.mockResolvedValueOnce(false); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).resolves.toMatchObject({ closed: true }); + expect(fake.deps.query).toHaveBeenCalledWith( + expect.stringMatching(/FROM compose_sessions[\s\S]+id=\$1[\s\S]+user_id=\$2/), + [CLAIMED_SESSION_ID, USER_ID], + ); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + }); + + it('keeps close claimed when token deletion reports a live mismatched row after acceptance', async () => { + const fake = closeDeps(closeSession({ subject: 'Synthetic subject' })); + const query = fake.deps.query.getMockImplementation(); + fake.deps.query.mockImplementation((sql, params) => ( + sql.includes('FROM compose_sessions') + ? { rows: [{ operation_state: 'closing', operation_token: 'different-token' }] } + : query(sql, params) + )); + fake.deps.deleteClaimedComposeSession.mockResolvedValueOnce(false); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_close_accepted_cleanup_pending', + status: 409, + expose: true, + }); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + }); + + it.each([ + ['empty new', closeSession()], + ['unchanged source', (() => { + const initial = closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + }); + return { ...initial, sourceInitialRevision: canonicalCompose(initial) }; + })()], + ])('safely releases %s close when token deletion fails before any external acceptance', async ( + _label, + session, + ) => { + const fake = closeDeps(session); + const failure = new Error('Synthetic token deletion failure'); + fake.deps.deleteClaimedComposeSession.mockRejectedValueOnce(failure); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).rejects.toBe(failure); + expect(fake.deps.draftService.saveDraft).not.toHaveBeenCalled(); + expect(fake.deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledOnce(); + }); + + it('normalizes an exposed missing-Drafts error, releases the claim, and emits no terminal success', async () => { + const fake = closeDeps(closeSession({ subject: 'Synthetic subject' })); + fake.deps.draftService.saveDraft.mockRejectedValueOnce(Object.assign( + new Error('No Drafts folder found for this account'), + { status: 422, expose: true }, + )); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_drafts_folder_not_found', + message: 'No Drafts folder is available for this account', + status: 422, + expose: true, + }); + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledOnce(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).toHaveBeenCalledTimes(1); + expect(fake.deps.broadcast.mock.calls[0][0].action).toBe('operation_released'); + }); + + it.each([ + ['already-coded exposed', Object.assign(new Error('Synthetic coded failure'), { + code: 'synthetic_coded', status: 409, expose: true, + })], + ['unknown non-exposed', new Error('Synthetic unknown failure')], + ])('preserves the original %s draft error while releasing the claim', async (_label, failure) => { + const fake = closeDeps(closeSession({ subject: 'Synthetic subject' })); + fake.deps.draftService.saveDraft.mockRejectedValueOnce(failure); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).rejects.toBe(failure); + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledOnce(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + }); + + it('releases the exact token and preserves the session when draft save fails', async () => { + const fake = closeDeps(closeSession({ subject: 'Synthetic subject' })); + const failure = Object.assign(new Error('Synthetic save failure'), { code: 'synthetic_save' }); + fake.deps.draftService.saveDraft.mockRejectedValueOnce(failure); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).rejects.toBe(failure); + + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + token: '00000000-0000-4000-8000-000000000020', + }, fake.deps); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).toHaveBeenCalledWith({ + type: 'compose_sessions_updated', + action: 'operation_released', + sessionId: CLAIMED_SESSION_ID, + slot: 3, + revision: 7, + }, USER_ID); + }); + + it('performs no external or token calls when the atomic claim conflicts', async () => { + const fake = closeDeps(); + const conflict = Object.assign(new Error('Synthetic conflict'), { + code: 'compose_conflict', status: 409, expose: true, + }); + fake.deps.claimComposeOperation.mockRejectedValueOnce(conflict); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 6, + changes: { subject: 'Conflicting synthetic subject' }, + }, fake.deps)).rejects.toBe(conflict); + + expect(fake.deps.query).not.toHaveBeenCalled(); + expect(fake.deps.imapManager.fetchAttachment).not.toHaveBeenCalled(); + expect(fake.deps.draftService.saveDraft).not.toHaveBeenCalled(); + expect(fake.deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + }); + + it('owner-scopes and materializes forwarded attachment bytes before saveDraft', async () => { + const forwardedMessageId = '00000000-0000-4000-8000-000000000030'; + const referencedAccount = { id: ACCOUNT_ID, email_address: 'sender@example.com' }; + const fake = closeDeps(closeSession({ + forwardedAttachments: [{ messageId: forwardedMessageId, part: '2' }], + })); + fake.deps.query.mockImplementation(async (sql, params) => { + if (sql.includes('FROM email_accounts')) return { rows: [fake.account] }; + if (sql.includes('FROM messages')) { + expect(params).toEqual([forwardedMessageId, USER_ID]); + return { rows: [{ + uid: 91, + folder: '[Synthetic]/Inbox', + attachments: [{ part: '2', filename: 'forwarded.txt', type: 'text/plain' }], + account: referencedAccount, + }] }; + } + throw new Error('Unexpected forwarded SQL'); + }); + fake.deps.imapManager.fetchAttachment.mockResolvedValueOnce(Buffer.from('forwarded bytes')); + + await closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps); + + expect(fake.deps.query).toHaveBeenCalledWith( + expect.stringMatching(/JOIN email_accounts[\s\S]+a\.user_id\s*=\s*\$2/), + [forwardedMessageId, USER_ID], + ); + expect(fake.deps.imapManager.fetchAttachment).toHaveBeenCalledWith( + referencedAccount, 91, '[Synthetic]/Inbox', '2', + ); + const draftInput = fake.deps.draftService.saveDraft.mock.calls[0][0]; + expect(draftInput.forwardedAttachments).toEqual([]); + expect(draftInput.attachments).toEqual([{ + filename: 'forwarded.txt', + content: Buffer.from('forwarded bytes').toString('base64'), + contentType: 'text/plain', + }]); + }); + + it.each([ + ['foreign reference', { rows: [] }, Buffer.from('unused')], + ['missing bytes', { rows: [{ + uid: 91, + folder: '[Synthetic]/Inbox', + attachments: [{ part: '2', filename: 'forwarded.txt', type: 'text/plain' }], + account: { id: ACCOUNT_ID }, + }] }, null], + ])('releases and preserves the session for %s', async (_label, messageResult, content) => { + const fake = closeDeps(closeSession({ + forwardedAttachments: [{ messageId: MESSAGE_ID, part: '2' }], + })); + fake.deps.query.mockImplementation(async (sql) => ( + sql.includes('FROM email_accounts') ? { rows: [fake.account] } : messageResult + )); + fake.deps.imapManager.fetchAttachment.mockResolvedValueOnce(content); + + await expect(closeComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + changes: {}, + }, fake.deps)).rejects.toMatchObject({ expose: true }); + expect(fake.deps.draftService.saveDraft).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + token: '00000000-0000-4000-8000-000000000020', + }, fake.deps); + }); +}); + +describe('sendComposeSession', () => { + it.each([ + ['account', sendSession({ accountId: null }), 'compose_account_required'], + ['recipient', sendSession({ to: [], cc: [], bcc: [] }), 'compose_recipients_required'], + ])('releases a claimed session with a missing %s before external send', async ( + _label, + session, + code, + ) => { + const fake = sendDeps(session); + + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).rejects.toMatchObject({ code, status: 400, expose: true }); + + expect(fake.deps.claimComposeOperation).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + operation: 'sending', + changes: {}, + }, fake.deps); + expect(fake.deps.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + token: '00000000-0000-4000-8000-000000000020', + }, fake.deps); + expect(fake.deps.broadcast).toHaveBeenCalledWith( + expect.objectContaining({ action: 'operation_released' }), + USER_ID, + ); + }); + + it('performs zero account, send, cleanup, or token calls when the claim conflicts', async () => { + const fake = sendDeps(); + const conflict = Object.assign(new Error('Compose session changed'), { + code: 'compose_conflict', status: 409, expose: true, + }); + fake.deps.claimComposeOperation.mockRejectedValueOnce(conflict); + + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 6, + idempotencyKey: 'synthetic-send-key', + }, fake.deps)).rejects.toBe(conflict); + + expect(fake.deps.query).not.toHaveBeenCalled(); + expect(fake.deps.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(fake.deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + }); + + it('preserves the complete immediate receipt and deletes the token only after accepted delivery', async () => { + const fake = sendDeps(); + const delivered = deferred(); + const result = { + ok: true, + messageId: '', + sentCopySaved: false, + receipt: { + subject: 'Synthetic send subject', + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + }, + }; + fake.deps.sendService.sendOrEnqueue.mockReturnValueOnce(delivered.promise); + + const sending = sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps); + await vi.waitFor(() => expect(fake.deps.sendService.sendOrEnqueue).toHaveBeenCalledOnce()); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + + delivered.resolve(result); + await expect(sending).resolves.toBe(result); + expect(fake.deps.outboxService.normalizeUndoWindow).toHaveBeenCalledWith(undefined, 0); + expect(fake.deps.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + userId: USER_ID, + account: fake.destinationAccount, + to: ['Recipient '], + subject: 'Synthetic send subject', + body: 'Synthetic send body', + bodyIsHtml: false, + plaintextEmail: true, + undoSeconds: 0, + idempotencyKey: `compose-session:${CLAIMED_SESSION_ID}`, + }), + fake.deps, + ); + expect(fake.deps.deleteClaimedComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + token: '00000000-0000-4000-8000-000000000020', + }, fake.deps); + expect(fake.deps.sendService.sendOrEnqueue.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.deleteClaimedComposeSession.mock.invocationCallOrder[0]); + expect(fake.deps.broadcast).toHaveBeenCalledWith({ + type: 'compose_sessions_updated', + action: 'sent', + sessionId: CLAIMED_SESSION_ID, + slot: 3, + revision: 7, + }, USER_ID); + }); + + it('durably enqueues with caller idempotency, source cleanup metadata, and no immediate draft delete', async () => { + const session = sendSession({ + accountId: DESTINATION_ACCOUNT_ID, + mode: 'reply', + replyAllRecipients: ['Synthetic Copied '], + attachments: [{ + id: '66666666-6666-4666-8666-666666666666', + filename: 'synthetic.txt', + contentType: 'text/plain', + byteCount: 12, + content: Buffer.from('server bytes'), + }], + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + sourceDraftMessageId: '', + sourceInitialRevision: { subject: 'Synthetic source' }, + }); + const fake = sendDeps(session, { preference: 30 }); + const result = { + queued: true, + outboxId: '00000000-0000-4000-8000-000000000030', + sendAt: new Date('2026-08-01T12:00:30.000Z'), + undoSeconds: 30, + }; + fake.deps.sendService.sendOrEnqueue.mockResolvedValueOnce(result); + const callerKey = 'k'.repeat(128); + + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + idempotencyKey: callerKey, + }, fake.deps)).resolves.toBe(result); + + expect(fake.deps.sendService.sendOrEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + account: fake.destinationAccount, + undoSeconds: 30, + idempotencyKey: expect.stringMatching( + new RegExp(`^compose-session:${CLAIMED_SESSION_ID}:[a-f0-9]{64}$`), + ), + deleteDraftOnSend: { + accountId: ACCOUNT_ID, + uid: 41, + folder: '[Synthetic]/Drafts', + }, + composeSessionRestore: { + version: 1, + originalSessionId: CLAIMED_SESSION_ID, + preferredSlot: 3, + changes: expect.objectContaining({ + mode: 'reply', + subject: 'Synthetic send subject', + }), + replyAllRecipients: ['Synthetic Copied '], + sourceDraft: { + accountId: ACCOUNT_ID, + folder: '[Synthetic]/Drafts', + uid: 41, + messageId: '', + initialRevision: { subject: 'Synthetic source' }, + }, + attachments: [{ + id: '66666666-6666-4666-8666-666666666666', + filename: 'synthetic.txt', + contentType: 'text/plain', + byteCount: 12, + contentSha256: '6fac1c9e222157d1baa07e669d6df5b6be7177dc362306c79acfc2c6f31dfd0b', + }], + }, + }), + fake.deps, + ); + expect(fake.deps.sendService.sendOrEnqueue.mock.calls[0][0].idempotencyKey) + .not.toContain(callerKey); + expect(fake.deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).toHaveBeenCalledOnce(); + expect(fake.deps.broadcast).toHaveBeenCalledWith( + expect.objectContaining({ action: 'queued' }), + USER_ID, + ); + }); + + it('derives stable per-session keys at immediate and queued send boundaries', async () => { + const callerKey = 'synthetic-shared-key'; + const firstImmediate = sendDeps(sendSession({ id: CLAIMED_SESSION_ID })); + const repeatedImmediate = sendDeps(sendSession({ id: CLAIMED_SESSION_ID })); + const secondQueued = sendDeps(sendSession({ id: SECOND_SESSION_ID })); + secondQueued.deps.sendService.sendOrEnqueue.mockResolvedValueOnce({ + queued: true, + outboxId: '00000000-0000-4000-8000-000000000030', + }); + + await sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + idempotencyKey: callerKey, + }, firstImmediate.deps); + await sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + idempotencyKey: callerKey, + }, repeatedImmediate.deps); + await sendComposeSession({ + userId: USER_ID, + id: SECOND_SESSION_ID, + expectedRevision: 7, + idempotencyKey: callerKey, + }, secondQueued.deps); + + const firstKey = firstImmediate.deps.sendService.sendOrEnqueue.mock.calls[0][0].idempotencyKey; + const repeatedKey = repeatedImmediate.deps.sendService.sendOrEnqueue.mock.calls[0][0].idempotencyKey; + const secondKey = secondQueued.deps.sendService.sendOrEnqueue.mock.calls[0][0].idempotencyKey; + expect(firstKey).toBe(repeatedKey); + expect(secondKey).not.toBe(firstKey); + expect(firstKey).toMatch( + new RegExp(`^compose-session:${CLAIMED_SESSION_ID}:[a-f0-9]{64}$`), + ); + expect(secondKey).toMatch( + new RegExp(`^compose-session:${SECOND_SESSION_ID}:[a-f0-9]{64}$`), + ); + expect(firstKey.length).toBeLessThanOrEqual(128); + expect(secondKey.length).toBeLessThanOrEqual(128); + expect(firstKey).not.toContain(callerKey); + expect(secondKey).not.toContain(callerKey); + }); + + it('deletes a claimed source through its owner only after immediate acceptance', async () => { + const session = sendSession({ + accountId: DESTINATION_ACCOUNT_ID, + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + }); + const fake = sendDeps(session); + const delivered = deferred(); + fake.deps.sendService.sendOrEnqueue.mockReturnValueOnce(delivered.promise); + + const sending = sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + idempotencyKey: 'synthetic-source-send', + }, fake.deps); + await vi.waitFor(() => expect(fake.deps.sendService.sendOrEnqueue).toHaveBeenCalledOnce()); + expect(fake.deps.draftService.deleteDraft).not.toHaveBeenCalled(); + + delivered.resolve({ ok: true, receipt: { subject: 'Synthetic send subject' } }); + await sending; + expect(fake.deps.draftService.deleteDraft).toHaveBeenCalledWith({ + account: fake.sourceAccount, + uid: 41, + folder: '[Synthetic]/Drafts', + }, fake.deps); + expect(fake.deps.sendService.sendOrEnqueue.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.draftService.deleteDraft.mock.invocationCallOrder[0]); + expect(fake.deps.draftService.deleteDraft.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.deleteClaimedComposeSession.mock.invocationCallOrder[0]); + }); + + it('releases SMTP, outbox, and malformed acceptance failures without deleting the session', async () => { + const failures = [ + Object.assign(new Error('Synthetic SMTP failure'), { code: 'synthetic_smtp' }), + Object.assign(new Error('Synthetic outbox failure'), { code: 'synthetic_outbox' }), + ]; + for (const failure of failures) { + const fake = sendDeps(); + fake.deps.sendService.sendOrEnqueue.mockRejectedValueOnce(failure); + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).rejects.toBe(failure); + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledOnce(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + } + + const malformed = sendDeps(); + malformed.deps.sendService.sendOrEnqueue.mockResolvedValueOnce({ queued: true }); + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, malformed.deps)).rejects.toMatchObject({ + code: 'compose_send_unaccepted', + status: 500, + expose: false, + }); + expect(malformed.deps.releaseComposeOperation).toHaveBeenCalledOnce(); + expect(malformed.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + + const ambiguous = sendDeps(); + ambiguous.deps.sendService.sendOrEnqueue.mockResolvedValueOnce({ + ok: true, + queued: true, + outboxId: '00000000-0000-4000-8000-000000000030', + }); + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, ambiguous.deps)).rejects.toMatchObject({ + code: 'compose_send_unaccepted', + status: 500, + expose: false, + }); + expect(ambiguous.deps.releaseComposeOperation).toHaveBeenCalledOnce(); + expect(ambiguous.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + }); + + it('treats immediate source cleanup as best effort after acceptance and never sends twice', async () => { + const session = sendSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + }); + const fake = sendDeps(session); + const accepted = { + ok: true, + messageId: '', + receipt: { subject: 'Synthetic send subject' }, + }; + const alreadyTerminal = Object.assign(new Error('Compose session not found'), { + code: 'compose_session_not_found', status: 404, expose: true, + }); + fake.deps.claimComposeOperation + .mockResolvedValueOnce(session) + .mockRejectedValueOnce(alreadyTerminal); + fake.deps.sendService.sendOrEnqueue.mockResolvedValue(accepted); + fake.deps.draftService.deleteDraft.mockRejectedValueOnce(Object.assign( + new Error('Synthetic cleanup failure with private diagnostics'), + { code: 'synthetic_cleanup' }, + )); + const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {}); + const input = { + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + idempotencyKey: 'caller-stable-key', + }; + + await expect(sendComposeSession(input, fake.deps)).resolves.toBe(accepted); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).toHaveBeenCalledOnce(); + expect(errorLog).toHaveBeenCalledWith( + 'Compose source cleanup failed after accepted send', + { code: 'synthetic_cleanup' }, + ); + expect(JSON.stringify(errorLog.mock.calls)).not.toContain('private diagnostics'); + + await expect(sendComposeSession(input, fake.deps)).rejects.toBe(alreadyTerminal); + expect(fake.deps.sendService.sendOrEnqueue).toHaveBeenCalledOnce(); + errorLog.mockRestore(); + }); + + it('keeps an accepted send claimed when terminal token deletion throws and blocks resend', async () => { + const session = sendSession(); + const fake = sendDeps(session); + const operationInProgress = Object.assign(new Error('Operation in progress'), { + code: 'compose_operation_in_progress', status: 409, expose: true, + }); + fake.deps.claimComposeOperation + .mockResolvedValueOnce(session) + .mockRejectedValueOnce(operationInProgress); + fake.deps.deleteClaimedComposeSession.mockRejectedValueOnce( + new Error('Synthetic database disconnect with private diagnostics'), + ); + + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_send_accepted_cleanup_pending', + status: 500, + expose: false, + message: 'The message was accepted but compose cleanup is still pending', + }); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).rejects.toBe(operationInProgress); + expect(fake.deps.sendService.sendOrEnqueue).toHaveBeenCalledOnce(); + }); + + it('treats an already-absent row as idempotent terminal cleanup after acceptance', async () => { + const fake = sendDeps(); + const query = fake.deps.query.getMockImplementation(); + fake.deps.query.mockImplementation((sql, params) => ( + sql.includes('FROM compose_sessions') ? { rows: [] } : query(sql, params) + )); + fake.deps.deleteClaimedComposeSession.mockResolvedValueOnce(false); + + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).resolves.toMatchObject({ ok: true }); + expect(fake.deps.query).toHaveBeenCalledWith( + expect.stringMatching(/FROM compose_sessions[\s\S]+id=\$1[\s\S]+user_id=\$2/), + [CLAIMED_SESSION_ID, USER_ID], + ); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).toHaveBeenCalledWith( + expect.objectContaining({ action: 'sent' }), + USER_ID, + ); + }); + + it('fails closed without release when a mismatched live claim remains after acceptance', async () => { + const fake = sendDeps(); + const query = fake.deps.query.getMockImplementation(); + fake.deps.query.mockImplementation((sql, params) => ( + sql.includes('FROM compose_sessions') + ? { rows: [{ operation_state: 'sending', operation_token: 'different-token' }] } + : query(sql, params) + )); + fake.deps.deleteClaimedComposeSession.mockResolvedValueOnce(false); + + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_send_accepted_cleanup_pending', + expose: false, + }); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + }); + + it('rejects distinct overlong same-prefix idempotency keys without sending or deleting', async () => { + const fake = sendDeps(); + const prefix = 'same-prefix-'.padEnd(128, 'x'); + + for (const key of [`${prefix}a`, `${prefix}b`]) { + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + idempotencyKey: key, + }, fake.deps)).rejects.toMatchObject({ + code: 'invalid_compose_idempotency_key', + status: 400, + expose: true, + }); + } + + expect(fake.deps.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledTimes(2); + }); + + it.each(['', ' ', 'key\nvalue', 'recipient@example.com', 'key/with/content'])( + 'rejects unsafe idempotency key %# and releases before send', + async (idempotencyKey) => { + const fake = sendDeps(); + + await expect(sendComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + idempotencyKey, + }, fake.deps)).rejects.toMatchObject({ + code: 'invalid_compose_idempotency_key', + status: 400, + expose: true, + }); + expect(fake.deps.sendService.sendOrEnqueue).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledOnce(); + }, + ); +}); + +describe('discardComposeSession', () => { + it('claims discarding and deletes a new session by the exact token', async () => { + const fake = closeDeps(closeSession({ operationState: 'discarding' })); + + await expect(discardComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).resolves.toEqual({ discarded: true, slot: 3 }); + + expect(fake.deps.claimComposeOperation).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + operation: 'discarding', + changes: {}, + }, fake.deps); + expect(fake.deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(fake.deps.deleteClaimedComposeSession).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + token: '00000000-0000-4000-8000-000000000020', + }, fake.deps); + expect(fake.deps.broadcast).toHaveBeenCalledWith({ + type: 'compose_sessions_updated', + action: 'discarded', + sessionId: CLAIMED_SESSION_ID, + slot: 3, + revision: 7, + }, USER_ID); + }); + + it('awaits source deletion through its owner-scoped account before token deletion', async () => { + const fake = closeDeps(closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + operationState: 'discarding', + })); + const deletion = deferred(); + fake.deps.draftService.deleteDraft.mockReturnValueOnce(deletion.promise); + + const discarding = discardComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps); + await vi.waitFor(() => expect(fake.deps.draftService.deleteDraft).toHaveBeenCalledOnce()); + expect(fake.deps.query).toHaveBeenCalledWith( + expect.stringMatching(/FROM email_accounts[\s\S]+id=\$1[\s\S]+user_id=\$2/), + [ACCOUNT_ID, USER_ID], + ); + expect(fake.deps.draftService.deleteDraft).toHaveBeenCalledWith({ + account: fake.account, + uid: 41, + folder: '[Synthetic]/Drafts', + reportDeletionAcceptance: true, + }, fake.deps); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + + deletion.resolve({ ok: true }); + await discarding; + expect(fake.deps.draftService.deleteDraft.mock.invocationCallOrder[0]) + .toBeLessThan(fake.deps.deleteClaimedComposeSession.mock.invocationCallOrder[0]); + }); + + it('releases a failed source deletion and preserves the session', async () => { + const fake = closeDeps(closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + operationState: 'discarding', + })); + const failure = Object.assign(new Error('Synthetic delete failure'), { + code: 'synthetic_delete', + }); + fake.deps.draftService.deleteDraft.mockRejectedValueOnce(failure); + + await expect(discardComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).rejects.toBe(failure); + + expect(fake.deps.releaseComposeOperation).toHaveBeenCalledWith({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + token: '00000000-0000-4000-8000-000000000020', + }, fake.deps); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).toHaveBeenCalledWith( + expect.objectContaining({ action: 'operation_released' }), + USER_ID, + ); + }); + + it('blocks discard retry when token deletion fails after source deletion was accepted', async () => { + const session = closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + operationState: 'discarding', + }); + const fake = closeDeps(session); + const operationInProgress = Object.assign(new Error('Operation in progress'), { + code: 'compose_operation_in_progress', status: 409, expose: true, + }); + fake.deps.claimComposeOperation + .mockResolvedValueOnce(session) + .mockRejectedValueOnce(operationInProgress); + fake.deps.deleteClaimedComposeSession.mockRejectedValueOnce( + new Error('Synthetic database disconnect'), + ); + const input = { + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }; + + await expect(discardComposeSession(input, fake.deps)).rejects.toMatchObject({ + code: 'compose_discard_accepted_cleanup_pending', + status: 409, + expose: true, + }); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + + await expect(discardComposeSession(input, fake.deps)).rejects.toBe(operationInProgress); + expect(fake.deps.draftService.deleteDraft).toHaveBeenCalledOnce(); + }); + + it('treats an owner-scoped absent row as completed discard cleanup after source deletion', async () => { + const fake = closeDeps(closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + operationState: 'discarding', + })); + const query = fake.deps.query.getMockImplementation(); + fake.deps.query.mockImplementation((sql, params) => ( + sql.includes('FROM compose_sessions') ? { rows: [] } : query(sql, params) + )); + fake.deps.deleteClaimedComposeSession.mockResolvedValueOnce(false); + + await expect(discardComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).resolves.toEqual({ discarded: true, slot: 3 }); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + }); + + it('keeps discard claimed when token deletion reports a live row after source deletion', async () => { + const fake = closeDeps(closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + operationState: 'discarding', + })); + const query = fake.deps.query.getMockImplementation(); + fake.deps.query.mockImplementation((sql, params) => ( + sql.includes('FROM compose_sessions') + ? { rows: [{ operation_state: 'discarding', operation_token: 'different-token' }] } + : query(sql, params) + )); + fake.deps.deleteClaimedComposeSession.mockResolvedValueOnce(false); + + await expect(discardComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_discard_accepted_cleanup_pending', + status: 409, + expose: true, + }); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + }); + + it('keeps discard claimed when IMAP deletion succeeded but local source cleanup is pending', async () => { + const fake = closeDeps(closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + operationState: 'discarding', + })); + fake.deps.draftService.deleteDraft.mockResolvedValueOnce({ + ok: true, + localCleanupPending: true, + }); + + await expect(discardComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_discard_accepted_cleanup_pending', + status: 409, + expose: true, + }); + expect(fake.deps.draftService.deleteDraft).toHaveBeenCalledWith({ + account: fake.account, + uid: 41, + folder: '[Synthetic]/Drafts', + reportDeletionAcceptance: true, + }, fake.deps); + expect(fake.deps.deleteClaimedComposeSession).not.toHaveBeenCalled(); + expect(fake.deps.releaseComposeOperation).not.toHaveBeenCalled(); + }); + + it('preserves the original external error when release also fails', async () => { + const fake = closeDeps(closeSession({ + sourceDraftAccountId: ACCOUNT_ID, + sourceDraftFolder: '[Synthetic]/Drafts', + sourceDraftUid: 41, + operationState: 'discarding', + })); + const original = Object.assign(new Error('Synthetic original failure'), { + code: 'synthetic_original', + }); + fake.deps.draftService.deleteDraft.mockRejectedValueOnce(original); + fake.deps.releaseComposeOperation.mockRejectedValueOnce(Object.assign( + new Error('Synthetic release failure'), + { code: 'synthetic_release' }, + )); + const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await expect(discardComposeSession({ + userId: USER_ID, + id: CLAIMED_SESSION_ID, + expectedRevision: 7, + }, fake.deps)).rejects.toBe(original); + + expect(errorLog).toHaveBeenCalledWith('Compose session operation release failed', { + originalCode: 'synthetic_original', + releaseCode: 'synthetic_release', + }); + expect(fake.deps.broadcast).not.toHaveBeenCalled(); + errorLog.mockRestore(); + }); +}); diff --git a/backend/src/services/composeSessionModel.js b/backend/src/services/composeSessionModel.js new file mode 100644 index 00000000..49bb1016 --- /dev/null +++ b/backend/src/services/composeSessionModel.js @@ -0,0 +1,144 @@ +import { normalizeRecipients, sanitizeHeaderValue } from './mail/addresses.js'; +import { UUID_RE } from '../utils/validation.js'; + +export const MAX_COMPOSE_SESSIONS = 9; +export const MAX_COMPOSE_ATTACHMENT_BYTES = 25 * 1024 * 1024; +export const MAX_COMPOSE_ATTACHMENTS = 100; +export const PATCHABLE_FIELDS = Object.freeze([ + 'accountId', 'aliasId', 'mode', 'to', 'cc', 'bcc', 'subject', 'body', + 'bodyIsHtml', 'quotedBody', 'quotedBodyHtml', 'editedSignature', + 'forwardedAttachments', 'priority', 'inReplyTo', 'references', 'fromChanged', +]); + +const RECIPIENT_FIELDS = new Set(['to', 'cc', 'bcc']); +const NULLABLE_UUID_FIELDS = new Set(['accountId', 'aliasId']); +const NULLABLE_TEXT_FIELDS = new Set([ + 'quotedBody', 'quotedBodyHtml', 'editedSignature', 'inReplyTo', +]); +const BOOLEAN_FIELDS = new Set(['bodyIsHtml', 'fromChanged']); +// Opaque transport correlation only: no whitespace, domains, paths, or free-form content. +const CLIENT_ID_RE = /^(?:[A-Za-z0-9_-]{1,64}|mcp:[A-Za-z0-9_-]{1,60})$/; + +export function composeSessionError(code, message, status = 400, details = {}) { + return Object.assign(new Error(message), { code, status, details, expose: true }); +} + +function invalidChanges(message) { + throw composeSessionError('invalid_compose_changes', message); +} + +function normalizeHeaderList(value, field) { + if (!Array.isArray(value)) invalidChanges(`${field} must be an array`); + return value.map((item, index) => { + if (typeof item !== 'string' || !item.trim()) { + invalidChanges(`${field}[${index}] must be a non-empty string`); + } + const normalized = sanitizeHeaderValue(item); + if (!normalized) invalidChanges(`${field}[${index}] must be a non-empty string`); + return normalized; + }); +} + +function normalizeForwardedAttachments(value) { + if (!Array.isArray(value)) invalidChanges('forwardedAttachments must be an array'); + return value.map((attachment, index) => { + if (!attachment || typeof attachment !== 'object' || Array.isArray(attachment) + || typeof attachment.messageId !== 'string' || !UUID_RE.test(attachment.messageId)) { + invalidChanges(`forwardedAttachments[${index}].messageId is invalid`); + } + if (typeof attachment.part !== 'string' || !attachment.part.trim()) { + invalidChanges(`forwardedAttachments[${index}].part is required`); + } + const part = sanitizeHeaderValue(attachment.part); + if (!part) invalidChanges(`forwardedAttachments[${index}].part is required`); + return { messageId: attachment.messageId, part }; + }); +} + +export function normalizeComposeChanges(input = {}) { + if (!input || typeof input !== 'object' || Array.isArray(input)) { + invalidChanges('changes must be an object'); + } + const output = {}; + for (const field of PATCHABLE_FIELDS) { + if (!Object.prototype.hasOwnProperty.call(input, field)) continue; + const value = input[field]; + if (RECIPIENT_FIELDS.has(field)) { + try { + output[field] = normalizeRecipients(value, field); + } catch (error) { + invalidChanges(error.message); + } + } else if (NULLABLE_UUID_FIELDS.has(field)) { + if (value !== null && (typeof value !== 'string' || !UUID_RE.test(value))) { + invalidChanges(`${field} must be a UUID or null`); + } + output[field] = value; + } else if (field === 'subject') { + if (typeof value !== 'string') invalidChanges('subject must be a string'); + output[field] = sanitizeHeaderValue(value); + } else if (field === 'body') { + if (typeof value !== 'string') invalidChanges('body must be a string'); + output[field] = value; + } else if (NULLABLE_TEXT_FIELDS.has(field)) { + if (value !== null && typeof value !== 'string') { + invalidChanges(`${field} must be a string or null`); + } + output[field] = field === 'inReplyTo' && value !== null + ? sanitizeHeaderValue(value) + : value; + } else if (BOOLEAN_FIELDS.has(field)) { + if (typeof value !== 'boolean') invalidChanges(`${field} must be a boolean`); + output[field] = value; + } else if (field === 'forwardedAttachments') { + output[field] = normalizeForwardedAttachments(value); + } else if (field === 'references') { + output[field] = normalizeHeaderList(value, field); + } else if (field === 'priority') { + if (!['low', 'normal', 'high'].includes(value)) { + invalidChanges('priority must be low, normal, or high'); + } + output[field] = value; + } else if (field === 'mode') { + if (!['new', 'reply', 'reply_all', 'forward'].includes(value)) { + invalidChanges('unsupported compose mode'); + } + output[field] = value; + } + } + return output; +} + +export function normalizeReplyAllRecipients(value = []) { + try { + return normalizeRecipients(value, 'replyAllRecipients'); + } catch (error) { + invalidChanges(error.message); + } +} + +export function normalizeComposeClientId(value) { + if (value === undefined || value === null || value === '') return undefined; + if (typeof value !== 'string' || !CLIENT_ID_RE.test(value)) { + throw composeSessionError( + 'invalid_client_id', + 'clientId must be 1-64 characters using letters, numbers, underscores, or hyphens', + ); + } + return value; +} + +export function meaningfulComposeSession(session = {}) { + const lists = ['to', 'cc', 'bcc', 'forwardedAttachments']; + if (lists.some(key => Array.isArray(session[key]) && session[key].length > 0)) return true; + if (['subject', 'body', 'quotedBody', 'quotedBodyHtml', 'editedSignature'] + .some(key => String(session[key] || '').trim() !== '')) return true; + if (Number(session.attachmentCount || 0) > 0) return true; + if (session.mode && session.mode !== 'new') return true; + if (session.inReplyTo || session.fromChanged) return true; + return session.priority != null && session.priority !== 'normal'; +} + +export function findComposeConflicts(fieldRevisions = {}, expectedRevision, fields = []) { + return fields.filter(field => Number(fieldRevisions[field] || 0) > Number(expectedRevision)); +} diff --git a/backend/src/services/composeSessionModel.test.js b/backend/src/services/composeSessionModel.test.js new file mode 100644 index 00000000..e3a17adc --- /dev/null +++ b/backend/src/services/composeSessionModel.test.js @@ -0,0 +1,164 @@ +import { describe, expect, it } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { + MAX_COMPOSE_SESSIONS, + findComposeConflicts, + meaningfulComposeSession, + normalizeComposeChanges, + normalizeComposeClientId, + normalizeReplyAllRecipients, +} from './composeSessionModel.js'; + +describe('composeSessionModel', () => { + it('relies on the unique user-slot constraint without a redundant index', () => { + const migration = readFileSync( + new URL('../../migrations/0053_compose_sessions.sql', import.meta.url), + 'utf8', + ); + expect(migration).toContain('UNIQUE (user_id, slot)'); + expect(migration).not.toContain('idx_compose_sessions_user'); + }); + + it('adds durable reply-all metadata and idempotent outbox restore linkage in 0054', () => { + const migration = readFileSync( + new URL('../../migrations/0054_compose_restore_reply_all.sql', import.meta.url), + 'utf8', + ); + expect(migration).toContain('reply_all_recipients JSONB'); + expect(migration).toContain('restored_compose_session_id UUID'); + expect(migration).toContain('ON DELETE SET NULL'); + }); + + it('pins the workspace to nine slots', () => { + expect(MAX_COMPOSE_SESSIONS).toBe(9); + }); + + it('normalizes editable fields without admitting ownership fields', () => { + expect(normalizeComposeChanges({ + subject: ' Hello\r\nBcc: injected@example.com ', + to: [' A '], + priority: 'high', + slot: 8, + user_id: 'attacker', + })).toEqual({ + subject: 'HelloBcc: injected@example.com', + to: ['A '], + priority: 'high', + }); + }); + + it.each([ + [null, 'changes must be an object'], + [[], 'changes must be an object'], + ['subject=x', 'changes must be an object'], + [{ accountId: 'not-a-uuid' }, 'accountId must be a UUID or null'], + [{ aliasId: 42 }, 'aliasId must be a UUID or null'], + [{ mode: 1 }, 'unsupported compose mode'], + [{ to: 'a@example.com' }, 'to must be an array'], + [{ cc: [42] }, 'cc[0] is empty or not a string'], + [{ bcc: ['missing-at'] }, 'bcc[0] is not a valid email address'], + [{ subject: false }, 'subject must be a string'], + [{ body: null }, 'body must be a string'], + [{ bodyIsHtml: 'true' }, 'bodyIsHtml must be a boolean'], + [{ quotedBody: 42 }, 'quotedBody must be a string or null'], + [{ quotedBodyHtml: [] }, 'quotedBodyHtml must be a string or null'], + [{ editedSignature: false }, 'editedSignature must be a string or null'], + [{ forwardedAttachments: {} }, 'forwardedAttachments must be an array'], + [{ forwardedAttachments: [{}] }, 'forwardedAttachments[0].messageId is invalid'], + [{ forwardedAttachments: [{ + messageId: '11111111-1111-4111-8111-111111111111', + part: '', + }] }, 'forwardedAttachments[0].part is required'], + [{ priority: 'urgent' }, 'priority must be low, normal, or high'], + [{ inReplyTo: [] }, 'inReplyTo must be a string or null'], + [{ references: '' }, 'references must be an array'], + [{ references: [42] }, 'references[0] must be a non-empty string'], + [{ fromChanged: 1 }, 'fromChanged must be a boolean'], + ])('rejects malformed patch input %#', (changes, message) => { + expect(() => normalizeComposeChanges(changes)).toThrowError( + expect.objectContaining({ + code: 'invalid_compose_changes', + message, + status: 400, + }), + ); + }); + + it('validates nullable UUIDs and sanitizes message-id headers', () => { + expect(normalizeComposeChanges({ + accountId: null, + aliasId: '11111111-1111-4111-8111-111111111111', + inReplyTo: ' \r\n ', + references: [' ', '\0'], + forwardedAttachments: [{ + messageId: '22222222-2222-4222-8222-222222222222', + part: ' 2.1\r\n ', + }], + })).toEqual({ + accountId: null, + aliasId: '11111111-1111-4111-8111-111111111111', + inReplyTo: '', + references: ['', ''], + forwardedAttachments: [{ + messageId: '22222222-2222-4222-8222-222222222222', + part: '2.1', + }], + }); + }); + + it.each([ + ['browser-a', 'browser-a'], + ['A_1-opaque', 'A_1-opaque'], + ['mcp:token-1', 'mcp:token-1'], + [`mcp:${'u'.repeat(60)}`, `mcp:${'u'.repeat(60)}`], + [undefined, undefined], + [null, undefined], + ['', undefined], + ])('normalizes an opaque client id %#', (value, expected) => { + expect(normalizeComposeClientId(value)).toBe(expected); + }); + + it.each([ + ['contains spaces'], + ['person@example.com'], + ['client.with.content'], + ['mcp:'], + ['mcp:user id'], + ['mcp:user/id'], + ['mcp:user\nnext'], + [`mcp:${'u'.repeat(61)}`], + ['x'.repeat(65)], + [{}], + ])('rejects malformed or content-bearing client ids %#', (value) => { + expect(() => normalizeComposeClientId(value)).toThrowError( + expect.objectContaining({ code: 'invalid_client_id', status: 400 }), + ); + }); + + it.each([ + [{ to: ['a@example.com'] }, true], + [{ subject: 'x' }, true], + [{ body: '

x

', bodyIsHtml: true }, true], + [{ attachmentCount: 1 }, true], + [{ mode: 'reply', inReplyTo: '' }, true], + [{ fromChanged: true }, true], + [{ accountId: 'default-account', mode: 'new' }, false], + ])('classifies meaningful state %#', (session, expected) => { + expect(meaningfulComposeSession(session)).toBe(expected); + }); + + it('reports only fields changed after the caller revision', () => { + expect(findComposeConflicts({ subject: 4, to: 2, body: 7 }, 3, ['subject', 'to'])) + .toEqual(['subject']); + }); + + it('sanitizes durable reply-all source recipients outside editable changes', () => { + expect(normalizeReplyAllRecipients([ + ' Synthetic Person ', + ])).toEqual(['Synthetic Person ']); + expect(normalizeComposeChanges({ + subject: 'Synthetic', + replyAllRecipients: ['ignored@example.com'], + })).toEqual({ subject: 'Synthetic' }); + }); +}); diff --git a/backend/src/services/composeSessionRestore.test.js b/backend/src/services/composeSessionRestore.test.js new file mode 100644 index 00000000..1bf11bb8 --- /dev/null +++ b/backend/src/services/composeSessionRestore.test.js @@ -0,0 +1,441 @@ +import { describe, expect, it, vi } from 'vitest'; +import { restoreQueuedComposeSession } from './composeSessionService.js'; +import { enqueue } from './outboxService.js'; + +const USER_ID = '11111111-1111-4111-8111-111111111111'; +const OUTBOX_ID = '22222222-2222-4222-8222-222222222222'; +const ACCOUNT_ID = '33333333-3333-4333-8333-333333333333'; +const SESSION_ID = '44444444-4444-4444-8444-444444444444'; +const ATTACHMENT_ID = '55555555-5555-4555-8555-555555555555'; + +function restorePayload() { + const payload = { + version: 1, + originalSessionId: SESSION_ID, + preferredSlot: 3, + changes: { + accountId: ACCOUNT_ID, + aliasId: null, + mode: 'reply', + to: ['Synthetic Recipient '], + cc: [], + bcc: [], + subject: 'Synthetic subject', + body: 'Synthetic body', + bodyIsHtml: false, + quotedBody: null, + quotedBodyHtml: null, + editedSignature: null, + forwardedAttachments: [], + priority: 'normal', + inReplyTo: '', + references: [''], + fromChanged: false, + }, + replyAllRecipients: ['Synthetic Copied '], + sourceDraft: { + accountId: ACCOUNT_ID, + folder: 'Drafts', + uid: 7, + messageId: '', + initialRevision: null, + }, + attachments: [{ + id: ATTACHMENT_ID, + filename: 'synthetic.txt', + contentType: 'text/plain', + byteCount: 12, + contentSha256: '6fac1c9e222157d1baa07e669d6df5b6be7177dc362306c79acfc2c6f31dfd0b', + }], + }; + payload.sourceDraft.initialRevision = { + ...structuredClone(payload.changes), + attachments: [{ + id: ATTACHMENT_ID, + filename: 'synthetic.txt', + contentType: 'text/plain', + byteCount: 12, + }], + }; + return payload; +} + +function fakeDependencies({ occupiedSlots = [] } = {}) { + const state = { + outbox: { + id: OUTBOX_ID, + user_id: USER_ID, + status: 'pending', + payload: { + composeSessionRestore: restorePayload(), + attachments: [{ + filename: 'synthetic.txt', + contentType: 'text/plain', + content: Buffer.from('server bytes').toString('base64'), + }], + }, + restored_compose_session_id: null, + idempotency_key: 'compose-session-original', + }, + sessions: occupiedSlots.map((slot, index) => ({ + id: `90000000-0000-4000-8000-${String(index + 1).padStart(12, '0')}`, + user_id: USER_ID, + slot, + })), + attachments: [], + }; + let lock = Promise.resolve(); + + const query = vi.fn(async (sql, params = []) => { + const normalized = sql.replace(/\s+/g, ' ').trim(); + if (normalized.includes('FROM outbox_messages') && normalized.includes('FOR UPDATE')) { + const [id, userId] = params; + return { rows: state.outbox.id === id && state.outbox.user_id === userId + ? [{ ...state.outbox }] + : [] }; + } + if (normalized.includes('pg_advisory_xact_lock')) return { rows: [{}] }; + if (normalized.includes('FROM email_accounts')) { + const [accountId, userId] = params; + return { rows: accountId === ACCOUNT_ID && userId === USER_ID ? [{ id: ACCOUNT_ID }] : [] }; + } + if (normalized.includes('generate_series(1, 9)')) { + const [, preferredSlot] = params; + const occupied = new Set(state.sessions.map(row => row.slot)); + const candidates = [preferredSlot, ...Array.from({ length: 9 }, (_, index) => index + 1)] + .filter((slot, index, all) => slot != null && all.indexOf(slot) === index); + const slot = candidates.find(candidate => !occupied.has(candidate)); + return { rows: slot == null ? [] : [{ slot }] }; + } + if (normalized.startsWith('INSERT INTO compose_sessions')) { + const row = { + id: params[0], user_id: params[1], slot: Number(params[2]), + account_id: params[3], alias_id: params[4], mode: params[5], + to_recipients: JSON.parse(params[6]), cc_recipients: JSON.parse(params[7]), + bcc_recipients: JSON.parse(params[8]), subject: params[9], body: params[10], + body_is_html: params[11], quoted_body: params[12], quoted_body_html: params[13], + edited_signature: params[14], forwarded_attachments: JSON.parse(params[15]), + priority: params[16], in_reply_to: params[17], thread_references: JSON.parse(params[18]), + from_changed: params[19], reply_all_recipients: JSON.parse(params[20]), + source_draft_account_id: params[21], source_draft_folder: params[22], + source_draft_uid: params[23], source_draft_message_id: params[24], + source_initial_revision: JSON.parse(params[25]), presentation_state: 'expanded', + operation_state: 'idle', operation_token: null, revision: 1, field_revisions: '{}', + last_focused_at: new Date('2026-08-01T00:00:00Z'), + created_at: new Date('2026-08-01T00:00:00Z'), updated_at: new Date('2026-08-01T00:00:00Z'), + }; + state.sessions.push(row); + return { rows: [{ ...row }] }; + } + if (normalized.startsWith('INSERT INTO compose_session_attachments')) { + const row = { + id: params[0], session_id: params[1], filename: params[2], content_type: params[3], + byte_count: params[4], content: Buffer.from(params[5]), + created_at: new Date('2026-08-01T00:00:00Z'), + }; + state.attachments.push(row); + return { rows: [{ ...row }] }; + } + if (normalized.startsWith('UPDATE outbox_messages')) { + state.outbox.status = 'cancelled'; + state.outbox.payload = {}; + state.outbox.restored_compose_session_id = params[2]; + if (normalized.includes('idempotency_key=NULL')) state.outbox.idempotency_key = null; + return { rows: [{ id: state.outbox.id }] }; + } + if (normalized.startsWith('INSERT INTO outbox_messages')) { + const [userId, accountId, payload, , , , , idempotencyKey] = params; + const existing = [state.outbox, state.newOutbox].find(row => ( + row?.user_id === userId && row.idempotency_key === idempotencyKey && idempotencyKey != null + )); + if (existing) return { rows: [{ id: existing.id, send_at: existing.send_at }] }; + state.newOutbox = { + id: '77777777-7777-4777-8777-777777777777', + user_id: userId, + account_id: accountId, + payload, + status: 'pending', + send_at: new Date('2026-08-01T00:00:30Z'), + idempotency_key: idempotencyKey, + }; + return { rows: [{ id: state.newOutbox.id, send_at: state.newOutbox.send_at }] }; + } + if (normalized.startsWith('SELECT * FROM compose_sessions')) { + const [id, userId] = params; + const row = state.sessions.find(item => item.id === id && item.user_id === userId); + return { rows: row ? [{ ...row }] : [] }; + } + if (normalized.includes('FROM compose_session_attachments')) { + return { rows: state.attachments + .filter(item => item.session_id === params[0]) + .map(item => ({ + id: item.id, + session_id: item.session_id, + filename: item.filename, + content_type: item.content_type, + byte_count: item.byte_count, + created_at: item.created_at, + })) }; + } + throw new Error(`Unexpected restore SQL: ${normalized}`); + }); + + const withTransaction = async (callback) => { + let release; + const prior = lock; + lock = new Promise(resolve => { release = resolve; }); + await prior; + const snapshot = structuredClone(state); + try { + return await callback({ query }); + } catch (error) { + Object.assign(state, snapshot); + throw error; + } finally { + release(); + } + }; + + async function claimWorker() { + return withTransaction(async (client) => { + const selected = await client.query( + 'SELECT * FROM outbox_messages WHERE id=$1 AND user_id=$2 FOR UPDATE', + [OUTBOX_ID, USER_ID], + ); + if (selected.rows[0]?.status !== 'pending') return false; + state.outbox.status = 'claimed'; + return true; + }); + } + + return { deps: { withTransaction, broadcast: vi.fn() }, state, claimWorker }; +} + +describe('restoreQueuedComposeSession', () => { + it('atomically restores attachments, wipes private payload, and replays without duplication', async () => { + const fake = fakeDependencies(); + + const first = await restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps); + const replay = await restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps); + + expect(first).toMatchObject({ restored: true, replayed: false, session: { + id: SESSION_ID, slot: 3, + replyAllRecipients: ['Synthetic Copied '], + attachments: [{ id: ATTACHMENT_ID, filename: 'synthetic.txt', byteCount: 12 }], + } }); + expect(first.session.attachments[0]).not.toHaveProperty('content'); + expect(replay).toMatchObject({ restored: true, replayed: true, session: { id: SESSION_ID } }); + expect(fake.state.sessions.filter(row => row.id === SESSION_ID)).toHaveLength(1); + expect(fake.state.attachments[0].content.equals(Buffer.from('server bytes'))).toBe(true); + expect(fake.state.outbox).toMatchObject({ + status: 'cancelled', payload: {}, restored_compose_session_id: SESSION_ID, + }); + }); + + it('rotates the restored send key so retry enqueue creates one new pending delivery', async () => { + const fake = fakeDependencies(); + await restoreQueuedComposeSession({ userId: USER_ID, outboxId: OUTBOX_ID }, fake.deps); + expect(fake.state.outbox.idempotency_key).toBeNull(); + const enqueueDeps = { + query: (...args) => fake.deps.withTransaction(client => client.query(...args)), + }; + const input = { + userId: USER_ID, + accountId: ACCOUNT_ID, + payload: { subject: 'New pending delivery' }, + undoSeconds: 30, + idempotencyKey: 'compose-session-original', + subject: 'New pending delivery', + toPreview: ['recipient@example.com'], + messageId: '', + }; + const [first, retry] = await Promise.all([ + enqueue(input, enqueueDeps), + enqueue(input, enqueueDeps), + ]); + expect(first.outbox_id).toBe('77777777-7777-4777-8777-777777777777'); + expect(retry.outbox_id).toBe(first.outbox_id); + expect(fake.state.newOutbox).toMatchObject({ + status: 'pending', + payload: expect.objectContaining({ subject: 'New pending delivery' }), + idempotency_key: 'compose-session-original', + }); + expect(fake.state.outbox).toMatchObject({ status: 'cancelled', payload: {} }); + await expect(restoreQueuedComposeSession({ userId: USER_ID, outboxId: OUTBOX_ID }, fake.deps)) + .resolves.toMatchObject({ restored: true, replayed: true, session: { id: SESSION_ID } }); + }); + + it('rolls back without cancelling or clearing when all nine slots are full', async () => { + const fake = fakeDependencies({ occupiedSlots: [1, 2, 3, 4, 5, 6, 7, 8, 9] }); + await expect(restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps)).rejects.toMatchObject({ code: 'compose_session_limit', status: 409 }); + expect(fake.state.outbox.status).toBe('pending'); + expect(fake.state.outbox.payload.attachments[0].content).toBeTruthy(); + expect(fake.state.sessions).toHaveLength(9); + }); + + it('uses the deterministic lowest free slot when the preferred slot is occupied', async () => { + const fake = fakeDependencies({ occupiedSlots: [1, 3] }); + await expect(restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps)).resolves.toMatchObject({ session: { slot: 2 } }); + }); + + it('rejects corrupt attachment content without clearing the pending payload', async () => { + const fake = fakeDependencies(); + fake.state.outbox.payload.attachments[0].content = 'not base64'; + await expect(restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps)).rejects.toMatchObject({ code: 'invalid_compose_restore_payload', status: 409 }); + expect(fake.state.outbox.status).toBe('pending'); + expect(fake.state.sessions).toHaveLength(0); + }); + + it.each([ + ['truncated', Buffer.from('server byte')], + ['expanded', Buffer.from('server bytes!')], + ])('rejects %s uploaded bytes when the canonical descriptor size is unchanged', async ( + _label, + content, + ) => { + const fake = fakeDependencies(); + fake.state.outbox.payload.composeSessionRestore.sourceDraft = null; + fake.state.outbox.payload.attachments[0].content = content.toString('base64'); + await expect(restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps)).rejects.toMatchObject({ code: 'invalid_compose_restore_payload', status: 409 }); + expect(fake.state.outbox).toMatchObject({ + status: 'pending', + idempotency_key: 'compose-session-original', + restored_compose_session_id: null, + }); + expect(fake.state.outbox.payload).not.toEqual({}); + expect(fake.state.sessions).toHaveLength(0); + }); + + it('rejects same-length substituted bytes without mutating the pending outbox row', async () => { + const fake = fakeDependencies(); + fake.state.outbox.payload.composeSessionRestore.sourceDraft = null; + fake.state.outbox.payload.attachments[0].content = Buffer.from('server bytez') + .toString('base64'); + const pendingBeforeRestore = structuredClone(fake.state.outbox); + + await expect(restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps)).rejects.toMatchObject({ code: 'invalid_compose_restore_payload', status: 409 }); + + expect(fake.state.outbox).toEqual(pendingBeforeRestore); + expect(fake.state.sessions).toHaveLength(0); + expect(fake.state.attachments).toHaveLength(0); + }); + + it('restores mixed source and later uploaded attachments with exact byte counts', async () => { + const fake = fakeDependencies(); + const uploadedId = '88888888-8888-4888-8888-888888888888'; + fake.state.outbox.payload.composeSessionRestore.attachments.push({ + id: uploadedId, + filename: 'later.txt', + contentType: 'text/plain', + byteCount: 5, + contentSha256: '1d9283d848ea941ace1fe0d2378ef8b70056a0d4d1648b95a322d90163e78285', + }); + fake.state.outbox.payload.attachments.push({ + filename: 'later.txt', + contentType: 'text/plain', + content: Buffer.from('later').toString('base64'), + }); + + const result = await restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps); + + expect(result.session.attachments).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: ATTACHMENT_ID, byteCount: 12 }), + expect.objectContaining({ id: uploadedId, byteCount: 5 }), + ])); + expect(result.session.sourceInitialRevision.attachments).toEqual([ + expect.objectContaining({ id: ATTACHMENT_ID, byteCount: 12 }), + ]); + }); + + it.each([ + ['missing version', capsule => { delete capsule.version; }], + ['unknown key', capsule => { capsule.unexpected = true; }], + ['missing canonical field', capsule => { delete capsule.changes.mode; }], + ['unknown canonical field', capsule => { capsule.changes.unexpected = true; }], + ['missing attachment byte count', capsule => { delete capsule.attachments[0].byteCount; }], + ['missing attachment digest', capsule => { delete capsule.attachments[0].contentSha256; }], + ['bad attachment digest', capsule => { capsule.attachments[0].contentSha256 = 'A'.repeat(64); }], + ['unknown attachment field', capsule => { capsule.attachments[0].unexpected = true; }], + ['unsafe attachment byte count', capsule => { + capsule.attachments[0].byteCount = Number.MAX_SAFE_INTEGER + 1; + }], + ['duplicate attachment ids', capsule => { capsule.attachments.push({ ...capsule.attachments[0] }); }], + ['duplicate source attachment ids', capsule => { + capsule.sourceDraft.initialRevision.attachments.push({ + ...capsule.sourceDraft.initialRevision.attachments[0], + }); + }], + ['invalid source attachment byte count', capsule => { + capsule.sourceDraft.initialRevision.attachments[0].byteCount = -1; + }], + ['null sending account', capsule => { capsule.changes.accountId = null; }], + ])('rejects exact-schema corruption: %s', async (_label, corrupt) => { + const fake = fakeDependencies(); + corrupt(fake.state.outbox.payload.composeSessionRestore); + if (fake.state.outbox.payload.composeSessionRestore.attachments.length > 1) { + fake.state.outbox.payload.attachments.push({ ...fake.state.outbox.payload.attachments[0] }); + } + await expect(restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps)).rejects.toMatchObject({ code: 'invalid_compose_restore_payload', status: 409 }); + expect(fake.state.outbox.status).toBe('pending'); + expect(fake.state.outbox.payload).not.toEqual({}); + expect(fake.state.sessions).toHaveLength(0); + }); + + it('revalidates the stored sending identity before restoring', async () => { + const fake = fakeDependencies(); + fake.state.outbox.payload.composeSessionRestore.changes.accountId = + 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; + await expect(restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps)).rejects.toMatchObject({ code: 'invalid_compose_restore_payload', status: 409 }); + expect(fake.state.outbox.status).toBe('pending'); + }); + + it('has exactly one winner against worker claim on the locked outbox row', async () => { + const fake = fakeDependencies(); + const [restored, claimed] = await Promise.all([ + restoreQueuedComposeSession({ userId: USER_ID, outboxId: OUTBOX_ID }, fake.deps) + .then(() => true, () => false), + fake.claimWorker(), + ]); + expect(Number(restored) + Number(claimed)).toBe(1); + }); + + it.each([ + ['claimed', 'compose_outbox_too_late', 409], + ['sent', 'compose_outbox_too_late', 409], + ['failed', 'compose_outbox_too_late', 409], + ['cancelled', 'compose_outbox_cancelled', 409], + ])('returns an explicit %s outcome', async (status, code, expectedStatus) => { + const fake = fakeDependencies(); + fake.state.outbox.status = status; + await expect(restoreQueuedComposeSession({ + userId: USER_ID, outboxId: OUTBOX_ID, + }, fake.deps)).rejects.toMatchObject({ code, status: expectedStatus }); + }); + + it('owner-scopes missing outbox rows', async () => { + const fake = fakeDependencies(); + await expect(restoreQueuedComposeSession({ + userId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', outboxId: OUTBOX_ID, + }, fake.deps)).rejects.toMatchObject({ code: 'compose_outbox_not_found', status: 404 }); + }); +}); diff --git a/backend/src/services/composeSessionService.js b/backend/src/services/composeSessionService.js new file mode 100644 index 00000000..976c2c96 --- /dev/null +++ b/backend/src/services/composeSessionService.js @@ -0,0 +1,1165 @@ +import { createHash, randomUUID, timingSafeEqual } from 'node:crypto'; +import { + MAX_COMPOSE_ATTACHMENT_BYTES, + MAX_COMPOSE_ATTACHMENTS, + PATCHABLE_FIELDS, + composeSessionError, + findComposeConflicts, + normalizeComposeChanges, + normalizeComposeClientId, + normalizeReplyAllRecipients, +} from './composeSessionModel.js'; +import { sanitizeHeaderValue } from './mail/addresses.js'; +import { UUID_RE } from '../utils/validation.js'; + +const FIELD_COLUMNS = Object.freeze({ + accountId: 'account_id', + aliasId: 'alias_id', + mode: 'mode', + to: 'to_recipients', + cc: 'cc_recipients', + bcc: 'bcc_recipients', + subject: 'subject', + body: 'body', + bodyIsHtml: 'body_is_html', + quotedBody: 'quoted_body', + quotedBodyHtml: 'quoted_body_html', + editedSignature: 'edited_signature', + forwardedAttachments: 'forwarded_attachments', + priority: 'priority', + inReplyTo: 'in_reply_to', + references: 'thread_references', + fromChanged: 'from_changed', +}); + +const JSON_FIELDS = new Set(['to', 'cc', 'bcc', 'forwardedAttachments', 'references']); +const MIME_TYPE_RE = /^[A-Za-z0-9!#$&^_.+-]+\/[A-Za-z0-9!#$&^_.+-]+$/; + +const NEW_SESSION_DEFAULTS = Object.freeze({ + accountId: null, + aliasId: null, + mode: 'new', + to: [], + cc: [], + bcc: [], + subject: '', + body: '', + bodyIsHtml: true, + quotedBody: null, + quotedBodyHtml: null, + editedSignature: null, + forwardedAttachments: [], + priority: 'normal', + inReplyTo: null, + references: [], + fromChanged: false, +}); + +function jsonValue(value, fallback) { + if (value == null) return fallback; + if (typeof value !== 'string') return value; + try { + return JSON.parse(value); + } catch { + return fallback; + } +} + +function mapSessionRow(row) { + return { + id: row.id, + slot: Number(row.slot), + accountId: row.account_id, + aliasId: row.alias_id, + mode: row.mode, + to: jsonValue(row.to_recipients, []), + cc: jsonValue(row.cc_recipients, []), + bcc: jsonValue(row.bcc_recipients, []), + subject: row.subject, + body: row.body, + bodyIsHtml: row.body_is_html, + quotedBody: row.quoted_body, + quotedBodyHtml: row.quoted_body_html, + editedSignature: row.edited_signature, + forwardedAttachments: jsonValue(row.forwarded_attachments, []), + priority: row.priority, + inReplyTo: row.in_reply_to, + references: jsonValue(row.thread_references, []), + fromChanged: row.from_changed, + replyAllRecipients: jsonValue(row.reply_all_recipients, []), + sourceDraftAccountId: row.source_draft_account_id, + sourceDraftFolder: row.source_draft_folder, + sourceDraftUid: row.source_draft_uid, + sourceDraftMessageId: row.source_draft_message_id, + sourceInitialRevision: jsonValue(row.source_initial_revision, null), + presentationState: row.presentation_state, + operationState: row.operation_state, + operationToken: row.operation_token, + revision: Number(row.revision), + fieldRevisions: jsonValue(row.field_revisions, {}), + lastFocusedAt: row.last_focused_at, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + +function mapSummaryRow(row) { + return { + id: row.id, + slot: Number(row.slot), + accountId: row.account_id, + aliasId: row.alias_id, + mode: row.mode, + subject: row.subject, + priority: row.priority, + presentationState: row.presentation_state, + operationState: row.operation_state, + revision: Number(row.revision), + lastFocusedAt: row.last_focused_at, + createdAt: row.created_at, + updatedAt: row.updated_at, + attachmentCount: Number(row.attachment_count || 0), + }; +} + +function mapAttachmentRow(row) { + return { + id: row.id, + filename: row.filename, + contentType: row.content_type, + byteCount: Number(row.byte_count), + createdAt: row.created_at, + }; +} + +function mapClaimedAttachmentRow(row) { + return { + ...mapAttachmentRow(row), + content: Buffer.from(row.content), + }; +} + +function sanitizeAttachmentFilename(filename) { + if (typeof filename !== 'string') { + throw composeSessionError( + 'invalid_attachment_filename', + 'Attachment filename must be a non-empty string', + ); + } + const sanitized = sanitizeHeaderValue(filename); + if (!sanitized || sanitized.length > 255) { + throw composeSessionError( + 'invalid_attachment_filename', + 'Attachment filename must be a non-empty string of at most 255 characters', + ); + } + return sanitized; +} + +function normalizeAttachmentInput(input) { + if (!Buffer.isBuffer(input.content)) { + throw composeSessionError('invalid_attachment_body', 'Attachment body must be raw bytes'); + } + const filename = sanitizeAttachmentFilename(input.filename); + const contentType = input.contentType ?? 'application/octet-stream'; + if (typeof contentType !== 'string' + || contentType.length > 127 + || !MIME_TYPE_RE.test(contentType)) { + throw composeSessionError( + 'invalid_attachment_content_type', + 'Attachment content type must be a valid MIME type', + ); + } + return { content: Buffer.from(input.content), filename, contentType }; +} + +function sessionNotFound() { + return composeSessionError('compose_session_not_found', 'Compose session not found', 404); +} + +function requireLocator(input) { + const hasId = input.id !== undefined && input.id !== null; + const hasSlot = input.slot !== undefined && input.slot !== null; + if (hasId === hasSlot) { + throw composeSessionError( + 'invalid_compose_locator', + 'Exactly one compose session id or slot is required', + 400, + ); + } + if (hasId && (typeof input.id !== 'string' || !UUID_RE.test(input.id))) { + throw composeSessionError( + 'invalid_compose_locator', + 'Compose session id must be a UUID', + 400, + ); + } + if (hasSlot && (!Number.isInteger(input.slot) || input.slot < 1 || input.slot > 9)) { + throw composeSessionError('invalid_compose_locator', 'slot must be an integer from 1 to 9', 400); + } + return hasId + ? { column: 'id', value: input.id } + : { column: 'slot', value: input.slot }; +} + +function requireAttachmentId(attachmentId) { + if (typeof attachmentId !== 'string' || !UUID_RE.test(attachmentId)) { + throw composeSessionError( + 'invalid_compose_attachment_id', + 'Compose attachment id must be a UUID', + 400, + ); + } + return attachmentId; +} + +function requireExpectedRevision(expectedRevision) { + if (!Number.isSafeInteger(expectedRevision) || expectedRevision < 1) { + throw composeSessionError( + 'invalid_compose_revision', + 'expectedRevision must be a positive integer', + 400, + ); + } + return expectedRevision; +} + +function requireOperation(operation) { + if (!['closing', 'discarding', 'sending'].includes(operation)) { + throw composeSessionError( + 'invalid_compose_operation', + 'operation must be closing, discarding, or sending', + 400, + ); + } + return operation; +} + +function requireOperationToken(token) { + if (typeof token !== 'string' || !UUID_RE.test(token)) { + throw composeSessionError( + 'invalid_compose_operation_token', + 'Compose operation token must be a UUID', + 400, + ); + } + return token; +} + +async function lockOwnedSession(client, input) { + const locator = requireLocator(input); + const result = await client.query( + `SELECT * FROM compose_sessions WHERE ${locator.column}=$1 AND user_id=$2 FOR UPDATE`, + [locator.value, input.userId], + ); + if (!result.rows.length) throw sessionNotFound(); + return { locator, row: result.rows[0] }; +} + +function ensureIdle(row) { + if (row.operation_state !== 'idle') { + throw composeSessionError( + 'compose_operation_in_progress', + 'A terminal compose operation is already in progress', + 409, + ); + } +} + +function remoteValues(row, fields) { + const session = mapSessionRow(row); + return Object.fromEntries(fields.map(field => [field, session[field]])); +} + +function ensureNoConflicts(row, expectedRevision, fields, requireExactRevision = false) { + const currentRevision = Number(row.revision); + let conflictingFields = findComposeConflicts( + jsonValue(row.field_revisions, {}), + expectedRevision, + fields, + ); + if (requireExactRevision && expectedRevision !== currentRevision) { + conflictingFields = ['revision']; + } else if (expectedRevision > currentRevision) { + conflictingFields = [...fields]; + } + if (!conflictingFields.length) return; + throw composeSessionError( + 'compose_conflict', + 'Compose session changed in the requested fields', + 409, + { + conflictingFields, + currentRevision, + remoteValues: remoteValues(row, conflictingFields), + }, + ); +} + +async function validateIdentity(client, userId, accountId, aliasId) { + if (accountId != null) { + const account = await client.query( + 'SELECT id FROM email_accounts WHERE id=$1 AND user_id=$2', + [accountId, userId], + ); + if (!account.rows.length) { + throw composeSessionError( + 'compose_account_not_found', + 'Compose account not found', + 404, + ); + } + } + + if (aliasId != null) { + if (accountId == null) { + throw composeSessionError('compose_alias_not_found', 'Compose alias not found', 404); + } + const alias = await client.query( + `SELECT aa.id + FROM account_aliases aa + JOIN email_accounts ea ON ea.id=aa.account_id + WHERE aa.id=$1 AND aa.account_id=$2 AND ea.user_id=$3`, + [aliasId, accountId, userId], + ); + if (!alias.rows.length) { + throw composeSessionError('compose_alias_not_found', 'Compose alias not found', 404); + } + } +} + +function broadcastInvalidation(deps, userId, action, session, clientId) { + if (typeof deps.broadcast !== 'function') return; + const payload = { + type: 'compose_sessions_updated', + action, + sessionId: session.id, + slot: session.slot, + revision: session.revision, + }; + if (clientId) payload.clientId = clientId; + deps.broadcast(payload, userId); +} + +async function advanceAttachmentRevision(client, locator, row, userId) { + const revision = Number(row.revision) + 1; + const fieldRevisions = { + ...jsonValue(row.field_revisions, {}), + attachments: revision, + }; + const updated = await client.query( + `UPDATE compose_sessions + SET field_revisions=$1::jsonb, revision=revision + 1, updated_at=NOW() + WHERE ${locator.column}=$2 AND user_id=$3 + RETURNING *`, + [JSON.stringify(fieldRevisions), locator.value, userId], + ); + return mapSessionRow(updated.rows[0]); +} + +export async function createComposeSession(input, deps) { + const requestedSlot = input.requestedSlot ?? null; + if (requestedSlot !== null + && (!Number.isInteger(requestedSlot) || requestedSlot < 1 || requestedSlot > 9)) { + throw composeSessionError( + 'invalid_compose_slot', + 'requestedSlot must be an integer from 1 to 9', + 400, + ); + } + + const changes = normalizeComposeChanges( + input.changes === undefined ? {} : input.changes, + ); + const clientId = normalizeComposeClientId(input.clientId); + const replyAllRecipients = normalizeReplyAllRecipients(input.replyAllRecipients ?? []); + const values = { ...NEW_SESSION_DEFAULTS, ...changes }; + const result = await deps.withTransaction(async (client) => { + await client.query( + 'SELECT pg_advisory_xact_lock(hashtext($1))', + [`compose-slots:${input.userId}`], + ); + await validateIdentity(client, input.userId, values.accountId, values.aliasId); + + const allocated = await client.query( + `SELECT candidate.slot + FROM generate_series(1, 9) AS candidate(slot) + WHERE ($2::smallint IS NULL OR candidate.slot=$2) + AND NOT EXISTS ( + SELECT 1 FROM compose_sessions existing + WHERE existing.user_id=$1 AND existing.slot=candidate.slot + ) + ORDER BY candidate.slot + LIMIT 1`, + [input.userId, requestedSlot], + ); + const slot = allocated.rows[0]?.slot; + if (!slot) { + if (requestedSlot !== null) { + throw composeSessionError( + 'compose_slot_occupied', + `Compose slot ${requestedSlot} is already occupied`, + 409, + ); + } + throw composeSessionError( + 'compose_session_limit', + 'Nine compose sessions are already open', + 409, + ); + } + + const touched = Object.keys(changes); + const fieldRevisions = Object.fromEntries(touched.map(field => [field, 1])); + const inserted = await client.query( + `INSERT INTO compose_sessions ( + user_id, slot, account_id, alias_id, mode, + to_recipients, cc_recipients, bcc_recipients, subject, body, + body_is_html, quoted_body, quoted_body_html, edited_signature, + forwarded_attachments, priority, in_reply_to, thread_references, + from_changed, field_revisions, reply_all_recipients + ) VALUES ( + $1, $2, $3, $4, $5, + $6::jsonb, $7::jsonb, $8::jsonb, $9, $10, + $11, $12, $13, $14, + $15::jsonb, $16, $17, $18::jsonb, + $19, $20::jsonb, $21::jsonb + ) + RETURNING *`, + [ + input.userId, + Number(slot), + values.accountId, + values.aliasId, + values.mode, + JSON.stringify(values.to), + JSON.stringify(values.cc), + JSON.stringify(values.bcc), + values.subject, + values.body, + values.bodyIsHtml, + values.quotedBody, + values.quotedBodyHtml, + values.editedSignature, + JSON.stringify(values.forwardedAttachments), + values.priority, + values.inReplyTo, + JSON.stringify(values.references), + values.fromChanged, + JSON.stringify(fieldRevisions), + JSON.stringify(replyAllRecipients), + ], + ); + return mapSessionRow(inserted.rows[0]); + }); + + broadcastInvalidation(deps, input.userId, 'created', result, clientId); + return result; +} + +export async function patchComposeSession(input, deps) { + requireExpectedRevision(input.expectedRevision); + const changes = normalizeComposeChanges( + input.changes === undefined ? {} : input.changes, + ); + const clientId = normalizeComposeClientId(input.clientId); + const result = await deps.withTransaction(async (client) => { + const { locator, row } = await lockOwnedSession(client, input); + ensureIdle(row); + const fields = Object.keys(changes); + if (!fields.length) return { session: mapSessionRow(row), changed: false }; + + ensureNoConflicts(row, input.expectedRevision, fields); + const accountId = Object.hasOwn(changes, 'accountId') + ? changes.accountId + : row.account_id; + const aliasId = Object.hasOwn(changes, 'aliasId') + ? changes.aliasId + : row.alias_id; + if (Object.hasOwn(changes, 'accountId') || Object.hasOwn(changes, 'aliasId')) { + await validateIdentity(client, input.userId, accountId, aliasId); + } + + const params = []; + const assignments = fields.map((field) => { + const column = FIELD_COLUMNS[field]; + const value = JSON_FIELDS.has(field) ? JSON.stringify(changes[field]) : changes[field]; + params.push(value); + return `${column}=$${params.length}${JSON_FIELDS.has(field) ? '::jsonb' : ''}`; + }); + const revision = Number(row.revision) + 1; + const fieldRevisions = { + ...jsonValue(row.field_revisions, {}), + ...Object.fromEntries(fields.map(field => [field, revision])), + }; + params.push(JSON.stringify(fieldRevisions)); + assignments.push(`field_revisions=$${params.length}::jsonb`); + assignments.push('revision=revision + 1', 'updated_at=NOW()'); + params.push(locator.value, input.userId); + + const updated = await client.query( + `UPDATE compose_sessions + SET ${assignments.join(', ')} + WHERE ${locator.column}=$${params.length - 1} AND user_id=$${params.length} + RETURNING *`, + params, + ); + return { session: mapSessionRow(updated.rows[0]), changed: true }; + }); + + if (result.changed) { + broadcastInvalidation(deps, input.userId, 'updated', result.session, clientId); + } + return result.session; +} + +export async function claimComposeOperation(input, deps) { + requireExpectedRevision(input.expectedRevision); + const operation = requireOperation(input.operation); + const changes = normalizeComposeChanges( + input.changes === undefined ? {} : input.changes, + ); + + return deps.withTransaction(async (client) => { + const { locator, row } = await lockOwnedSession(client, input); + ensureIdle(row); + const fields = Object.keys(changes); + ensureNoConflicts(row, input.expectedRevision, fields, fields.length === 0); + + const accountId = Object.hasOwn(changes, 'accountId') + ? changes.accountId + : row.account_id; + const aliasId = Object.hasOwn(changes, 'aliasId') + ? changes.aliasId + : row.alias_id; + if (Object.hasOwn(changes, 'accountId') || Object.hasOwn(changes, 'aliasId')) { + await validateIdentity(client, input.userId, accountId, aliasId); + } + + const params = []; + const assignments = fields.map((field) => { + const column = FIELD_COLUMNS[field]; + const value = JSON_FIELDS.has(field) ? JSON.stringify(changes[field]) : changes[field]; + params.push(value); + return `${column}=$${params.length}${JSON_FIELDS.has(field) ? '::jsonb' : ''}`; + }); + if (fields.length) { + const revision = Number(row.revision) + 1; + const fieldRevisions = { + ...jsonValue(row.field_revisions, {}), + ...Object.fromEntries(fields.map(field => [field, revision])), + }; + params.push(JSON.stringify(fieldRevisions)); + assignments.push(`field_revisions=$${params.length}::jsonb`, 'revision=revision + 1'); + } + + const token = randomUUID(); + params.push(operation, token); + assignments.push( + `operation_state=$${params.length - 1}`, + `operation_token=$${params.length}`, + 'updated_at=NOW()', + ); + params.push(locator.value, input.userId); + const updated = await client.query( + `UPDATE compose_sessions + SET ${assignments.join(', ')} + WHERE ${locator.column}=$${params.length - 1} AND user_id=$${params.length} + RETURNING *`, + params, + ); + const session = mapSessionRow(updated.rows[0]); + const attachmentResult = await client.query( + `SELECT id, filename, content_type, byte_count, content, created_at + FROM compose_session_attachments + WHERE session_id=$1 + ORDER BY created_at, id`, + [session.id], + ); + return { + ...session, + attachments: attachmentResult.rows.map(mapClaimedAttachmentRow), + }; + }); +} + +export async function releaseComposeOperation(input, deps) { + const locator = requireLocator(input); + const token = requireOperationToken(input.token); + const result = await deps.query( + `UPDATE compose_sessions + SET operation_state='idle', operation_token=NULL, updated_at=NOW() + WHERE ${locator.column}=$1 AND user_id=$2 AND operation_token=$3 + RETURNING id`, + [locator.value, input.userId, token], + ); + return result.rows.length > 0; +} + +export async function deleteClaimedComposeSession(input, deps) { + const locator = requireLocator(input); + const token = requireOperationToken(input.token); + const result = await deps.query( + `DELETE FROM compose_sessions + WHERE ${locator.column}=$1 AND user_id=$2 AND operation_token=$3 + RETURNING id`, + [locator.value, input.userId, token], + ); + return result.rows.length > 0; +} + +export async function listComposeSessions({ userId }, deps) { + const result = await deps.query( + `SELECT cs.id, cs.slot, cs.account_id, cs.alias_id, cs.mode, cs.subject, + cs.priority, cs.presentation_state, cs.operation_state, cs.revision, + cs.last_focused_at, cs.created_at, cs.updated_at, + COUNT(csa.id)::int AS attachment_count + FROM compose_sessions cs + LEFT JOIN compose_session_attachments csa ON csa.session_id=cs.id + WHERE cs.user_id=$1 + GROUP BY cs.id + ORDER BY cs.slot`, + [userId], + ); + return result.rows.map(mapSummaryRow); +} + +export async function getComposeSession(input, deps) { + const locator = requireLocator(input); + const result = await deps.query( + `SELECT * FROM compose_sessions WHERE ${locator.column}=$1 AND user_id=$2`, + [locator.value, input.userId], + ); + if (!result.rows.length) throw sessionNotFound(); + const session = mapSessionRow(result.rows[0]); + const attachmentResult = await deps.query( + `SELECT id, filename, content_type, byte_count, created_at + FROM compose_session_attachments + WHERE session_id=$1 + ORDER BY created_at, id`, + [session.id], + ); + return { ...session, attachments: attachmentResult.rows.map(mapAttachmentRow) }; +} + +export async function setComposePresentation(input, deps) { + requireExpectedRevision(input.expectedRevision); + const clientId = normalizeComposeClientId(input.clientId); + const state = input.state ?? input.presentationState; + if (!['expanded', 'minimized'].includes(state)) { + throw composeSessionError( + 'invalid_presentation_state', + 'state must be expanded or minimized', + 400, + ); + } + + const session = await deps.withTransaction(async (client) => { + const { locator, row } = await lockOwnedSession(client, input); + ensureIdle(row); + ensureNoConflicts(row, input.expectedRevision, ['presentationState']); + const revision = Number(row.revision) + 1; + const fieldRevisions = { + ...jsonValue(row.field_revisions, {}), + presentationState: revision, + }; + const params = [state, JSON.stringify(fieldRevisions), locator.value, input.userId]; + const focusAssignment = state === 'expanded' ? ', last_focused_at=NOW()' : ''; + const updated = await client.query( + `UPDATE compose_sessions + SET presentation_state=$1, field_revisions=$2::jsonb, + revision=revision + 1${focusAssignment}, updated_at=NOW() + WHERE ${locator.column}=$3 AND user_id=$4 + RETURNING *`, + params, + ); + return mapSessionRow(updated.rows[0]); + }); + + broadcastInvalidation(deps, input.userId, 'presentation', session, clientId); + return session; +} + +export async function addComposeAttachment(input, deps) { + requireExpectedRevision(input.expectedRevision); + const { content, filename, contentType } = normalizeAttachmentInput(input); + const clientId = normalizeComposeClientId(input.clientId); + + const result = await deps.withTransaction(async (client) => { + const { locator, row } = await lockOwnedSession(client, input); + ensureIdle(row); + ensureNoConflicts(row, input.expectedRevision, ['attachments']); + + const aggregate = await client.query( + `SELECT COUNT(*)::int AS attachment_count, + COALESCE(SUM(byte_count), 0)::bigint AS total_bytes + FROM compose_session_attachments + WHERE session_id=$1`, + [row.id], + ); + const attachmentCount = Number(aggregate.rows[0]?.attachment_count || 0); + const totalBytes = Number(aggregate.rows[0]?.total_bytes || 0); + if (attachmentCount >= MAX_COMPOSE_ATTACHMENTS) { + throw composeSessionError( + 'attachment_count_limit', + 'Compose sessions support at most 100 attachments', + 413, + ); + } + if (totalBytes + content.length > MAX_COMPOSE_ATTACHMENT_BYTES) { + throw composeSessionError( + 'attachment_limit', + 'Compose attachments exceed the 25 MiB limit', + 413, + ); + } + + const inserted = await client.query( + `INSERT INTO compose_session_attachments ( + session_id, filename, content_type, byte_count, content + ) VALUES ($1, $2, $3, $4, $5) + RETURNING id, filename, content_type, byte_count, created_at`, + [row.id, filename, contentType, content.length, content], + ); + const session = await advanceAttachmentRevision(client, locator, row, input.userId); + return { attachment: mapAttachmentRow(inserted.rows[0]), session }; + }); + + broadcastInvalidation( + deps, + input.userId, + 'attachment_added', + result.session, + clientId, + ); + return { + sessionId: result.session.id, + slot: result.session.slot, + revision: result.session.revision, + attachment: result.attachment, + }; +} + +export async function removeComposeAttachment(input, deps) { + requireExpectedRevision(input.expectedRevision); + requireAttachmentId(input.attachmentId); + const clientId = normalizeComposeClientId(input.clientId); + const result = await deps.withTransaction(async (client) => { + const { locator, row } = await lockOwnedSession(client, input); + ensureIdle(row); + const existing = await client.query( + `SELECT id FROM compose_session_attachments + WHERE id=$1 AND session_id=$2`, + [input.attachmentId, row.id], + ); + if (!existing.rows.length) return { changed: false, session: mapSessionRow(row) }; + ensureNoConflicts(row, input.expectedRevision, ['attachments']); + + const deleted = await client.query( + `DELETE FROM compose_session_attachments csa + USING compose_sessions cs + WHERE csa.id=$1 + AND csa.session_id=cs.id + AND cs.id=$2 + AND cs.user_id=$3 + RETURNING csa.id`, + [input.attachmentId, row.id, input.userId], + ); + if (!deleted.rows.length) return { changed: false, session: mapSessionRow(row) }; + const session = await advanceAttachmentRevision(client, locator, row, input.userId); + return { changed: true, session }; + }); + + if (result.changed) { + broadcastInvalidation( + deps, + input.userId, + 'attachment_removed', + result.session, + clientId, + ); + } + return { + sessionId: result.session.id, + slot: result.session.slot, + revision: result.session.revision, + removedAttachmentId: input.attachmentId, + }; +} + +function restoreOutboxError(code, message, status) { + return composeSessionError(code, message, status); +} + +function invalidRestorePayload() { + return restoreOutboxError( + 'invalid_compose_restore_payload', + 'Queued compose restore payload is invalid', + 409, + ); +} + +function hasExactKeys(value, keys) { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + return actual.length === expected.length + && actual.every((key, index) => key === expected[index]); +} + +function canonicalBase64Bytes(value) { + if (typeof value !== 'string' + || value.length % 4 !== 0 + || !/^[A-Za-z0-9+/]*={0,2}$/.test(value)) { + throw invalidRestorePayload(); + } + const content = Buffer.from(value, 'base64'); + if (content.toString('base64') !== value) { + throw invalidRestorePayload(); + } + return content; +} + +function normalizeRestoreAttachmentFingerprints(value) { + if (!Array.isArray(value) || value.length > MAX_COMPOSE_ATTACHMENTS) { + throw invalidRestorePayload(); + } + const ids = new Set(); + let totalBytes = 0; + const attachments = value.map((attachment) => { + if (!hasExactKeys(attachment, ['id', 'filename', 'contentType', 'byteCount']) + || typeof attachment.id !== 'string' || !UUID_RE.test(attachment.id) + || ids.has(attachment.id) + || !Number.isSafeInteger(attachment.byteCount) || attachment.byteCount < 0) { + throw invalidRestorePayload(); + } + ids.add(attachment.id); + totalBytes += attachment.byteCount; + let normalized; + try { + normalized = normalizeAttachmentInput({ + filename: attachment.filename, + contentType: attachment.contentType, + content: Buffer.alloc(0), + }); + } catch { + throw invalidRestorePayload(); + } + return { + id: attachment.id, + filename: normalized.filename, + contentType: normalized.contentType, + byteCount: attachment.byteCount, + }; + }); + if (totalBytes > MAX_COMPOSE_ATTACHMENT_BYTES) throw invalidRestorePayload(); + return attachments.sort((left, right) => left.id.localeCompare(right.id)); +} + +function normalizeRestoreSource(value) { + if (value == null) return { + accountId: null, folder: null, uid: null, messageId: null, initialRevision: null, + }; + if (!hasExactKeys(value, [ + 'accountId', 'folder', 'uid', 'messageId', 'initialRevision', + ]) + || typeof value.accountId !== 'string' || !UUID_RE.test(value.accountId) + || typeof value.folder !== 'string' || !value.folder.trim() + || value.folder.length > 500 || /[\r\n\0]/.test(value.folder) + || !Number.isSafeInteger(value.uid) || value.uid < 1 + || (value.messageId != null && typeof value.messageId !== 'string') + || value.initialRevision == null + || !hasExactKeys(value.initialRevision, [...PATCHABLE_FIELDS, 'attachments'])) { + throw invalidRestorePayload(); + } + let initialChanges; + try { + initialChanges = normalizeComposeChanges(value.initialRevision); + } catch { + throw invalidRestorePayload(); + } + if (initialChanges.accountId !== value.accountId) throw invalidRestorePayload(); + const attachments = normalizeRestoreAttachmentFingerprints( + value.initialRevision.attachments, + ); + return { + accountId: value.accountId, + folder: value.folder, + uid: value.uid, + messageId: value.messageId == null ? null : sanitizeHeaderValue(value.messageId), + initialRevision: { ...initialChanges, attachments }, + }; +} + +function normalizeRestoreCapsule(value, queuedAttachments) { + if (!hasExactKeys(value, [ + 'version', 'originalSessionId', 'preferredSlot', 'changes', + 'replyAllRecipients', 'sourceDraft', 'attachments', + ]) + || value.version !== 1 + || typeof value.originalSessionId !== 'string' || !UUID_RE.test(value.originalSessionId) + || !Number.isInteger(value.preferredSlot) + || value.preferredSlot < 1 || value.preferredSlot > 9 + || !Array.isArray(value.attachments) + || !Array.isArray(queuedAttachments) + || queuedAttachments.length !== value.attachments.length) { + throw invalidRestorePayload(); + } + if (value.attachments.length > MAX_COMPOSE_ATTACHMENTS) { + throw invalidRestorePayload(); + } + if (!hasExactKeys(value.changes, PATCHABLE_FIELDS) + || typeof value.changes.accountId !== 'string' + || !UUID_RE.test(value.changes.accountId)) throw invalidRestorePayload(); + let changes; + let replyAllRecipients; + try { + changes = normalizeComposeChanges(value.changes); + replyAllRecipients = normalizeReplyAllRecipients(value.replyAllRecipients); + } catch { + throw invalidRestorePayload(); + } + const attachmentIds = new Set(); + const attachments = value.attachments.map((attachment, index) => { + const queued = queuedAttachments[index]; + if (!hasExactKeys( + attachment, + ['id', 'filename', 'contentType', 'byteCount', 'contentSha256'], + ) + || typeof attachment.id !== 'string' || !UUID_RE.test(attachment.id) + || attachmentIds.has(attachment.id) + || !Number.isSafeInteger(attachment.byteCount) || attachment.byteCount < 0 + || attachment.byteCount > MAX_COMPOSE_ATTACHMENT_BYTES + || typeof attachment.contentSha256 !== 'string' + || !/^[a-f0-9]{64}$/.test(attachment.contentSha256) + || !queued || typeof queued !== 'object' || Array.isArray(queued) + || queued.filename !== attachment.filename + || (queued.contentType ?? 'application/octet-stream') + !== (attachment.contentType ?? 'application/octet-stream')) { + throw invalidRestorePayload(); + } + attachmentIds.add(attachment.id); + try { + const content = canonicalBase64Bytes(queued.content); + if (content.length !== attachment.byteCount) throw invalidRestorePayload(); + const expectedDigest = Buffer.from(attachment.contentSha256, 'hex'); + const actualDigest = createHash('sha256').update(content).digest(); + if (!timingSafeEqual(actualDigest, expectedDigest)) throw invalidRestorePayload(); + return { + id: attachment.id, + ...normalizeAttachmentInput({ + filename: attachment.filename, + contentType: attachment.contentType, + content, + }), + }; + } catch { + throw invalidRestorePayload(); + } + }); + const totalBytes = attachments.reduce((total, attachment) => total + attachment.content.length, 0); + if (totalBytes > MAX_COMPOSE_ATTACHMENT_BYTES) { + throw invalidRestorePayload(); + } + return { + originalSessionId: value.originalSessionId, + preferredSlot: value.preferredSlot, + changes, + replyAllRecipients, + sourceDraft: normalizeRestoreSource(value.sourceDraft), + attachments, + }; +} + +async function restoredSessionSnapshot(client, id, userId) { + const result = await client.query( + 'SELECT * FROM compose_sessions WHERE id=$1 AND user_id=$2', + [id, userId], + ); + if (!result.rows.length) return null; + const session = mapSessionRow(result.rows[0]); + const attachmentResult = await client.query( + `SELECT id, filename, content_type, byte_count, created_at + FROM compose_session_attachments + WHERE session_id=$1 + ORDER BY created_at, id`, + [id], + ); + return { ...session, attachments: attachmentResult.rows.map(mapAttachmentRow) }; +} + +export async function restoreQueuedComposeSession(input, deps) { + if (typeof input.outboxId !== 'string' || !UUID_RE.test(input.outboxId)) { + throw composeSessionError( + 'invalid_compose_outbox_id', + 'Outbox id must be a UUID', + 400, + ); + } + + const outcome = await deps.withTransaction(async (client) => { + const selected = await client.query( + `SELECT id, status, payload, restored_compose_session_id + FROM outbox_messages + WHERE id=$1 AND user_id=$2 + FOR UPDATE`, + [input.outboxId, input.userId], + ); + if (!selected.rows.length) { + throw restoreOutboxError( + 'compose_outbox_not_found', + 'Queued compose message not found', + 404, + ); + } + const row = selected.rows[0]; + if (row.status === 'cancelled') { + if (row.restored_compose_session_id) { + const session = await restoredSessionSnapshot( + client, + row.restored_compose_session_id, + input.userId, + ); + if (session) return { restored: true, replayed: true, session }; + } + throw restoreOutboxError( + 'compose_outbox_cancelled', + 'Queued message was already cancelled', + 409, + ); + } + if (row.status !== 'pending') { + throw restoreOutboxError( + 'compose_outbox_too_late', + 'Queued message can no longer be restored', + 409, + ); + } + + const payload = jsonValue(row.payload, {}); + const capsule = normalizeRestoreCapsule( + payload?.composeSessionRestore, + payload?.attachments, + ); + const values = { ...NEW_SESSION_DEFAULTS, ...capsule.changes }; + await client.query( + 'SELECT pg_advisory_xact_lock(hashtext($1))', + [`compose-slots:${input.userId}`], + ); + try { + await validateIdentity(client, input.userId, values.accountId, values.aliasId); + if (capsule.sourceDraft.accountId + && capsule.sourceDraft.accountId !== values.accountId) { + await validateIdentity(client, input.userId, capsule.sourceDraft.accountId, null); + } + } catch { + throw invalidRestorePayload(); + } + + const allocated = await client.query( + `SELECT candidate.slot + FROM generate_series(1, 9) AS candidate(slot) + WHERE NOT EXISTS ( + SELECT 1 FROM compose_sessions existing + WHERE existing.user_id=$1 AND existing.slot=candidate.slot + ) + ORDER BY CASE WHEN candidate.slot=$2 THEN 0 ELSE 1 END, candidate.slot + LIMIT 1`, + [input.userId, capsule.preferredSlot], + ); + const slot = allocated.rows[0]?.slot; + if (!slot) { + throw composeSessionError( + 'compose_session_limit', + 'Nine compose sessions are already open', + 409, + ); + } + + const inserted = await client.query( + `INSERT INTO compose_sessions ( + id, user_id, slot, account_id, alias_id, mode, + to_recipients, cc_recipients, bcc_recipients, subject, body, + body_is_html, quoted_body, quoted_body_html, edited_signature, + forwarded_attachments, priority, in_reply_to, thread_references, + from_changed, reply_all_recipients, + source_draft_account_id, source_draft_folder, source_draft_uid, + source_draft_message_id, source_initial_revision + ) VALUES ( + $1, $2, $3, $4, $5, $6, + $7::jsonb, $8::jsonb, $9::jsonb, $10, $11, + $12, $13, $14, $15, + $16::jsonb, $17, $18, $19::jsonb, + $20, $21::jsonb, + $22, $23, $24, $25, $26::jsonb + ) RETURNING *`, + [ + capsule.originalSessionId, + input.userId, + Number(slot), + values.accountId, + values.aliasId, + values.mode, + JSON.stringify(values.to), + JSON.stringify(values.cc), + JSON.stringify(values.bcc), + values.subject, + values.body, + values.bodyIsHtml, + values.quotedBody, + values.quotedBodyHtml, + values.editedSignature, + JSON.stringify(values.forwardedAttachments), + values.priority, + values.inReplyTo, + JSON.stringify(values.references), + values.fromChanged, + JSON.stringify(capsule.replyAllRecipients), + capsule.sourceDraft.accountId, + capsule.sourceDraft.folder, + capsule.sourceDraft.uid, + capsule.sourceDraft.messageId, + JSON.stringify(capsule.sourceDraft.initialRevision), + ], + ); + const attachmentRows = []; + for (const attachment of capsule.attachments) { + const stored = await client.query( + `INSERT INTO compose_session_attachments ( + id, session_id, filename, content_type, byte_count, content + ) VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id, filename, content_type, byte_count, created_at`, + [ + attachment.id, + inserted.rows[0].id, + attachment.filename, + attachment.contentType, + attachment.content.length, + attachment.content, + ], + ); + attachmentRows.push(stored.rows[0]); + } + await client.query( + `UPDATE outbox_messages + SET status='cancelled', payload='{}'::jsonb, idempotency_key=NULL, + restored_compose_session_id=$3, updated_at=NOW() + WHERE id=$1 AND user_id=$2 AND status='pending' + RETURNING id`, + [input.outboxId, input.userId, inserted.rows[0].id], + ); + return { + restored: true, + replayed: false, + session: { + ...mapSessionRow(inserted.rows[0]), + attachments: attachmentRows.map(mapAttachmentRow), + }, + }; + }); + + if (!outcome.replayed) { + broadcastInvalidation(deps, input.userId, 'restored', outcome.session); + } + return outcome; +} diff --git a/backend/src/services/composeSessionService.test.js b/backend/src/services/composeSessionService.test.js new file mode 100644 index 00000000..c250bd45 --- /dev/null +++ b/backend/src/services/composeSessionService.test.js @@ -0,0 +1,1278 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + addComposeAttachment, + claimComposeOperation, + createComposeSession, + deleteClaimedComposeSession, + getComposeSession, + listComposeSessions, + patchComposeSession, + releaseComposeOperation, + removeComposeAttachment, + setComposePresentation, +} from './composeSessionService.js'; + +const USER_A = '11111111-1111-4111-8111-111111111111'; +const USER_B = '22222222-2222-4222-8222-222222222222'; +const ACCOUNT_A = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const ACCOUNT_A2 = 'abababab-abab-4bab-8bab-abababababab'; +const ACCOUNT_B = 'acacacac-acac-4cac-8cac-acacacacacac'; +const ALIAS_A = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; +const CREATED_AT = new Date('2026-08-01T00:00:00.000Z'); +const ATTACHMENT_AT = new Date('2026-08-01T00:01:00.000Z'); +const MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024; + +function parseJson(value) { + return typeof value === 'string' ? JSON.parse(value) : value; +} + +function fakeDependencies() { + const sessions = new Map(); + const attachments = []; + const accountOwners = new Map([ + [ACCOUNT_A, USER_A], + [ACCOUNT_A2, USER_A], + [ACCOUNT_B, USER_B], + ]); + const aliasAccounts = new Map([[ALIAS_A, ACCOUNT_A]]); + let nextSession = 1; + let nextAttachment = 1; + + const query = vi.fn(async (sql, params = []) => { + const normalized = sql.replace(/\s+/g, ' ').trim(); + + if (normalized.includes('pg_advisory_xact_lock')) return { rows: [{}] }; + + if (normalized.includes('FROM account_aliases')) { + const [aliasId, accountId, userId] = params; + const valid = aliasAccounts.get(aliasId) === accountId + && accountOwners.get(accountId) === userId; + return { rows: valid ? [{ id: aliasId }] : [] }; + } + + if (normalized.includes('FROM email_accounts')) { + const [accountId, userId] = params; + return { rows: accountOwners.get(accountId) === userId ? [{ id: accountId }] : [] }; + } + + if (normalized.includes('generate_series(1, 9)')) { + const [userId, requestedSlot] = params; + const occupied = new Set( + [...sessions.values()].filter(row => row.user_id === userId).map(row => row.slot), + ); + const candidates = requestedSlot == null + ? Array.from({ length: 9 }, (_, index) => index + 1) + : [requestedSlot]; + const slot = candidates.find(candidate => !occupied.has(candidate)); + return { rows: slot == null ? [] : [{ slot }] }; + } + + if (normalized.startsWith('INSERT INTO compose_sessions')) { + const [ + userId, slot, accountId, aliasId, mode, to, cc, bcc, subject, body, + bodyIsHtml, quotedBody, quotedBodyHtml, editedSignature, forwardedAttachments, + priority, inReplyTo, references, fromChanged, fieldRevisions, + replyAllRecipients, + ] = params; + const id = `00000000-0000-4000-8000-${String(nextSession++).padStart(12, '0')}`; + const row = { + id, + user_id: userId, + slot, + account_id: accountId, + alias_id: aliasId, + mode, + to_recipients: parseJson(to), + cc_recipients: parseJson(cc), + bcc_recipients: parseJson(bcc), + subject, + body, + body_is_html: bodyIsHtml, + quoted_body: quotedBody, + quoted_body_html: quotedBodyHtml, + edited_signature: editedSignature, + forwarded_attachments: parseJson(forwardedAttachments), + priority, + in_reply_to: inReplyTo, + thread_references: parseJson(references), + from_changed: fromChanged, + presentation_state: 'expanded', + operation_state: 'idle', + operation_token: null, + revision: 1, + field_revisions: parseJson(fieldRevisions), + reply_all_recipients: parseJson(replyAllRecipients), + last_focused_at: CREATED_AT, + created_at: CREATED_AT, + updated_at: CREATED_AT, + }; + sessions.set(id, row); + return { rows: [{ ...row }] }; + } + + if (normalized.includes('FROM compose_sessions cs') + && normalized.includes('attachment_count')) { + const [userId] = params; + const rows = [...sessions.values()] + .filter(row => row.user_id === userId) + .sort((left, right) => left.slot - right.slot) + .map(row => ({ + id: row.id, + slot: row.slot, + account_id: row.account_id, + alias_id: row.alias_id, + mode: row.mode, + subject: row.subject, + priority: row.priority, + presentation_state: row.presentation_state, + operation_state: row.operation_state, + revision: row.revision, + last_focused_at: row.last_focused_at, + created_at: row.created_at, + updated_at: row.updated_at, + attachment_count: attachments.filter(item => item.session_id === row.id).length, + })); + return { rows }; + } + + if (normalized.startsWith('SELECT COUNT(*)')) { + const [sessionId] = params; + const matching = attachments.filter(item => item.session_id === sessionId); + const totalBytes = matching + .reduce((total, item) => total + item.byte_count, 0); + return { rows: [{ attachment_count: matching.length, total_bytes: totalBytes }] }; + } + + if (normalized.startsWith('INSERT INTO compose_session_attachments')) { + const [sessionId, filename, contentType, byteCount, content] = params; + const id = `10000000-0000-4000-8000-${String(nextAttachment++).padStart(12, '0')}`; + const row = { + id, + session_id: sessionId, + filename, + content_type: contentType, + byte_count: byteCount, + content, + created_at: ATTACHMENT_AT, + }; + attachments.push(row); + return { rows: [{ ...row }] }; + } + + if (normalized.startsWith('DELETE FROM compose_session_attachments csa')) { + const [attachmentId, sessionId, userId] = params; + const session = sessions.get(sessionId); + const index = attachments.findIndex(item => ( + item.id === attachmentId && item.session_id === sessionId + )); + if (!session || session.user_id !== userId || index < 0) return { rows: [] }; + const [deleted] = attachments.splice(index, 1); + return { rows: [{ id: deleted.id }] }; + } + + if (normalized.startsWith('DELETE FROM compose_sessions')) { + const whereMatch = normalized.match( + /WHERE (id|slot)=\$(\d+) AND user_id=\$(\d+) AND operation_token=\$(\d+)/, + ); + const locator = params[Number(whereMatch[2]) - 1]; + const userId = params[Number(whereMatch[3]) - 1]; + const token = params[Number(whereMatch[4]) - 1]; + const row = [...sessions.values()].find(candidate => ( + candidate.user_id === userId + && candidate.operation_token === token + && (whereMatch[1] === 'id' ? candidate.id === locator : candidate.slot === locator) + )); + if (!row) return { rows: [] }; + sessions.delete(row.id); + return { rows: [{ id: row.id }] }; + } + + if (normalized.startsWith('SELECT id FROM compose_session_attachments')) { + const [attachmentId, sessionId] = params; + const attachment = attachments.find(item => ( + item.id === attachmentId && item.session_id === sessionId + )); + return { rows: attachment ? [{ id: attachment.id }] : [] }; + } + + if (normalized.includes('FROM compose_session_attachments')) { + const [sessionId] = params; + return { + rows: attachments + .filter(item => item.session_id === sessionId) + .map(item => ({ + id: item.id, + filename: item.filename, + content_type: item.content_type, + byte_count: item.byte_count, + created_at: item.created_at, + ...(normalized.includes('content') ? { content: item.content } : {}), + })), + }; + } + + if (normalized.startsWith('SELECT * FROM compose_sessions')) { + const [locator, userId] = params; + const byId = normalized.includes('WHERE id=$1'); + const row = [...sessions.values()].find(candidate => ( + candidate.user_id === userId + && (byId ? candidate.id === locator : candidate.slot === locator) + )); + return { rows: row ? [{ ...row }] : [] }; + } + + if (normalized.startsWith('UPDATE compose_sessions')) { + const whereMatch = normalized.match(/WHERE (id|slot)=\$(\d+) AND user_id=\$(\d+)/); + const locator = params[Number(whereMatch[2]) - 1]; + const userId = params[Number(whereMatch[3]) - 1]; + const row = [...sessions.values()].find(candidate => ( + candidate.user_id === userId + && (whereMatch[1] === 'id' ? candidate.id === locator : candidate.slot === locator) + )); + if (!row) return { rows: [] }; + const tokenMatch = normalized.match(/AND operation_token=\$(\d+)/); + if (tokenMatch && row.operation_token !== params[Number(tokenMatch[1]) - 1]) { + return { rows: [] }; + } + + const setClause = normalized.match(/SET (.+) WHERE/)[1]; + for (const assignment of setClause.split(', ')) { + const valueMatch = assignment.match(/^([a-z_]+)=\$(\d+)/); + if (valueMatch) { + const [, column, index] = valueMatch; + const value = params[Number(index) - 1]; + row[column] = assignment.includes('::jsonb') ? parseJson(value) : value; + } else if (assignment === 'revision=revision + 1') { + row.revision += 1; + } else if (assignment === 'last_focused_at=NOW()') { + row.last_focused_at = new Date('2026-08-01T00:05:00.000Z'); + } else if (assignment === 'updated_at=NOW()') { + row.updated_at = new Date('2026-08-01T00:05:00.000Z'); + } else if (assignment === "operation_state='idle'") { + row.operation_state = 'idle'; + } else if (assignment === 'operation_token=NULL') { + row.operation_token = null; + } + } + return { rows: [{ ...row }] }; + } + + throw new Error(`Unexpected SQL in fake: ${normalized}`); + }); + + return { + deps: { + query, + withTransaction: vi.fn(async callback => callback({ query })), + broadcast: vi.fn(), + }, + deleteSession(id) { sessions.delete(id); }, + setOperationState(id, operationState) { sessions.get(id).operation_state = operationState; }, + getSession(id) { return sessions.get(id); }, + addAttachment(id, metadata) { + attachments.push({ session_id: id, content: Buffer.from('not returned'), ...metadata }); + }, + getAttachment(id) { return attachments.find(item => item.id === id); }, + }; +} + +function expectExactInvalidationKeys(broadcast) { + for (const [payload, userId] of broadcast.mock.calls) { + expect(userId).toBe(USER_A); + const keys = [ + 'action', 'revision', 'sessionId', 'slot', 'type', + ]; + if (Object.hasOwn(payload, 'clientId')) keys.push('clientId'); + expect(Object.keys(payload).sort()).toEqual(keys.sort()); + expect(JSON.stringify(payload).length).toBeLessThanOrEqual(256); + expect(payload).not.toHaveProperty('subject'); + expect(payload).not.toHaveProperty('body'); + expect(payload).not.toHaveProperty('to'); + if (payload.clientId !== undefined) { + expect(payload.clientId).toMatch(/^[A-Za-z0-9_-]{1,64}$/); + } + } +} + +describe('compose session persistence', () => { + it('locks the user allocation key before selecting a free slot', async () => { + const { deps } = fakeDependencies(); + await createComposeSession({ userId: USER_A, changes: {} }, deps); + + const lockIndex = deps.query.mock.calls.findIndex(([sql]) => ( + sql === 'SELECT pg_advisory_xact_lock(hashtext($1))' + )); + const allocationIndex = deps.query.mock.calls.findIndex(([sql]) => ( + sql.includes('generate_series(1, 9)') + )); + expect(lockIndex).toBeGreaterThanOrEqual(0); + expect(deps.query.mock.calls[lockIndex]).toEqual([ + 'SELECT pg_advisory_xact_lock(hashtext($1))', + [`compose-slots:${USER_A}`], + ]); + expect(allocationIndex).toBeGreaterThan(lockIndex); + }); + + it('allocates, patches, merges disjoint stale fields, and reports same-field conflicts', async () => { + const { deps } = fakeDependencies(); + const created = await createComposeSession({ + userId: USER_A, + changes: { accountId: ACCOUNT_A, aliasId: ALIAS_A, subject: 'hello' }, + clientId: 'browser-a', + }, deps); + expect(created).toMatchObject({ slot: 1, revision: 1, subject: 'hello' }); + + await expect(createComposeSession({ + userId: USER_A, + requestedSlot: 1, + changes: {}, + }, deps)).rejects.toMatchObject({ code: 'compose_slot_occupied', status: 409 }); + + const patched = await patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { subject: 'new subject' }, + clientId: 'mcp-token-1', + }, deps); + expect(patched).toMatchObject({ revision: 2, subject: 'new subject' }); + + const merged = await patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { body: 'A disjoint edit' }, + }, deps); + expect(merged).toMatchObject({ revision: 3, subject: 'new subject', body: 'A disjoint edit' }); + + await expect(patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { subject: 'stale subject' }, + }, deps)).rejects.toMatchObject({ + code: 'compose_conflict', + status: 409, + details: { + conflictingFields: ['subject'], + currentRevision: 3, + remoteValues: { subject: 'new subject' }, + }, + }); + + expectExactInvalidationKeys(deps.broadcast); + expect(deps.broadcast.mock.calls).toEqual([ + [{ + type: 'compose_sessions_updated', + action: 'created', + sessionId: created.id, + slot: 1, + revision: 1, + clientId: 'browser-a', + }, USER_A], + [{ + type: 'compose_sessions_updated', + action: 'updated', + sessionId: created.id, + slot: 1, + revision: 2, + clientId: 'mcp-token-1', + }, USER_A], + [{ + type: 'compose_sessions_updated', + action: 'updated', + sessionId: created.id, + slot: 1, + revision: 3, + }, USER_A], + ]); + }); + + it('reuses the lowest free slot and rejects a tenth session', async () => { + const fake = fakeDependencies(); + const created = []; + for (let requestedSlot = 1; requestedSlot <= 3; requestedSlot += 1) { + created.push(await createComposeSession({ + userId: USER_A, + requestedSlot, + changes: {}, + }, fake.deps)); + } + fake.deleteSession(created[1].id); + await expect(createComposeSession({ userId: USER_A, changes: {} }, fake.deps)) + .resolves.toMatchObject({ slot: 2 }); + for (let requestedSlot = 4; requestedSlot <= 9; requestedSlot += 1) { + await createComposeSession({ userId: USER_A, requestedSlot, changes: {} }, fake.deps); + } + await expect(createComposeSession({ userId: USER_A, changes: {} }, fake.deps)) + .rejects.toMatchObject({ code: 'compose_session_limit', status: 409 }); + }); + + it('scopes UUID operations and account identities to the owner', async () => { + const { deps } = fakeDependencies(); + const created = await createComposeSession({ + userId: USER_A, + changes: { accountId: ACCOUNT_A }, + }, deps); + + await expect(getComposeSession({ userId: USER_B, id: created.id }, deps)) + .rejects.toMatchObject({ code: 'compose_session_not_found', status: 404 }); + await expect(patchComposeSession({ + userId: USER_B, + id: created.id, + expectedRevision: 1, + changes: { subject: 'cross-owner edit' }, + }, deps)).rejects.toMatchObject({ code: 'compose_session_not_found', status: 404 }); + await expect(createComposeSession({ + userId: USER_B, + changes: { accountId: ACCOUNT_A }, + }, deps)).rejects.toMatchObject({ code: 'compose_account_not_found', status: 404 }); + }); + + it('round-trips non-editable reply-all source recipients across reloads', async () => { + const { deps } = fakeDependencies(); + const created = await createComposeSession({ + userId: USER_A, + changes: { mode: 'reply', to: ['Sender '] }, + replyAllRecipients: ['Copied '], + }, deps); + expect(created.replyAllRecipients).toEqual(['Copied ']); + + const patched = await patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { subject: 'Synthetic reply', replyAllRecipients: ['ignored@example.com'] }, + }, deps); + expect(patched.replyAllRecipients).toEqual(['Copied ']); + await expect(getComposeSession({ userId: USER_A, id: created.id }, deps)) + .resolves.toMatchObject({ replyAllRecipients: ['Copied '] }); + }); + + it('rejects aliases from another account or owner', async () => { + const { deps } = fakeDependencies(); + + await expect(createComposeSession({ + userId: USER_A, + changes: { accountId: ACCOUNT_A2, aliasId: ALIAS_A }, + }, deps)).rejects.toMatchObject({ code: 'compose_alias_not_found', status: 404 }); + await expect(createComposeSession({ + userId: USER_B, + changes: { accountId: ACCOUNT_B, aliasId: ALIAS_A }, + }, deps)).rejects.toMatchObject({ code: 'compose_alias_not_found', status: 404 }); + }); + + it('rejects edits while a terminal operation owns the row', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, fake.deps); + await patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { subject: 'newer server value' }, + }, fake.deps); + fake.setOperationState(created.id, 'sending'); + + await expect(patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { subject: 'too late' }, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_operation_in_progress', + status: 409, + }); + }); + + it.each([ + ['not-an-object'], + [null], + [[{ subject: 'array is not changes' }]], + ])('rejects non-object changes before opening a transaction %#', async (changes) => { + const { deps } = fakeDependencies(); + + await expect(createComposeSession({ userId: USER_A, changes }, deps)) + .rejects.toMatchObject({ code: 'invalid_compose_changes', status: 400 }); + expect(deps.withTransaction).not.toHaveBeenCalled(); + }); + + it('rejects malformed patch field types without changing the session', async () => { + const { deps } = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, deps); + + await expect(patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { to: 'recipient@example.com', bodyIsHtml: 'yes' }, + }, deps)).rejects.toMatchObject({ code: 'invalid_compose_changes', status: 400 }); + await expect(getComposeSession({ userId: USER_A, id: created.id }, deps)) + .resolves.toMatchObject({ revision: 1, to: [], bodyIsHtml: true }); + }); + + it.each([ + ['person@example.com'], + ['contains spaces'], + ['x'.repeat(65)], + [{ private: 'content' }], + ])('rejects malformed client ids before persistence %#', async (clientId) => { + const { deps } = fakeDependencies(); + + await expect(createComposeSession({ + userId: USER_A, + changes: {}, + clientId, + }, deps)).rejects.toMatchObject({ code: 'invalid_client_id', status: 400 }); + expect(deps.withTransaction).not.toHaveBeenCalled(); + expect(deps.broadcast).not.toHaveBeenCalled(); + }); + + it('builds patch SQL only from the fixed editable-field map', async () => { + const { deps } = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, deps); + const injectedField = 'subject=$1; DROP TABLE compose_sessions; --'; + + await patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { subject: 'Allowed value', [injectedField]: 'not SQL' }, + }, deps); + + const updateSql = deps.query.mock.calls.find(([sql]) => sql.startsWith('UPDATE compose_sessions'))[0]; + expect(updateSql).toContain('subject=$1'); + expect(updateSql).not.toContain('DROP TABLE'); + expect(updateSql).not.toContain(injectedField); + }); + + it('returns content-minimal summaries and metadata-only snapshots by id or slot', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ + userId: USER_A, + changes: { + to: ['Recipient '], + subject: 'Summary title', + body: 'Private body', + quotedBody: 'Private quote', + }, + }, fake.deps); + fake.addAttachment(created.id, { + id: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + filename: 'synthetic.txt', + content_type: 'text/plain', + byte_count: 12, + created_at: ATTACHMENT_AT, + }); + + const [summary] = await listComposeSessions({ userId: USER_A }, fake.deps); + expect(summary).toStrictEqual({ + id: created.id, + slot: 1, + accountId: null, + aliasId: null, + mode: 'new', + subject: 'Summary title', + priority: 'normal', + presentationState: 'expanded', + operationState: 'idle', + revision: 1, + lastFocusedAt: CREATED_AT, + createdAt: CREATED_AT, + updatedAt: CREATED_AT, + attachmentCount: 1, + }); + + const byId = await getComposeSession({ userId: USER_A, id: created.id }, fake.deps); + const bySlot = await getComposeSession({ userId: USER_A, slot: 1 }, fake.deps); + expect(byId).toStrictEqual(bySlot); + expect(byId).toStrictEqual({ + id: created.id, + slot: 1, + accountId: null, + aliasId: null, + mode: 'new', + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Summary title', + body: 'Private body', + bodyIsHtml: true, + quotedBody: 'Private quote', + quotedBodyHtml: null, + editedSignature: null, + forwardedAttachments: [], + priority: 'normal', + inReplyTo: null, + references: [], + fromChanged: false, + replyAllRecipients: [], + sourceDraftAccountId: undefined, + sourceDraftFolder: undefined, + sourceDraftUid: undefined, + sourceDraftMessageId: undefined, + sourceInitialRevision: null, + presentationState: 'expanded', + operationState: 'idle', + operationToken: null, + revision: 1, + fieldRevisions: { to: 1, subject: 1, body: 1, quotedBody: 1 }, + lastFocusedAt: CREATED_AT, + createdAt: CREATED_AT, + updatedAt: CREATED_AT, + attachments: [{ + id: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + filename: 'synthetic.txt', + contentType: 'text/plain', + byteCount: 12, + createdAt: ATTACHMENT_AT, + }], + }); + + const listSql = fake.deps.query.mock.calls + .find(([sql]) => sql.includes('attachment_count'))[0]; + expect(listSql.replace(/\s+/g, ' ').trim()).toBe( + 'SELECT cs.id, cs.slot, cs.account_id, cs.alias_id, cs.mode, cs.subject, ' + + 'cs.priority, cs.presentation_state, cs.operation_state, cs.revision, ' + + 'cs.last_focused_at, cs.created_at, cs.updated_at, ' + + 'COUNT(csa.id)::int AS attachment_count FROM compose_sessions cs ' + + 'LEFT JOIN compose_session_attachments csa ON csa.session_id=cs.id ' + + 'WHERE cs.user_id=$1 GROUP BY cs.id ORDER BY cs.slot', + ); + const attachmentSql = fake.deps.query.mock.calls + .find(([sql]) => sql.includes('FROM compose_session_attachments'))[0]; + expect(attachmentSql.replace(/\s+/g, ' ').trim()).toBe( + 'SELECT id, filename, content_type, byte_count, created_at ' + + 'FROM compose_session_attachments WHERE session_id=$1 ORDER BY created_at, id', + ); + }); + + it('requires exactly one owner-scoped locator', async () => { + const { deps } = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, deps); + + await expect(getComposeSession({ userId: USER_A }, deps)) + .rejects.toMatchObject({ code: 'invalid_compose_locator', status: 400 }); + await expect(getComposeSession({ userId: USER_A, id: created.id, slot: 1 }, deps)) + .rejects.toMatchObject({ code: 'invalid_compose_locator', status: 400 }); + await expect(patchComposeSession({ + userId: USER_A, + expectedRevision: 1, + changes: { subject: 'missing locator' }, + }, deps)).rejects.toMatchObject({ code: 'invalid_compose_locator', status: 400 }); + await expect(getComposeSession({ userId: USER_A, id: 'not-a-uuid' }, deps)) + .rejects.toMatchObject({ code: 'invalid_compose_locator', status: 400 }); + await expect(removeComposeAttachment({ + userId: USER_A, + id: created.id, + attachmentId: 'not-a-uuid', + expectedRevision: 1, + }, deps)).rejects.toMatchObject({ code: 'invalid_compose_attachment_id', status: 400 }); + }); + + it('revision-checks presentation changes and focuses only expanded sessions', async () => { + const { deps } = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, deps); + + const minimized = await setComposePresentation({ + userId: USER_A, + slot: 1, + expectedRevision: 1, + state: 'minimized', + }, deps); + expect(minimized).toMatchObject({ presentationState: 'minimized', revision: 2 }); + const minimizeSql = deps.query.mock.calls.filter(([sql]) => ( + sql.startsWith('UPDATE compose_sessions') + )).at(-1)[0]; + expect(minimizeSql).not.toContain('last_focused_at=NOW()'); + + const expanded = await setComposePresentation({ + userId: USER_A, + id: created.id, + expectedRevision: 2, + state: 'expanded', + }, deps); + expect(expanded).toMatchObject({ presentationState: 'expanded', revision: 3 }); + const expandSql = deps.query.mock.calls.filter(([sql]) => ( + sql.startsWith('UPDATE compose_sessions') + )).at(-1)[0]; + expect(expandSql).toContain('last_focused_at=NOW()'); + expectExactInvalidationKeys(deps.broadcast); + expect(deps.broadcast.mock.calls.slice(1)).toEqual([ + [{ + type: 'compose_sessions_updated', + action: 'presentation', + sessionId: created.id, + slot: 1, + revision: 2, + }, USER_A], + [{ + type: 'compose_sessions_updated', + action: 'presentation', + sessionId: created.id, + slot: 1, + revision: 3, + }, USER_A], + ]); + + await expect(setComposePresentation({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + state: 'minimized', + }, deps)).rejects.toMatchObject({ + code: 'compose_conflict', + details: { conflictingFields: ['presentationState'], currentRevision: 3 }, + }); + await expect(setComposePresentation({ + userId: USER_A, + id: created.id, + expectedRevision: 3, + state: 'hidden', + }, deps)).rejects.toMatchObject({ code: 'invalid_presentation_state', status: 400 }); + }); + + it('adds attachment bytes with a sanitized filename and metadata-only result', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, fake.deps); + const content = Buffer.from([0, 1, 2, 254, 255]); + + const added = await addComposeAttachment({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + filename: 'synthetic\r\n\0report.bin', + contentType: 'application/octet-stream', + content, + clientId: 'browser-attachment', + }, fake.deps); + + expect(added).toStrictEqual({ + sessionId: created.id, + slot: 1, + revision: 2, + attachment: { + id: '10000000-0000-4000-8000-000000000001', + filename: 'syntheticreport.bin', + contentType: 'application/octet-stream', + byteCount: 5, + createdAt: ATTACHMENT_AT, + }, + }); + expect(Object.keys(added.attachment).sort()).toEqual([ + 'byteCount', 'contentType', 'createdAt', 'filename', 'id', + ]); + const stored = fake.getAttachment(added.attachment.id); + expect(Buffer.isBuffer(stored.content)).toBe(true); + expect(stored.content).toEqual(content); + expect(stored.content).not.toBe(content); + expect(stored.byte_count).toBe(content.length); + + const snapshot = await getComposeSession({ userId: USER_A, id: created.id }, fake.deps); + expect(snapshot).toMatchObject({ revision: 2, fieldRevisions: { attachments: 2 } }); + expectExactInvalidationKeys(fake.deps.broadcast); + expect(fake.deps.broadcast.mock.calls.at(-1)).toEqual([{ + type: 'compose_sessions_updated', + action: 'attachment_added', + sessionId: created.id, + slot: 1, + revision: 2, + clientId: 'browser-attachment', + }, USER_A]); + }); + + it.each([ + [{ content: 'not-a-buffer' }, 'invalid_attachment_body'], + [{ filename: '' }, 'invalid_attachment_filename'], + [{ filename: 42 }, 'invalid_attachment_filename'], + [{ contentType: 'not-a-mime-type' }, 'invalid_attachment_content_type'], + [{ contentType: 'text/plain\r\nX-Injected: yes' }, 'invalid_attachment_content_type'], + ])('rejects malformed attachment input %#', async (override, code) => { + const fake = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, fake.deps); + + await expect(addComposeAttachment({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + filename: 'synthetic.bin', + contentType: 'application/octet-stream', + content: Buffer.from('synthetic'), + ...override, + }, fake.deps)).rejects.toMatchObject({ code, status: 400 }); + expect(fake.deps.broadcast).toHaveBeenCalledTimes(1); + }); + + it('accepts an attachment exactly at the aggregate 25 MiB boundary', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, fake.deps); + + await expect(addComposeAttachment({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + filename: 'exact-boundary.bin', + contentType: 'application/octet-stream', + content: Buffer.alloc(MAX_ATTACHMENT_BYTES), + }, fake.deps)).resolves.toMatchObject({ + revision: 2, + attachment: { byteCount: MAX_ATTACHMENT_BYTES }, + }); + }); + + it('rejects a 101st attachment before storing more bytes', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, fake.deps); + for (let index = 0; index < 100; index += 1) { + fake.addAttachment(created.id, { + id: `20000000-0000-4000-8000-${String(index).padStart(12, '0')}`, + filename: `synthetic-${index}.txt`, + content_type: 'text/plain', + byte_count: 1, + created_at: ATTACHMENT_AT, + }); + } + + await expect(addComposeAttachment({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + filename: 'one-too-many.txt', + contentType: 'text/plain', + content: Buffer.from('x'), + }, fake.deps)).rejects.toMatchObject({ code: 'attachment_count_limit', status: 413 }); + expect(fake.deps.query.mock.calls.some(([sql]) => ( + sql.startsWith('INSERT INTO compose_session_attachments') + ))).toBe(false); + }); + + it('rejects attachment bytes above the aggregate 25 MiB limit', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, fake.deps); + fake.addAttachment(created.id, { + id: 'dddddddd-dddd-4ddd-8ddd-dddddddddddd', + filename: 'existing.bin', + content_type: 'application/octet-stream', + byte_count: MAX_ATTACHMENT_BYTES - 1, + created_at: ATTACHMENT_AT, + }); + + await expect(addComposeAttachment({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + filename: 'too-big.bin', + contentType: 'application/octet-stream', + content: Buffer.alloc(2), + }, fake.deps)).rejects.toMatchObject({ code: 'attachment_limit', status: 413 }); + + expect(fake.deps.query.mock.calls.some(([sql]) => ( + sql.includes('COALESCE(SUM(byte_count), 0)') + ))).toBe(true); + expect(fake.deps.query.mock.calls.some(([sql]) => ( + sql.startsWith('INSERT INTO compose_session_attachments') + ))).toBe(false); + expect(fake.deps.broadcast).toHaveBeenCalledTimes(1); + }); + + it('owner-scopes attachment mutations and applies the operation guard', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, fake.deps); + + await expect(addComposeAttachment({ + userId: USER_B, + id: created.id, + expectedRevision: 1, + filename: 'out-of-scope.txt', + contentType: 'text/plain', + content: Buffer.from('synthetic'), + }, fake.deps)).rejects.toMatchObject({ code: 'compose_session_not_found', status: 404 }); + + fake.setOperationState(created.id, 'sending'); + await expect(removeComposeAttachment({ + userId: USER_A, + id: created.id, + attachmentId: 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee', + expectedRevision: 1, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_operation_in_progress', + status: 409, + }); + }); + + it('removes attachments idempotently and increments revision only when present', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, fake.deps); + const added = await addComposeAttachment({ + userId: USER_A, + slot: 1, + expectedRevision: 1, + filename: 'remove-me.txt', + contentType: 'text/plain', + content: Buffer.from('reserved test content'), + }, fake.deps); + + const removed = await removeComposeAttachment({ + userId: USER_A, + id: created.id, + attachmentId: added.attachment.id, + expectedRevision: 2, + clientId: 'browser-remove', + }, fake.deps); + expect(removed).toStrictEqual({ + sessionId: created.id, + slot: 1, + revision: 3, + removedAttachmentId: added.attachment.id, + }); + expect(fake.getAttachment(added.attachment.id)).toBeUndefined(); + expect(fake.deps.broadcast.mock.calls.at(-1)).toEqual([{ + type: 'compose_sessions_updated', + action: 'attachment_removed', + sessionId: created.id, + slot: 1, + revision: 3, + clientId: 'browser-remove', + }, USER_A]); + + const broadcastsAfterRemoval = fake.deps.broadcast.mock.calls.length; + const absent = await removeComposeAttachment({ + userId: USER_A, + id: created.id, + attachmentId: added.attachment.id, + expectedRevision: 2, + }, fake.deps); + expect(absent).toStrictEqual({ + sessionId: created.id, + slot: 1, + revision: 3, + removedAttachmentId: added.attachment.id, + }); + expect(fake.deps.broadcast).toHaveBeenCalledTimes(broadcastsAfterRemoval); + + const deleteSql = fake.deps.query.mock.calls.find(([sql]) => ( + sql.startsWith('DELETE FROM compose_session_attachments csa') + ))[0].replace(/\s+/g, ' ').trim(); + expect(deleteSql).toContain('USING compose_sessions cs'); + expect(deleteSql).toContain('csa.session_id=cs.id'); + expect(deleteSql).toContain('cs.user_id=$3'); + }); + + it('does not remove an attachment through another owner or compose session', async () => { + const fake = fakeDependencies(); + const first = await createComposeSession({ + userId: USER_A, + requestedSlot: 1, + changes: {}, + }, fake.deps); + const second = await createComposeSession({ + userId: USER_A, + requestedSlot: 2, + changes: {}, + }, fake.deps); + const added = await addComposeAttachment({ + userId: USER_A, + id: first.id, + expectedRevision: 1, + filename: 'owned-by-first.txt', + contentType: 'text/plain', + content: Buffer.from('synthetic'), + }, fake.deps); + + await expect(removeComposeAttachment({ + userId: USER_B, + id: first.id, + attachmentId: added.attachment.id, + expectedRevision: 2, + }, fake.deps)).rejects.toMatchObject({ code: 'compose_session_not_found', status: 404 }); + await expect(removeComposeAttachment({ + userId: USER_A, + id: second.id, + attachmentId: added.attachment.id, + expectedRevision: 1, + }, fake.deps)).resolves.toMatchObject({ + sessionId: second.id, + slot: 2, + revision: 1, + removedAttachmentId: added.attachment.id, + }); + expect(fake.getAttachment(added.attachment.id)).toBeDefined(); + }); + + it('atomically claims an operation with final changes and complete attachment bytes', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ + userId: USER_A, + changes: { accountId: ACCOUNT_A, subject: 'Initial subject' }, + }, fake.deps); + const added = await addComposeAttachment({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + filename: 'claim.txt', + contentType: 'text/plain', + content: Buffer.from('synthetic claim bytes'), + }, fake.deps); + + const claimed = await claimComposeOperation({ + userId: USER_A, + id: created.id, + expectedRevision: 2, + operation: 'closing', + changes: { subject: 'Final subject' }, + }, fake.deps); + + expect(claimed).toMatchObject({ + id: created.id, + slot: 1, + subject: 'Final subject', + revision: 3, + fieldRevisions: { accountId: 1, subject: 3, attachments: 2 }, + operationState: 'closing', + }); + expect(claimed.operationToken).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + expect(claimed.attachments).toStrictEqual([{ + id: added.attachment.id, + filename: 'claim.txt', + contentType: 'text/plain', + byteCount: 21, + content: Buffer.from('synthetic claim bytes'), + createdAt: ATTACHMENT_AT, + }]); + + await expect(claimComposeOperation({ + userId: USER_A, + slot: 1, + expectedRevision: 3, + operation: 'sending', + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_operation_in_progress', + status: 409, + }); + + await expect(patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 3, + changes: { body: 'blocked while claimed' }, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_operation_in_progress', + status: 409, + }); + }); + + it('rejects a stale conflicting operation final patch without claiming the row', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ + userId: USER_A, + changes: { subject: 'Base subject' }, + }, fake.deps); + await patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { subject: 'Remote subject' }, + }, fake.deps); + + await expect(claimComposeOperation({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + operation: 'closing', + changes: { subject: 'Stale final subject' }, + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_conflict', + status: 409, + details: { + conflictingFields: ['subject'], + currentRevision: 2, + remoteValues: { subject: 'Remote subject' }, + }, + }); + expect(fake.getSession(created.id)).toMatchObject({ + operation_state: 'idle', + operation_token: null, + revision: 2, + subject: 'Remote subject', + }); + }); + + it('rejects a stale no-change operation claim without claiming the row', async () => { + const fake = fakeDependencies(); + const stale = await createComposeSession({ + userId: USER_A, + requestedSlot: 1, + changes: { subject: 'Base subject' }, + }, fake.deps); + await patchComposeSession({ + userId: USER_A, + id: stale.id, + expectedRevision: 1, + changes: { subject: 'New server subject' }, + }, fake.deps); + + await expect(claimComposeOperation({ + userId: USER_A, + id: stale.id, + expectedRevision: 1, + operation: 'sending', + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_conflict', + status: 409, + details: { + conflictingFields: ['revision'], + currentRevision: 2, + remoteValues: { revision: 2 }, + }, + }); + expect(fake.getSession(stale.id)).toMatchObject({ + operation_state: 'idle', + operation_token: null, + revision: 2, + }); + }); + + it('rejects a future no-change operation claim without claiming the row', async () => { + const fake = fakeDependencies(); + const future = await createComposeSession({ + userId: USER_A, + changes: {}, + }, fake.deps); + await expect(claimComposeOperation({ + userId: USER_A, + slot: 1, + expectedRevision: 2, + operation: 'discarding', + }, fake.deps)).rejects.toMatchObject({ + code: 'compose_conflict', + status: 409, + details: { + conflictingFields: ['revision'], + currentRevision: 1, + remoteValues: { revision: 1 }, + }, + }); + expect(fake.getSession(future.id)).toMatchObject({ + operation_state: 'idle', + operation_token: null, + revision: 1, + }); + }); + + it('accepts a stale disjoint operation final patch and advances its field revision', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ + userId: USER_A, + changes: { subject: 'Base subject' }, + }, fake.deps); + await patchComposeSession({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + changes: { subject: 'New server subject' }, + }, fake.deps); + + const claimed = await claimComposeOperation({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + operation: 'closing', + changes: { body: 'Disjoint final body' }, + }, fake.deps); + expect(claimed).toMatchObject({ + revision: 3, + subject: 'New server subject', + body: 'Disjoint final body', + operationState: 'closing', + fieldRevisions: { subject: 2, body: 3 }, + }); + }); + + it('validates operation claims before opening a transaction', async () => { + const { deps } = fakeDependencies(); + + await expect(claimComposeOperation({ + userId: USER_A, + id: '00000000-0000-4000-8000-000000000001', + expectedRevision: 1, + operation: 'idle', + }, deps)).rejects.toMatchObject({ code: 'invalid_compose_operation', status: 400 }); + await expect(claimComposeOperation({ + userId: USER_A, + id: '00000000-0000-4000-8000-000000000001', + expectedRevision: 0, + operation: 'closing', + }, deps)).rejects.toMatchObject({ code: 'invalid_compose_revision', status: 400 }); + expect(deps.withTransaction).not.toHaveBeenCalled(); + }); + + it('releases and deletes operations only for the owner locator and matching token', async () => { + const fake = fakeDependencies(); + const created = await createComposeSession({ userId: USER_A, changes: {} }, fake.deps); + const firstClaim = await claimComposeOperation({ + userId: USER_A, + id: created.id, + expectedRevision: 1, + operation: 'closing', + }, fake.deps); + const wrongToken = 'ffffffff-ffff-4fff-8fff-ffffffffffff'; + + await expect(releaseComposeOperation({ + userId: USER_A, + id: created.id, + token: wrongToken, + }, fake.deps)).resolves.toBe(false); + expect(fake.getSession(created.id)).toMatchObject({ + operation_state: 'closing', + operation_token: firstClaim.operationToken, + }); + await expect(releaseComposeOperation({ + userId: USER_B, + id: created.id, + token: firstClaim.operationToken, + }, fake.deps)).resolves.toBe(false); + await expect(releaseComposeOperation({ + userId: USER_A, + id: created.id, + token: firstClaim.operationToken, + }, fake.deps)).resolves.toBe(true); + expect(fake.getSession(created.id)).toMatchObject({ + operation_state: 'idle', + operation_token: null, + }); + + const secondClaim = await claimComposeOperation({ + userId: USER_A, + slot: 1, + expectedRevision: 1, + operation: 'discarding', + }, fake.deps); + await expect(deleteClaimedComposeSession({ + userId: USER_A, + slot: 1, + token: wrongToken, + }, fake.deps)).resolves.toBe(false); + expect(fake.getSession(created.id)).toBeDefined(); + await expect(deleteClaimedComposeSession({ + userId: USER_B, + slot: 1, + token: secondClaim.operationToken, + }, fake.deps)).resolves.toBe(false); + await expect(deleteClaimedComposeSession({ + userId: USER_A, + slot: 1, + token: secondClaim.operationToken, + }, fake.deps)).resolves.toBe(true); + expect(fake.getSession(created.id)).toBeUndefined(); + + const releaseSql = fake.deps.query.mock.calls.find(([sql]) => ( + sql.startsWith('UPDATE compose_sessions') && sql.includes("operation_state='idle'") + ))[0].replace(/\s+/g, ' ').trim(); + expect(releaseSql).toContain('WHERE id=$1 AND user_id=$2 AND operation_token=$3'); + const deleteSql = fake.deps.query.mock.calls.find(([sql]) => ( + sql.startsWith('DELETE FROM compose_sessions') + ))[0].replace(/\s+/g, ' ').trim(); + expect(deleteSql).toContain('WHERE slot=$1 AND user_id=$2 AND operation_token=$3'); + }); +}); diff --git a/backend/src/services/connectionErrors.js b/backend/src/services/connectionErrors.js new file mode 100644 index 00000000..d624ee55 --- /dev/null +++ b/backend/src/services/connectionErrors.js @@ -0,0 +1,38 @@ +// Map IMAP, SMTP, and network connection errors to user-friendly messages that +// do not expose server internals. SMTP categories and messages intentionally +// match sanitizeSmtpError so the connection probe remains consistent with send. +export function sanitizeConnectionError(err) { + const msg = err?.message || ''; + const code = err?.code || ''; + const responseStatus = err?.responseStatus || ''; + const combined = `${code} ${responseStatus} ${msg}`; + + if (/ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ECONNRESET|EHOSTUNREACH/i.test(combined)) { + return 'Could not connect to the mail server. Check your SMTP settings.'; + } + if ( + err?.authenticationFailed || + /AUTHENTICATIONFAILED|535|534|530|invalid.?login|authentication.?fail|bad.*credentials|username.*password|password.*username|login denied/i.test(combined) + ) { + return 'Authentication failed. Check your email account credentials.'; + } + if (/throttl|rate.?limit|too many|4\.2\.|4\.7\.94/i.test(msg)) { + return 'The mail server is rate limiting sends. Please try again shortly.'; + } + if (/550|5\.[13]\.|reject|blacklist|spam|not.?accept/i.test(msg)) { + return 'Message was rejected by the mail server.'; + } + if (/TLS|SSL|certificate|handshake/i.test(msg)) { + return 'Secure connection to the mail server failed. Check your TLS settings.'; + } + if (/connection test timed out/i.test(msg)) { + return 'Connection test timed out. Check your mail server settings.'; + } + if (/greeting|capabilit|IMAP4rev/i.test(msg)) { + return 'Could not establish an IMAP connection. Check your IMAP settings.'; + } + if (/^(NO|BAD)$/i.test(responseStatus) || /^(NO|BAD)\b/i.test(msg)) { + return 'The IMAP server rejected the connection. Check your IMAP settings.'; + } + return 'Failed to send message. Please try again.'; +} diff --git a/backend/src/services/connectionErrors.test.js b/backend/src/services/connectionErrors.test.js new file mode 100644 index 00000000..bfe73722 --- /dev/null +++ b/backend/src/services/connectionErrors.test.js @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; +import { sanitizeConnectionError } from './connectionErrors.js'; + +describe('sanitizeConnectionError', () => { + const fixtures = [ + { + error: new Error('connect ECONNREFUSED 203.0.113.1:587'), + expectedMessage: 'Could not connect to the mail server. Check your SMTP settings.', + }, + { + error: new Error('535 5.7.8 Authentication failed: invalid login'), + expectedMessage: 'Authentication failed. Check your email account credentials.', + }, + { + error: new Error('454 4.7.94 rate limit exceeded'), + expectedMessage: 'The mail server is rate limiting sends. Please try again shortly.', + }, + { + error: new Error('550 5.1.1 rejected as spam'), + expectedMessage: 'Message was rejected by the mail server.', + }, + { + error: new Error('TLS certificate handshake failed'), + expectedMessage: 'Secure connection to the mail server failed. Check your TLS settings.', + }, + { + error: new Error('unexpected SMTP response'), + expectedMessage: 'Failed to send message. Please try again.', + }, + { + error: { authenticationFailed: true, message: 'Command failed' }, + expectedMessage: 'Authentication failed. Check your email account credentials.', + }, + { + error: { code: 'AUTHENTICATIONFAILED', responseStatus: 'NO', message: 'NO Login denied' }, + expectedMessage: 'Authentication failed. Check your email account credentials.', + }, + { + error: { responseStatus: 'BAD', message: 'BAD invalid command during connection' }, + expectedMessage: 'The IMAP server rejected the connection. Check your IMAP settings.', + }, + { + error: new Error('Failed to receive greeting from server'), + expectedMessage: 'Could not establish an IMAP connection. Check your IMAP settings.', + }, + { + error: new Error('Server did not advertise IMAP4rev1 capability'), + expectedMessage: 'Could not establish an IMAP connection. Check your IMAP settings.', + }, + { + error: new Error('Connection test timed out after 10 seconds'), + expectedMessage: 'Connection test timed out. Check your mail server settings.', + }, + { + error: { responseStatus: 'NO', message: 'NO mailbox unavailable' }, + expectedMessage: 'The IMAP server rejected the connection. Check your IMAP settings.', + }, + ]; + + it.each(fixtures)('maps $error to a safe message', ({ error, expectedMessage }) => { + expect(sanitizeConnectionError(error)).toBe(expectedMessage); + }); +}); diff --git a/backend/src/services/connectionTest.js b/backend/src/services/connectionTest.js new file mode 100644 index 00000000..5fafb699 --- /dev/null +++ b/backend/src/services/connectionTest.js @@ -0,0 +1,74 @@ +import { ImapFlow } from 'imapflow'; +import { makeClientCfg } from './imapManager.js'; +import { resolveForConnection } from './hostValidation.js'; +import { getConnectionPolicy } from './connectionPolicy.js'; +import { buildSmtpTransport } from './mail/smtp.js'; +import { sanitizeConnectionError } from './connectionErrors.js'; + +const CONNECTION_TIMEOUT_MS = 10_000; + +function withTimeout(operation, onTimeout) { + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + try { + onTimeout?.(); + } finally { + reject(new Error('Connection test timed out after 10 seconds')); + } + }, CONNECTION_TIMEOUT_MS); + }); + return Promise.race([operation, timeout]).finally(() => clearTimeout(timer)); +} + +async function probeImap(account) { + let client; + let connected = false; + try { + const policy = await getConnectionPolicy(); + const resolved = await resolveForConnection(account.imap_host, { + allowPrivate: policy.allowPrivateHosts, + }); + const cfg = makeClientCfg(account, resolved, { + enableIdle: false, + policy, + }); + client = new ImapFlow(cfg); + await withTimeout(client.connect(), () => client.close()); + connected = true; + return { ok: true }; + } catch (err) { + return { ok: false, error: sanitizeConnectionError(err) }; + } finally { + if (connected) { + try { + await client.logout(); + } catch { + client.close(); + } + } else { + client?.close(); + } + } +} + +async function probeSmtp(account) { + let transport; + try { + ({ transport } = await buildSmtpTransport(account)); + await withTimeout(transport.verify(), () => transport.close?.()); + return { ok: true }; + } catch (err) { + return { ok: false, error: sanitizeConnectionError(err) }; + } finally { + transport?.close?.(); + } +} + +export async function testConnection(account) { + const [imap, smtp] = await Promise.all([ + probeImap(account), + probeSmtp(account), + ]); + return { imap, smtp }; +} diff --git a/backend/src/services/connectionTest.test.js b/backend/src/services/connectionTest.test.js new file mode 100644 index 00000000..30645ccf --- /dev/null +++ b/backend/src/services/connectionTest.test.js @@ -0,0 +1,159 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +const { + ImapFlow, + makeClientCfg, + buildSmtpTransport, + resolveForConnection, + getConnectionPolicy, + imapConnect, + imapLogout, + imapClose, + smtpVerify, + smtpClose, +} = vi.hoisted(() => { + const imapConnect = vi.fn(); + const imapLogout = vi.fn(); + const imapClose = vi.fn(); + return { + ImapFlow: vi.fn(function MockImapFlow() { + this.connect = imapConnect; + this.logout = imapLogout; + this.close = imapClose; + }), + makeClientCfg: vi.fn(), + buildSmtpTransport: vi.fn(), + resolveForConnection: vi.fn(), + getConnectionPolicy: vi.fn(), + imapConnect, + imapLogout, + imapClose, + smtpVerify: vi.fn(), + smtpClose: vi.fn(), + }; +}); + +vi.mock('imapflow', () => ({ ImapFlow })); +vi.mock('./imapManager.js', () => ({ makeClientCfg })); +vi.mock('./mail/smtp.js', () => ({ buildSmtpTransport })); +vi.mock('./hostValidation.js', () => ({ resolveForConnection })); +vi.mock('./connectionPolicy.js', () => ({ getConnectionPolicy })); + +import { testConnection } from './connectionTest.js'; + +const account = { + imap_host: 'imap.example.com', + smtp_host: 'smtp.example.com', + imap_port: 993, + smtp_port: 587, +}; + +beforeEach(() => { + ImapFlow.mockClear(); + makeClientCfg.mockReset().mockReturnValue({ host: '203.0.113.10' }); + resolveForConnection.mockReset().mockResolvedValue({ + host: '203.0.113.10', + servername: 'imap.example.com', + }); + getConnectionPolicy.mockReset().mockResolvedValue({ + allowPrivateHosts: false, + allowInsecureTls: false, + }); + imapConnect.mockReset().mockResolvedValue(undefined); + imapLogout.mockReset().mockResolvedValue(undefined); + imapClose.mockReset(); + smtpVerify.mockReset().mockResolvedValue(true); + smtpClose.mockReset(); + buildSmtpTransport.mockReset().mockResolvedValue({ + transport: { verify: smtpVerify, close: smtpClose }, + account, + }); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('testConnection', () => { + it('returns success for both probes', async () => { + await expect(testConnection(account)).resolves.toEqual({ + imap: { ok: true }, + smtp: { ok: true }, + }); + expect(imapLogout).toHaveBeenCalledOnce(); + expect(smtpVerify).toHaveBeenCalledOnce(); + }); + + it('keeps SMTP successful when IMAP fails', async () => { + imapConnect.mockRejectedValueOnce(new Error('authenticationFailed: invalid login')); + + const result = await testConnection(account); + + expect(result.imap).toEqual({ + ok: false, + error: 'Authentication failed. Check your email account credentials.', + }); + expect(result.smtp).toEqual({ ok: true }); + expect(smtpVerify).toHaveBeenCalledOnce(); + }); + + it('keeps IMAP successful when SMTP fails', async () => { + smtpVerify.mockRejectedValueOnce(new Error('connect ECONNREFUSED')); + + const result = await testConnection(account); + + expect(result.imap).toEqual({ ok: true }); + expect(result.smtp).toEqual({ + ok: false, + error: 'Could not connect to the mail server. Check your SMTP settings.', + }); + }); + + it('force-closes a timed-out IMAP probe without hanging SMTP', async () => { + vi.useFakeTimers(); + imapConnect.mockReturnValueOnce(new Promise(() => {})); + + const pending = testConnection(account); + await vi.advanceTimersByTimeAsync(10_000); + const result = await pending; + + expect(imapClose).toHaveBeenCalled(); + expect(result.imap).toEqual({ + ok: false, + error: 'Connection test timed out. Check your mail server settings.', + }); + expect(result.smtp).toEqual({ ok: true }); + }); + + it('force-closes a timed-out SMTP probe without hanging IMAP', async () => { + vi.useFakeTimers(); + smtpVerify.mockReturnValueOnce(new Promise(() => {})); + + const pending = testConnection(account); + await vi.advanceTimersByTimeAsync(10_000); + const result = await pending; + + expect(smtpClose).toHaveBeenCalled(); + expect(result.imap).toEqual({ ok: true }); + expect(result.smtp).toEqual({ + ok: false, + error: 'Connection test timed out. Check your mail server settings.', + }); + }); + + it('imports only the named makeClientCfg helper from imapManager', () => { + const source = readFileSync( + fileURLToPath(new URL('./connectionTest.js', import.meta.url)), + 'utf8' + ); + const imapManagerImport = source.match( + /import\s+([^;]+)\s+from\s+['"]\.\/imapManager\.js['"]/ + ); + + expect(imapManagerImport?.[1]).toMatch(/^\{\s*makeClientCfg\s*\}$/); + expect(source).not.toMatch(/import\s+imapManager\b/); + expect(source.replace(/^import .*$/gm, '')).not.toMatch(/\bimapManager\./); + }); +}); diff --git a/backend/src/services/draftService.js b/backend/src/services/draftService.js new file mode 100644 index 00000000..adc40701 --- /dev/null +++ b/backend/src/services/draftService.js @@ -0,0 +1,231 @@ +import { randomBytes as defaultRandomBytes } from 'crypto'; +import { sanitizeSignature } from './emailSanitizer.js'; +import { embedInlineDataImages as defaultEmbedInlineDataImages } from '../utils/inlineImages.js'; +import { mapRecipientList, sanitizeHeaderValue } from './mail/addresses.js'; +import { resolveFromIdentity as defaultResolveFromIdentity } from './mail/identity.js'; +import { + bodyToHtml, + bodyToPlain, + buildMailOptions as defaultBuildMailOptions, + renderRaw as defaultRenderRaw, + sigToPlainText, + textToHtml, +} from './mail/mimeBuilder.js'; + +function serviceError(message, status) { + return Object.assign(new Error(message), { status, expose: true }); +} + +async function resolveDraftsFolder(account, deps) { + const mapped = account.folder_mappings?.drafts; + if (mapped) return mapped; + const result = await deps.query( + "SELECT path FROM folders WHERE account_id = $1 AND special_use = '\\Drafts' LIMIT 1", + [account.id], + ); + return result.rows[0]?.path || null; +} + +export async function buildRawDraft(input, deps) { + if (!input.account) throw serviceError('Account not found', 404); + + const resolveFromIdentity = deps.resolveFromIdentity || defaultResolveFromIdentity; + const embedInlineDataImages = deps.embedInlineDataImages || defaultEmbedInlineDataImages; + const buildMailOptions = deps.buildMailOptions || defaultBuildMailOptions; + const renderRaw = deps.renderRaw || defaultRenderRaw; + const makeRandomBytes = deps.randomBytes || defaultRandomBytes; + + const identity = await resolveFromIdentity( + input.account, + { aliasId: input.aliasId, aliasEmail: input.aliasEmail }, + deps, + ); + + const rawSignature = input.editedSignature !== undefined + ? (input.editedSignature || null) + : identity.signature; + const effectiveSignature = rawSignature ? sanitizeSignature(rawSignature) : null; + const signatureText = effectiveSignature ? sigToPlainText(effectiveSignature) : null; + const bodyText = bodyToPlain(input.body || '', input.bodyIsHtml); + const bodyHtml = bodyToHtml(input.body || '', input.bodyIsHtml); + const rawHtml = bodyHtml + + (effectiveSignature + ? `
${effectiveSignature}
` + : '') + + (input.quotedBodyHtml || (input.quotedBody ? textToHtml(input.quotedBody) : '')); + const embedded = embedInlineDataImages(rawHtml); + + const uploadedAttachments = Array.isArray(input.attachments) + ? input.attachments.map(attachment => ({ + filename: sanitizeHeaderValue(attachment.filename || 'attachment'), + content: Buffer.isBuffer(attachment.content) + ? attachment.content + : Buffer.from(attachment.content || '', 'base64'), + contentType: typeof attachment.contentType === 'string' + ? attachment.contentType + : 'application/octet-stream', + })) + : []; + const attachments = [...embedded.attachments, ...uploadedAttachments]; + + // Stable Message-ID so the appended MIME and the local DB row reference the same message. + const domain = identity.fromEmail.split('@')[1] || 'mailflow.local'; + const messageId = `<${makeRandomBytes(16).toString('hex')}@${domain}>`; + const text = signatureText + ? `${bodyText}\n\n-- \n${signatureText}${input.quotedBody || ''}` + : `${bodyText}${input.quotedBody || ''}`; + const replyTo = input.replyTo !== undefined ? input.replyTo : identity.fromReplyTo; + const mailOptions = buildMailOptions({ + messageId, + fromName: identity.fromName, + fromEmail: identity.fromEmail, + replyTo, + to: Array.isArray(input.to) ? input.to : [input.to], + cc: Array.isArray(input.cc) ? input.cc : [], + bcc: Array.isArray(input.bcc) ? input.bcc : [], + subject: input.subject || '', + priority: input.priority, + text, + html: embedded.html, + inReplyTo: input.inReplyTo, + references: input.references, + attachments, + }); + const rawMessage = await renderRaw(mailOptions); + const snippet = text.replace(/\s+/g, ' ').trim().slice(0, 200); + + // rawHtml (before CID embedding) is what the composer should reopen with so data: + // image URIs remain editable. + return { + rawMessage, + account: input.account, + meta: { + messageId, + fromName: identity.fromName, + fromEmail: identity.fromEmail, + bodyHtml: rawHtml, + bodyText: text, + snippet, + inReplyTo: input.inReplyTo || null, + references: input.references || null, + }, + }; +} + +export async function saveDraft(input, deps) { + const { rawMessage, account, meta } = await buildRawDraft(input, deps); + const draftsFolder = await resolveDraftsFolder(account, deps); + if (!draftsFolder) throw serviceError('No Drafts folder found for this account', 422); + + // APPEND the new draft first so a failed update can never lose the message. + const { uid } = await deps.imapManager.appendToFolder( + account, + draftsFolder, + rawMessage, + ['\\Draft', '\\Seen'], + ); + + // Local persistence is non-fatal because IMAP already holds the new draft. + if (uid != null) { + try { + await deps.imapManager.upsertDraftMessageRecord(account, draftsFolder, uid, { + messageId: meta.messageId, + subject: input.subject, + fromName: meta.fromName, + fromEmail: meta.fromEmail, + to: mapRecipientList(input.to), + cc: mapRecipientList(input.cc), + inReplyTo: meta.inReplyTo, + references: meta.references, + snippet: meta.snippet, + bodyHtml: meta.bodyHtml, + bodyText: meta.bodyText, + }); + } catch (rowErr) { + console.error(`Draft: failed to persist local row uid=${uid}: ${rowErr.message}`); + } + } + + // Delete the old draft only after append and local upsert have completed. + let sourceDraftDeleted; + if (input.existingUid && input.existingFolder) { + try { + await deps.imapManager.permanentDeleteMessage(account, input.existingUid, input.existingFolder); + await deps.query( + 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', + [account.id, input.existingUid, input.existingFolder], + ); + sourceDraftDeleted = true; + } catch (delErr) { + sourceDraftDeleted = false; + console.error(`Draft: failed to delete old uid=${input.existingUid}: ${delErr.message}`); + } + } + + return { + uid, + folder: draftsFolder, + messageId: meta.messageId, + ...(input.reportSourceDraftDeletion === true && sourceDraftDeleted !== undefined + ? { sourceDraftDeleted } + : {}), + }; +} + +export async function deleteDraft({ account, uid, folder, reportDeletionAcceptance = false }, deps) { + await deps.imapManager.permanentDeleteMessage(account, uid, folder); + try { + await deps.query( + 'DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3', + [account.id, uid, folder], + ); + } catch (error) { + if (reportDeletionAcceptance) return { ok: true, localCleanupPending: true }; + throw error; + } + return { ok: true }; +} + +export async function listDrafts({ userId, account, limit = 50, offset = 0 }, deps) { + const folder = await resolveDraftsFolder(account, deps); + if (!folder) return { drafts: [], total: 0 }; + const safeLimit = Math.min(Math.max(parseInt(limit, 10) || 50, 1), 500); + const safeOffset = Math.max(parseInt(offset, 10) || 0, 0); + const countResult = await deps.query( + `SELECT COUNT(*)::int AS n FROM messages m + WHERE m.account_id = $1 AND m.folder = $3 AND m.is_deleted = false + AND NOT EXISTS ( + SELECT 1 FROM compose_sessions cs + WHERE cs.user_id = $2 + AND cs.source_draft_account_id = m.account_id + AND cs.source_draft_folder = m.folder + AND cs.source_draft_uid = m.uid + )`, + [account.id, userId, folder], + ); + const result = await deps.query( + `SELECT m.* FROM messages m + WHERE m.account_id = $1 AND m.folder = $3 AND m.is_deleted = false + AND NOT EXISTS ( + SELECT 1 FROM compose_sessions cs + WHERE cs.user_id = $2 + AND cs.source_draft_account_id = m.account_id + AND cs.source_draft_folder = m.folder + AND cs.source_draft_uid = m.uid + ) + ORDER BY m.date DESC + LIMIT $4 OFFSET $5`, + [account.id, userId, folder, safeLimit, safeOffset], + ); + return { drafts: result.rows, total: countResult.rows[0]?.n ?? 0 }; +} + +export async function getDraft({ account, uid, folder }, deps) { + const result = await deps.query( + `SELECT * FROM messages + WHERE account_id = $1 AND uid = $2 AND folder = $3 AND is_deleted = false + LIMIT 1`, + [account.id, uid, folder], + ); + return result.rows[0] || null; +} diff --git a/backend/src/services/draftService.test.js b/backend/src/services/draftService.test.js new file mode 100644 index 00000000..3c58c2c0 --- /dev/null +++ b/backend/src/services/draftService.test.js @@ -0,0 +1,372 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + buildRawDraft, + deleteDraft, + getDraft, + listDrafts, + saveDraft, +} from './draftService.js'; +import { sendMessage } from './sendService.js'; + +const account = { + id: 'account-1', + email_address: 'sender@example.com', + name: 'Sender', + sender_name: null, + signature: null, + folder_mappings: {}, +}; + +function draftInput(overrides = {}) { + return { + userId: 'user-1', + account, + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Draft subject', + body: 'Draft body', + bodyIsHtml: false, + ...overrides, + }; +} + +function buildDeps(overrides = {}) { + return { + query: vi.fn(), + imapManager: {}, + resolveFromIdentity: vi.fn().mockResolvedValue({ + fromName: 'Sender', + fromEmail: 'sender@example.com', + fromReplyTo: 'reply@example.com', + signature: null, + aliasId: null, + }), + embedInlineDataImages: vi.fn(html => ({ html, attachments: [] })), + buildMailOptions: vi.fn(options => options), + renderRaw: vi.fn().mockResolvedValue(Buffer.from('raw draft')), + randomBytes: vi.fn(() => Buffer.alloc(16, 3)), + ...overrides, + }; +} + +describe('buildRawDraft', () => { + it('uses the already-scoped account and carries reply, attachment, and priority fields into MIME', async () => { + const deps = buildDeps(); + + const result = await buildRawDraft(draftInput({ + aliasId: 'alias-1', + inReplyTo: '', + references: ' ', + replyTo: 'explicit-reply@example.com', + priority: 'high', + attachments: [{ + filename: 'draft.txt', + content: Buffer.from('draft attachment').toString('base64'), + contentType: 'text/plain', + }], + }), deps); + + expect(deps.resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasId: 'alias-1', aliasEmail: undefined }, + deps, + ); + expect(deps.query).not.toHaveBeenCalled(); + expect(deps.buildMailOptions).toHaveBeenCalledWith(expect.objectContaining({ + replyTo: 'explicit-reply@example.com', + inReplyTo: '', + references: ' ', + priority: 'high', + attachments: [expect.objectContaining({ + filename: 'draft.txt', + content: Buffer.from('draft attachment'), + })], + })); + expect(result).toMatchObject({ + rawMessage: Buffer.from('raw draft'), + account, + meta: { + fromName: 'Sender', + fromEmail: 'sender@example.com', + bodyHtml: expect.stringContaining('Draft body'), + bodyText: expect.stringContaining('Draft body'), + inReplyTo: '', + references: ' ', + }, + }); + expect(result.meta.messageId).toMatch(/^<03030303.+@example\.com>$/); + }); +}); + +describe('saveDraft', () => { + it('persists reply threading metadata unchanged so a reopened draft can send in the same thread', async () => { + const inReplyTo = ''; + const references = ' '; + let persistedRow; + const imapManager = { + appendToFolder: vi.fn().mockResolvedValue({ uid: 5 }), + upsertDraftMessageRecord: vi.fn(async (_account, _folder, uid, meta) => { + persistedRow = { + uid, + in_reply_to: meta.inReplyTo, + thread_references: meta.references, + }; + }), + permanentDeleteMessage: vi.fn(), + }; + const query = vi.fn(async (sql) => { + if (sql.includes("special_use = '\\Drafts'")) return { rows: [{ path: 'Drafts' }] }; + if (sql.includes('SELECT * FROM messages')) return { rows: [persistedRow] }; + return { rows: [] }; + }); + const deps = buildDeps({ imapManager, query }); + + await saveDraft(draftInput({ inReplyTo, references }), deps); + const reopened = await getDraft({ account, uid: 5, folder: 'Drafts' }, deps); + + expect(reopened).toMatchObject({ + in_reply_to: inReplyTo, + thread_references: references, + }); + + const delivered = []; + const makeSendDeps = () => { + const transport = { + sendMail: vi.fn(async mailOptions => { delivered.push(mailOptions); }), + }; + return { + query: vi.fn(), + imapManager: {}, + resolveFromIdentity: vi.fn().mockResolvedValue({ + fromName: 'Sender', + fromEmail: 'sender@example.com', + fromReplyTo: 'reply@example.com', + signature: null, + aliasId: null, + }), + buildSmtpTransport: vi.fn().mockResolvedValue({ transport, account }), + buildMailOptions: vi.fn(options => options), + renderRaw: vi.fn().mockResolvedValue(Buffer.from('raw mime')), + embedInlineDataImages: vi.fn(html => ({ html, attachments: [] })), + learnSentRecipients: vi.fn(), + resolveSentFolder: vi.fn().mockResolvedValue('Sent'), + persistSentCopy: vi.fn().mockResolvedValue({ sentCopySaved: true }), + randomBytes: vi.fn(() => Buffer.alloc(16, 1)), + }; + }; + + await sendMessage(draftInput({ inReplyTo, references }), makeSendDeps()); + await sendMessage(draftInput({ + inReplyTo: reopened.in_reply_to, + references: reopened.thread_references, + }), makeSendDeps()); + + expect(delivered).toHaveLength(2); + expect({ + inReplyTo: delivered[1].inReplyTo, + references: delivered[1].references, + }).toEqual({ + inReplyTo: delivered[0].inReplyTo, + references: delivered[0].references, + }); + }); + + it('keeps append → upsert → delete-old ordering for crash safety', async () => { + const events = []; + const imapManager = { + appendToFolder: vi.fn(async () => { + events.push('append'); + return { uid: 5 }; + }), + upsertDraftMessageRecord: vi.fn(async () => { events.push('upsert'); }), + permanentDeleteMessage: vi.fn(async () => { events.push('delete-imap'); }), + }; + const query = vi.fn(async (sql) => { + if (sql.includes("special_use = '\\Drafts'")) return { rows: [{ path: 'Drafts' }] }; + if (sql.startsWith('DELETE FROM messages')) events.push('delete-db'); + return { rows: [] }; + }); + const deps = buildDeps({ imapManager, query }); + + const result = await saveDraft(draftInput({ + existingUid: 4, + existingFolder: 'Drafts', + }), deps); + + expect(events).toEqual(['append', 'upsert', 'delete-imap', 'delete-db']); + expect(imapManager.upsertDraftMessageRecord).toHaveBeenCalledWith( + account, + 'Drafts', + 5, + expect.objectContaining({ + subject: 'Draft subject', + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + }), + ); + expect(result).toMatchObject({ uid: 5, folder: 'Drafts' }); + expect(result.messageId).toMatch(/^<03030303.+@example\.com>$/); + }); + + it('reports an accepted replacement with failed source cleanup only when opted in', async () => { + const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {}); + const makeDeps = () => buildDeps({ + imapManager: { + appendToFolder: vi.fn().mockResolvedValue({ uid: 5 }), + upsertDraftMessageRecord: vi.fn().mockResolvedValue(undefined), + permanentDeleteMessage: vi.fn().mockRejectedValue(new Error('Synthetic delete failure')), + }, + query: vi.fn().mockResolvedValue({ rows: [{ path: 'Drafts' }] }), + }); + const replacement = draftInput({ existingUid: 4, existingFolder: 'Drafts' }); + + const strictDeps = makeDeps(); + await expect(saveDraft({ + ...replacement, + reportSourceDraftDeletion: true, + }, strictDeps)).resolves.toMatchObject({ + uid: 5, + folder: 'Drafts', + sourceDraftDeleted: false, + }); + expect(strictDeps.imapManager.appendToFolder).toHaveBeenCalledOnce(); + + const legacyDeps = makeDeps(); + const legacy = await saveDraft(replacement, legacyDeps); + expect(legacy).not.toHaveProperty('sourceDraftDeleted'); + expect(legacyDeps.imapManager.appendToFolder).toHaveBeenCalledOnce(); + errorLog.mockRestore(); + }); + + it('keeps local persistence non-fatal after a successful append', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + const imapManager = { + appendToFolder: vi.fn().mockResolvedValue({ uid: 5 }), + upsertDraftMessageRecord: vi.fn().mockRejectedValue(new Error('db down')), + permanentDeleteMessage: vi.fn(), + }; + const deps = buildDeps({ + imapManager, + query: vi.fn().mockResolvedValue({ rows: [{ path: 'Drafts' }] }), + }); + + await expect(saveDraft(draftInput(), deps)).resolves.toMatchObject({ uid: 5, folder: 'Drafts' }); + }); + + it('does not upsert without a reliable uid and errors when no Drafts folder resolves', async () => { + const imapManager = { + appendToFolder: vi.fn().mockResolvedValue({ uid: null }), + upsertDraftMessageRecord: vi.fn(), + permanentDeleteMessage: vi.fn(), + }; + const deps = buildDeps({ + imapManager, + query: vi.fn().mockResolvedValue({ rows: [{ path: 'Drafts' }] }), + }); + await saveDraft(draftInput(), deps); + expect(imapManager.upsertDraftMessageRecord).not.toHaveBeenCalled(); + + const missingDeps = buildDeps({ + imapManager, + query: vi.fn().mockResolvedValue({ rows: [] }), + }); + await expect(saveDraft(draftInput(), missingDeps)).rejects.toMatchObject({ + message: 'No Drafts folder found for this account', + status: 422, + expose: true, + }); + }); +}); + +describe('draft persistence helpers', () => { + it('deletes from IMAP before removing the local row', async () => { + const events = []; + const deps = { + imapManager: { + permanentDeleteMessage: vi.fn(async () => { events.push('imap'); }), + }, + query: vi.fn(async () => { + events.push('db'); + return { rows: [] }; + }), + }; + await expect(deleteDraft({ account, uid: 9, folder: 'Drafts' }, deps)).resolves.toEqual({ ok: true }); + expect(events).toEqual(['imap', 'db']); + }); + + it('reports accepted IMAP deletion with pending local cleanup only when opted in', async () => { + const makeDeps = () => ({ + imapManager: { + permanentDeleteMessage: vi.fn().mockResolvedValue(undefined), + }, + query: vi.fn().mockRejectedValue(new Error('Synthetic local cleanup failure')), + }); + + const strictDeps = makeDeps(); + await expect(deleteDraft({ + account, + uid: 9, + folder: 'Drafts', + reportDeletionAcceptance: true, + }, strictDeps)).resolves.toEqual({ ok: true, localCleanupPending: true }); + expect(strictDeps.imapManager.permanentDeleteMessage).toHaveBeenCalledOnce(); + + const legacyDeps = makeDeps(); + await expect(deleteDraft({ + account, + uid: 9, + folder: 'Drafts', + }, legacyDeps)).rejects.toThrow('Synthetic local cleanup failure'); + }); + + it('lists and gets drafts from the already-scoped account', async () => { + const query = vi.fn() + .mockResolvedValueOnce({ rows: [{ n: 1 }] }) + .mockResolvedValueOnce({ rows: [{ uid: 7, subject: 'Draft' }] }) + .mockResolvedValueOnce({ rows: [{ uid: 7, subject: 'Draft' }] }); + + const mappedAccount = { ...account, folder_mappings: { drafts: 'Drafts' } }; + await expect(listDrafts({ + userId: 'user-1', + account: mappedAccount, + limit: 10, + offset: 0, + }, { query })) + .resolves.toEqual({ drafts: [{ uid: 7, subject: 'Draft' }], total: 1 }); + await expect(getDraft({ account: mappedAccount, uid: 7, folder: 'Drafts' }, { query })) + .resolves.toEqual({ uid: 7, subject: 'Draft' }); + + expect(query.mock.calls[0][1][0]).toBe('account-1'); + expect(query.mock.calls[2][1]).toEqual(['account-1', 7, 'Drafts']); + }); + + it('owner-scopes claimed source filtering in both draft count and page queries', async () => { + const query = vi.fn() + .mockResolvedValueOnce({ rows: [{ n: 1 }] }) + .mockResolvedValueOnce({ rows: [{ uid: 8, subject: 'Unclaimed draft' }] }); + const mappedAccount = { ...account, folder_mappings: { drafts: 'Drafts' } }; + + await expect(listDrafts({ + userId: 'user-1', + account: mappedAccount, + limit: 25, + offset: 5, + }, { query })).resolves.toEqual({ + drafts: [{ uid: 8, subject: 'Unclaimed draft' }], + total: 1, + }); + + for (const [sql] of query.mock.calls) { + expect(sql).toContain('FROM messages m'); + expect(sql).toContain(`AND NOT EXISTS ( + SELECT 1 FROM compose_sessions cs + WHERE cs.user_id = $2 + AND cs.source_draft_account_id = m.account_id + AND cs.source_draft_folder = m.folder + AND cs.source_draft_uid = m.uid + )`); + } + expect(query.mock.calls[0][1]).toEqual(['account-1', 'user-1', 'Drafts']); + expect(query.mock.calls[1][1]).toEqual(['account-1', 'user-1', 'Drafts', 25, 5]); + }); +}); diff --git a/backend/src/services/emailSanitizer.js b/backend/src/services/emailSanitizer.js index 1d068bd4..af927499 100644 --- a/backend/src/services/emailSanitizer.js +++ b/backend/src/services/emailSanitizer.js @@ -1,5 +1,10 @@ import sanitizeHtml from 'sanitize-html'; +// Reject strings that contain characters that could inject extra email headers. +export function hasHeaderInjectionChars(str) { + return typeof str === 'string' && /[\r\n\0]/.test(str); +} + // Strip the element from email HTML, preserving any \nWorld', maxChars: 1000, + cfg: { stripHTML: true, collapseWhitespace: true }, want: 'Hello\n\nWorld', wantTrunc: false }, + { name: 'StripHTMLDropsScriptBlock', subject: '', body: 'Pre\n\nPost', maxChars: 1000, + cfg: { stripHTML: true, collapseWhitespace: true }, want: 'Pre\n\nPost', wantTrunc: false }, + { name: 'StripBase64DropsDataURI', subject: '', body: 'Before image data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg== After image', maxChars: 1000, + cfg: { stripBase64: true, collapseWhitespace: true }, want: 'Before image After image', wantTrunc: false }, + { name: 'StripBase64DropsBareBlob', subject: '', body: 'Hello ' + 'A'.repeat(250) + ' world', maxChars: 1000, + cfg: { stripBase64: true, collapseWhitespace: true }, want: 'Hello world', wantTrunc: false }, + { name: 'StripBase64DropsBareBlobWithSlashes', subject: '', body: 'Before ' + 'abcdefghij/'.repeat(30) + ' After', maxChars: 1000, + cfg: { stripBase64: true, collapseWhitespace: true }, want: 'Before After', wantTrunc: false }, + { name: 'StripBase64KeepsShortAlphanumeric', subject: '', body: 'Token=abc123XYZ check this hash 0123456789abcdef0123456789abcdef.', maxChars: 1000, + cfg: { stripBase64: true }, want: 'Token=abc123XYZ check this hash 0123456789abcdef0123456789abcdef.', wantTrunc: false }, + { name: 'StripURLTrackingDropsKnownParams', subject: '', body: 'Visit https://example.com/page?utm_source=newsletter&utm_medium=email&fbclid=xyz&id=42 today', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'Visit https://example.com/page?id=42 today', wantTrunc: false }, + { name: 'StripURLTrackingHandlesHubSpotMixedCase', subject: '', body: 'Click https://hub.example.com/cta?hsCtaTracking=abc&id=7 now', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'Click https://hub.example.com/cta?id=7 now', wantTrunc: false }, + { name: 'StripURLTrackingPreservesTrailingPunctuation', subject: '', body: 'See https://example.com/x?utm_source=foo&keep=1.', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'See https://example.com/x?keep=1.', wantTrunc: false }, + { name: 'StripURLTrackingLeavesCleanURLAlone', subject: '', body: 'See https://example.com/page?id=42 and ftp://elsewhere/path', maxChars: 1000, + cfg: { stripURLTracking: true }, want: 'See https://example.com/page?id=42 and ftp://elsewhere/path', wantTrunc: false }, + { name: 'CollapseWhitespaceShrinksRuns', subject: '', body: 'Line one\n\n\n\nLine two with big gaps', maxChars: 1000, + cfg: { collapseWhitespace: true }, want: 'Line one\n\nLine two with big gaps', wantTrunc: false }, + { name: 'StripBase64KeepsLongURLPath', subject: '', body: 'https://example.com/' + 'a/b'.repeat(80) + '/end', maxChars: 2000, + cfg: { stripBase64: true }, want: 'https://example.com/' + 'a/b'.repeat(80) + '/end', wantTrunc: false }, + { name: 'StripPipelineSweepsOversizedImgTag', subject: '', body: 'Before ' + 'x' + ' After', maxChars: 5000, + cfg: { stripHTML: true, stripBase64: true, collapseWhitespace: true }, want: 'Before After', wantTrunc: false }, + { name: 'CollapseWhitespaceNormalizesCRLF', subject: '', body: 'Line one \r\n\r\n\r\n\r\nLine two', maxChars: 1000, + cfg: { collapseWhitespace: true }, want: 'Line one\n\nLine two', wantTrunc: false }, + { name: 'PipelineRemovesPollutionEndToEnd', subject: 'Newsletter', + body: '

Hello

\n\n\n' + 'data:image/gif;base64,R0lGODlhAQABAAAAACw= ' + '\n\n\nClick ' + 'https://example.com/?utm_source=x&keep=y' + '\n\n-- \nSig', + maxChars: 1000, cfg: { stripQuotes: true, stripSignatures: true, stripHTML: true, stripBase64: true, stripURLTracking: true, collapseWhitespace: true }, + want: 'Subject: Newsletter\n\nHello\n\nClick https://example.com/?keep=y', wantTrunc: false }, +]; + +describe('preprocess (msgvault fixture parity)', () => { + for (const tt of cases) { + it(tt.name, () => { + const { text, truncated } = preprocess(tt.subject, tt.body, tt.maxChars, tt.cfg); + if (tt.want !== undefined) expect(text).toBe(tt.want); + if (tt.lenLE) expect(Buffer.byteLength(text, 'utf8')).toBeLessThanOrEqual(tt.lenLE); + expect(truncated).toBe(tt.wantTrunc); + }); + } +}); diff --git a/backend/src/services/embeddings/rankingQuality.test.js b/backend/src/services/embeddings/rankingQuality.test.js new file mode 100644 index 00000000..5bc06aa9 --- /dev/null +++ b/backend/src/services/embeddings/rankingQuality.test.js @@ -0,0 +1,83 @@ +import { describe, it, expect, afterAll } from 'vitest'; +import { hasTestDb, withTestDb, closeTestDb } from './__testdb__.js'; +import { fusedSearch } from './vectorStore.js'; + +afterAll(async () => { await closeTestDb(); }); + +const DIM = 8; +function vec(axis, jitter = 0) { const v = new Array(DIM).fill(0); v[axis] = 1; if (jitter) v[(axis + 1) % DIM] = jitter; return v; } + +// 8 concepts. Each has a keyword doc and a synonym-only doc that shares the +// vector neighbourhood but NOT the keyword. Queries ask by keyword; the gold +// hit is the synonym-only doc that lexical alone would miss. +const CONCEPTS = [ + { kw: 'invoice', syn: 'billing statement', axis: 0 }, + { kw: 'flight', syn: 'boarding pass itinerary', axis: 1 }, + { kw: 'invoice2', syn: 'amount due remittance', axis: 2 }, + { kw: 'meeting', syn: 'calendar sync standup', axis: 3 }, + { kw: 'refund', syn: 'money back reversal', axis: 4 }, + { kw: 'password', syn: 'credential reset link', axis: 5 }, + { kw: 'shipment', syn: 'parcel tracking dispatch', axis: 6 }, + { kw: 'contract', syn: 'signed agreement terms', axis: 7 }, +]; +// 20 labeled queries (concepts reused with different jitter for the extra 12). +const QUERIES = []; +for (let i = 0; i < 20; i++) { + const c = CONCEPTS[i % CONCEPTS.length]; + QUERIES.push({ text: c.kw, qvec: vec(c.axis, (i >= 8 ? 0.15 : 0)), goldAxis: c.axis }); +} + +describe.skipIf(!hasTestDb())('ranking quality: hybrid must not lose to lexical', () => { + it('hybrid recall of semantic-only gold hits >= lexical, and never lower', async () => { + await withTestDb(async (pool) => { + const u = (await pool.query(`INSERT INTO users (username,password_hash) VALUES ('r','x') RETURNING id`)).rows[0].id; + const acc = (await pool.query( + `INSERT INTO email_accounts (user_id,name,email_address,enabled) VALUES ($1,'A','a@x.test',true) RETURNING id`, [u])).rows[0].id; + const gen = (await pool.query( + `INSERT INTO index_generations (model,dimension,fingerprint,started_at,state) VALUES ('m',$1,'m:8:test',$2,'active') RETURNING id`, + [DIM, Math.floor(Date.now() / 1000)])).rows[0].id; + + const goldByAxis = {}; + let uid = 0; + for (const c of CONCEPTS) { + // keyword doc (axis vector + keyword in subject) + await insertDoc(pool, acc, gen, ++uid, `${c.kw} notice`, vec(c.axis), DIM); + // synonym-only doc: gold semantic hit, NO keyword token + const goldId = await insertDoc(pool, acc, gen, ++uid, c.syn, vec(c.axis, 0.05), DIM); + goldByAxis[c.axis] = goldId; + } + + let hybridWins = 0, lexicalWins = 0; + for (const q of QUERIES) { + const goldId = goldByAxis[q.goldAxis]; + const lex = await fusedSearch({ ftsQuery: q.text, queryVec: null, generation: { id: gen, dimension: DIM }, + accountIds: [acc], buildFilters: () => [], rrfK: 60, kPerSignal: 100, limit: 10 }, { client: pool }); + const hyb = await fusedSearch({ ftsQuery: q.text, queryVec: q.qvec, generation: { id: gen, dimension: DIM }, + accountIds: [acc], buildFilters: () => [], rrfK: 60, kPerSignal: 100, limit: 10 }, { client: pool }); + const lexHit = lex.hits.some(h => h.message_id === goldId); + const hybHit = hyb.hits.some(h => h.message_id === goldId); + if (hybHit && !lexHit) hybridWins++; + if (lexHit && !hybHit) lexicalWins++; + // Never lose: whatever lexical surfaced, hybrid still surfaces. + for (const h of lex.hits) { + expect(hyb.hits.some(x => x.message_id === h.message_id)).toBe(true); + } + } + // Semantic-only gold hits: hybrid recovers them, lexical cannot. + expect(hybridWins).toBeGreaterThanOrEqual(QUERIES.length); + expect(lexicalWins).toBe(0); + console.log(`[ranking-quality] hybridWins=${hybridWins} lexicalWins=${lexicalWins} of ${QUERIES.length}`); + }); + }); +}); + +async function insertDoc(pool, acc, gen, uid, subject, embedding, dim) { + const id = (await pool.query( + `INSERT INTO messages (account_id,uid,folder,subject,from_name,from_email,date,snippet,body_text,is_deleted) + VALUES ($1,$2,'INBOX',$3,'S','s@x.test',now(),$3,$3,false) RETURNING id`, [acc, uid, subject])).rows[0].id; + await pool.query( + `INSERT INTO embeddings (generation_id,message_id,chunk_index,embedded_at,source_char_len,embedding,dimension) + VALUES ($1,$2,0,$3,$4,$5::vector,$6)`, + [gen, id, Math.floor(Date.now() / 1000), subject.length, `[${embedding.join(',')}]`, dim]); + return id; +} diff --git a/backend/src/services/embeddings/scheduler.js b/backend/src/services/embeddings/scheduler.js new file mode 100644 index 00000000..9afe7c23 --- /dev/null +++ b/backend/src/services/embeddings/scheduler.js @@ -0,0 +1,91 @@ +import { resolveEmbedConfig, generationFingerprint } from './config.js'; +import { isVectorAvailable } from './vectorStore.js'; +import * as store from './vectorStore.js'; +import * as generations from './generations.js'; +const { buildingGeneration, activeGeneration, retireGeneration } = generations; +import { EmbeddingClient } from './client.js'; +import { EmbeddingWorker } from './worker.js'; +import { tryAcquireEmbedRun, releaseEmbedRun } from './embedRunLock.js'; + +let _timer = null; +let _running = false; +let _lastBackstop = 0; +let _backstopIntervalMs = 86_400_000; +// Latch for the once-per-fingerprint "paused" log below — holds the config +// fingerprint we last warned about so a mismatched active generation does not +// spam the log every tick. Cleared whenever a matching run proceeds. +let _pausedForFingerprint = null; + +// One scheduler pass: drive an existing building-or-active generation. No-ops when +// vector is unavailable, embeddings are disabled, or no generation exists. Exported +// for unit testing. +export async function runSchedulerTick(nowMs) { + if (!isVectorAvailable()) return; + let cfg; + try { cfg = await resolveEmbedConfig(); } catch { return; } + if (!cfg || !cfg.enabled || !cfg.endpoint || !cfg.model || !(cfg.dimension > 0)) return; + + const building = await buildingGeneration(); + const gen = building || await activeGeneration(); + if (!gen) return; + + // Single-flight: skip this tick if a manual build — or another run — already + // holds the shared lock, so the scheduler never double-drives a generation. + if (!tryAcquireEmbedRun()) return; + try { + // Fingerprint guard: the worker embeds with a client built from the *current* + // config, so it must only drive a generation whose fingerprint matches that + // config. If the admin changed model/dimension/preprocess mid-build, the + // resolved generation is stale — driving it would embed with the new client + // into the old generation and fail the dimension check forever. + const cfgFingerprint = generationFingerprint(cfg); + if (gen.fingerprint !== cfgFingerprint) { + if (building) { + // A building gen is superseded by the config change — retire it (deletes its + // rows; generations never mix) so a fresh, correctly-fingerprinted build can + // start. Done under the single-flight lock so it can't race an embed run. + await retireGeneration(gen.id); + console.log(`[embed-scheduler] retired building generation ${gen.id}: fingerprint '${gen.fingerprint}' superseded by config '${cfgFingerprint}'`); + } else if (_pausedForFingerprint !== cfgFingerprint) { + // An active gen can't be retired from under live search — pause incremental + // embedding until a rebuild lands. Log once per config fingerprint, not every tick. + _pausedForFingerprint = cfgFingerprint; + console.log(`[embed-scheduler] active generation ${gen.id} fingerprint '${gen.fingerprint}' != config '${cfgFingerprint}' — pausing incremental embedding until a rebuild`); + } + return; + } + _pausedForFingerprint = null; // a matching run clears the paused-log latch + + const client = new EmbeddingClient({ endpoint: cfg.endpoint, apiKey: cfg.apiKey, model: cfg.model, dimension: cfg.dimension }); + // Pass `generations` so the worker can promote a fully-covered building + // generation to active at its shared run-completion seam (worker.js). + const worker = new EmbeddingWorker({ store, client, generations, preprocessCfg: cfg.preprocess, maxInputChars: cfg.maxInputChars, batchSize: cfg.batchSize }); + + if (nowMs - _lastBackstop >= _backstopIntervalMs) { + _lastBackstop = nowMs; + await worker.runBackstop(gen.id); + } else { + await worker.runOnce(gen.id); + } + } finally { + releaseEmbedRun(); + } +} + +export function startEmbeddingScheduler({ intervalMs = 60000, backstopIntervalMs = 86_400_000 } = {}) { + if (_timer) return; + _backstopIntervalMs = backstopIntervalMs; + _lastBackstop = Date.now(); // first backstop one interval out, not on boot + _timer = setInterval(async () => { + if (_running) return; // never overlap ticks + _running = true; + try { await runSchedulerTick(Date.now()); } + catch (err) { console.error(`Embedding scheduler tick error: ${err.message}`); } + finally { _running = false; } + }, intervalMs); + _timer.unref?.(); +} + +export function stopEmbeddingScheduler() { + if (_timer) { clearInterval(_timer); _timer = null; } +} diff --git a/backend/src/services/embeddings/scheduler.test.js b/backend/src/services/embeddings/scheduler.test.js new file mode 100644 index 00000000..0ca9f7b7 --- /dev/null +++ b/backend/src/services/embeddings/scheduler.test.js @@ -0,0 +1,118 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('./config.js', () => ({ resolveEmbedConfig: vi.fn(), generationFingerprint: vi.fn(() => 'fp') })); +vi.mock('./vectorStore.js', () => ({ isVectorAvailable: vi.fn(() => true) })); +vi.mock('./generations.js', () => ({ buildingGeneration: vi.fn(), activeGeneration: vi.fn(), retireGeneration: vi.fn() })); +const runOnce = vi.fn().mockResolvedValue({ claimed: 0 }); +const runBackstop = vi.fn().mockResolvedValue({ claimed: 0 }); +// Regular (constructable) function impls — arrow-function impls throw +// "is not a constructor" when the module under test does `new EmbeddingWorker(...)`. +vi.mock('./worker.js', () => ({ EmbeddingWorker: vi.fn(function () { return { runOnce, runBackstop }; }) })); +vi.mock('./client.js', () => ({ EmbeddingClient: vi.fn(function () {}) })); + +const { resolveEmbedConfig } = await import('./config.js'); +const { isVectorAvailable } = await import('./vectorStore.js'); +const { buildingGeneration, activeGeneration, retireGeneration } = await import('./generations.js'); +const { runSchedulerTick } = await import('./scheduler.js'); +// Real single-flight lock (not mocked) — the scheduler tick must respect it. +const { tryAcquireEmbedRun, releaseEmbedRun, isEmbedRunActive } = await import('./embedRunLock.js'); + +const enabledCfg = { enabled: true, endpoint: 'http://h/v1', model: 'm', dimension: 4, maxInputChars: 32768, batchSize: 32, preprocess: {} }; + +beforeEach(() => { + releaseEmbedRun(); // ensure the shared single-flight lock starts free each test + // Reset the once-queues so an early-returning test can't leave an unconsumed + // mockResolvedValueOnce that poisons a later test's expectations. + resolveEmbedConfig.mockReset(); + buildingGeneration.mockReset(); + activeGeneration.mockReset(); + retireGeneration.mockReset(); + isVectorAvailable.mockReset().mockReturnValue(true); + runOnce.mockClear(); + runBackstop.mockClear(); +}); + +describe('runSchedulerTick', () => { + it('no-ops when vector is unavailable', async () => { + isVectorAvailable.mockReturnValueOnce(false); + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + }); + + it('no-ops when embeddings config is disabled', async () => { + resolveEmbedConfig.mockResolvedValueOnce({ ...enabledCfg, enabled: false }); + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + }); + + it('no-ops when there is no building or active generation', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce(null); + activeGeneration.mockResolvedValueOnce(null); + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + }); + + it('runs the worker against the building generation', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce({ id: '9', dimension: 4, fingerprint: 'fp' }); + await runSchedulerTick(1000); + expect(runOnce).toHaveBeenCalledWith('9'); + }); + + it('skips when a run is already active (manual build in flight) — C1', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce({ id: '9', dimension: 4, fingerprint: 'fp' }); + expect(tryAcquireEmbedRun()).toBe(true); // simulate a manual build holding the lock + await runSchedulerTick(1000); + expect(runOnce).not.toHaveBeenCalled(); + releaseEmbedRun(); + }); + + it('releases the single-flight lock after running (does not starve later runs)', async () => { + resolveEmbedConfig.mockResolvedValueOnce(enabledCfg); + buildingGeneration.mockResolvedValueOnce({ id: '9', dimension: 4, fingerprint: 'fp' }); + await runSchedulerTick(1000); + expect(runOnce).toHaveBeenCalledWith('9'); + expect(isEmbedRunActive()).toBe(false); + }); + + // Fingerprint guard: the config's fingerprint must match the generation the worker + // would drive, or a model/dimension change mid-build embeds with the new client into + // the old generation and wedges forever. + it('retires a building generation whose fingerprint no longer matches the config', async () => { + resolveEmbedConfig.mockResolvedValue(enabledCfg); + buildingGeneration.mockResolvedValue({ id: '9', dimension: 4, fingerprint: 'old-fp' }); + await runSchedulerTick(1000); + expect(retireGeneration).toHaveBeenCalledWith('9'); + expect(runOnce).not.toHaveBeenCalled(); + expect(isEmbedRunActive()).toBe(false); // lock released on the retire-and-return path + }); + + it('skips a mismatched active generation and logs once per fingerprint (no retire)', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + try { + resolveEmbedConfig.mockResolvedValue(enabledCfg); + buildingGeneration.mockResolvedValue(null); + // A matching active run first clears the once-per-fingerprint latch so this test + // is independent of whatever ran before it. + activeGeneration.mockResolvedValueOnce({ id: '7', dimension: 4, fingerprint: 'fp' }); + await runSchedulerTick(1000); + runOnce.mockClear(); + + // Two mismatched ticks: skip both, never retire an active gen, log exactly once. + activeGeneration.mockResolvedValue({ id: '7', dimension: 4, fingerprint: 'old-fp' }); + await runSchedulerTick(2000); + await runSchedulerTick(3000); + + expect(runOnce).not.toHaveBeenCalled(); + expect(retireGeneration).not.toHaveBeenCalled(); + expect(isEmbedRunActive()).toBe(false); + const pauseLogs = logSpy.mock.calls.filter(([m]) => /pausing incremental/i.test(String(m))); + expect(pauseLogs).toHaveLength(1); + } finally { + logSpy.mockRestore(); + } + }); +}); diff --git a/backend/src/services/embeddings/stampSkipped.integration.test.js b/backend/src/services/embeddings/stampSkipped.integration.test.js new file mode 100644 index 00000000..41da2215 --- /dev/null +++ b/backend/src/services/embeddings/stampSkipped.integration.test.js @@ -0,0 +1,76 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +// L2: stamp (embed_gen) + stale-vector prune must be ONE transaction. store.stampSkipped +// CAS-stamps rows with a last_modified token, unconditionally stamps rows without one, +// and deletes embeddings only for ids whose stamp actually landed — all atomically. +d('store.stampSkipped (single-tx stamp + prune)', () => { + let store, client, acctId, userId, gen; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + await store.ensureVectorSchema(); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'stamp')); + const gi = await client.query(`INSERT INTO index_generations (model, dimension, fingerprint, started_at, state) VALUES ('m',4,'fp',0,'building') RETURNING id`); + gen = gi.rows[0].id; + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + async function seedMsgWithVector(uid) { + const m = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1,$2,'INBOX','x') RETURNING id, last_modified::text lm`, [acctId, uid]); + const id = m.rows[0].id; + await client.query(`INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, dimension, embedding) VALUES ($1,$2,0,0,1,4,'[1,0,0,0]')`, [gen, id]); + // Keep message_count honest for the seeded vector so the decrement + // assertions below observe real deltas (upsert normally maintains this). + await client.query('UPDATE index_generations SET message_count = message_count + 1 WHERE id = $1', [gen]); + return { id, lm: m.rows[0].lm }; + } + + const messageCount = async () => + Number((await client.query('SELECT message_count FROM index_generations WHERE id = $1', [gen])).rows[0].message_count); + + it('CAS-stamps and prunes the vector when last_modified matches', async () => { + const { id, lm } = await seedMsgWithVector(70001); + const before = await messageCount(); + const missed = await store.stampSkipped(gen, [{ id, lastModified: lm }], []); + expect(missed).toEqual([]); + const stamp = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [id]); + expect(String(stamp.rows[0].embed_gen)).toBe(String(gen)); + const emb = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, id]); + expect(emb.rows[0].n).toBe(0); // pruned in the same tx + expect(await messageCount()).toBe(before - 1); // Fix 7: the prune decrements the generation + }); + + it('a CAS miss decrements nothing (the vector stays)', async () => { + const { id } = await seedMsgWithVector(70004); + const before = await messageCount(); + await store.stampSkipped(gen, [{ id, lastModified: '1999-01-01 00:00:00+00' }], []); + expect(await messageCount()).toBe(before); + }); + + it('on a CAS miss leaves BOTH the stamp and the vector untouched', async () => { + const { id } = await seedMsgWithVector(70002); + const missed = await store.stampSkipped(gen, [{ id, lastModified: '1999-01-01 00:00:00+00' }], []); + expect(missed).toEqual([id]); + const stamp = await client.query('SELECT embed_gen FROM messages WHERE id = $1', [id]); + expect(stamp.rows[0].embed_gen).toBeNull(); + const emb = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, id]); + expect(emb.rows[0].n).toBe(1); // not pruned — the row still needs work + }); + + it('unconditionally stamps + prunes a plain (missing-row) id', async () => { + const { id } = await seedMsgWithVector(70003); + const missed = await store.stampSkipped(gen, [], [id]); + expect(missed).toEqual([]); + const emb = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, id]); + expect(emb.rows[0].n).toBe(0); + }); +}); diff --git a/backend/src/services/embeddings/testSupport.js b/backend/src/services/embeddings/testSupport.js new file mode 100644 index 00000000..88b0afc3 --- /dev/null +++ b/backend/src/services/embeddings/testSupport.js @@ -0,0 +1,19 @@ +// Shared IT/dev-script fixture: seed and tear down a throwaway user + email account +// against a real DB. `db` is anything with .query(text, params) — a pg.Client, a +// pooled client, or the app pool. Deleting the user cascades to email_accounts → +// messages (ON DELETE CASCADE), so cleanupAccount removes everything in one step. +import { randomUUID } from 'crypto'; + +export async function seedAccount(db, label = 'it') { + const u = await db.query('INSERT INTO users (username) VALUES ($1) RETURNING id', [`${label}-${randomUUID()}`]); + const userId = u.rows[0].id; + const a = await db.query( + 'INSERT INTO email_accounts (user_id, name, email_address) VALUES ($1, $2, $3) RETURNING id', + [userId, label, `${label}-${randomUUID()}@example.com`], + ); + return { userId, accountId: a.rows[0].id }; +} + +export async function cleanupAccount(db, userId) { + if (userId) await db.query('DELETE FROM users WHERE id = $1', [userId]); +} diff --git a/backend/src/services/embeddings/vectorErrors.js b/backend/src/services/embeddings/vectorErrors.js new file mode 100644 index 00000000..e6516b8d --- /dev/null +++ b/backend/src/services/embeddings/vectorErrors.js @@ -0,0 +1,5 @@ +// Leaf module: imports nothing from vectorStore/hybrid so Phase 3's loadVector +// and Phase 5's MCP handlers can import this without a cycle. +export class VectorUnavailableError extends Error { + constructor(reason) { super(reason); this.name = 'VectorUnavailableError'; this.reason = reason; } +} diff --git a/backend/src/services/embeddings/vectorSchema.integration.test.js b/backend/src/services/embeddings/vectorSchema.integration.test.js new file mode 100644 index 00000000..f3501a24 --- /dev/null +++ b/backend/src/services/embeddings/vectorSchema.integration.test.js @@ -0,0 +1,45 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('ensureVectorSchema (pgvector image)', () => { + let store, client; + beforeAll(async () => { + // Point db.js's pool at the test DB via env before importing the module graph. + const u = new URL(DSN); + process.env.DB_HOST = u.hostname; + process.env.DB_PORT = u.port; + process.env.DB_NAME = u.pathname.slice(1); + process.env.DB_USER = u.username; + process.env.DB_PASSWORD = u.password; + store = await import('./vectorStore.js'); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + }); + afterAll(async () => { await client.end(); }); + + it('creates the vector schema and reports available', async () => { + const { vectorAvailable } = await store.ensureVectorSchema(); + expect(vectorAvailable).toBe(true); + expect(store.isVectorAvailable()).toBe(true); + const t = await client.query( + "SELECT to_regclass('embeddings') e, to_regclass('index_generations') g, to_regclass('embed_watermark') w, to_regclass('embed_runs') r" + ); + expect(t.rows[0].e).not.toBeNull(); + expect(t.rows[0].g).not.toBeNull(); + expect(t.rows[0].w).not.toBeNull(); + expect(t.rows[0].r).not.toBeNull(); + }); + + it('is idempotent (second call does not throw)', async () => { + await expect(store.ensureVectorSchema()).resolves.toMatchObject({ vectorAvailable: true }); + }); + + it('builds a partial HNSW index for a dimension on the empty table', async () => { + await store.ensureVectorIndex(4); + const idx = await client.query("SELECT indexname FROM pg_indexes WHERE tablename='embeddings' AND indexname='idx_embeddings_hnsw_d4'"); + expect(idx.rows.length).toBe(1); + }); +}); diff --git a/backend/src/services/embeddings/vectorSchema.unit.test.js b/backend/src/services/embeddings/vectorSchema.unit.test.js new file mode 100644 index 00000000..a1376c68 --- /dev/null +++ b/backend/src/services/embeddings/vectorSchema.unit.test.js @@ -0,0 +1,32 @@ +import { describe, it, expect, vi } from 'vitest'; + +// Mock the module graph so ensureVectorSchema runs without a real DB. The dedicated +// client (withDedicatedClient → new pg.Client) and the pool both resolve; the only +// variable is whether ensureVectorIndex(dim) throws (a non-integer dimension throws +// synchronously), which is exactly the "flag vs return" disagreement path. +const clientMock = { connect: vi.fn().mockResolvedValue(), query: vi.fn().mockResolvedValue({}), end: vi.fn().mockResolvedValue() }; +// Regular (constructable) function impl — `new pg.Client(...)` needs a constructor. +vi.mock('pg', () => ({ default: { Client: vi.fn(function () { return clientMock; }) } })); +vi.mock('../db.js', () => ({ pool: { query: vi.fn().mockResolvedValue({}) }, withTransaction: vi.fn() })); +vi.mock('./config.js', () => ({ resolveEmbedConfig: vi.fn() })); + +const { resolveEmbedConfig } = await import('./config.js'); +const { ensureVectorSchema, isVectorAvailable } = await import('./vectorStore.js'); + +describe('ensureVectorSchema flag/return agreement', () => { + it('reports available and sets the flag when the whole bring-up succeeds', async () => { + resolveEmbedConfig.mockResolvedValueOnce({ dimension: 4, skipExtensionCreate: false }); + const r = await ensureVectorSchema(); + expect(r.vectorAvailable).toBe(true); + expect(isVectorAvailable()).toBe(true); + }); + + it('leaves isVectorAvailable() false when ensureVectorIndex throws after the schema builds', async () => { + // dimension 2.5 passes the `> 0` guard but ensureVectorIndex throws on the + // non-integer, AFTER the extension + schema succeed — the disagreement window. + resolveEmbedConfig.mockResolvedValueOnce({ dimension: 2.5, skipExtensionCreate: false }); + const r = await ensureVectorSchema(); + expect(r.vectorAvailable).toBe(false); + expect(isVectorAvailable()).toBe(false); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.fused.test.js b/backend/src/services/embeddings/vectorStore.fused.test.js new file mode 100644 index 00000000..1f8cf489 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.fused.test.js @@ -0,0 +1,296 @@ +import { describe, it, expect, afterAll } from 'vitest'; +import { hasTestDb, withTestDb, closeTestDb } from './__testdb__.js'; +import { fusedSearch } from './vectorStore.js'; +import { searchLexical } from '../search/lexicalRepo.js'; + +afterAll(async () => { await closeTestDb(); }); + +describe.skipIf(!hasTestDb())('pgvector test DB', () => { + it('boots ensureVectorSchema and exposes the vector extension', async () => { + await withTestDb(async (pool) => { + const { rows } = await pool.query(`SELECT extname FROM pg_extension WHERE extname = 'vector'`); + expect(rows).toHaveLength(1); + }); + }); +}); + +function unitVec(dim, axis) { + const v = new Array(dim).fill(0); + v[axis] = 1; + return v; +} + +// Seeds one user + account + N messages (trigger fills search_fts) and an active +// generation with one chunk per message. Returns { accountId, generation, ids }. +async function seedThree(pool) { + const u = (await pool.query( + `INSERT INTO users (username, password_hash) VALUES ('t','x') RETURNING id`)).rows[0].id; + const acc = (await pool.query( + `INSERT INTO email_accounts (user_id, name, email_address, enabled) + VALUES ($1,'A','a@x.test',true) RETURNING id`, [u])).rows[0].id; + const base = Date.UTC(2025, 0, 15, 12, 0, 0); + const rows = [ + ['alpha quantum project update', 'discussing the quantum roadmap', 0, false], + ['beta vector indexing notes', 'notes about hybrid search and ranking', 1, true], + ['gamma project retrospective', 'retro covering the quantum milestone', 2, false], + ]; + const ids = []; + for (let i = 0; i < rows.length; i++) { + const [subject, body, , hasAtt] = rows[i]; + const id = (await pool.query( + `INSERT INTO messages (account_id, uid, folder, subject, from_name, from_email, + date, snippet, body_text, has_attachments, is_deleted) + VALUES ($1,$2,'INBOX',$3,'Sender','s@x.test',$4,$5,$6,$7,false) RETURNING id`, + [acc, 100 + i, subject, new Date(base + i * 86400000).toISOString(), body, body, hasAtt] + )).rows[0].id; + ids.push(id); + } + const gen = (await pool.query( + `INSERT INTO index_generations (model, dimension, fingerprint, started_at, state) + VALUES ('m', 4, 'm:4:test', $1, 'active') RETURNING id`, + [Math.floor(Date.now() / 1000)])).rows[0].id; + for (let i = 0; i < ids.length; i++) { + await pool.query( + `INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, embedding, dimension) + VALUES ($1,$2,0,$3,4,$4::vector,4)`, + [gen, ids[i], Math.floor(Date.now() / 1000), `[${unitVec(4, i).join(',')}]`]); + } + return { accountId: acc, generation: { id: gen, dimension: 4 }, ids }; +} + +const base = { rrfK: 60, kPerSignal: 10, limit: 10, buildFilters: () => [] }; + +describe.skipIf(!hasTestDb())('fusedSearch', () => { + it('FTS-only: returns the two quantum messages, bm25 set / vector null, RRF descending', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits, poolSaturated } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(poolSaturated).toBe(false); + expect(hits).toHaveLength(2); + const set = new Set(hits.map(h => h.message_id)); + expect(set.has(s.ids[0]) && set.has(s.ids[2])).toBe(true); + for (const h of hits) { + expect(h.vector_score).toBeNull(); + expect(h.bm25_score).not.toBeNull(); + expect(h.rrf_score).toBeGreaterThan(0); + } + for (let i = 1; i < hits.length; i++) { + expect(hits[i - 1].rrf_score).toBeGreaterThanOrEqual(hits[i].rrf_score); + } + }); + }); + + it('prefix consistency: the fused FTS leg matches the same hit set as the lexical path for a prefix term (Opus review fix-round)', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + // A message whose only match for "invo" is a prefix of "invoice" — none + // of seedThree's quantum-related docs share that prefix, so a correct + // prefix match (and only a correct prefix match) finds exactly this one. + const invoiceId = (await pool.query( + `INSERT INTO messages (account_id, uid, folder, subject, from_name, from_email, + date, snippet, body_text, is_deleted) + VALUES ($1,103,'INBOX','Invoice reminder','Sender','s@x.test',now(),'inv','an invoice is attached',false) + RETURNING id`, + [s.accountId])).rows[0].id; + + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'invo', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + const lex = await searchLexical((text, params) => pool.query(text, params), { + parsed: { filters: [], terms: [{ value: 'invo', negate: false }] }, + accountIds: [s.accountId], folderScope: null, folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + + const fusedIds = new Set(hits.map(h => h.message_id)); + const lexIds = new Set(lex.rows.map(r => r.id)); + expect(fusedIds.has(invoiceId)).toBe(true); + expect(fusedIds).toEqual(lexIds); + }); + }); + + it('ANN-only: top hit is the on-axis message, bm25 null / vector set', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits, poolSaturated } = await fusedSearch( + { ...base, ftsQuery: null, queryVec: unitVec(4, 0), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(poolSaturated).toBe(false); + expect(hits[0].message_id).toBe(s.ids[0]); + for (const h of hits) { + expect(h.bm25_score).toBeNull(); + expect(h.vector_score).not.toBeNull(); + } + }); + }); + + it('hybrid: union of the FTS pair and the ANN-only third message (3 hits), RRF descending', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: unitVec(4, 1), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(hits).toHaveLength(3); + for (let i = 1; i < hits.length; i++) { + expect(hits[i - 1].rrf_score).toBeGreaterThanOrEqual(hits[i].rrf_score); + } + }); + }); + + it('saturation: kPerSignal below the pool size flips poolSaturated and trims', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits, poolSaturated } = await fusedSearch( + { ...base, kPerSignal: 1, ftsQuery: 'quantum', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(poolSaturated).toBe(true); + expect(hits).toHaveLength(1); + }); + }); + + it('tenant scope: a different account never leaks into the pool', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: null, generation: s.generation, + accountIds: ['00000000-0000-0000-0000-000000000000'] }, + { client: pool }); + expect(hits).toHaveLength(0); + }); + }); + + it('multi-chunk dedup: a message with a close and a far chunk appears once at its MIN distance', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + // Give the winning (chunk_index=0, on-axis) chunk distinctive offsets so + // Task 5b's best_char_start assertion can tell it apart from the far one. + await pool.query( + `UPDATE embeddings SET chunk_char_start = 6, chunk_char_end = 40 + WHERE generation_id = $1 AND message_id = $2 AND chunk_index = 0`, + [s.generation.id, s.ids[0]]); + // give ids[0] a second, far chunk on axis 2, with DIFFERENT offsets + await pool.query( + `INSERT INTO embeddings (generation_id, message_id, chunk_index, embedded_at, source_char_len, chunk_char_start, chunk_char_end, embedding, dimension) + VALUES ($1,$2,1,$3,4,100,140,$4::vector,4)`, + [s.generation.id, s.ids[0], Math.floor(Date.now() / 1000), `[${unitVec(4, 2).join(',')}]`]); + const { hits } = await fusedSearch( + { ...base, ftsQuery: null, queryVec: unitVec(4, 0), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + const counts = {}; + for (const h of hits) counts[h.message_id] = (counts[h.message_id] || 0) + 1; + expect(counts[s.ids[0]]).toBe(1); + const top = hits.find(h => h.message_id === s.ids[0]); + expect(top.vector_score).toBeCloseTo(1.0, 6); // 1 - MIN(distance)=0 + // The winning (close) chunk's offsets ride through, not the far chunk's. + expect(top.best_chunk_index).toBe(0); + expect(top.best_char_start).toBe(6); + expect(top.best_char_end).toBe(40); + }); + }); + + it('rejects an empty request', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + await expect(fusedSearch( + { ...base, ftsQuery: null, queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool })).rejects.toThrow(); + }); + }); +}); + +// Wave D Fix 1, verified against the live pgvector container: english stopwords +// normalize to an EMPTY tsquery under 'english' (numnode = 0) and `@@ ''` is +// FALSE — pre-fix, ONE stopword in the AND'd term chain zeroed every backfilled +// result ("waiting for invoice" → 0 hits because of "for"). +describe.skipIf(!hasTestDb())('stopword terms never zero a search (Fix 1)', () => { + async function seedWaiting(pool, s) { + return (await pool.query( + `INSERT INTO messages (account_id, uid, folder, subject, from_name, from_email, + date, snippet, body_text, is_deleted) + VALUES ($1,104,'INBOX','Waiting for invoice','Sender','s@x.test',now(),'w', + 'we are waiting for the invoice payment',false) RETURNING id`, + [s.accountId])).rows[0].id; + } + const lex = (pool, terms, s, ordering = 'relevance') => + searchLexical((text, params) => pool.query(text, params), { + parsed: { filters: [], terms }, + accountIds: [s.accountId], folderScope: null, folderFuzzy: false, ordering, limit: 50, offset: 0, + }); + + it('lexical: "waiting for invoice" finds the message despite the stopword', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const id = await seedWaiting(pool, s); + const res = await lex(pool, [ + { value: 'waiting', negate: false }, + { value: 'for', negate: false }, + { value: 'invoice', negate: false }, + ], s); + expect(res.rows.map((r) => r.id)).toContain(id); + }); + }); + + it('lexical: a stopword-ONLY query degrades to filter-only/date-order — never zero-by-stopword', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const res = await lex(pool, [{ value: 'the', negate: false }], s); + expect(res.hasCondition).toBe(true); + expect(res.rows).toHaveLength(3); // the whole (trash-excluded) scope + const dates = res.rows.map((r) => new Date(r.date).getTime()); + expect(dates).toEqual([...dates].sort((a, b) => b - a)); // rank 0 ⇒ date tiebreak + }); + }); + + it('lexical: a NEGATED stopword contributes nothing instead of excluding everything', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const id = await seedWaiting(pool, s); + const res = await lex(pool, [ + { value: 'invoice', negate: false }, + { value: 'the', negate: true }, // body contains "the" — must NOT exclude + ], s); + expect(res.rows.map((r) => r.id)).toContain(id); + }); + }); + + it('fused BM25 leg: "waiting for invoice" matches the same set the lexical path finds', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const id = await seedWaiting(pool, s); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'waiting for invoice', queryVec: null, generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + const res = await lex(pool, [ + { value: 'waiting', negate: false }, + { value: 'for', negate: false }, + { value: 'invoice', negate: false }, + ], s); + expect(hits.map((h) => h.message_id)).toContain(id); + expect(new Set(hits.map((h) => h.message_id))).toEqual(new Set(res.rows.map((r) => r.id))); + }); + }); + + it('fused: an ALL-stopword ftsQuery leaves the BM25 leg empty — pure-ANN ranking, silence not noise', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'the for you', queryVec: unitVec(4, 0), generation: s.generation, accountIds: [s.accountId] }, + { client: pool }); + expect(hits.length).toBeGreaterThan(0); // ANN still answers + expect(hits.every((h) => h.bm25_score === null)).toBe(true); // FTS contributed nothing + }); + }); + +}); + +describe.skipIf(!hasTestDb())('fusedSearch input validation', () => { + it('rejects a dimension mismatch', async () => { + await withTestDb(async (pool) => { + const s = await seedThree(pool); + await expect(fusedSearch( + { ...base, ftsQuery: null, queryVec: [1, 2, 3], generation: s.generation, accountIds: [s.accountId] }, + { client: pool })).rejects.toThrow(); + }); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.fusedSql.test.js b/backend/src/services/embeddings/vectorStore.fusedSql.test.js new file mode 100644 index 00000000..4b714a61 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.fusedSql.test.js @@ -0,0 +1,140 @@ +import { describe, it, expect } from 'vitest'; +import { fusedSearch, FUSED_ANN_CHUNKS_PER_MESSAGE, HNSW_EF_SEARCH_MAX } from './vectorStore.js'; + +// SQL-text pins for fusedSearch, driven through an injected fake client (a +// plain object — NOT a pg.Pool — so withEfSearch uses it verbatim and every +// statement it issues is recorded in order). +// +// `annPools` scripts the ann_pool_size the fused SELECT reports on each +// successive attempt, so the widening loop's exits are testable. +function fakeDb({ chunkCount = 5000, filteredMessages = 3000, annPools = [999], ftsPool = 5, delegation } = {}) { + const calls = []; + let attempt = -1; + return { + calls, + fusedRuns() { return calls.filter((c) => /^WITH /.test(c.text) && /FROM fused f/.test(c.text)); }, + query: async (text, params) => { + calls.push({ text, params }); + if (/count\(\*\)::int AS n FROM embeddings/.test(text)) return { rows: [{ n: chunkCount }] }; + if (/count\(DISTINCT e\.message_id\)/.test(text)) return { rows: [{ n: filteredMessages }] }; + if (/FROM fused f/.test(text)) { + attempt = Math.min(attempt + 1, annPools.length - 1); + return { + rows: [{ + message_id: 'm1', subject: 's', + fts_pool_size: ftsPool, ann_pool_size: annPools[attempt], + ...(delegation === undefined ? {} : { delegation }), + }], + }; + } + return { rows: [] }; + }, + }; +} + +const gen = { id: 7, dimension: 4 }; +const base = { + generation: gen, accountIds: ['a1'], rrfK: 60, kPerSignal: 10, limit: 10, + buildFilters: () => [], +}; + +describe('fusedSearch BM25 leg — stopword-safe combined tsquery (Fix 1)', () => { + it('matches AND ranks with ONE `&&`-combined tsquery so an empty (stopword) operand drops out', async () => { + const db = fakeDb(); + await fusedSearch({ ...base, ftsQuery: 'waiting for invoice', queryVec: [1, 0, 0, 0] }, { client: db }); + const sql = db.fusedRuns()[0].text; + const combined = + "(to_tsquery('english', quote_literal($2) || ':*') && " + + "to_tsquery('english', quote_literal($3) || ':*') && " + + "to_tsquery('english', quote_literal($4) || ':*'))"; + // Single @@ against the combined query — `&&` drops an empty-normalizing + // operand ("for"), so a stopword can no longer zero the whole leg the way + // the old per-term `@@ ... AND @@ ...` chain did. + expect(sql).toContain(`m.search_fts @@ ${combined}`); + // The rank arg is the SAME combined construction (match and rank can't diverge). + expect(sql).toContain(`ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, ${combined}, 32)`); + // No per-term AND chain remains. + expect(sql).not.toMatch(/@@ to_tsquery\('english', quote_literal\(\$\d+\) \|\| ':\*'\) AND/); + const params = db.fusedRuns()[0].params; + expect(params.slice(1, 4)).toEqual(['waiting', 'for', 'invoice']); + }); +}); + +describe('fusedSearch delegation projection', () => { + it('returns owner-scoped delegation metadata for hybrid results', async () => { + const delegation = JSON.stringify({ + contact_id: 'contact-1', + display_name: 'Example Teammate', + primary_email: 'teammate@example.test', + }); + const db = fakeDb({ delegation }); + + const { hits } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, + { client: db }, + ); + + const sql = db.fusedRuns()[0].text; + expect(sql).toContain('delegation_meta.delegation AS delegation'); + expect(sql).toContain('gd.user_id = a.user_id'); + expect(sql).toContain('gd.account_id = m.account_id'); + expect(sql).toContain('gd.thread_key = m.thread_key'); + expect(hits[0].delegation).toEqual({ + contact_id: 'contact-1', + display_name: 'Example Teammate', + primary_email: 'teammate@example.test', + }); + }); +}); + +describe('fusedSearch ANN leg — hnsw.ef_search per attempt (Fix 2)', () => { + it('runs each ANN attempt in a transaction that SET LOCALs ef_search to the inner LIMIT', async () => { + const db = fakeDb(); + await fusedSearch({ ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + const texts = db.calls.map((c) => c.text); + const inner = (base.kPerSignal + 1) * FUSED_ANN_CHUNKS_PER_MESSAGE; // 88 + const begin = texts.indexOf('BEGIN'); + const guc = texts.indexOf(`SET LOCAL hnsw.ef_search = ${inner}`); + const run = texts.findIndex((t) => /FROM fused f/.test(t)); + const commit = texts.indexOf('COMMIT'); + // BEGIN → SET LOCAL → fused SELECT → COMMIT, in that order, on one client. + expect(begin).toBeGreaterThanOrEqual(0); + expect(guc).toBeGreaterThan(begin); + expect(run).toBeGreaterThan(guc); + expect(commit).toBeGreaterThan(run); + }); + + it(`caps the GUC at HNSW_EF_SEARCH_MAX (${HNSW_EF_SEARCH_MAX}) — pgvector rejects larger values`, async () => { + const db = fakeDb(); + // kPerSignal=200 → inner LIMIT (201*8=1608) exceeds the pgvector cap. + await fusedSearch({ ...base, kPerSignal: 200, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + expect(db.calls.some((c) => c.text === `SET LOCAL hnsw.ef_search = ${HNSW_EF_SEARCH_MAX}`)).toBe(true); + expect(db.calls.some((c) => /SET LOCAL hnsw\.ef_search = 1608/.test(c.text))).toBe(false); + }); + + it('re-issues a LARGER ef_search when the widening loop grows the inner LIMIT', async () => { + // First attempt dedups to 5 (< kPerSignal+1 = 11, < filteredCeiling), + // second grows to 11 and exits. + const db = fakeDb({ annPools: [5, 11] }); + await fusedSearch({ ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + const gucs = db.calls.filter((c) => /^SET LOCAL hnsw\.ef_search = /.test(c.text)).map((c) => c.text); + expect(gucs).toEqual(['SET LOCAL hnsw.ef_search = 88', 'SET LOCAL hnsw.ef_search = 176']); + expect(db.fusedRuns()).toHaveLength(2); + }); + + it('breaks the widening loop when the ann pool stops growing between attempts', async () => { + // The pool sticks at 5 forever; without the no-growth break the loop + // would double 88 → … → 5000 (the chunk ceiling) re-running for nothing. + const db = fakeDb({ annPools: [5, 5] }); + const { poolSaturated } = await fusedSearch( + { ...base, ftsQuery: 'quantum', queryVec: [1, 0, 0, 0] }, { client: db }); + expect(db.fusedRuns()).toHaveLength(2); + expect(poolSaturated).toBe(false); + }); + + it('an FTS-only request issues no transaction and no GUC (nothing to tune)', async () => { + const db = fakeDb(); + await fusedSearch({ ...base, ftsQuery: 'quantum', queryVec: null }, { client: db }); + expect(db.calls.some((c) => /hnsw\.ef_search|^BEGIN$/.test(c.text))).toBe(false); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.integration.test.js b/backend/src/services/embeddings/vectorStore.integration.test.js new file mode 100644 index 00000000..fef87f58 --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.integration.test.js @@ -0,0 +1,85 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('vectorStore ANN', () => { + let store, client, gen, mA, mB, userId; + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + await store.ensureVectorSchema(); + await store.ensureVectorIndex(4); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + // Re-runnable: clear generation state before seeding. embed_runs FKs + // index_generations without ON DELETE CASCADE (generations are never hard-deleted + // in production), so clear it before the parent to keep the reset re-runnable. + await client.query('DELETE FROM embeddings'); + await client.query('DELETE FROM embed_runs'); + await client.query('DELETE FROM index_generations'); + let acctId; + ({ accountId: acctId, userId } = await seedAccount(client, 'ann')); + const a = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 81001, 'INBOX', 'A') RETURNING id`, [acctId]); + const b = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 81002, 'INBOX', 'B') RETURNING id`, [acctId]); + mA = a.rows[0].id; mB = b.rows[0].id; + const now = Math.floor(Date.now() / 1000); + const gi = await client.query(`INSERT INTO index_generations (model, dimension, fingerprint, started_at, state) VALUES ('m',4,'fp',$1,'building') RETURNING id`, [now]); + gen = gi.rows[0].id; + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('returns the nearest message first', async () => { + await store.upsert(gen, [ + { messageId: mA, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + { messageId: mB, chunkIndex: 0, vector: [0, 1, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + ]); + const hits = await store.annSearch(gen, [1, 0, 0, 0], 2); + expect(hits[0].messageId).toBe(mA); + expect(hits[0].rank).toBe(1); + expect(hits[0].score).toBeGreaterThan(hits[1].score); + }); + + it('re-upsert is idempotent (PK replace, no duplicate chunks)', async () => { + await store.upsert(gen, [{ messageId: mA, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }]); + const c = await client.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1 AND message_id = $2', [gen, mA]); + expect(c.rows[0].n).toBe(1); + }); + + it('dedups multi-chunk messages by best (MIN) distance', async () => { + // mB gets a far chunk (0,1,0,0) and a near chunk (1,0,0,0); querying [1,0,0,0] + // should rank mB by its BEST chunk, ahead of a mid message. + await store.upsert(gen, [ + { messageId: mB, chunkIndex: 0, vector: [0, 1, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + { messageId: mB, chunkIndex: 1, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 4, chunkCharEnd: 8, truncated: false }, + ]); + const hits = await store.annSearch(gen, [1, 0, 0, 0], 5); + const mbHit = hits.find((h) => h.messageId === mB); + expect(mbHit.score).toBeGreaterThan(0.9); // best chunk is an exact match + // one hit per message + expect(new Set(hits.map((h) => h.messageId)).size).toBe(hits.length); + }); + + it('filter.accountIds scopes ANN to one account (find_similar_messages needs this)', async () => { + // A second account whose message is an EXACT match for the query — without the + // filter it would surface; with the account filter it must be excluded, and the + // widening still returns the in-scope match. + const other = await seedAccount(client, 'ann-other'); + const oc = await client.query(`INSERT INTO messages (account_id, uid, folder, subject) VALUES ($1, 81003, 'INBOX', 'C') RETURNING id`, [other.accountId]); + const mC = oc.rows[0].id; + await store.upsert(gen, [ + { messageId: mA, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + { messageId: mC, chunkIndex: 0, vector: [1, 0, 0, 0], sourceCharLen: 4, chunkCharStart: 0, chunkCharEnd: 4, truncated: false }, + ]); + const firstAcct = (await client.query('SELECT account_id FROM messages WHERE id = $1', [mA])).rows[0].account_id; + const scoped = await store.annSearch(gen, [1, 0, 0, 0], 5, { filter: { accountIds: [firstAcct] } }); + expect(scoped.some((h) => h.messageId === mC)).toBe(false); + expect(scoped.some((h) => h.messageId === mA)).toBe(true); + const unscoped = await store.annSearch(gen, [1, 0, 0, 0], 5); + expect(unscoped.some((h) => h.messageId === mC)).toBe(true); + await cleanupAccount(client, other.userId); + }); +}); diff --git a/backend/src/services/embeddings/vectorStore.js b/backend/src/services/embeddings/vectorStore.js new file mode 100644 index 00000000..2b35456c --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.js @@ -0,0 +1,757 @@ +import pg from 'pg'; +import { pool, withTransaction } from '../db.js'; +import { resolveEmbedConfig } from './config.js'; +import { LEXICAL_RANK_SQL, ftsTermQueryArg, hasSearchableToken } from '../search/lexicalRepo.js'; +import { DELEGATION_SELECT_SQL, delegationJoinSql, mapDelegationRow } from '../gtdDelegations.js'; + +export const ZERO_UUID = '00000000-0000-0000-0000-000000000000'; + +let _vectorAvailable = false; +export function isVectorAvailable() { return _vectorAvailable; } + +// Build a short-lived connection WITHOUT the pool's `-c statement_timeout=30000` +// startup option, then explicitly disable statement_timeout, so a slow DDL build +// (HNSW over a repopulated table after the alpine→Debian image swap) is not killed +// at 30s. Caller closes nothing — this helper owns the connect/end lifecycle. +async function withDedicatedClient(fn) { + const client = new pg.Client({ + host: process.env.DB_HOST || 'postgres', + port: Number(process.env.DB_PORT) || 5432, + database: process.env.DB_NAME || 'mailflow', + user: process.env.DB_USER || 'mailflow', + password: process.env.DB_PASSWORD, + }); + await client.connect(); + try { + await client.query('SET statement_timeout = 0'); + return await fn(client); + } finally { + await client.end().catch(() => {}); + } +} + +const SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS index_generations ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + model TEXT NOT NULL, + dimension INTEGER NOT NULL, + fingerprint TEXT NOT NULL, + started_at BIGINT NOT NULL, + seeded_at BIGINT, + completed_at BIGINT, + activated_at BIGINT, + state TEXT NOT NULL, + message_count BIGINT NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_generations_active ON index_generations(state) WHERE state = 'active'; +CREATE UNIQUE INDEX IF NOT EXISTS idx_generations_building ON index_generations(state) WHERE state = 'building'; + +CREATE TABLE IF NOT EXISTS embeddings ( + generation_id BIGINT NOT NULL REFERENCES index_generations(id) ON DELETE CASCADE, + message_id UUID NOT NULL, + chunk_index INTEGER NOT NULL DEFAULT 0, + embedded_at BIGINT NOT NULL, + source_char_len INTEGER NOT NULL, + chunk_char_start INTEGER NOT NULL DEFAULT 0, + chunk_char_end INTEGER NOT NULL DEFAULT 0, + truncated BOOLEAN NOT NULL DEFAULT FALSE, + dimension INTEGER NOT NULL, + embedding vector NOT NULL, + PRIMARY KEY (generation_id, message_id, chunk_index) +); +CREATE INDEX IF NOT EXISTS idx_embeddings_msg ON embeddings(message_id); +CREATE INDEX IF NOT EXISTS idx_embeddings_dim ON embeddings(dimension); + +CREATE TABLE IF NOT EXISTS embed_runs ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + generation_id BIGINT NOT NULL REFERENCES index_generations(id), + started_at BIGINT NOT NULL, + ended_at BIGINT, + claimed INTEGER NOT NULL DEFAULT 0, + succeeded INTEGER NOT NULL DEFAULT 0, + failed INTEGER NOT NULL DEFAULT 0, + truncated INTEGER NOT NULL DEFAULT 0, + error TEXT +); + +CREATE TABLE IF NOT EXISTS embed_watermark ( + generation_id BIGINT PRIMARY KEY, + watermark_id UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000' +); +`; + +// Best-effort startup routine (pattern: encryptExistingCredentials in db.js). Never +// throws into boot: on any failure it logs, sets vector_available=false, and returns. +export async function ensureVectorSchema() { + _vectorAvailable = false; + let cfg = null; + try { cfg = await resolveEmbedConfig(); } catch { /* ai_config unreadable — treat as unset */ } + const skipExtension = cfg?.skipExtensionCreate === true; + try { + if (!skipExtension) { + await pool.query('CREATE EXTENSION IF NOT EXISTS vector'); + } + // Apply the schema on a dedicated no-timeout connection: on a legacy populated DB + // the idx_embeddings_* builds can exceed the pool's 30s cap (migrate.go). + await withDedicatedClient((client) => client.query(SCHEMA_SQL)); + if (cfg?.dimension > 0) { + await ensureVectorIndex(cfg.dimension); + } + // Set the flag only after the WHOLE bring-up (extension + schema + per-dimension + // HNSW) succeeds, so isVectorAvailable() never disagrees with the return value — + // e.g. an ensureVectorIndex failure must leave the flag false. + _vectorAvailable = true; + console.log('Vector schema ready — semantic search available'); + return { vectorAvailable: true }; + } catch (err) { + _vectorAvailable = false; + console.warn(`Vector disabled: ${err.message} — lexical search unaffected`); + return { vectorAvailable: false }; + } +} + +// Partial per-dimension HNSW cosine index, created while the table is empty so it is +// maintained incrementally (README invariant; port of migrate.go EnsureVectorIndex). +// The `WHERE dimension = N` guard lets generations of different dims coexist. +export async function ensureVectorIndex(dim) { + if (!Number.isInteger(dim) || dim <= 0) throw new Error(`invalid dimension ${dim}`); + const stmt = `CREATE INDEX IF NOT EXISTS idx_embeddings_hnsw_d${dim} + ON embeddings USING hnsw ((embedding::vector(${dim})) vector_cosine_ops) + WHERE dimension = ${dim}`; + await withDedicatedClient((client) => client.query(stmt)); +} + +const LIVE_MESSAGES_WHERE = 'is_deleted = false'; // Mailflow live-message predicate +const ANN_OVERFETCH = 4; // backend.go annOverFetchFactor + +// pgvector's hnsw.ef_search GUC defaults to 40 and hard-caps at 1000: the HNSW +// scan visits at most ef_search candidates, so any inner ANN `ORDER BY <=> +// LIMIT` above it is silently truncated to ~ef_search rows and widening the +// LIMIT re-runs an identical plan. msgvault sizes a per-connection GUC to 1000 +// (store.go HNSWEfSearch): >= the worst-case fused inner LIMIT, +// (kPerSignal+1)*FUSED_ANN_CHUNKS_PER_MESSAGE ≈ 808 at the default +// kPerSignal=100, with headroom, while keeping per-query latency bounded. +// Mailflow's pool sets no per-connection GUCs, so every ANN-issuing statement +// runs in a transaction that `SET LOCAL`s the GUC to its own inner LIMIT, +// capped here (pgvector REJECTS values above 1000; beyond the cap, recall is +// best-effort — the same trade msgvault documents). +export const HNSW_EF_SEARCH_MAX = 1000; + +// Run fn(client) in a transaction whose hnsw.ef_search covers `efSearch` +// candidates (capped at HNSW_EF_SEARCH_MAX). A pg.Pool is pinned to one +// client first — BEGIN / SET LOCAL / queries must share a session; an +// injected single client (tests) is used as-is. +async function withEfSearch(db, efSearch, fn) { + const pinned = db instanceof pg.Pool ? await db.connect() : db; + try { + await pinned.query('BEGIN'); + await pinned.query(`SET LOCAL hnsw.ef_search = ${Math.min(Math.max(1, Math.floor(efSearch)), HNSW_EF_SEARCH_MAX)}`); + const out = await fn(pinned); + await pinned.query('COMMIT'); + return out; + } catch (err) { + await pinned.query('ROLLBACK').catch(() => {}); + throw err; + } finally { + if (pinned !== db) pinned.release(); + } +} + +export function vectorLiteral(vec) { + return `[${vec.map((f) => Number(f).toString()).join(',')}]`; +} + +// Upsert chunks for one generation. Idempotent per message: clears the message's +// prior chunks (chunk count is not stable across re-embeds) then inserts the new set, +// all in one tx. Maintains index_generations.message_count by distinct-message delta. +export async function upsert(gen, chunks) { + if (!chunks.length) return; + await withTransaction(async (client) => { + // Row-lock the generation (serializes against activate/retire) and read its dim. + const g = await client.query('SELECT dimension, state FROM index_generations WHERE id = $1 FOR UPDATE', [gen]); + if (!g.rows.length) throw new Error(`unknown generation ${gen}`); + if (g.rows[0].state === 'retired') { const e = new Error(`generation retired ${gen}`); e.code = 'GEN_RETIRED'; throw e; } + const dim = g.rows[0].dimension; + for (const c of chunks) { + if (c.vector.length !== dim) throw new Error(`dimension mismatch: chunk for msg ${c.messageId} has ${c.vector.length}, gen has ${dim}`); + } + const ids = [...new Set(chunks.map((c) => c.messageId))]; + const pre = await client.query( + 'SELECT COUNT(DISTINCT message_id)::int n FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', + [gen, ids], + ); + await client.query('DELETE FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', [gen, ids]); + const now = Math.floor(Date.now() / 1000); + for (const c of chunks) { + await client.query( + `INSERT INTO embeddings + (generation_id, message_id, chunk_index, embedded_at, source_char_len, + chunk_char_start, chunk_char_end, truncated, dimension, embedding) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10::vector)`, + [gen, c.messageId, c.chunkIndex, now, c.sourceCharLen, c.chunkCharStart, c.chunkCharEnd, c.truncated, dim, vectorLiteral(c.vector)], + ); + } + const delta = ids.length - pre.rows[0].n; + if (delta !== 0) { + await client.query('UPDATE index_generations SET message_count = message_count + $1 WHERE id = $2', [delta, gen]); + } + }); +} + +export async function chunkCount(gen) { + const r = await pool.query('SELECT COUNT(*)::int n FROM embeddings WHERE generation_id = $1', [gen]); + return r.rows[0].n; +} + +// ANN search over one generation. Inner ORDER BY <=> LIMIT uses the partial HNSW index +// (dimension embedded as a literal); outer GROUP BY collapses multi-chunk messages to +// their best (MIN) distance. Widens the inner LIMIT until k distinct messages survive the +// dedup. Score = 1 - cosine_distance. +// +// An optional structured `filter` narrows the candidate set by pushing predicates INTO +// the inner liveness EXISTS (a Postgres-native join on messages — the equivalent of +// msgvault's json_each resolved-id set, backend.go filtered path). This is REQUIRED for +// account-scoped callers (e.g. plan-phase5 find_similar_messages in a multi-user DB): +// filtering after k-NN can return zero in-scope rows, whereas widening WITH the filter in +// SQL keeps pulling candidates until k in-scope messages survive (or the generation is +// exhausted). filter fields (all optional, AND-combined): +// accountIds: string[] (UUIDs) → m.account_id = ANY($n::uuid[]) +// after / before: timestamptz-comparable (ISO string or Date) → m.date >= / < $n +// hasAttachment: boolean → m.has_attachments = true/false +// With no filter the SQL + params are byte-identical to the fast path (filter clauses and +// their binds simply don't appear). Note: with a filter present the planner may fall back +// from the HNSW index to a scan within the filtered set — the same trade msgvault accepts. +// Append the structured filter's predicates to `args` (relative to messages alias `m`) +// and return the joined WHERE string. Built with a fresh `args` per query so each +// statement's $N ordinals resolve independently (msgvault's bind-closure pattern). +function buildAnnFilter(filter, args) { + const where = [LIVE_MESSAGES_WHERE]; + if (filter) { + if (filter.accountIds?.length) { args.push(filter.accountIds); where.push(`m.account_id = ANY($${args.length}::uuid[])`); } + if (filter.after) { args.push(filter.after); where.push(`m.date >= $${args.length}`); } + // before is EXCLUSIVE (<) — msgvault filter.go parity, and the same + // bound lexicalRepo's before: operator applies (one convention everywhere). + if (filter.before) { args.push(filter.before); where.push(`m.date < $${args.length}`); } + if (filter.hasAttachment === true) where.push('m.has_attachments = true'); + else if (filter.hasAttachment === false) where.push('m.has_attachments = false'); + } + return where.join(' AND '); +} + +// Doubling-widen the inner ANN LIMIT until at least k distinct messages survive the +// outer dedup, the distinct-message early exit is reached, or the candidate ceiling is +// hit. `ceiling` counts CHUNKS (bounds the inner LIMIT); `distinctEarlyExit` counts the +// distinct MESSAGES that can possibly appear (equals k on the empty-filter path, a +// no-op; equals the filtered distinct-message count on the filtered path, so a +// selective filter stops as soon as every in-scope message is surfaced instead of +// widening up to the whole generation's chunk count — msgvault searchWiden parity). +// Port of backend.go searchWiden. Exported for unit testing. +export async function searchWiden(k, ceiling, distinctEarlyExit, run) { + let innerLimit = Math.max(k * ANN_OVERFETCH, k); + for (;;) { + if (innerLimit > ceiling) innerLimit = ceiling; + const hits = await run(innerLimit); + if (hits.length >= k || hits.length >= distinctEarlyExit || innerLimit >= ceiling) { + return hits.slice(0, k).map((h, i) => ({ ...h, rank: i + 1 })); + } + innerLimit *= 2; + } +} + +export async function annSearch(gen, queryVec, k, { efSearch = 100, filter = null } = {}) { + if (!queryVec.length) throw new Error('annSearch: empty query vector'); + const g = await pool.query('SELECT dimension FROM index_generations WHERE id = $1', [gen]); + if (!g.rows.length) throw new Error(`unknown generation ${gen}`); + const dim = g.rows[0].dimension; + if (queryVec.length !== dim) throw new Error(`dimension mismatch: query ${queryVec.length}, gen ${dim}`); + const lit = vectorLiteral(queryVec); + + // Widening bounds. Filtered: count only the in-scope candidate set (chunks = ceiling, + // distinct messages = early exit) with the SAME EXISTS predicate, so the loop stops + // once every in-scope message is surfaced. Unfiltered: whole-generation chunk count as + // the ceiling and k as the (no-op) early exit — byte-identical to the original fast path. + let ceiling, distinctEarlyExit; + if (filter) { + const cArgs = [gen]; + const cWhere = buildAnnFilter(filter, cArgs); + const cnt = await pool.query( + `SELECT COUNT(*)::int chunks, COUNT(DISTINCT e.message_id)::int messages + FROM embeddings e + WHERE e.generation_id = $1 + AND EXISTS (SELECT 1 FROM messages m WHERE m.id = e.message_id AND ${cWhere})`, + cArgs, + ); + ceiling = cnt.rows[0].chunks; + distinctEarlyExit = cnt.rows[0].messages; + } else { + ceiling = await chunkCount(gen); + distinctEarlyExit = k; + } + if (ceiling === 0) return []; + + // $1 = query vector, $2 = generation; filter binds (if any) take $3.. ; the two LIMITs + // are appended per widening run as the trailing two params. + const args = [lit, gen]; + const where = buildAnnFilter(filter, args); + const innerArg = args.length + 1; + const outerArg = args.length + 2; + const sql = ` + SELECT ann.message_id, MIN(ann.distance) AS distance + FROM ( + SELECT e.message_id, (e.embedding::vector(${dim})) <=> $1::vector AS distance + FROM embeddings e + WHERE e.generation_id = $2 AND e.dimension = ${dim} + AND EXISTS (SELECT 1 FROM messages m WHERE m.id = e.message_id AND ${where}) + ORDER BY e.embedding::vector(${dim}) <=> $1::vector + LIMIT $${innerArg} + ) ann + GROUP BY ann.message_id + ORDER BY distance, ann.message_id + LIMIT $${outerArg}`; + const client = await pool.connect(); + try { + await client.query('BEGIN'); + const hits = await searchWiden(k, ceiling, distinctEarlyExit, async (innerLimit) => { + // Re-issue the GUC per widening attempt: ef_search must cover the inner + // LIMIT or the HNSW scan truncates it and the widening loop is a no-op + // (see HNSW_EF_SEARCH_MAX). `efSearch` stays the floor for small limits. + await client.query(`SET LOCAL hnsw.ef_search = ${Math.min(Math.max(Number(efSearch), innerLimit), HNSW_EF_SEARCH_MAX)}`); + const r = await client.query(sql, [...args, innerLimit, k]); + return r.rows.map((row, i) => ({ messageId: row.message_id, score: 1 - Number(row.distance), rank: i + 1 })); + }); + await client.query('COMMIT'); + return hits; + } catch (err) { + await client.query('ROLLBACK').catch(() => {}); + throw err; + } finally { + client.release(); + } +} + +// Return the chunk_index=0 vector for messageId in the active generation. +export async function loadVector(messageId) { + const active = await pool.query("SELECT id, dimension FROM index_generations WHERE state = 'active'"); + if (!active.rows.length) throw new Error('no active generation'); + const r = await pool.query( + 'SELECT embedding::text lit FROM embeddings WHERE generation_id = $1 AND message_id = $2 AND chunk_index = 0', + [active.rows[0].id, messageId], + ); + if (!r.rows.length) throw new Error(`no embedding for message ${messageId}`); + return r.rows[0].lit.slice(1, -1).split(',').map(Number); +} + +// Forward scan by UUID for live messages needing work, resuming above afterId. The +// predicate is `embed_gen IS NULL` (NOT the OR with `embed_gen <> target`) so the +// partial index idx_messages_embed_pending (migration 0045) drives an O(pending) scan +// even on a huge, fully-covered mailbox — an OR forces a full seq scan. This is +// correct because createGeneration resets every live row's stamp to NULL when a NEW +// generation is created, so a rebuild's prior-generation rows surface as NULL here too: +// no live row ever carries a non-null stamp for a generation OTHER than the current +// target. `target` is kept in the signature (callers pass it) but is not needed in the +// predicate given that invariant. +export async function scanForEmbedding(target, afterId, limit) { + const r = await pool.query( + `SELECT id FROM messages + WHERE embed_gen IS NULL AND ${LIVE_MESSAGES_WHERE} AND id > $1 + ORDER BY id LIMIT $2`, + [afterId, limit], + ); + return r.rows.map((row) => row.id); +} + +export async function setEmbedGen(ids, target) { + if (!ids.length) return; + await pool.query('UPDATE messages SET embed_gen = $1 WHERE id = ANY($2::uuid[])', [target, ids]); +} + +// Optimistic CAS stamp: only stamp rows whose last_modified text token is unchanged +// since the worker read it. Returns the ids that MISSED (last_modified moved) — not +// stamped; the backstop recovers them. Bind the token back as ::timestamptz for +// exact-equality (JS Date would lose the microseconds pg stores). +export async function setEmbedGenIfUnchanged(items, target) { + const missed = []; + await withTransaction(async (client) => { + for (const it of items) { + const res = await client.query( + 'UPDATE messages SET embed_gen = $1 WHERE id = $2 AND last_modified = $3::timestamptz', + [target, it.id, it.lastModified], + ); + if (res.rowCount === 0) missed.push(it.id); + } + }); + return missed; +} + +// Skip-mark rows (empty/missing) and prune their now-stale vectors in ONE transaction +// (parity with msgvault worker.go stampSkipped). CAS-stamps `casItems` (rows with a +// last_modified token) and unconditionally stamps `plainIds` (missing rows with no row +// to guard), then deletes embeddings for every id whose stamp actually landed — a +// CAS-missed id keeps both its NULL stamp and its vector so it is re-found later. +// index_generations.message_count is decremented by the DISTINCT messages the delete +// actually removes (msgvault backend.go:1118-1167 counts, deletes, and applies the +// delta under the generation row lock; this used to delete without decrementing, so +// the count drifted upward on every skipped re-embed). Returns the CAS-missed ids. +// Bind the token as ::timestamptz for exact equality. +export async function stampSkipped(gen, casItems, plainIds) { + const missed = []; + if (!casItems.length && !plainIds.length) return missed; + await withTransaction(async (client) => { + // Lock the generation row FIRST — the same order upsert() takes it (and + // msgvault's Delete, which locks before touching embeddings precisely to + // avoid an ABBA asymmetry with those writers) — so the decrement below is + // serialized against concurrent upsert/activate/retire. + const g = await client.query('SELECT id FROM index_generations WHERE id = $1 FOR UPDATE', [gen]); + if (!g.rows.length) throw new Error(`unknown generation ${gen}`); + for (const it of casItems) { + const res = await client.query( + 'UPDATE messages SET embed_gen = $1 WHERE id = $2 AND last_modified = $3::timestamptz', + [gen, it.id, it.lastModified], + ); + if (res.rowCount === 0) missed.push(it.id); + } + if (plainIds.length) { + await client.query('UPDATE messages SET embed_gen = $1 WHERE id = ANY($2::uuid[])', [gen, plainIds]); + } + const missedSet = new Set(missed); + const stamped = [...casItems.map((c) => c.id), ...plainIds].filter((id) => !missedSet.has(id)); + if (stamped.length) { + const pre = await client.query( + 'SELECT COUNT(DISTINCT message_id)::int n FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', + [gen, stamped], + ); + await client.query('DELETE FROM embeddings WHERE generation_id = $1 AND message_id = ANY($2::uuid[])', [gen, stamped]); + if (pre.rows[0].n > 0) { + await client.query('UPDATE index_generations SET message_count = message_count - $1 WHERE id = $2', [pre.rows[0].n, gen]); + } + } + }); + return missed; +} + +export async function getWatermark(gen) { + const r = await pool.query('SELECT watermark_id FROM embed_watermark WHERE generation_id = $1', [gen]); + return r.rows.length ? r.rows[0].watermark_id : ZERO_UUID; +} + +export async function setWatermark(gen, id) { + await pool.query( + `INSERT INTO embed_watermark (generation_id, watermark_id) VALUES ($1, $2) + ON CONFLICT (generation_id) DO UPDATE SET watermark_id = EXCLUDED.watermark_id`, + [gen, id], + ); +} + +export async function resetWatermark(gen) { + await setWatermark(gen, ZERO_UUID); +} + +// Fetch subject + inline bodies + the last_modified CAS token (as text) for a +// batch of message ids. Bodies are inline in messages: no message_bodies table. +export async function fetchForEmbedding(ids) { + if (!ids.length) return []; + const r = await pool.query( + `SELECT id, + COALESCE(subject, '') AS subject, + COALESCE(body_text, '') AS "bodyText", + COALESCE(body_html, '') AS "bodyHtml", + last_modified::text AS "lastModified" + FROM messages WHERE id = ANY($1::uuid[])`, + [ids], + ); + return r.rows; +} + +export async function startEmbedRun(gen) { + const now = Math.floor(Date.now() / 1000); + const r = await pool.query('INSERT INTO embed_runs (generation_id, started_at) VALUES ($1, $2) RETURNING id', [gen, now]); + return r.rows[0].id; +} + +export async function finalizeEmbedRun(runId, res, err) { + if (!runId) return; + const now = Math.floor(Date.now() / 1000); + await pool.query( + `UPDATE embed_runs SET ended_at = $1, claimed = $2, succeeded = $3, failed = $4, truncated = $5, error = $6 WHERE id = $7`, + [now, res.claimed, res.succeeded, res.failed, res.truncated, err ? String(err.message || err) : null, runId], + ).catch(() => {}); +} + +// ── Fused RRF search — port of internal/vector/pgvector/fused.go ── + +export const FUSED_ANN_CHUNKS_PER_MESSAGE = 8; + +const DISPLAY_COLS = ` + m.id AS message_id, m.uid, m.folder, m.subject, m.from_name, m.from_email, + m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, + a.name AS account_name, a.email_address AS account_email, a.color AS account_color, + ${DELEGATION_SELECT_SQL}`; + +async function fusedChunkCount(db, generation) { + const { rows } = await db.query( + `SELECT count(*)::int AS n FROM embeddings WHERE generation_id = $1 AND dimension = $2`, + [generation.id, generation.dimension]); + return rows[0].n; +} + +async function filteredChunkMessageCount(db, { generation, accountIds, buildFilters }) { + const args = [generation.id, generation.dimension, accountIds]; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + const filters = buildFilters(bind).map(c => ` AND ${c}`).join(''); + const { rows } = await db.query( + `SELECT count(DISTINCT e.message_id)::int AS n + FROM embeddings e + JOIN messages m ON m.id = e.message_id + WHERE e.generation_id = $1 AND e.dimension = $2 + AND m.account_id = ANY($3) AND m.is_deleted = false${filters}`, + args); + return rows[0].n; +} + +// Single-query hybrid RRF fusion: BM25 leg (weighted ts_rank_cd over +// search_fts, reusing LEXICAL_RANK_SQL) + ANN leg (cosine distance +// over one generation's embeddings), FULL OUTER JOIN'd and combined via +// reciprocal-rank fusion. Either leg can be omitted (ftsQuery/queryVec null) +// for lexical-only or vector-only pools. `buildFilters(bind) → string[]` +// applies the SAME structured operator predicates to both legs (README "one +// search seam" — lexicalRepo remains the single owner of those predicates). +// Each hit also carries best_chunk_index/best_char_start/best_char_end — +// the ANN leg's winning (min-distance) chunk's offsets, null on an +// FTS-only hit. These are CODE POINTS into the PREPROCESSED text (README +// Unicode contract), not raw body_text byte offsets — chunkmatch.js owns +// turning them into a raw-body byte snippet. +export async function fusedSearch(req, { client } = {}) { + const db = client || pool; + const { generation, accountIds, rrfK, kPerSignal, limit } = req; + const buildFilters = req.buildFilters || (() => []); + // Tokenize once (terms don't change across the widening loop below) and + // apply the SAME hygiene the lexical path applies (searchLexical): + // drop sub-2-char and punctuation-only tokens rather than handing Postgres + // a term that would normalize to zero lexemes. This is also what makes + // `useFTS` mean "the FTS leg actually has something to match on", not just + // "a non-empty string was passed" — an all-punctuation ftsQuery degrades to + // ANN-only (or throws below, same as if ftsQuery were absent, if ANN is + // also unavailable). + const ftsTerms = typeof req.ftsQuery === 'string' + ? req.ftsQuery.trim().split(/\s+/).filter(t => t.length >= 2 && hasSearchableToken(t)) + : []; + const useFTS = ftsTerms.length > 0; + const useANN = Array.isArray(req.queryVec) && req.queryVec.length > 0; + if (!useFTS && !useANN) throw new Error('fusedSearch: neither ftsQuery nor queryVec provided'); + if (useANN && req.queryVec.length !== generation.dimension) { + throw new Error(`fusedSearch: dimension mismatch (query ${req.queryVec.length}, generation ${generation.dimension})`); + } + const dim = generation.dimension; + const kPlus1 = kPerSignal + 1; + + let chunkCeiling = 0; + let filteredCeiling = 0; + if (useANN) { + chunkCeiling = await fusedChunkCount(db, generation); + filteredCeiling = await filteredChunkMessageCount(db, { generation, accountIds, buildFilters }); + } + + async function runFused(exec, innerChunks) { + const args = []; + const bind = (v) => { args.push(v); return `$${args.length}`; }; + + const accArg = bind(accountIds); + const live = `m.account_id = ANY(${accArg}) AND m.is_deleted = false`; + const filterAnd = buildFilters(bind).map(c => ` AND ${c}`).join(''); + + const ctes = []; + if (useFTS) { + // One bind per term, `&&`-combined into a SINGLE tsquery used for BOTH + // the @@ match and the ts_rank_cd rank — the same per-term prefix-or- + // phrase construction lexicalRepo.js's lexical path uses + // (ftsTermQueryArg), so "invo" matches "invoice" here exactly as it + // does via searchLexical, and msgvault's fused.go shape (one combined + // BuildFTSTerm arg for match and rank alike). `@@ (a && b)` is + // equivalent to `@@ a AND @@ b`, and `&&` DROPS an empty operand + // (verified against pgvector/pg16) — which is what keeps an english + // stopword ("waiting for invoice") from zeroing the whole BM25 leg: + // the stopword's tsquery normalizes empty and simply vanishes from the + // combined query. An ALL-stopword ftsQuery combines to an empty tsquery + // that matches nothing, so the leg contributes silence (not noise) and + // the fused ranking degrades to pure ANN. ftsTermQueryArg wants the raw + // placeholder NUMBER (it prepends its own `$`, matching lexicalRepo.js's + // own internal convention) — push directly onto `args` rather than + // through `bind`, which returns an already-`$`-prefixed string. + const termArgs = ftsTerms.map((term) => { args.push(term); return args.length; }); + const tsquery = `(${ftsTerms.map((term, i) => ftsTermQueryArg(termArgs[i], term)).join(' && ')})`; + const matchWhere = `m.search_fts @@ ${tsquery}`; + const kp1 = bind(kPlus1); + const k = bind(kPerSignal); + const rank = LEXICAL_RANK_SQL('m.search_fts', tsquery); + ctes.push(`fts_pool AS ( + SELECT m.id AS message_id, ${rank} AS bm25 + FROM messages m + WHERE ${matchWhere} + AND ${live}${filterAnd} + ORDER BY bm25 DESC + LIMIT ${kp1} +)`); + ctes.push(`fts_ranked AS ( + SELECT message_id, bm25, + ROW_NUMBER() OVER (ORDER BY bm25 DESC, message_id ASC) AS rnk + FROM fts_pool + ORDER BY bm25 DESC, message_id ASC + LIMIT ${k} +)`); + } + if (useANN) { + const vecArg = bind(vectorLiteral(req.queryVec)); + const genArg = bind(generation.id); + const innerArg = bind(innerChunks); + const kp1 = bind(kPlus1); + const k = bind(kPerSignal); + // DISTINCT ON (message_id), ordered by distance, picks the SAME min-distance + // chunk per message as the MIN(distance)/GROUP BY form — but also + // keeps that winning chunk's offsets, which the excerpt seam + // needs. Offsets are code points into the PREPROCESSED text, not raw + // body_text (README Unicode contract) — phase 5's chunkmatch.js owns + // turning them into a raw-body byte snippet. + ctes.push(`ann_pool AS ( + SELECT d.message_id, d.distance, d.chunk_index, d.chunk_char_start, d.chunk_char_end + FROM ( + SELECT DISTINCT ON (ann.message_id) + ann.message_id, ann.distance, ann.chunk_index, ann.chunk_char_start, ann.chunk_char_end + FROM ( + SELECT e.message_id, e.chunk_index, e.chunk_char_start, e.chunk_char_end, + (e.embedding::vector(${dim})) <=> ${vecArg}::vector AS distance + FROM embeddings e + WHERE e.generation_id = ${genArg} AND e.dimension = ${dim} + AND EXISTS (SELECT 1 FROM messages m WHERE m.id = e.message_id AND ${live}${filterAnd}) + ORDER BY e.embedding::vector(${dim}) <=> ${vecArg}::vector + LIMIT ${innerArg} + ) ann + ORDER BY ann.message_id, ann.distance + ) d + ORDER BY d.distance + LIMIT ${kp1} +)`); + ctes.push(`ann_ranked AS ( + SELECT message_id, distance, chunk_index, chunk_char_start, chunk_char_end, + ROW_NUMBER() OVER (ORDER BY distance ASC, message_id ASC) AS rnk + FROM ann_pool + ORDER BY distance ASC, message_id ASC + LIMIT ${k} +)`); + } + + const poolArgsLen = args.length; // rrfk/limit are appended after this + const poolCTEs = ctes.slice(); + const rrfkArg = bind(rrfK); + const limitArg = bind(limit); + + // 1.0 is a `numeric` literal in Postgres; numeric / bigint (rnk) stays + // numeric, and node-postgres returns numeric columns as strings (to avoid + // silent precision loss). Cast to double precision so rrf_score/bm25_score + // come back as JS numbers, matching msgvault's float64 RRF score. + let fused; + if (useFTS && useANN) { + fused = `fused AS ( + SELECT COALESCE(b.message_id, v.message_id) AS message_id, + COALESCE(1.0::float8 / (${rrfkArg} + b.rnk), 0.0) + COALESCE(1.0::float8 / (${rrfkArg} + v.rnk), 0.0) AS rrf_score, + b.bm25::float8 AS bm25_score, + CASE WHEN v.distance IS NULL THEN NULL ELSE 1.0::float8 - v.distance END AS vector_score, + v.chunk_index AS best_chunk_index, v.chunk_char_start AS best_char_start, v.chunk_char_end AS best_char_end + FROM fts_ranked b + FULL OUTER JOIN ann_ranked v USING (message_id) +)`; + } else if (useFTS) { + fused = `fused AS ( + SELECT b.message_id, 1.0::float8 / (${rrfkArg} + b.rnk) AS rrf_score, + b.bm25::float8 AS bm25_score, CAST(NULL AS double precision) AS vector_score, + CAST(NULL AS int) AS best_chunk_index, CAST(NULL AS int) AS best_char_start, CAST(NULL AS int) AS best_char_end + FROM fts_ranked b +)`; + } else { + fused = `fused AS ( + SELECT v.message_id, 1.0::float8 / (${rrfkArg} + v.rnk) AS rrf_score, + CAST(NULL AS double precision) AS bm25_score, 1.0::float8 - v.distance AS vector_score, + v.chunk_index AS best_chunk_index, v.chunk_char_start AS best_char_start, v.chunk_char_end AS best_char_end + FROM ann_ranked v +)`; + } + ctes.push(fused); + + const ftsPoolExpr = useFTS ? '(SELECT count(*) FROM fts_pool)' : '0'; + const annPoolExpr = useANN ? '(SELECT count(*) FROM ann_pool)' : '0'; + + const sql = `WITH ${ctes.join(',\n')} +SELECT ${DISPLAY_COLS}, + f.rrf_score, f.bm25_score, f.vector_score, + f.best_chunk_index, f.best_char_start, f.best_char_end, + ${ftsPoolExpr} AS fts_pool_size, + ${annPoolExpr} AS ann_pool_size + FROM fused f + JOIN messages m ON m.id = f.message_id + JOIN email_accounts a ON a.id = m.account_id + ${delegationJoinSql('m', 'a')} + ORDER BY f.rrf_score DESC, f.message_id ASC + LIMIT ${limitArg}`; + + const { rows } = await exec.query(sql, args); + let ftsPoolSize = 0; + let annPoolSize = 0; + if (rows.length > 0) { + ftsPoolSize = rows[0].fts_pool_size; + annPoolSize = rows[0].ann_pool_size; + } else { + // Empty result: the pool-size subqueries never fired (they ride the row + // stream). Re-run a prefix-only count over just the pool CTEs and their + // args (drop the trailing rrfk/limit args). Port of fused.go:322-342. + const prefix = `WITH ${poolCTEs.join(',\n')}\n`; + const prefixArgs = args.slice(0, poolArgsLen); + if (useFTS) { + ftsPoolSize = (await exec.query(prefix + 'SELECT count(*)::int AS n FROM fts_pool', prefixArgs)).rows[0].n; + } + if (useANN) { + annPoolSize = (await exec.query(prefix + 'SELECT count(*)::int AS n FROM ann_pool', prefixArgs)).rows[0].n; + } + } + const hits = rows.map((row) => { + const h = { ...row, delegation: mapDelegationRow(row) }; + delete h.fts_pool_size; + delete h.ann_pool_size; + return h; + }); + return { hits, ftsPoolSize, annPoolSize }; + } + + // Candidate-widening loop (port of fused.go:346-380). Start wide enough that + // the common single-chunk case is one query; grow innerChunks (doubling, + // capped by chunkCeiling) only while the ANN dedup collapses the pool below + // kPerSignal+1 and more chunks remain. FTS never collapses, so only ANN drives it. + let innerChunks = kPlus1 * FUSED_ANN_CHUNKS_PER_MESSAGE; + let result; + let prevAnnPool = -1; + for (;;) { + if (useANN && chunkCeiling > 0 && innerChunks > chunkCeiling) innerChunks = chunkCeiling; + // Each ANN attempt sets hnsw.ef_search to its OWN inner LIMIT (capped at + // HNSW_EF_SEARCH_MAX): at the pgvector default of 40 the HNSW scan + // truncated every attempt to ~40 chunks and this loop re-ran an identical + // plan up to the ceiling. FTS-only requests skip the transaction — no ANN + // scan, nothing to tune. + result = useANN + ? await withEfSearch(db, innerChunks, (tx) => runFused(tx, innerChunks)) + : await runFused(db, innerChunks); + if (!useANN || + result.annPoolSize >= kPlus1 || + result.annPoolSize >= filteredCeiling || + innerChunks >= chunkCeiling) break; + // A widened re-run that failed to GROW the pool will never grow it (the + // graph/ef_search budget is saturated) — stop instead of doubling toward + // the ceiling for identical results. + if (result.annPoolSize <= prevAnnPool) break; + prevAnnPool = result.annPoolSize; + let next = innerChunks * 2; + if (chunkCeiling > 0 && next > chunkCeiling) next = chunkCeiling; + if (next === innerChunks) break; + innerChunks = next; + } + + const poolSaturated = result.ftsPoolSize > kPerSignal || result.annPoolSize > kPerSignal; + return { hits: result.hits, poolSaturated, generation }; +} diff --git a/backend/src/services/embeddings/vectorStore.unit.test.js b/backend/src/services/embeddings/vectorStore.unit.test.js new file mode 100644 index 00000000..2fdd838e --- /dev/null +++ b/backend/src/services/embeddings/vectorStore.unit.test.js @@ -0,0 +1,92 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +vi.mock('../db.js', () => ({ pool: {}, withTransaction: vi.fn() })); +import { withTransaction } from '../db.js'; +import { vectorLiteral, searchWiden, stampSkipped } from './vectorStore.js'; + +describe('vectorLiteral', () => { + it('formats a float vector as pgvector text', () => { + expect(vectorLiteral([1, 2.5, -3.25])).toBe('[1,2.5,-3.25]'); + }); + it('emits an empty-brackets literal for an empty vector', () => { + expect(vectorLiteral([])).toBe('[]'); + }); +}); + +describe('searchWiden (A1 filtered early-exit)', () => { + it('early-exits once distinctEarlyExit is reached (filtered path, one run)', async () => { + const runs = []; + const run = async (n) => { runs.push(n); return [{ messageId: 'a' }, { messageId: 'b' }]; }; + const hits = await searchWiden(5, 1000, 2, run); // k=5, ceiling=1000, earlyExit=2 + expect(runs).toEqual([20]); // did NOT widen up to the 1000-chunk ceiling + expect(hits.map((h) => h.rank)).toEqual([1, 2]); + }); + + it('widens by doubling until k distinct survive (no-filter: earlyExit=k)', async () => { + const runs = []; + const run = async (n) => { runs.push(n); return Array.from({ length: Math.min(Math.floor(n / 10), 5) }, (_, i) => ({ messageId: 'm' + i })); }; + const hits = await searchWiden(5, 100, 5, run); + expect(runs).toEqual([20, 40, 80]); // 2 < 5, 4 < 5, then 8 → sliced to 5 + expect(hits).toHaveLength(5); + }); + + it('stops at the ceiling even when k is never reached', async () => { + const runs = []; + const run = async (n) => { runs.push(n); return [{ messageId: 'only' }]; }; + const hits = await searchWiden(5, 30, 5, run); + expect(runs).toEqual([20, 30]); // clamps the doubled 40 to the 30 ceiling, then stops + expect(hits).toHaveLength(1); + }); +}); + +describe('stampSkipped (Wave D Fix 7 — message_count decrement)', () => { + beforeEach(() => { withTransaction.mockReset(); }); + + function scriptedClient({ deletedDistinct = 2 } = {}) { + const calls = []; + const client = { + calls, + query: vi.fn(async (text, params) => { + calls.push({ text, params }); + if (/FROM index_generations WHERE id = \$1 FOR UPDATE/.test(text)) return { rows: [{ id: params[0] }] }; + if (/^UPDATE messages SET embed_gen = \$1 WHERE id = \$2 AND/.test(text)) { + return { rowCount: params[1] === 'cas-missed' ? 0 : 1 }; // CAS stamp + } + if (/COUNT\(DISTINCT message_id\)/.test(text)) return { rows: [{ n: deletedDistinct }] }; + return { rows: [], rowCount: 1 }; + }), + }; + withTransaction.mockImplementation(async (fn) => fn(client)); + return client; + } + + it('decrements message_count by the DISTINCT messages actually deleted, in the same tx, under the generation row lock', async () => { + const client = scriptedClient({ deletedDistinct: 2 }); + const missed = await stampSkipped(9, + [{ id: 'cas-ok', lastModified: 't1' }, { id: 'cas-missed', lastModified: 't2' }], + ['plain-1']); + expect(missed).toEqual(['cas-missed']); + const texts = client.calls.map((c) => c.text); + // Generation row locked FIRST — same lock order as upsert()/msgvault Delete. + expect(texts[0]).toMatch(/SELECT id FROM index_generations WHERE id = \$1 FOR UPDATE/); + // Vectors deleted only for ids whose stamp landed (CAS-missed keeps its vector)… + const del = client.calls.find((c) => /DELETE FROM embeddings/.test(c.text)); + expect(del.params).toEqual([9, ['cas-ok', 'plain-1']]); + // …counted BEFORE the delete, and the count decrements the generation. + expect(texts.findIndex((t) => /COUNT\(DISTINCT message_id\)/.test(t))) + .toBeLessThan(texts.findIndex((t) => /DELETE FROM embeddings/.test(t))); + const upd = client.calls.find((c) => /UPDATE index_generations SET message_count = message_count - \$1/.test(c.text)); + expect(upd.params).toEqual([2, 9]); + }); + + it('skips the decrement when the stamped ids had no vectors to delete', async () => { + const client = scriptedClient({ deletedDistinct: 0 }); + await stampSkipped(9, [{ id: 'cas-ok', lastModified: 't1' }], []); + expect(client.calls.some((c) => /UPDATE index_generations SET message_count/.test(c.text))).toBe(false); + expect(client.calls.some((c) => /DELETE FROM embeddings/.test(c.text))).toBe(true); + }); + + it('is a no-op (no transaction) for empty inputs', async () => { + expect(await stampSkipped(9, [], [])).toEqual([]); + expect(withTransaction).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/services/embeddings/worker.integration.test.js b/backend/src/services/embeddings/worker.integration.test.js new file mode 100644 index 00000000..353d2573 --- /dev/null +++ b/backend/src/services/embeddings/worker.integration.test.js @@ -0,0 +1,58 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import pg from 'pg'; +import { seedAccount, cleanupAccount } from './testSupport.js'; + +const DSN = process.env.VECTOR_IT_DB; +const d = DSN ? describe : describe.skip; + +d('worker end-to-end (pgvector)', () => { + let store, generations, worker, client, acctId, gen, userId; + const DIM = 4; + const fakeClient = { async embed(inputs) { return inputs.map((_, i) => [0.1, 0.2, 0.3, 0.4 + i * 1e-6]); } }; + + beforeAll(async () => { + const u = new URL(DSN); + Object.assign(process.env, { DB_HOST: u.hostname, DB_PORT: u.port, DB_NAME: u.pathname.slice(1), DB_USER: u.username, DB_PASSWORD: u.password }); + store = await import('./vectorStore.js'); + generations = await import('./generations.js'); + const { EmbeddingWorker } = await import('./worker.js'); + await store.ensureVectorSchema(); + await store.ensureVectorIndex(DIM); + client = new pg.Client({ connectionString: DSN }); + await client.connect(); + // embed_runs FKs index_generations without ON DELETE CASCADE, so clear it first. + await client.query('DELETE FROM embeddings'); await client.query('DELETE FROM embed_runs'); await client.query('DELETE FROM index_generations'); + // runOnce scans EVERY live embed_gen-IS-NULL message, so stray rows left by + // other IT files (withTestDb truncates before each test, not after the last) + // would inflate the coverage counts — clear messages like the tables above. + await client.query('DELETE FROM messages'); + ({ accountId: acctId, userId } = await seedAccount(client, 'wrk')); + for (let i = 0; i < 40; i++) { + await client.query(`INSERT INTO messages (account_id, uid, folder, subject, body_text) VALUES ($1,$2,'INBOX',$3,$4)`, + [acctId, 83000 + i, `subject ${i}`, `body content number ${i}`]); + } + gen = await generations.createGeneration('fake', DIM, 'fp-e2e'); + worker = new EmbeddingWorker({ store, client: fakeClient, generations, preprocessCfg: { stripHTML: true, collapseWhitespace: true }, maxInputChars: 32768, batchSize: 8 }); + }); + afterAll(async () => { await cleanupAccount(client, userId); await client.end(); }); + + it('drives the corpus to full coverage and auto-activates the building generation', async () => { + // The worker was constructed with `generations` in its deps (above), so + // draining the scan for the building generation promotes it to active at the + // worker seam — NO manual activateGeneration call here. This is the regression + // guard for the wiring that was missing in production. + const res = await worker.runOnce(gen); + expect(res.succeeded).toBe(40); + const pending = await client.query('SELECT COUNT(*)::int n FROM messages WHERE account_id = $1 AND embed_gen IS DISTINCT FROM $2', [acctId, gen]); + expect(pending.rows[0].n).toBe(0); + const emb = await client.query('SELECT COUNT(DISTINCT message_id)::int n FROM embeddings WHERE generation_id = $1', [gen]); + expect(emb.rows[0].n).toBe(40); + expect((await generations.activeGeneration()).id).toBe(gen); + expect(await generations.buildingGeneration()).toBeNull(); // promoted, no longer building + }); + + it('re-run is a no-op (idempotent coverage)', async () => { + const res = await worker.runOnce(gen); + expect(res.claimed).toBe(0); + }); +}); diff --git a/backend/src/services/embeddings/worker.js b/backend/src/services/embeddings/worker.js new file mode 100644 index 00000000..8ea2caf2 --- /dev/null +++ b/backend/src/services/embeddings/worker.js @@ -0,0 +1,354 @@ +// Port of internal/vector/embed/worker.go RunOnce/RunBackstop (scan-and-fill). +import { preprocess } from './preprocess.js'; +import { chunkText, chunkOverlapFor, MAX_SPANS } from './chunk.js'; +import { isPermanent4xx } from './client.js'; + +// worker.go rawBodyMultiplier. Exported for chunkmatch.js, whose read-path re-preprocess +// must derive the identical maxBodyRunes cap or chunk offsets misalign on huge bodies. +export const RAW_BODY_MULT = 16; + +export class EmbeddingWorker { + constructor(deps) { + this.deps = { batchSize: 32, maxConsecutiveFailures: 5, log: console, ...deps }; + } + + runOnce(gen) { return this._run(gen, false); } + runBackstop(gen) { return this._run(gen, true); } + + async _run(gen, backstop) { + const { store } = this.deps; + const res = { claimed: 0, succeeded: 0, failed: 0, truncated: 0 }; + let runId; + let runErr = null; + try { runId = (await store.startEmbedRun?.(gen)) || 0; } catch { runId = 0; } + try { + let consecutiveFailures = 0; + let afterId = backstop ? store.ZERO_UUID : await store.getWatermark(gen); + for (;;) { + const ids = await store.scanForEmbedding(gen, afterId, this.deps.batchSize); + if (!ids.length) { + if (!backstop) await store.resetWatermark(gen); // re-scan from start next tick + // Coverage reached: the shared driver seam (build route + scheduler) that + // promotes a fully-embedded 'building' generation to 'active'. + await this._activateIfBuildingCovered(gen); + return res; + } + res.claimed += ids.length; + const batchMax = ids[ids.length - 1]; + + let eb; + try { + eb = await this._embedBatch(gen, ids); + } catch (err) { + consecutiveFailures++; + this.deps.log.warn?.(`embed batch failed (gen ${gen}): ${err.message}`); + + if (isPermanent4xx(err)) { + // Downshift to size=1 and drain (worker.go RunOnce ErrPermanent4xx branch): + // embed what succeeds, stamp-drop confirmed per-message 4xx offenders, and + // leave rows unstamped only on an all-drop where the endpoint embedded nothing. + this.deps.log.info?.(`embed: downshifting to size=1 to drain failing batch (gen ${gen}, ${ids.length})`); + const dr = await this._downshiftDrain(gen, ids, res); + res.succeeded += dr.embedded; + // Reset the cap on embeddedOK (endpoint embedded+upserted something) — a + // CAS-missed stamp still proves the endpoint is healthy. + if (dr.embeddedOK > 0) consecutiveFailures = 0; + // Advance ONLY past the contiguously-stamped prefix so an unstamped + // straggler is never skipped (safeAdvanceID == batchMax on a clean drain). + if (dr.safeAdvanceID && dr.safeAdvanceID > afterId) { + afterId = dr.safeAdvanceID; + if (!backstop) await store.setWatermark(gen, dr.safeAdvanceID); + } + if (dr.drainErr) { + if (dr.drainErr.code === 'GEN_RETIRED') { this.deps.log.info?.(`generation ${gen} retired mid-drain; stopping`); return res; } + // A transient (non-4xx) error during the drain is a hard abort (an earlier + // singleton may have stamped, so the watermark stopped at the contiguous + // prefix and the next run re-finds the rest). + if (!isPermanent4xx(dr.drainErr)) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${dr.drainErr.message}`, { cause: err }); + } + // All-drop 4xx: the rows stay unstamped; let the failure cap trip. + if (consecutiveFailures >= this.deps.maxConsecutiveFailures) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${dr.drainErr.message}`, { cause: err }); + } + } + continue; + } + + // Non-4xx (transient) error: leave the batch unstamped, do not advance the + // cursor, so the failure cap short-circuits a persistent fault. + res.failed += ids.length; + if (consecutiveFailures >= this.deps.maxConsecutiveFailures) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${err.message}`, { cause: err }); + } + continue; // do not advance cursor; next scan re-finds the batch + } + res.truncated += eb.truncated; + const skipIds = [...eb.missing, ...eb.empty]; + + if (!eb.chunks.length) { + if (skipIds.length) await this._stampSkipped(gen, skipIds, eb.lastModified); + consecutiveFailures = 0; + afterId = batchMax; + if (!backstop) await store.setWatermark(gen, batchMax); + continue; + } + + // Step 1: upsert embeddings FIRST (idempotent; crash-safe ordering). + try { + await store.upsert(gen, eb.chunks); + } catch (err) { + if (err.code === 'GEN_RETIRED') { this.deps.log.info?.(`generation ${gen} retired mid-run; stopping`); return res; } + consecutiveFailures++; + res.failed += eb.embeddedIds.length; + if (consecutiveFailures >= this.deps.maxConsecutiveFailures) { + throw new Error(`embed worker aborting after ${consecutiveFailures} consecutive failures: ${err.message}`, { cause: err }); + } + continue; + } + + // Step 2: skip-mark empty/missing, then CAS-stamp embedded rows. + if (skipIds.length) await this._stampSkipped(gen, skipIds, eb.lastModified); + const missed = await store.setEmbedGenIfUnchanged( + eb.embeddedIds.map((id) => ({ id, lastModified: eb.lastModified.get(id) })), gen, + ); + if (missed.length) this.deps.log.info?.(`embed_gen CAS misses (concurrent edit): ${missed.length} — backstop recovers`); + res.succeeded += eb.embeddedIds.length - missed.length; + consecutiveFailures = 0; + // Advance the watermark even on a whole-batch CAS miss (backstop recovers misses). + afterId = batchMax; + if (!backstop) await store.setWatermark(gen, batchMax); + this.deps.onProgress?.({ done: res.succeeded, claimed: res.claimed, truncated: res.truncated }); + } + } catch (err) { + runErr = err; + throw err; + } finally { + try { await store.finalizeEmbedRun?.(runId, res, runErr); } catch { /* observability only */ } + } + } + + // Activation-on-coverage seam (worker.go parity — see divergence note below). + // A run reaches here only after scanForEmbedding drains, i.e. no live message + // still needs embedding for `gen` (poison messages count as covered — they are + // stamped, not skipped). When `gen` is the CURRENT building generation, promote + // it to 'active'. This is the sole production caller of activateGeneration: + // without it a completed build stays 'building' forever and hybrid search never + // leaves its index_building fallback. + // + // No-ops when: + // - no generations collaborator is wired (a bare worker with nothing to promote), or + // - `gen` is not the building generation — an active generation's incremental + // or backstop run must never re-activate (README: generations never mix). + // + // Tolerates the two lifecycle races activateGeneration encodes, so a completed + // build is never lost and activation never fails the embed run: + // - "still has messages needing embedding": a late content edit re-NULLed a row + // between the drain and the activate (the coverage gate re-asserted inside the + // activate tx caught it) — leave it building; the next scheduler tick re-drains + // and retries. + // - "not in 'building' state": a concurrent activate/retire already moved it — no-op. + // Any other error is logged, not thrown: activation is a post-coverage promotion, + // not part of the run result. + // + // Divergence from msgvault worker.go: there activation lives in the DRIVERS + // (scheduler embed_job.go + the CLI embed_vector.go each re-check coverage and + // call ActivateGeneration). We fold it into this shared worker seam instead so + // BOTH Mailflow drivers (the fire-and-forget build route and the scheduler tick) + // get it from one place and cannot drift out of sync — the exact failure that + // left the live build wedged in 'building'. The coverage precondition (scan + // drained) and the no-force, gate-re-asserted activate call match worker.go. + async _activateIfBuildingCovered(gen) { + const gens = this.deps.generations; + if (!gens?.activateGeneration || !gens?.buildingGeneration) return; + let building; + try { + building = await gens.buildingGeneration(); + } catch (err) { + this.deps.log.warn?.(`embed: could not read building generation to activate ${gen}: ${err.message}`); + return; + } + if (!building || building.id !== gen) return; // active/incremental run — nothing to promote + try { + await gens.activateGeneration(gen); // no force — activateGeneration re-asserts the coverage gate atomically + this.deps.log.info?.(`embed: generation ${gen} fully covered — activated`); + } catch (err) { + this.deps.log.warn?.(`embed: generation ${gen} not activated (${err.message}); scheduler will retry next tick if still building`); + } + } + + async _embedBatch(gen, ids) { + const { store } = this.deps; + const rows = await store.fetchForEmbedding(ids); + const fetched = new Set(); + const lastModified = new Map(); + const msgs = []; + const empty = []; + for (const r of rows) { + fetched.add(r.id); + lastModified.set(r.id, r.lastModified); + const body = r.bodyText && r.bodyText.trim() !== '' ? r.bodyText : (r.bodyHtml || ''); + const ppCfg = { ...this.deps.preprocessCfg }; + if (!ppCfg.maxBodyRunes && this.deps.maxInputChars > 0) { + ppCfg.maxBodyRunes = this.deps.maxInputChars * MAX_SPANS * RAW_BODY_MULT; + } + const { text, truncated } = preprocess(r.subject || '', body, 0, ppCfg); + if (text.trim() === '') { empty.push(r.id); continue; } + msgs.push({ id: r.id, text, bodyTruncated: truncated }); + } + const missing = ids.filter((id) => !fetched.has(id)); + if (!msgs.length) return { chunks: [], embeddedIds: [], missing, empty, truncated: 0, lastModified }; + + const window = this.deps.maxInputChars; + const overlap = chunkOverlapFor(window); + const pieces = []; + const inputs = []; + const truncatedMsg = new Set(); + for (const m of msgs) { + const { spans, tailDropped } = chunkText(m.text, window, overlap, MAX_SPANS); + const msgTrunc = m.bodyTruncated || tailDropped; + spans.forEach((sp, j) => { + const hardCut = window > 0 && (sp.charEnd - sp.charStart) === window && j < spans.length - 1; + const trunc = msgTrunc || hardCut; + if (trunc) truncatedMsg.add(m.id); + pieces.push({ id: m.id, chunkIndex: j, text: sp.text, chars: sp.charEnd - sp.charStart, charStart: sp.charStart, charEnd: sp.charEnd, trunc }); + inputs.push(sp.text); + }); + } + + const vecs = []; + try { + for (let i = 0; i < inputs.length; i += this.deps.batchSize) { + const got = await this.deps.client.embed(inputs.slice(i, i + this.deps.batchSize)); + vecs.push(...got); + } + } catch (err) { + // Attach the CAS tokens fetched before the embed call so a downshift drain can + // CAS-drop a per-message 4xx offender with the last_modified read at fetch time. + err.lastModified = lastModified; + throw err; + } + if (vecs.length !== pieces.length) throw new Error(`embedder returned ${vecs.length} vectors for ${pieces.length} chunk inputs`); + + const chunks = []; + const embeddedIds = []; + const seen = new Set(); + for (let i = 0; i < pieces.length; i++) { + const p = pieces[i]; + chunks.push({ messageId: p.id, chunkIndex: p.chunkIndex, vector: vecs[i], sourceCharLen: p.chars, chunkCharStart: p.charStart, chunkCharEnd: p.charEnd, truncated: p.trunc }); + if (!seen.has(p.id)) { seen.add(p.id); embeddedIds.push(p.id); } + } + return { chunks, embeddedIds, missing, empty, truncated: truncatedMsg.size, lastModified }; + } + + // Port of worker.go downshiftDrain: walk a 4xx-failing batch one message at a time. + // Embed + upsert + CAS-stamp what succeeds; skip-mark empty/missing; and DEFER + // per-message 4xx offenders — stamp-dropping them only when some sibling embedded + // (proving the endpoint is healthy, so the 4xx is message-specific). On an all-drop + // (endpoint embedded nothing) the deferred ids are left UNSTAMPED and a permanent-4xx + // error is returned so a misconfigured endpoint never silently loses work. Returns + // { embedded, embeddedOK, stamped, safeAdvanceID, drainErr }; safeAdvanceID is the + // highest CONTIGUOUSLY-stamped id (batchMax on a clean drain) so the caller never + // advances the watermark past an unstamped straggler. + async _downshiftDrain(gen, ids, res) { + const { store } = this.deps; + let embedded = 0; + let embeddedOK = 0; + let stamped = 0; + let contiguousStampedID = null; + let brokeContiguity = false; + const deferredDrops = []; + let lastDeferredErr = null; + const lm = new Map(); + const advance = (id, didStamp) => { + if (didStamp) { if (!brokeContiguity) contiguousStampedID = id; } + else brokeContiguity = true; + }; + + for (const id of ids) { + let eb; + try { + eb = await this._embedBatch(gen, [id]); + } catch (e) { + if (isPermanent4xx(e)) { + if (e.lastModified) for (const [k, v] of e.lastModified) lm.set(k, v); + deferredDrops.push(id); + lastDeferredErr = e; + brokeContiguity = true; // a deferred id breaks the stamped-from-start prefix + continue; + } + // Transient error: leave this id unstamped and abort the drain; the watermark + // stays at the contiguous stamped prefix so the next run re-finds it. + return { embedded, embeddedOK, stamped, safeAdvanceID: contiguousStampedID, drainErr: e }; + } + for (const [k, v] of eb.lastModified) lm.set(k, v); + + if (!eb.chunks.length) { + // Missing/empty singleton — skip-mark it. + const skip = [...eb.missing, ...eb.empty]; + let didStamp = true; + if (skip.length) { + const missed = await this._stampSkipped(gen, skip, eb.lastModified); + const stampedSkip = skip.length - missed.length; + stamped += stampedSkip; + didStamp = stampedSkip > 0; // a CAS-missed skip leaves the row unstamped + } + advance(id, didStamp); + continue; + } + + try { + await store.upsert(gen, eb.chunks); + } catch (uerr) { + return { embedded, embeddedOK, stamped, safeAdvanceID: contiguousStampedID, drainErr: uerr }; + } + embeddedOK++; // endpoint demonstrably embedded + upserted this singleton + const missed = await store.setEmbedGenIfUnchanged( + eb.embeddedIds.map((mid) => ({ id: mid, lastModified: eb.lastModified.get(mid) })), gen, + ); + const stampedHere = eb.embeddedIds.length - missed.length; + res.truncated += eb.truncated; + embedded += stampedHere; + stamped += stampedHere; + advance(id, stampedHere > 0); + } + + // Clean drain: every id is resolved (stamped, skip-marked, or a deferred 4xx below). + const safeAdvanceID = ids[ids.length - 1]; + + if (!deferredDrops.length) return { embedded, embeddedOK, stamped, safeAdvanceID, drainErr: null }; + + if (embeddedOK > 0) { + // The endpoint embedded something, so the 4xxs are message-specific — stamp-drop them. + for (const id of deferredDrops) { + this.deps.log.warn?.(`stamping (dropping) message after singleton 4xx (gen ${gen}, id ${id}): ${lastDeferredErr?.message}`); + } + const missed = await this._stampSkipped(gen, deferredDrops, lm); + stamped += deferredDrops.length - missed.length; + return { embedded, embeddedOK, stamped, safeAdvanceID, drainErr: null }; + } + + // embeddedOK === 0: endpoint embedded nothing — can't distinguish an endpoint-wide + // failure from a batch where every message is unembeddable. Leave the deferred ids + // UNSTAMPED and surface the 4xx (marked permanent so the caller's cap, not a hard + // abort, governs) so a misconfigured endpoint does not silently drop work. + const err = new Error(`downshift all-drop: every singleton returned non-retryable 4xx (left ${deferredDrops.length} row(s) unstamped): ${lastDeferredErr?.message}`); + err.permanent4xx = true; + return { embedded, embeddedOK, stamped, safeAdvanceID: contiguousStampedID, drainErr: err }; + } + + // Skip-mark empty/missing rows (drops them from the next scan) and remove any stale + // vectors for the rows that were actually stamped — the stamp + prune happen in ONE + // transaction inside store.stampSkipped (msgvault worker.go parity). CAS-protected + // for rows with a last_modified token; unconditional for missing rows. + async _stampSkipped(gen, ids, lastModified) { + const { store } = this.deps; + const cas = []; + const plain = []; + for (const id of ids) { + if (lastModified.has(id)) cas.push({ id, lastModified: lastModified.get(id) }); + else plain.push(id); + } + return store.stampSkipped(gen, cas, plain); + } +} diff --git a/backend/src/services/embeddings/worker.test.js b/backend/src/services/embeddings/worker.test.js new file mode 100644 index 00000000..e61692db --- /dev/null +++ b/backend/src/services/embeddings/worker.test.js @@ -0,0 +1,250 @@ +import { describe, it, expect, vi } from 'vitest'; +import { EmbeddingWorker } from './worker.js'; + +const ZERO = '00000000-0000-0000-0000-000000000000'; +const uid = (n) => `00000000-0000-0000-0000-${String(n).padStart(12, '0')}`; + +function makeStore(messages, opts = {}) { + const watermark = new Map(); + const state = { upsertCalls: 0, resetCalls: 0 }; + return { + ZERO_UUID: ZERO, + _messages: messages, + _state: state, + async scanForEmbedding(target, afterId, limit) { + return messages + .filter((m) => (m.embedGen == null || m.embedGen !== target) && !m.isDeleted && m.id > afterId) + .sort((a, b) => (a.id < b.id ? -1 : 1)) + .slice(0, limit) + .map((m) => m.id); + }, + async fetchForEmbedding(ids) { + return messages.filter((m) => ids.includes(m.id)).map((m) => ({ + id: m.id, subject: m.subject || '', bodyText: m.bodyText || '', bodyHtml: m.bodyHtml || '', lastModified: m.lastModified, + })); + }, + async upsert() { state.upsertCalls++; }, + async setEmbedGen(ids, target) { for (const id of ids) { const m = messages.find((x) => x.id === id); if (m) m.embedGen = target; } }, + async setEmbedGenIfUnchanged(items, target) { + const missed = []; + for (const it of items) { + const m = messages.find((x) => x.id === it.id); + if (opts.casMiss?.has(it.id)) { missed.push(it.id); continue; } + if (m && m.lastModified === it.lastModified) m.embedGen = target; else missed.push(it.id); + } + return missed; + }, + async stampSkipped(target, casItems, plainIds) { + const missed = []; + for (const it of casItems) { + const m = messages.find((x) => x.id === it.id); + if (opts.casMiss?.has(it.id)) { missed.push(it.id); continue; } + if (m && m.lastModified === it.lastModified) m.embedGen = target; else missed.push(it.id); + } + for (const id of plainIds) { const m = messages.find((x) => x.id === id); if (m) m.embedGen = target; } + return missed; + }, + async getWatermark(gen) { state.getWatermarkCalled = true; return watermark.get(gen) || ZERO; }, + async setWatermark(gen, id) { watermark.set(gen, id); }, + async resetWatermark(gen) { state.resetCalls++; watermark.set(gen, ZERO); }, + }; +} +const fakeClient = { async embed(inputs) { return inputs.map(() => [0.1, 0.2, 0.3, 0.4]); } }; +const deps = (store, over = {}) => ({ store, client: fakeClient, preprocessCfg: {}, maxInputChars: 32768, batchSize: 32, ...over }); + +// A generations collaborator stub for the activation seam. `building` is what +// buildingGeneration() returns (null ⇒ the run's gen is NOT building — an active +// or incremental run); `activate` is the activateGeneration spy. +function makeGenerations({ building = null, activate } = {}) { + return { + buildingGeneration: vi.fn(async () => building), + activateGeneration: activate || vi.fn(async () => {}), + }; +} + +describe('EmbeddingWorker.runOnce', () => { + it('embeds pending messages and stamps embed_gen', async () => { + const msgs = [ + { id: uid(1), subject: 'hello', bodyText: 'world', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 'foo', bodyText: 'bar', lastModified: 't2', embedGen: null }, + ]; + const store = makeStore(msgs); + const w = new EmbeddingWorker(deps(store)); + const res = await w.runOnce('7'); + expect(res.claimed).toBe(2); + expect(res.succeeded).toBe(2); + expect(msgs.every((m) => m.embedGen === '7')).toBe(true); + }); + + it('excludes a CAS miss from succeeded but still advances the watermark', async () => { + const msgs = [ + { id: uid(1), subject: 's1', bodyText: 'b1', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 's2', bodyText: 'b2', lastModified: 't2', embedGen: null }, + ]; + const store = makeStore(msgs, { casMiss: new Set([uid(2)]) }); + const w = new EmbeddingWorker(deps(store)); + const res = await w.runOnce('7'); + expect(res.succeeded).toBe(1); + const m2 = msgs.find((m) => m.id === uid(2)); + expect(m2.embedGen).toBeNull(); // recovered by backstop later + }); + + it('is idempotent — a second run finds nothing', async () => { + const msgs = [{ id: uid(1), subject: 's', bodyText: 'b', lastModified: 't1', embedGen: null }]; + const store = makeStore(msgs); + const w = new EmbeddingWorker(deps(store)); + await w.runOnce('7'); + const res2 = await w.runOnce('7'); + expect(res2.claimed).toBe(0); + }); + + it('resets the watermark on scan exhaustion (P2)', async () => { + const store = makeStore([]); + const w = new EmbeddingWorker(deps(store)); + await w.runOnce('7'); + expect(store._state.resetCalls).toBeGreaterThan(0); + }); + + it('aborts after maxConsecutiveFailures on a persistent embed failure (does not loop)', async () => { + const msgs = Array.from({ length: 10 }, (_, i) => ({ id: uid(i + 1), subject: 's', bodyText: 'b', lastModified: `t${i}`, embedGen: null })); + const store = makeStore(msgs); + const scanSpy = vi.spyOn(store, 'scanForEmbedding'); + const failingClient = { async embed() { throw new Error('unreachable'); } }; + const w = new EmbeddingWorker(deps(store, { client: failingClient, maxConsecutiveFailures: 3 })); + await expect(w.runOnce('7')).rejects.toThrow(/aborting after 3 consecutive failures/); + expect(scanSpy.mock.calls.length).toBe(3); // bounded, not infinite + }); + + it('downshifts on a permanent 4xx: isolates the poison message and embeds the rest (W1)', async () => { + const msgs = [ + { id: uid(1), subject: 'good one', bodyText: 'hello', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 'poison here', bodyText: 'x', lastModified: 't2', embedGen: null }, + { id: uid(3), subject: 'good two', bodyText: 'world', lastModified: 't3', embedGen: null }, + ]; + const store = makeStore(msgs); + // 4xx whenever the batch contains the poison text; succeeds otherwise. + const poisonClient = { + async embed(inputs) { + if (inputs.some((t) => t.includes('poison'))) { const e = new Error('unembeddable input'); e.permanent4xx = true; throw e; } + return inputs.map(() => [0.1, 0.2, 0.3, 0.4]); + }, + }; + const w = new EmbeddingWorker(deps(store, { client: poisonClient, maxConsecutiveFailures: 3 })); + const res = await w.runOnce('7'); + expect(msgs.find((m) => m.id === uid(1)).embedGen).toBe('7'); // embedded + expect(msgs.find((m) => m.id === uid(3)).embedGen).toBe('7'); // embedded + expect(msgs.find((m) => m.id === uid(2)).embedGen).toBe('7'); // stamp-dropped (leaves the scan) + expect(res.succeeded).toBe(2); // only the two good ones count as embedded + }); + + it('all-4xx batch leaves rows unstamped and aborts after maxConsecutiveFailures (no silent drop) (W1)', async () => { + const msgs = Array.from({ length: 4 }, (_, i) => ({ id: uid(i + 1), subject: 's', bodyText: 'b', lastModified: `t${i}`, embedGen: null })); + const store = makeStore(msgs); + const scanSpy = vi.spyOn(store, 'scanForEmbedding'); + const allBadClient = { async embed() { const e = new Error('always bad'); e.permanent4xx = true; throw e; } }; + const w = new EmbeddingWorker(deps(store, { client: allBadClient, maxConsecutiveFailures: 3 })); + await expect(w.runOnce('7')).rejects.toThrow(/aborting after 3 consecutive failures/); + expect(msgs.every((m) => m.embedGen === null)).toBe(true); // nothing silently dropped + expect(scanSpy.mock.calls.length).toBe(3); // bounded + }); +}); + +// The activation seam: the SHARED worker-run completion point that both drivers +// (the manual build route and the scheduler) reach. When a run drains the scan +// for the generation that is currently 'building', the worker promotes it to +// 'active' — the production wiring that was missing (activateGeneration had no +// non-test caller, so completed builds stayed 'building' forever). +describe('EmbeddingWorker activation seam', () => { + it('activates the building generation once, without force, after the scan drains', async () => { + const gen = '7'; + const msgs = [ + { id: uid(1), subject: 'a', bodyText: 'b', lastModified: 't1', embedGen: null }, + { id: uid(2), subject: 'c', bodyText: 'd', lastModified: 't2', embedGen: null }, + ]; + const store = makeStore(msgs); + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + const res = await w.runOnce(gen); + expect(res.succeeded).toBe(2); + expect(generations.buildingGeneration).toHaveBeenCalled(); + expect(activateSpy).toHaveBeenCalledTimes(1); + expect(activateSpy.mock.calls[0]).toEqual([gen]); // called with the gen only — no force + }); + + it('does not activate when the run aborts before draining the scan', async () => { + const gen = '7'; + const msgs = Array.from({ length: 6 }, (_, i) => ({ id: uid(i + 1), subject: 's', bodyText: 'b', lastModified: `t${i}`, embedGen: null })); + const store = makeStore(msgs); + const failingClient = { async embed() { throw new Error('unreachable'); } }; + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { client: failingClient, maxConsecutiveFailures: 2, generations })); + await expect(w.runOnce(gen)).rejects.toThrow(/aborting after/); + expect(activateSpy).not.toHaveBeenCalled(); // the scan never drained → no coverage → no activation + }); + + it('never activates an active generation on an incremental run (no building generation)', async () => { + const gen = '7'; + const store = makeStore([]); // nothing pending — scan drains immediately + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: null, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await w.runOnce(gen); + expect(activateSpy).not.toHaveBeenCalled(); + }); + + it('never activates an active generation on a backstop run', async () => { + const gen = '7'; + const store = makeStore([]); + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: null, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await w.runBackstop(gen); + expect(activateSpy).not.toHaveBeenCalled(); + }); + + it('does not activate the running generation when a DIFFERENT generation is building (rebuild in progress)', async () => { + const activeGen = '7'; + const store = makeStore([]); + const activateSpy = vi.fn(async () => {}); + const generations = makeGenerations({ building: { id: '8' }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await w.runOnce(activeGen); + expect(activateSpy).not.toHaveBeenCalled(); + }); + + it('swallows the "still has messages" activation race and leaves the run result intact', async () => { + const gen = '7'; + const msgs = [{ id: uid(1), subject: 'a', bodyText: 'b', lastModified: 't1', embedGen: null }]; + const store = makeStore(msgs); + const activateSpy = vi.fn(async () => { throw new Error(`generation ${gen} still has messages needing embedding; pass force to override`); }); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + const res = await w.runOnce(gen); // must NOT reject — activation is a post-coverage promotion + expect(res.succeeded).toBe(1); + expect(activateSpy).toHaveBeenCalledTimes(1); + }); + + it('swallows the "not in building state" activation race (already active/retired)', async () => { + const gen = '7'; + const store = makeStore([]); + const activateSpy = vi.fn(async () => { throw new Error(`generation ${gen} not in 'building' state`); }); + const generations = makeGenerations({ building: { id: gen }, activate: activateSpy }); + const w = new EmbeddingWorker(deps(store, { generations })); + await expect(w.runOnce(gen)).resolves.toBeTruthy(); + expect(activateSpy).toHaveBeenCalledTimes(1); + }); +}); + +describe('EmbeddingWorker.runBackstop', () => { + it('ignores the watermark and finds a sub-watermark straggler', async () => { + const msgs = [{ id: uid(1), subject: 's', bodyText: 'b', lastModified: 't1', embedGen: null }]; + const store = makeStore(msgs); + await store.setWatermark('7', uid(999)); // high watermark that would hide id 1 + const w = new EmbeddingWorker(deps(store)); + const res = await w.runBackstop('7'); + expect(res.succeeded).toBe(1); + expect(msgs[0].embedGen).toBe('7'); + }); +}); diff --git a/backend/src/services/gtd/actions.js b/backend/src/services/gtd/actions.js new file mode 100644 index 00000000..ec19fe90 --- /dev/null +++ b/backend/src/services/gtd/actions.js @@ -0,0 +1,204 @@ +import { query } from '../db.js'; +import { archiveInboxCopy } from '../archiveInbox.js'; +import { getGtdConfig, resolveGtdStateFolder } from '../gtdConfig.js'; +import { reconcileDelegatedRemovals } from '../gtdDelegations.js'; +import { fanOutReadToSiblings } from '../../utils/mailUtils.js'; + +export function classifyTarget({ enabled, folders, state }) { + if (!enabled) return { status: 400, error: 'GTD is not enabled for this account' }; + const folder = resolveGtdStateFolder(state, folders); + if (!folder) return { status: 400, error: `Unknown GTD state: ${state}` }; + return { folder }; +} + +export function resolveDoneFolders({ enabled, folders, states, existing }) { + if (!enabled) return { status: 400, error: 'GTD is not enabled for this account' }; + if (states === 'all') { + const present = new Set(Array.isArray(existing) ? existing : []); + const resolved = []; + for (const folder of Object.values(folders || {})) { + if (present.has(folder) && !resolved.includes(folder)) resolved.push(folder); + } + return { folders: resolved }; + } + if (!Array.isArray(states) || states.length === 0) { + return { status: 400, error: 'states must be a non-empty array' }; + } + const resolved = []; + for (const state of states) { + const folder = resolveGtdStateFolder(state, folders); + if (!folder) return { status: 400, error: `Unknown GTD state: ${state}` }; + if (!resolved.includes(folder)) resolved.push(folder); + } + return { folders: resolved }; +} + +async function loadOwnedMessage(userId, accountIds, messageId) { + const accountClause = accountIds == null ? '' : ' AND m.account_id = ANY($3::uuid[])'; + const params = accountIds == null + ? [messageId, userId] + : [messageId, userId, accountIds]; + const result = await query( + `SELECT m.* + FROM messages m + JOIN email_accounts a ON a.id = m.account_id + WHERE m.id = $1 AND a.user_id = $2${accountClause}`, + params + ); + return result.rows[0] || null; +} + +async function resolveCopyUid(msg, folder) { + if (msg.folder === folder) return msg.uid; + const sib = await query( + 'SELECT uid FROM messages WHERE account_id = $1 AND folder = $2 AND message_id = $3 AND is_deleted = false LIMIT 1', + [msg.account_id, folder, msg.message_id] + ); + return sib.rows[0]?.uid ?? null; +} + +export async function gtdClassify(imapManager, { userId, accountIds, messageId, state }) { + const msg = await loadOwnedMessage(userId, accountIds, messageId); + if (!msg) return { ok: false, status: 404, error: 'Message not found' }; + + const { enabled, folders } = await getGtdConfig(msg.account_id); + const target = classifyTarget({ enabled, folders, state }); + if (target.error) return { ok: false, status: target.status, error: target.error }; + const toFolder = target.folder; + + if (msg.folder === toFolder) return { ok: true, folder: toFolder }; + + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); + const account = accountResult.rows[0]; + + try { + await imapManager.ensureFolder(account, toFolder); + await imapManager.copyMessage(msg.account_id, msg.uid, msg.folder, toFolder); + } catch (err) { + console.error(`GTD classify failed for message ${messageId} -> ${toFolder}:`, err.message); + return { ok: false, status: 500, error: 'Failed to apply GTD label' }; + } + + return { ok: true, folder: toFolder }; +} + +export async function gtdUnclassify(imapManager, { userId, accountIds, messageId, state }) { + const msg = await loadOwnedMessage(userId, accountIds, messageId); + if (!msg) return { ok: false, status: 404, error: 'Message not found' }; + + const { enabled, folders } = await getGtdConfig(msg.account_id); + const target = classifyTarget({ enabled, folders, state }); + if (target.error) return { ok: false, status: target.status, error: target.error }; + const stateFolder = target.folder; + + if (msg.folder !== stateFolder && !msg.message_id) { + return { ok: false, status: 400, error: 'Message has no Message-ID — cannot resolve GTD copy' }; + } + const siblingUid = await resolveCopyUid(msg, stateFolder); + if (siblingUid == null) { + if (state === 'delegated' && msg.thread_key) { + await reconcileDelegatedRemovals({ + userId, accountId: msg.account_id, + delegatedFolder: stateFolder, threadKeys: [msg.thread_key], + }); + } + return { ok: true, removed: false }; + } + + try { + await imapManager.removeMessageCopy(msg.account_id, siblingUid, stateFolder); + if (state === 'delegated' && msg.thread_key) { + await reconcileDelegatedRemovals({ + userId, accountId: msg.account_id, + delegatedFolder: stateFolder, threadKeys: [msg.thread_key], + }); + } + } catch (err) { + console.error(`GTD unclassify failed for message ${messageId} in ${stateFolder}:`, err.message); + return { ok: false, status: 500, error: 'Failed to remove GTD label' }; + } + + return { ok: true, removed: true, folder: stateFolder }; +} + +export async function gtdDone(imapManager, { userId, accountIds, id, states }) { + const msg = await loadOwnedMessage(userId, accountIds, id); + if (!msg) return { ok: false, status: 404, error: 'Message not found' }; + if (!msg.message_id) { + return { ok: false, status: 400, error: 'Message has no Message-ID — cannot mark done' }; + } + + const { enabled, folders } = await getGtdConfig(msg.account_id); + const allStates = states == null || states === 'all'; + let existing; + if (allStates && enabled) { + const copies = await query( + 'SELECT DISTINCT folder FROM messages WHERE account_id = $1 AND message_id = $2 AND is_deleted = false', + [msg.account_id, msg.message_id] + ); + existing = copies.rows.map(r => r.folder); + } + const target = allStates + ? resolveDoneFolders({ enabled, folders, states: 'all', existing }) + : resolveDoneFolders({ enabled, folders, states }); + if (target.error) return { ok: false, status: target.status, error: target.error }; + + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); + const account = accountResult.rows[0]; + + const inbox = await query( + 'SELECT id, uid, is_read FROM messages WHERE account_id = $1 AND folder = $2 AND message_id = $3 AND is_deleted = false LIMIT 1', + [msg.account_id, 'INBOX', msg.message_id] + ); + const inboxCopy = inbox.rows[0] || null; + try { + await fanOutReadToSiblings(msg.account_id, msg.message_id, true); + if (inboxCopy && !inboxCopy.is_read) { + await imapManager.setFlag(account, inboxCopy.uid, 'INBOX', '\\Seen', true); + } + } catch (err) { + console.warn(`GTD done: mark-read for ${id} degraded:`, err.message); + } + + const stripOrder = [ + ...target.folders.filter(f => f !== msg.folder), + ...target.folders.filter(f => f === msg.folder), + ]; + const removed = []; + try { + for (const folder of stripOrder) { + const uid = await resolveCopyUid(msg, folder); + if (uid != null) { + await imapManager.removeMessageCopy(msg.account_id, uid, folder); + removed.push(folder); + } + if (folder === folders.delegated && msg.thread_key) { + await reconcileDelegatedRemovals({ + userId, accountId: msg.account_id, + delegatedFolder: folder, threadKeys: [msg.thread_key], + }); + } + } + } catch (err) { + console.error(`GTD done: label strip for ${id} failed:`, err.message); + return { ok: false, status: 500, error: 'Failed to mark done' }; + } + + let archived = false; + let noArchiveFolder = false; + let archiveFailed = false; + if (inboxCopy) { + try { + const result = await archiveInboxCopy(imapManager, account, inboxCopy); + archived = result.archived; + noArchiveFolder = result.noArchiveFolder; + } catch (err) { + console.error(`GTD done: archive of INBOX copy for ${id} failed:`, err.message); + archiveFailed = true; + } + } + + imapManager.broadcast({ type: 'gtd_sections_updated', accountId: msg.account_id }, account.user_id); + + return { ok: true, removed, archived, noArchiveFolder, archiveFailed }; +} diff --git a/backend/src/services/gtd/actions.test.js b/backend/src/services/gtd/actions.test.js new file mode 100644 index 00000000..c35dcb0a --- /dev/null +++ b/backend/src/services/gtd/actions.test.js @@ -0,0 +1,146 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../gtdConfig.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, getGtdConfig: vi.fn() }; +}); +vi.mock('../archiveInbox.js', () => ({ archiveInboxCopy: vi.fn() })); +vi.mock('../../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, fanOutReadToSiblings: vi.fn() }; +}); + +import { query } from '../db.js'; +import { archiveInboxCopy } from '../archiveInbox.js'; +import { DEFAULT_GTD_FOLDERS, getGtdConfig } from '../gtdConfig.js'; +import { + gtdClassify, + gtdDone, + gtdUnclassify, +} from './actions.js'; + +const ID = '11111111-1111-4111-8111-111111111111'; +const msg = { + id: ID, + account_id: 'a1', + uid: 10, + folder: 'INBOX', + message_id: '', + is_read: true, +}; +const account = { id: 'a1', user_id: 'u1' }; + +function manager() { + return { + ensureFolder: vi.fn().mockResolvedValue(undefined), + copyMessage: vi.fn().mockResolvedValue(undefined), + removeMessageCopy: vi.fn().mockResolvedValue(undefined), + setFlag: vi.fn().mockResolvedValue(undefined), + broadcast: vi.fn(), + }; +} + +beforeEach(() => { + query.mockReset(); + getGtdConfig.mockReset(); + archiveInboxCopy.mockReset(); + getGtdConfig.mockResolvedValue({ enabled: true, folders: DEFAULT_GTD_FOLDERS }); +}); + +describe('gtdClassify', () => { + it('narrows message ownership to accountIds before IMAP work', async () => { + query.mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await gtdClassify(imap, { + userId: 'u1', + accountIds: ['a1'], + messageId: ID, + state: 'todo', + }); + + expect(result).toEqual({ ok: false, status: 404, error: 'Message not found' }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($3::uuid[])'); + expect(params).toEqual([ID, 'u1', ['a1']]); + expect(imap.copyMessage).not.toHaveBeenCalled(); + }); + + it('preserves the ensure-and-copy receipt', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('FROM messages m')) return { rows: [msg] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = manager(); + + const result = await gtdClassify(imap, { + userId: 'u1', + accountIds: null, + messageId: ID, + state: 'todo', + }); + + expect(result).toEqual({ ok: true, folder: 'Todo' }); + expect(imap.ensureFolder).toHaveBeenCalledWith(account, 'Todo'); + expect(imap.copyMessage).toHaveBeenCalledWith('a1', 10, 'INBOX', 'Todo'); + }); +}); + +describe('gtdUnclassify', () => { + it('removes the resolved label-folder copy', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('FROM messages m')) return { rows: [msg] }; + if (sql.startsWith('SELECT uid FROM messages')) return { rows: [{ uid: 42 }] }; + return { rows: [] }; + }); + const imap = manager(); + + const result = await gtdUnclassify(imap, { + userId: 'u1', + accountIds: null, + messageId: ID, + state: 'todo', + }); + + expect(result).toEqual({ ok: true, removed: true, folder: 'Todo' }); + expect(imap.removeMessageCopy).toHaveBeenCalledWith('a1', 42, 'Todo'); + }); +}); + +describe('gtdDone', () => { + it('keeps strip-success/archive-failure as a 200-shaped partial success', async () => { + const watchMsg = { ...msg, folder: 'Watch' }; + const inboxCopy = { id: 'inbox-id', uid: 77, is_read: true }; + query.mockImplementation(async (sql) => { + if (sql.includes('FROM messages m') && sql.includes('JOIN email_accounts')) return { rows: [watchMsg] }; + if (sql.startsWith('SELECT * FROM email_accounts')) return { rows: [account] }; + if (sql.startsWith('SELECT id, uid, is_read FROM messages')) return { rows: [inboxCopy] }; + if (sql.startsWith('SELECT uid FROM messages')) return { rows: [{ uid: 10 }] }; + return { rows: [] }; + }); + archiveInboxCopy.mockRejectedValue(new Error('archive failed')); + const imap = manager(); + + const result = await gtdDone(imap, { + userId: 'u1', + accountIds: null, + id: ID, + states: ['watch'], + }); + + expect(result).toEqual({ + ok: true, + removed: ['Watch'], + archived: false, + noArchiveFolder: false, + archiveFailed: true, + }); + expect(imap.removeMessageCopy).toHaveBeenCalled(); + expect(imap.broadcast).toHaveBeenCalledWith( + { type: 'gtd_sections_updated', accountId: 'a1' }, + 'u1', + ); + }); +}); diff --git a/backend/src/services/gtdDelegations.js b/backend/src/services/gtdDelegations.js new file mode 100644 index 00000000..edb5e082 --- /dev/null +++ b/backend/src/services/gtdDelegations.js @@ -0,0 +1,266 @@ +import { query } from './db.js'; +import { getGtdConfig, resolveGtdStateFolder } from './gtdConfig.js'; +import { createKeyedSerializer } from '../utils/keyedSerializer.js'; + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const serializeDelegation = createKeyedSerializer(); + +export class GtdDelegationError extends Error { + constructor(code, status, message = code) { + super(message); + this.code = code; + this.status = status; + } +} + +export function mapDelegationRow(row) { + if (!row?.delegation) return null; + return typeof row.delegation === 'string' ? JSON.parse(row.delegation) : row.delegation; +} + +export async function loadOwnedContactSnapshot(userId, contactId) { + const { rows } = await query(` + SELECT c.id, COALESCE(c.display_name, c.primary_email, 'Unknown contact') AS display_name, + c.primary_email + FROM contacts c + WHERE c.id = $1 AND c.user_id = $2 + `, [contactId, userId]); + if (!rows[0]) throw new GtdDelegationError('contact_not_found', 404); + return rows[0]; +} + +export async function upsertDelegation({ userId, accountId, threadKey, contact }) { + const { rows } = await query(` + INSERT INTO gtd_delegations ( + user_id, account_id, thread_key, contact_id, + contact_display_name_snapshot, contact_primary_email_snapshot + ) VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (user_id, account_id, thread_key) DO UPDATE SET + contact_id = EXCLUDED.contact_id, + contact_display_name_snapshot = EXCLUDED.contact_display_name_snapshot, + contact_primary_email_snapshot = EXCLUDED.contact_primary_email_snapshot, + delegated_at = CASE + WHEN gtd_delegations.contact_id IS DISTINCT FROM EXCLUDED.contact_id THEN NOW() + ELSE gtd_delegations.delegated_at + END, + updated_at = NOW() + RETURNING contact_id, + contact_display_name_snapshot AS display_name, + contact_primary_email_snapshot AS primary_email, + delegated_at, updated_at + `, [userId, accountId, threadKey, contact.id, contact.display_name, contact.primary_email]); + return rows[0]; +} + +export async function clearDelegations({ userId, accountId, threadKeys }) { + const unique = [...new Set(threadKeys.filter(Boolean))]; + if (unique.length === 0) return 0; + const result = await query(` + DELETE FROM gtd_delegations + WHERE user_id = $1 AND account_id = $2 AND thread_key = ANY($3::text[]) + `, [userId, accountId, unique]); + return result.rowCount; +} + +export async function reconcileDelegatedRemovals({ userId, accountId, delegatedFolder, threadKeys }) { + const unique = [...new Set(threadKeys.filter(Boolean))]; + if (!unique.length) return 0; + const result = await query(` + DELETE FROM gtd_delegations gd + WHERE gd.user_id = $1 + AND gd.account_id = $2 + AND gd.thread_key = ANY($3::text[]) + AND NOT EXISTS ( + SELECT 1 FROM messages m + WHERE m.account_id = gd.account_id + AND m.thread_key = gd.thread_key + AND m.folder = $4 + AND m.is_deleted = false + ) + `, [userId, accountId, unique, delegatedFolder]); + return result.rowCount; +} + +export async function sweepStaleDelegations({ userId, accountId, delegatedFolder }) { + const result = await query(` + DELETE FROM gtd_delegations gd + WHERE gd.user_id = $1 + AND gd.account_id = $2 + AND NOT EXISTS ( + SELECT 1 FROM messages m + WHERE m.account_id = gd.account_id + AND m.thread_key = gd.thread_key + AND m.folder = $3 + AND m.is_deleted = false + ) + `, [userId, accountId, delegatedFolder]); + return result.rowCount; +} + +export const DELEGATION_SELECT_SQL = 'delegation_meta.delegation AS delegation'; + +export function delegationJoinSql(messageAlias = 'm', accountAlias = 'a') { + if (![messageAlias, accountAlias].every(alias => /^[a-z][a-z0-9_]*$/i.test(alias))) { + throw new TypeError('Invalid SQL alias'); + } + return `LEFT JOIN LATERAL ( + SELECT jsonb_build_object( + 'contact_id', gd.contact_id, + 'display_name', COALESCE(dc.display_name, gd.contact_display_name_snapshot), + 'primary_email', COALESCE(dc.primary_email, gd.contact_primary_email_snapshot), + 'delegated_at', gd.delegated_at, + 'updated_at', gd.updated_at + ) AS delegation + FROM gtd_delegations gd + LEFT JOIN contacts dc ON dc.id = gd.contact_id AND dc.user_id = gd.user_id + WHERE gd.user_id = ${accountAlias}.user_id + AND gd.account_id = ${messageAlias}.account_id + AND gd.thread_key = ${messageAlias}.thread_key + ) delegation_meta ON TRUE`; +} + +async function loadOwnedTargets(userId, messageIds) { + const { rows } = await query(` + SELECT m.id, m.account_id, m.uid, m.folder, m.message_id, m.thread_key + FROM messages m + JOIN email_accounts a ON a.id = m.account_id + WHERE a.user_id = $1 AND m.id = ANY($2::uuid[]) AND m.is_deleted = false + `, [userId, messageIds]); + return rows; +} + +async function loadAccount(accountId, userId) { + const { rows } = await query( + 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2 AND enabled = true', + [accountId, userId], + ); + return rows[0] || null; +} + +async function liveDelegatedCopies(target, folder) { + const { rows } = await query(` + SELECT uid FROM messages + WHERE account_id = $1 AND thread_key = $2 AND folder = $3 AND is_deleted = false + ORDER BY date DESC NULLS LAST + `, [target.account_id, target.thread_key, folder]); + return rows; +} + +async function compensateCopy({ target, folder, copiedUid, imapManager }) { + try { + if (copiedUid == null) return false; + await imapManager.removeMessageCopy(target.account_id, copiedUid, folder); + return true; + } catch { + return false; + } +} + +const publicFailure = (messageId, code = 'operation_failed', compensated = false) => ({ + messageId, + ok: false, + error: { code, message: code === 'not_found' ? 'Message not found' : 'Delegation failed' }, + compensated, +}); + +export async function delegateMessages({ userId, messageIds, contactId, imapManager }) { + if (!Array.isArray(messageIds)) throw new GtdDelegationError('invalid_request', 400); + const inputIds = [...new Set(messageIds)]; + if (inputIds.length < 1 || inputIds.length > 100 || inputIds.some(id => !UUID_RE.test(id))) { + throw new GtdDelegationError('invalid_request', 400); + } + if (contactId !== null && !UUID_RE.test(contactId || '')) { + throw new GtdDelegationError('invalid_request', 400); + } + const contact = contactId === null ? null : await loadOwnedContactSnapshot(userId, contactId); + const rows = await loadOwnedTargets(userId, inputIds); + const byId = new Map(rows.map(row => [row.id, row])); + const outcomes = new Map(inputIds.filter(id => !byId.has(id)).map(id => [id, publicFailure(id, 'not_found')])); + const threads = new Map(); + for (const id of inputIds) { + const row = byId.get(id); + if (!row) continue; + if (!row.thread_key) { + outcomes.set(id, publicFailure(id)); + continue; + } + const key = `${row.account_id}\u0000${row.thread_key}`; + const item = threads.get(key) || { target: row, ids: [] }; + item.ids.push(id); + threads.set(key, item); + } + + for (const [threadKey, { target, ids }] of threads) { + await serializeDelegation(threadKey, async () => { + let copyAttempted = false; + let copiedUid = null; + let account; + let delegatedFolder = null; + try { + account = await loadAccount(target.account_id, userId); + if (!account) throw new Error('account unavailable'); + const config = await getGtdConfig(target.account_id); + delegatedFolder = config.enabled ? resolveGtdStateFolder('delegated', config.folders) : null; + if (!delegatedFolder) throw new Error('delegated folder unavailable'); + await imapManager.ensureFolder(account, delegatedFolder); + let existing = await liveDelegatedCopies(target, delegatedFolder); + if (!existing.length) { + await imapManager.syncFolderOnDemand(account, delegatedFolder); + existing = await liveDelegatedCopies(target, delegatedFolder); + } + if (!existing.length) { + // The remote COPY can succeed before a later local insert fails, so the attempt + // must be marked before awaiting it. The catch path then reconciles the folder + // and removes any copy whose outcome was ambiguous. + copyAttempted = true; + copiedUid = await imapManager.copyMessage( + target.account_id, target.uid, target.folder, delegatedFolder, + ); + if (copiedUid == null) { + // Non-UIDPLUS COPY is materialized by destination sync. Await the shared + // in-flight sync before releasing this thread's serializer, otherwise an + // immediate retry could issue a second remote COPY. + await imapManager.syncFolderOnDemand(account, delegatedFolder); + existing = await liveDelegatedCopies(target, delegatedFolder); + if (!existing.length) throw new Error('delegated copy did not reconcile'); + // Attribute a destination UID only when reconciliation found exactly one. + // Multiple copies can mean a concurrent external client also labeled the + // thread, and compensation must never guess which one belongs to this call. + if (existing.length === 1) copiedUid = existing[0].uid; + } + } + const delegation = contact + ? await upsertDelegation({ + userId, accountId: target.account_id, threadKey: target.thread_key, contact, + }) + : (await clearDelegations({ + userId, accountId: target.account_id, threadKeys: [target.thread_key], + }), null); + for (const messageId of ids) outcomes.set(messageId, { + messageId, + ok: true, + accountId: target.account_id, + threadKey: target.thread_key, + delegation, + }); + } catch (error) { + if (copiedUid == null && error?.copiedUid != null) copiedUid = error.copiedUid; + const compensated = copyAttempted && delegatedFolder && copiedUid != null + ? await compensateCopy({ target, folder: delegatedFolder, copiedUid, imapManager }) + : false; + const code = error instanceof GtdDelegationError ? error.code : 'operation_failed'; + for (const messageId of ids) outcomes.set(messageId, publicFailure(messageId, code, compensated)); + } + }); + } + + const results = inputIds.map(id => outcomes.get(id)); + const successCount = results.filter(result => result.ok).length; + const failureCount = results.length - successCount; + return { + status: failureCount === 0 ? 'success' : successCount === 0 ? 'failed' : 'partial', + successCount, + failureCount, + results, + }; +} diff --git a/backend/src/services/gtdDelegations.test.js b/backend/src/services/gtdDelegations.test.js new file mode 100644 index 00000000..a2fcc0e8 --- /dev/null +++ b/backend/src/services/gtdDelegations.test.js @@ -0,0 +1,224 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('./db.js', () => ({ query: vi.fn() })); +vi.mock('./gtdConfig.js', () => ({ + getGtdConfig: vi.fn(), + resolveGtdStateFolder: vi.fn((_state, folders) => folders.delegated), +})); + +import { query } from './db.js'; +import { getGtdConfig } from './gtdConfig.js'; +import { + delegateMessages, + delegationJoinSql, + loadOwnedContactSnapshot, + mapDelegationRow, + reconcileDelegatedRemovals, + sweepStaleDelegations, + upsertDelegation, +} from './gtdDelegations.js'; + +const USER = '11111111-1111-4111-8111-111111111111'; +const ACCOUNT = '22222222-2222-4222-8222-222222222222'; +const MESSAGE_A = '33333333-3333-4333-8333-333333333333'; +const MESSAGE_B = '44444444-4444-4444-8444-444444444444'; +const CONTACT = '55555555-5555-4555-8555-555555555555'; +const contact = { id: CONTACT, display_name: 'Casey Rivera', primary_email: 'casey@example.test' }; +const row = id => ({ + id, account_id: ACCOUNT, uid: id === MESSAGE_A ? 10 : 11, folder: 'INBOX', + message_id: `<${id}@example.test>`, thread_key: 'thread@example.test', +}); + +const imapManager = () => ({ + ensureFolder: vi.fn(), copyMessage: vi.fn().mockResolvedValue(77), + removeMessageCopy: vi.fn(), syncFolderOnDemand: vi.fn(), +}); + +function stubSuccess({ messages = [row(MESSAGE_A)], existing = false, insertError = null } = {}) { + query.mockImplementation(async sql => { + if (sql.includes('FROM contacts c')) return { rows: [contact] }; + if (sql.includes('FROM messages m') && sql.includes('ANY($2::uuid[])')) return { rows: messages }; + if (sql.startsWith('SELECT * FROM email_accounts')) return { rows: [{ id: ACCOUNT, user_id: USER }] }; + if (sql.includes('SELECT uid FROM messages')) return { rows: existing ? [{ uid: 88 }] : [] }; + if (sql.includes('INSERT INTO gtd_delegations')) { + if (insertError) throw insertError; + return { rows: [{ contact_id: CONTACT, display_name: contact.display_name, primary_email: contact.primary_email }] }; + } + if (sql.includes('DELETE FROM gtd_delegations')) return { rows: [], rowCount: 1 }; + return { rows: [], rowCount: 0 }; + }); +} + +beforeEach(() => { + query.mockReset(); + getGtdConfig.mockReset(); + getGtdConfig.mockResolvedValue({ enabled: true, folders: { delegated: 'Delegated' } }); +}); + +describe('delegation persistence primitives', () => { + it('rejects a contact owned by another user without exposing it', async () => { + query.mockResolvedValueOnce({ rows: [] }); + await expect(loadOwnedContactSnapshot(USER, CONTACT)) + .rejects.toMatchObject({ code: 'contact_not_found', status: 404 }); + }); + + it('resets the age anchor only when the contact changes', async () => { + query.mockResolvedValueOnce({ rows: [contact] }); + await upsertDelegation({ userId: USER, accountId: ACCOUNT, threadKey: 'thread', contact }); + expect(query.mock.calls[0][0]).toContain('contact_id IS DISTINCT FROM EXCLUDED.contact_id'); + expect(query.mock.calls[0][0]).toContain('THEN NOW()'); + expect(query.mock.calls[0][0]).toContain('updated_at = NOW()'); + }); + + it('maps JSON and validates reusable join aliases', () => { + expect(mapDelegationRow({ delegation: '{"contact_id":null}' })).toEqual({ contact_id: null }); + expect(delegationJoinSql('m', 'a')).toContain('gd.user_id = a.user_id'); + expect(() => delegationJoinSql('m;drop', 'a')).toThrow(TypeError); + }); +}); + +describe('delegateMessages', () => { + it('normalizes duplicate IDs and performs one copy for one logical thread', async () => { + stubSuccess({ messages: [row(MESSAGE_A), row(MESSAGE_B)] }); + const imap = imapManager(); + const result = await delegateMessages({ + userId: USER, messageIds: [MESSAGE_A, MESSAGE_A, MESSAGE_B], contactId: CONTACT, imapManager: imap, + }); + expect(result).toMatchObject({ status: 'success', successCount: 2, failureCount: 0 }); + expect(result.results.map(item => item.messageId)).toEqual([MESSAGE_A, MESSAGE_B]); + expect(imap.copyMessage).toHaveBeenCalledTimes(1); + }); + + it('clears a previous person when contactId is null while retaining the label', async () => { + stubSuccess({ existing: true }); + const result = await delegateMessages({ + userId: USER, messageIds: [MESSAGE_A], contactId: null, imapManager: imapManager(), + }); + expect(result.results[0]).toMatchObject({ ok: true, delegation: null }); + expect(query.mock.calls.some(([sql]) => sql.includes('DELETE FROM gtd_delegations'))).toBe(true); + }); + + it('removes a newly copied label when persistence fails', async () => { + stubSuccess({ insertError: new Error('write failed') }); + const imap = imapManager(); + const result = await delegateMessages({ + userId: USER, messageIds: [MESSAGE_A], contactId: CONTACT, imapManager: imap, + }); + expect(imap.removeMessageCopy).toHaveBeenCalledWith(ACCOUNT, 77, 'Delegated'); + expect(result.results[0]).toMatchObject({ ok: false, compensated: true }); + }); + + it('does not remove a pre-existing delegated label when persistence fails', async () => { + stubSuccess({ existing: true, insertError: new Error('write failed') }); + const imap = imapManager(); + await delegateMessages({ userId: USER, messageIds: [MESSAGE_A], contactId: CONTACT, imapManager: imap }); + expect(imap.removeMessageCopy).not.toHaveBeenCalled(); + }); + + it('serializes simultaneous requests for one thread so only one remote copy is made', async () => { + let copied = false; + query.mockImplementation(async sql => { + if (sql.includes('FROM contacts c')) return { rows: [contact] }; + if (sql.includes('FROM messages m') && sql.includes('ANY($2::uuid[])')) return { rows: [row(MESSAGE_A)] }; + if (sql.startsWith('SELECT * FROM email_accounts')) return { rows: [{ id: ACCOUNT, user_id: USER }] }; + if (sql.includes('SELECT uid FROM messages')) return { rows: copied ? [{ uid: 77 }] : [] }; + if (sql.includes('INSERT INTO gtd_delegations')) return { rows: [contact] }; + return { rows: [], rowCount: 0 }; + }); + const imap = imapManager(); + imap.copyMessage.mockImplementation(async () => { copied = true; return 77; }); + + const [first, retry] = await Promise.all([ + delegateMessages({ userId: USER, messageIds: [MESSAGE_A], contactId: CONTACT, imapManager: imap }), + delegateMessages({ userId: USER, messageIds: [MESSAGE_A], contactId: CONTACT, imapManager: imap }), + ]); + + expect(first.status).toBe('success'); + expect(retry.status).toBe('success'); + expect(imap.copyMessage).toHaveBeenCalledTimes(1); + }); + + it('awaits non-UIDPLUS destination reconciliation before reporting success', async () => { + let syncCount = 0; + query.mockImplementation(async sql => { + if (sql.includes('FROM contacts c')) return { rows: [contact] }; + if (sql.includes('FROM messages m') && sql.includes('ANY($2::uuid[])')) return { rows: [row(MESSAGE_A)] }; + if (sql.startsWith('SELECT * FROM email_accounts')) return { rows: [{ id: ACCOUNT, user_id: USER }] }; + if (sql.includes('SELECT uid FROM messages')) return { rows: syncCount >= 2 ? [{ uid: 91 }] : [] }; + if (sql.includes('INSERT INTO gtd_delegations')) return { rows: [contact] }; + return { rows: [], rowCount: 0 }; + }); + const imap = imapManager(); + imap.copyMessage.mockResolvedValue(null); + imap.syncFolderOnDemand.mockImplementation(async () => { syncCount += 1; }); + + const result = await delegateMessages({ + userId: USER, messageIds: [MESSAGE_A], contactId: CONTACT, imapManager: imap, + }); + + expect(result.status).toBe('success'); + expect(imap.syncFolderOnDemand).toHaveBeenCalledTimes(2); + }); + + it('compensates the exact UID when COPY succeeds remotely but local completion throws', async () => { + query.mockImplementation(async sql => { + if (sql.includes('FROM contacts c')) return { rows: [contact] }; + if (sql.includes('FROM messages m') && sql.includes('ANY($2::uuid[])')) return { rows: [row(MESSAGE_A)] }; + if (sql.startsWith('SELECT * FROM email_accounts')) return { rows: [{ id: ACCOUNT, user_id: USER }] }; + if (sql.includes('SELECT uid FROM messages')) return { rows: [] }; + return { rows: [], rowCount: 0 }; + }); + const imap = imapManager(); + imap.copyMessage.mockImplementation(async () => { + const error = new Error('local sibling insert failed'); + error.copiedUid = 93; + throw error; + }); + + const result = await delegateMessages({ + userId: USER, messageIds: [MESSAGE_A], contactId: CONTACT, imapManager: imap, + }); + + expect(imap.removeMessageCopy).toHaveBeenCalledWith(ACCOUNT, 93, 'Delegated'); + expect(result.results[0]).toMatchObject({ ok: false, compensated: true }); + }); + + it('does not guess at compensation when an ambiguous COPY failure has no exact UID', async () => { + stubSuccess(); + const imap = imapManager(); + imap.copyMessage.mockRejectedValue(new Error('connection lost after COPY')); + + const result = await delegateMessages({ + userId: USER, messageIds: [MESSAGE_A], contactId: CONTACT, imapManager: imap, + }); + + expect(imap.removeMessageCopy).not.toHaveBeenCalled(); + expect(result.results[0]).toMatchObject({ ok: false, compensated: false }); + }); + + it('rejects more than 100 IDs before querying', async () => { + const ids = Array.from({ length: 101 }, (_, index) => `${String(index).padStart(8, '0')}-0000-4000-8000-000000000000`); + await expect(delegateMessages({ userId: USER, messageIds: ids, contactId: null, imapManager: imapManager() })) + .rejects.toMatchObject({ code: 'invalid_request', status: 400 }); + expect(query).not.toHaveBeenCalled(); + }); +}); + +it('reconciles only delegation rows without a surviving folder copy', async () => { + query.mockResolvedValueOnce({ rowCount: 2, rows: [] }); + await expect(reconcileDelegatedRemovals({ + userId: USER, accountId: ACCOUNT, delegatedFolder: 'Delegated', threadKeys: ['a', 'a', 'b'], + })).resolves.toBe(2); + expect(query.mock.calls[0][1]).toEqual([USER, ACCOUNT, ['a', 'b'], 'Delegated']); + expect(query.mock.calls[0][0]).toContain('NOT EXISTS'); +}); + +it('sweeps stale rows even when the original removed thread is no longer available', async () => { + query.mockResolvedValueOnce({ rowCount: 3, rows: [] }); + await expect(sweepStaleDelegations({ + userId: USER, accountId: ACCOUNT, delegatedFolder: 'Delegated', + })).resolves.toBe(3); + expect(query.mock.calls[0][1]).toEqual([USER, ACCOUNT, 'Delegated']); + expect(query.mock.calls[0][0]).toContain('NOT EXISTS'); + expect(query.mock.calls[0][0]).not.toContain('ANY('); +}); diff --git a/backend/src/services/gtdSections.js b/backend/src/services/gtdSections.js index 31ce41e9..d947f069 100644 --- a/backend/src/services/gtdSections.js +++ b/backend/src/services/gtdSections.js @@ -1,6 +1,7 @@ import { query } from './db.js'; import { getGtdConfig, GTD_STATES } from './gtdConfig.js'; import { resolveAllDraftsPaths } from '../utils/mailUtils.js'; +import { DELEGATION_SELECT_SQL, delegationJoinSql, mapDelegationRow } from './gtdDelegations.js'; // States the frontend merges into the single "Waiting" section (utils/gtd.js). Their // counts must dedupe a thread holding BOTH labels; see the waiting_agg CTE below. @@ -34,8 +35,11 @@ const SECTION_SQL = ` ), msg AS ( SELECT m.id, m.account_id, m.thread_key, m.message_id, m.folder, - m.subject, m.from_name, m.from_email, m.date, m.snippet, m.is_read, m.is_starred, m.uid, m.gtd_gist + m.subject, m.from_name, m.from_email, m.date, m.snippet, m.is_read, m.is_starred, m.uid, m.gtd_gist, + ${DELEGATION_SELECT_SQL} FROM messages m + JOIN email_accounts a ON a.id = m.account_id + ${delegationJoinSql('m', 'a')} WHERE m.account_id = $1 AND m.is_deleted = false AND m.folder <> ALL($4::text[]) @@ -51,7 +55,7 @@ const SECTION_SQL = ` head AS ( SELECT DISTINCT ON (account_id, thread_key) thread_key, account_id, message_id, folder, - subject, from_name, from_email, date, snippet, is_starred, uid, id, gtd_gist + subject, from_name, from_email, date, snippet, is_starred, uid, id, gtd_gist, delegation FROM msg -- Prefer a row that lives in a GTD label folder: that copy's id is stable for as long -- as the thread is in a section, whereas a transient INBOX copy (archived/purged out @@ -85,7 +89,7 @@ const SECTION_SQL = ` ranked AS ( SELECT ts.state, h.thread_key, h.account_id, h.message_id, h.folder, - h.subject, h.from_name, h.from_email, h.date, h.snippet, h.is_starred, h.uid, h.id, h.gtd_gist, + h.subject, h.from_name, h.from_email, h.date, h.snippet, h.is_starred, h.uid, h.id, h.gtd_gist, h.delegation, fa.folders, fa.in_inbox, fa.thread_unread, COUNT(*) OVER (PARTITION BY ts.state) AS total, COUNT(*) FILTER (WHERE fa.thread_unread) OVER (PARTITION BY ts.state) AS unread, @@ -95,7 +99,7 @@ const SECTION_SQL = ` JOIN folders_agg fa ON fa.thread_key = ts.thread_key ) SELECT state, thread_key, account_id, message_id, folder, - subject, from_name, from_email, date, snippet, is_starred, uid, id, gtd_gist, + subject, from_name, from_email, date, snippet, is_starred, uid, id, gtd_gist, delegation, folders, in_inbox, thread_unread, total::int AS total, unread::int AS unread, waiting_total::int AS waiting_total, waiting_unread::int AS waiting_unread FROM ranked @@ -133,6 +137,7 @@ function mapHead(row) { // AI-condensed one-line gist for waiting rows, when cached on this head. // Null until lazily generated; the client falls back to the raw snippet. gist: row.gtd_gist || null, + delegation: mapDelegationRow(row), }; } diff --git a/backend/src/services/gtdSections.test.js b/backend/src/services/gtdSections.test.js index cd5dd2e1..655d68d1 100644 --- a/backend/src/services/gtdSections.test.js +++ b/backend/src/services/gtdSections.test.js @@ -82,6 +82,19 @@ describe('getGtdSections — account resolution', () => { }); describe('getGtdSections — section folding', () => { + it('projects normalized delegation metadata and snapshot fallbacks', async () => { + const delegation = { + contact_id: null, display_name: 'Casey Rivera', primary_email: 'casey@example.test', + delegated_at: '2026-07-01T12:00:00.000Z', updated_at: '2026-07-02T12:00:00.000Z', + }; + query + .mockResolvedValueOnce({ rows: [{ id: 'acc-1', folder_mappings: null }] }) + .mockResolvedValueOnce({ rows: [headRow({ state: 'delegated', delegation: JSON.stringify(delegation) })] }); + const { sections } = await getGtdSections({ userId: 'u1' }); + expect(sections.delegated.threads[0].delegation).toEqual(delegation); + expect(query.mock.calls[1][0]).toContain('gtd_delegations'); + }); + it('places a multi-folder thread in every state section it belongs to, once each, preserving in_inbox', async () => { query .mockResolvedValueOnce({ rows: [{ id: 'acc-1', folder_mappings: null }] }) // accounts diff --git a/backend/src/services/gtdTransitions.js b/backend/src/services/gtdTransitions.js index e16f7451..81b89ca1 100644 --- a/backend/src/services/gtdTransitions.js +++ b/backend/src/services/gtdTransitions.js @@ -2,6 +2,7 @@ import { query } from './db.js'; import { getGtdConfig } from './gtdConfig.js'; import { resolveAllDraftsPaths } from '../utils/mailUtils.js'; import { logger } from './logger.js'; +import { reconcileDelegatedRemovals } from './gtdDelegations.js'; // Transition rules for auto-stripping a GTD label once a thread's state has moved on, // evaluated per thread against its LAST non-draft message. Designed to match the @@ -148,7 +149,7 @@ export async function runGtdTransitions(imapManager, account, threadKeys) { let anyStripped = false; - for (const [, threadRows] of byThread) { + for (const [threadKey, threadRows] of byThread) { const nonDraft = threadRows.filter((r) => !draftPaths.has(r.folder)); if (nonDraft.length === 0) continue; @@ -160,6 +161,8 @@ export async function runGtdTransitions(imapManager, account, threadKeys) { if (diff > 0 || (diff === 0 && String(r.id) > String(newest.id))) newest = r; } const isSelf = owner.has(normalizeAddress(newest.from_email)); + let delegatedRemoved = false; + let delegatedRemovalFailed = false; for (const [state, folder] of Object.entries(stateFolder)) { const rule = STRIP_RULE[state]; @@ -170,7 +173,9 @@ export async function runGtdTransitions(imapManager, account, threadKeys) { anyStripped = true; try { await imapManager.removeMessageCopy(account.id, copy.uid, copy.folder); + if (state === 'delegated') delegatedRemoved = true; } catch (err) { + if (state === 'delegated') delegatedRemovalFailed = true; // An external automation may strip the same label concurrently, so the copy // can already be gone on the server. Treat a failed removal as a successful // strip and move on; the stale DB row reconciles on the next sync. @@ -178,6 +183,12 @@ export async function runGtdTransitions(imapManager, account, threadKeys) { } } } + if (delegatedRemoved && !delegatedRemovalFailed) { + await reconcileDelegatedRemovals({ + userId: account.user_id, accountId: account.id, + delegatedFolder: stateFolder.delegated, threadKeys: [threadKey], + }); + } } // One batched emit per run (not per stripped copy) so the rail converges once. diff --git a/backend/src/services/gtdTransitions.test.js b/backend/src/services/gtdTransitions.test.js index eabed8e9..49604a18 100644 --- a/backend/src/services/gtdTransitions.test.js +++ b/backend/src/services/gtdTransitions.test.js @@ -4,6 +4,7 @@ vi.mock('./db.js', () => ({ query: vi.fn() })); vi.mock('./gtdConfig.js', () => ({ getGtdConfig: vi.fn() })); vi.mock('../utils/mailUtils.js', () => ({ resolveAllDraftsPaths: vi.fn() })); vi.mock('./logger.js', () => ({ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); +vi.mock('./gtdDelegations.js', () => ({ reconcileDelegatedRemovals: vi.fn() })); import { getOwnerAddresses, @@ -16,6 +17,7 @@ import { import { query } from './db.js'; import { getGtdConfig } from './gtdConfig.js'; import { resolveAllDraftsPaths } from '../utils/mailUtils.js'; +import { reconcileDelegatedRemovals } from './gtdDelegations.js'; const DEFAULT_FOLDERS = { todo: 'Todo', watch: 'Watch', delegated: 'Delegated', someday: 'Someday', reference: 'Reference' }; const account = { id: 'acct-1', user_id: 'user-1', email_address: 'me@example.com', folder_mappings: {} }; @@ -89,6 +91,7 @@ describe('runGtdTransitions', () => { query.mockReset(); getGtdConfig.mockReset(); resolveAllDraftsPaths.mockReset(); + reconcileDelegatedRemovals.mockReset(); invalidateOwnerAddressesCache('acct-1'); getGtdConfig.mockResolvedValue({ enabled: true, folders: DEFAULT_FOLDERS }); resolveAllDraftsPaths.mockResolvedValue(new Set(['Drafts'])); @@ -119,6 +122,21 @@ describe('runGtdTransitions', () => { expect(mgr.removeMessageCopy).toHaveBeenCalledWith('acct-1', 22, 'Watch'); expect(mgr.removeMessageCopy).toHaveBeenCalledWith('acct-1', 23, 'Delegated'); expect(mgr.removeMessageCopy).not.toHaveBeenCalledWith('acct-1', 21, 'Todo'); + expect(reconcileDelegatedRemovals).toHaveBeenCalledWith({ + userId: 'user-1', accountId: 'acct-1', + delegatedFolder: 'Delegated', threadKeys: ['t1'], + }); + }); + + it('retains delegation metadata when the Delegated IMAP removal fails', async () => { + mockQuery({ rows: [ + { thread_key: 't1', uid: 20, folder: 'INBOX', from_email: 'them@other.com', date: '2026-07-09T12:00:00Z', id: 'r1' }, + { thread_key: 't1', uid: 23, folder: 'Delegated', from_email: 'them@other.com', date: '2026-07-09T12:00:00Z', id: 'r2' }, + ] }); + const mgr = fakeManager(); + mgr.removeMessageCopy.mockRejectedValue(new Error('remove failed')); + await runGtdTransitions(mgr, account, ['t1']); + expect(reconcileDelegatedRemovals).not.toHaveBeenCalled(); }); it('treats an alias sender as the owner (self-strips Todo)', async () => { diff --git a/backend/src/services/imapManager.js b/backend/src/services/imapManager.js index 433ae55a..52687b0f 100644 --- a/backend/src/services/imapManager.js +++ b/backend/src/services/imapManager.js @@ -16,6 +16,8 @@ import { getConnectionPolicy } from './connectionPolicy.js'; import { applyInboxRules, applyBlockList } from './inboxRules.js'; import { generateVCard } from '../utils/vcard.js'; import { randomUUID } from 'crypto'; +import { reconcileDelegatedRemovals, sweepStaleDelegations } from './gtdDelegations.js'; +import { restoreSnoozedRow } from './mailbox/snooze.js'; // Shorthand for log lines — keeps domain visible while masking the local part. @@ -526,6 +528,10 @@ function safeDate(d) { // flagPollEveryTicks: for non-push flag providers, poll flags every N successful sync ticks. // snippetIndex: run the background snippet indexer after backfill. // Disabled for providers that throttle body fetches too aggressively. +// bodyBackfill: run the background body-materialization drainer +// (services/bodyBackfill.js) for this provider. Enabled only for +// providers that tolerate sustained background BODY[] fetches; +// Gmail/PurelyMail/Microsoft stay off in v1 (unproven / throttle-hostile). // skipFolderPatterns: folder path substrings to skip during backfill (label-view dedup). // skipFolderNames: exact folder paths to skip (non-selectable namespace containers). // batchSize/Delay/errorDelay/batchesPerConn: backfill rate-limit tuning. @@ -540,6 +546,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: false, snippetIndex: false, + bodyBackfill: false, speculativeFetch: false, skipFolderPatterns: ['all mail', '[gmail]/starred', '[gmail]/important'], // [Gmail] is a namespace container — not a selectable mailbox. It must be @@ -551,6 +558,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: true, speculativeFetch: false, skipFolderPatterns: [], skipFolderNames: [], @@ -561,6 +569,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: true, speculativeFetch: true, skipFolderPatterns: [], skipFolderNames: [], @@ -570,6 +579,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: false, speculativeFetch: true, skipFolderPatterns: [], skipFolderNames: [], @@ -601,6 +611,7 @@ const PROVIDERS = { idleKeepaliveMs: 4 * 60 * 1000, // re-issue IDLE every 4 min (Apple Mail-style) so the connection never goes deaf pushesFlags: false, // IDLE 'flags' handles most changes; keep the periodic flag poll as a backstop snippetIndex: false, + bodyBackfill: false, speculativeFetch: false, preferFreshBodyFetch: true, freshInboxSync: false, // IDLE push + backstop poll on the persistent connection replaces fresh-login-per-tick @@ -619,6 +630,7 @@ const PROVIDERS = { fetchBody: false, pushesFlags: true, snippetIndex: true, + bodyBackfill: true, speculativeFetch: true, skipFolderPatterns: [], skipFolderNames: [], @@ -764,6 +776,13 @@ export async function emitGtdSectionsRefreshIfEnabled(mgr, account, changedCount // is identical either way. export const emitGtdSectionsRefreshOnDelete = emitGtdSectionsRefreshIfEnabled; +export function delegationSnapshotIsComplete(serverUids, localRows) { + const localUids = new Set(localRows.map(row => Number(row.uid))); + const normalizedServerUids = new Set([...serverUids].map(Number)); + return localUids.size === normalizedServerUids.size + && [...normalizedServerUids].every(uid => localUids.has(uid)); +} + // One GTD tick's body: sync each designated label folder for a connected gtd_enabled // account, then broadcast a single gtd_sections_updated if any folder actually changed. // Folders are synced one at a time (not in parallel) so a multi-folder account doesn't @@ -788,15 +807,23 @@ export async function runGtdSyncTick(mgr, account) { const key = `${account.id}:${folder}`; if (mgr.onDemandSyncing.has(key)) continue; // a user-triggered sync owns this folder mgr.onDemandSyncing.add(key); - try { + mgr.onDemandSyncPromises ||= new Map(); + const syncPromise = (async () => { const before = await mgr._gtdFolderFingerprint(account.id, folder); await mgr._gtdSyncFolder(account, folder); const after = await mgr._gtdFolderFingerprint(account.id, folder); - if (before !== after) changedFolders.push(folder); + return before !== after; + })(); + mgr.onDemandSyncPromises.set(key, syncPromise); + try { + if (await syncPromise) changedFolders.push(folder); } catch (err) { console.warn(`GTD sync error ${logAccount(account)}/${folder}:`, err.message); } finally { mgr.onDemandSyncing.delete(key); + if (mgr.onDemandSyncPromises.get(key) === syncPromise) { + mgr.onDemandSyncPromises.delete(key); + } } } @@ -1309,6 +1336,7 @@ export class ImapManager { this._backfillSem = createKeyedSemaphore(BACKFILL_MAX_PER_HOST); // cap concurrent backfills per provider host this._connectCooldown = new Map(); // accountId -> { until: ms, failures: number } after connection refusals this.onDemandSyncing = new Set(); // `${accountId}:${folder}` — prevent duplicate on-demand syncs + this.onDemandSyncPromises = new Map(); // same key -> awaitable user-triggered sync this.syncingAccounts = new Set(); // prevent overlapping interval syncs this.syncStartedAt = new Map(); // accountId -> ms when the current sync tick began (hung-sync detection) this.syncThrottleSkips = new Map(); // accountId -> remaining ticks to skip when throttled @@ -3783,6 +3811,7 @@ export class ImapManager { to = [], cc = [], inReplyTo = null, + references = null, snippet = '', bodyHtml = null, bodyText = null, @@ -3794,9 +3823,9 @@ export class ImapManager { INSERT INTO messages ( account_id, uid, folder, message_id, subject, from_name, from_email, to_addresses, cc_addresses, - in_reply_to, date, snippet, is_read, is_starred, has_attachments, + in_reply_to, thread_references, date, snippet, is_read, is_starred, has_attachments, flags, body_html, body_text, thread_id - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb,$9::jsonb,$10,$11,$12,true,false,false,$13::jsonb,$14,$15,$16) + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb,$9::jsonb,$10,$11,$12,$13,true,false,false,$14::jsonb,$15,$16,$17) ON CONFLICT (account_id, uid, folder) DO UPDATE SET message_id = COALESCE(EXCLUDED.message_id, messages.message_id), subject = CASE @@ -3811,6 +3840,7 @@ export class ImapManager { WHEN EXCLUDED.cc_addresses::text IS NOT NULL AND EXCLUDED.cc_addresses::text <> '[]' THEN EXCLUDED.cc_addresses ELSE messages.cc_addresses END, in_reply_to = COALESCE(EXCLUDED.in_reply_to, messages.in_reply_to), + thread_references = COALESCE(EXCLUDED.thread_references, messages.thread_references), date = EXCLUDED.date, snippet = CASE WHEN EXCLUDED.snippet <> '' THEN EXCLUDED.snippet ELSE messages.snippet END, flags = EXCLUDED.flags, @@ -3821,7 +3851,7 @@ export class ImapManager { sanitizeStr(subject || '(no subject)'), sanitizeStr(fromName || ''), sanitizeStr(fromEmail || ''), JSON.stringify(Array.isArray(to) ? to : []), JSON.stringify(Array.isArray(cc) ? cc : []), - inReplyTo || null, safeDate(date), sanitizeStr(snippet || ''), + inReplyTo || null, references || null, safeDate(date), sanitizeStr(snippet || ''), JSON.stringify(['\\Draft', '\\Seen']), bodyHtml != null ? sanitizeStr(bodyHtml) : null, bodyText != null ? sanitizeStr(bodyText) : null, @@ -3850,30 +3880,63 @@ export class ImapManager { // Uses a pooled connection — does NOT touch the main sync connection. async syncFolderOnDemand(account, folder) { const key = `${account.id}:${folder}`; + this.onDemandSyncPromises ||= new Map(); + const inFlight = this.onDemandSyncPromises.get(key); + if (inFlight) return inFlight; if (this.onDemandSyncing.has(key)) { console.log(`syncFolderOnDemand skipped (already running): ${logAccount(account)}/${folder}`); return; } this.onDemandSyncing.add(key); - console.log(`syncFolderOnDemand start: ${logAccount(account)}/${folder}`); - try { - await withFreshClient(account, async (client) => { - await this.syncMessages(account, client, folder, 100, false, true); - }); - console.log(`syncFolderOnDemand done: ${logAccount(account)}/${folder}`); - // sync_complete fires mailflow:refresh in the frontend, reloading the message list - this.broadcast({ type: 'sync_complete', accountId: account.id }, account.user_id); - } catch (err) { - console.error(`On-demand sync error ${logAccount(account)}/${folder}:`, err.message); - } finally { - this.onDemandSyncing.delete(key); - } + const syncPromise = (async () => { + console.log(`syncFolderOnDemand start: ${logAccount(account)}/${folder}`); + try { + await withFreshClient(account, async (client) => { + await this.syncMessages(account, client, folder, 100, false, true); + }); + console.log(`syncFolderOnDemand done: ${logAccount(account)}/${folder}`); + // sync_complete fires mailflow:refresh in the frontend, reloading the message list + this.broadcast({ type: 'sync_complete', accountId: account.id }, account.user_id); + } catch (err) { + console.error(`On-demand sync error ${logAccount(account)}/${folder}:`, err.message); + throw err; + } finally { + this.onDemandSyncing.delete(key); + this.onDemandSyncPromises.delete(key); + } + })(); + this.onDemandSyncPromises.set(key, syncPromise); + return syncPromise; } // Pre-fetch and cache the body for newly arrived messages immediately after sync. // Called in the background (via setImmediate) so it doesn't block the sync path. // By the time the user clicks the email (typically 2–10s later), the body is already // in the DB and the click returns instantly without a live IMAP round-trip. + // Shared fetch+sanitize+snippet+UPDATE step behind every body-materialization path + // (opportunistic new-mail prefetch, on-view folder prefetch, and the body-backfill + // drainer). Kept as one helper so the UPDATE shape that fires the search_fts/last_modified + // triggers never drifts between callers. Returns true if a body was actually written. + // Private (underscore-prefixed, matching this file's convention for internal helpers like + // _enqueueFlagPush) — not new public surface, so it doesn't count against the + // one-narrow-method invariant that governs fetchBodiesForMessages. + async _fetchAndStoreBody(account, msg) { + const { html, text, attachments } = await this.fetchMessageBody( + account, msg.uid, msg.folder || 'INBOX' + ); + const safeHtml = html ? sanitizeEmail(html) : null; + if (!safeHtml && !text) return false; + const snip = snippetFromBody(text, safeHtml || html); + await query( + `UPDATE messages + SET body_html = $1, body_text = $2, attachments = $3, + snippet = CASE WHEN $5 != '' THEN $5 ELSE snippet END + WHERE id = $4`, + [sanitizeStr(safeHtml), sanitizeStr(text), JSON.stringify(attachments || []), msg.id, sanitizeStr(snip)] + ); + return true; + } + async prefetchNewMessageBodies(account, messages) { for (const msg of messages) { try { @@ -3884,26 +3947,58 @@ export class ImapManager { ); if (existing.rows.length) continue; - const { html, text, attachments } = await this.fetchMessageBody( - account, msg.uid, msg.folder || 'INBOX' - ); - const safeHtml = html ? sanitizeEmail(html) : null; - if (safeHtml || text) { - const snip = snippetFromBody(text, safeHtml || html); - await query( - `UPDATE messages - SET body_html = $1, body_text = $2, attachments = $3, - snippet = CASE WHEN $5 != '' THEN $5 ELSE snippet END - WHERE id = $4`, - [sanitizeStr(safeHtml), sanitizeStr(text), JSON.stringify(attachments || []), msg.id, sanitizeStr(snip)] - ); - } + await this._fetchAndStoreBody(account, msg); } catch (err) { console.warn(`Body prefetch failed for uid ${msg.uid}:`, err.message); } } } + // Narrow entry point for the body-materialization drainer (services/bodyBackfill.js). + // Given a set of message IDs, fetch each body over IMAP and persist it with the same + // UPDATE the prefetch path uses. This is the ONLY method the drainer calls into the + // singleton — all batching, pacing, provider gating, and progress live in bodyBackfill.js + // (imapManager invariant, specs/search-overhaul/README.md). Returns the number of bodies + // actually written. + // + // A single poison message (deleted/renamed folder, a per-message server-side rejection) + // must not wedge the whole batch — bodyBackfill.js's keyset cursor only advances past ids + // this method actually processed, and its circuit breaker only trips on a batch that made + // zero progress. So each message's fetch+store is individually caught and skipped here; + // only when the WHOLE batch fails (zero successes and at least one error — almost always a + // dead connection, not a bad message) do we rethrow, so the drainer's breaker can still back + // a genuinely broken account off. + async fetchBodiesForMessages(accountId, messageIds) { + if (!messageIds.length) return { fetched: 0 }; + + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); + if (!accountResult.rows.length) return { fetched: 0 }; + const account = accountResult.rows[0]; + + // Only touch rows that still lack a body — a concurrent open/prefetch may have filled + // some since the drainer selected them. + const rowsResult = await query( + `SELECT id, uid, folder FROM messages + WHERE id = ANY($1::uuid[]) AND body_html IS NULL AND body_text IS NULL`, + [messageIds] + ); + + let fetched = 0; + let errors = 0; + let lastError = null; + for (const msg of rowsResult.rows) { + try { + if (await this._fetchAndStoreBody(account, msg)) fetched++; + } catch (err) { + errors++; + lastError = err; + console.warn(`Body backfill: skipped message ${msg.id} (uid ${msg.uid}):`, err.message); + } + } + if (fetched === 0 && errors > 0) throw lastError; + return { fetched }; + } + // Background body prefetch for messages currently visible in a folder. // Called after GET /messages responds so the user gets a fast first impression // without waiting for this work. Respects the quiet window — pauses between @@ -3937,18 +4032,7 @@ export class ImapManager { ); if (existing.rows.length) continue; - const { html, text, attachments } = await this.fetchMessageBody(account, msg.uid, msg.folder); - const safeHtml = html ? sanitizeEmail(html) : null; - if (safeHtml || text) { - const snip = snippetFromBody(text, safeHtml || html); - await query( - `UPDATE messages - SET body_html = $1, body_text = $2, attachments = $3, - snippet = CASE WHEN $5 != '' THEN $5 ELSE snippet END - WHERE id = $4`, - [sanitizeStr(safeHtml), sanitizeStr(text), JSON.stringify(attachments || []), msg.id, sanitizeStr(snip)] - ); - } + await this._fetchAndStoreBody(account, msg); } catch (err) { console.warn(`Folder body prefetch failed for uid ${msg.uid}:`, err.message); } @@ -4464,7 +4548,14 @@ export class ImapManager { return null; } - await insertCopiedSibling(accountId, uid, fromFolder, toFolder, newUid); + try { + await insertCopiedSibling(accountId, uid, fromFolder, toFolder, newUid); + } catch (err) { + // The remote UIDPLUS COPY already succeeded. Preserve its exact destination UID so + // the caller can compensate without guessing among same-thread copies. + if (err && typeof err === 'object') err.copiedUid = newUid; + throw err; + } return newUid; } @@ -4808,69 +4899,16 @@ export class ImapManager { WHERE sm.snooze_until <= NOW() `); + const restoreImapManager = Object.assign(Object.create(this), { + _withFreshClient: withFreshClient, + }); + for (const row of due.rows) { try { - const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [row.account_id]); - if (!accountResult.rows.length) continue; - const account = accountResult.rows[0]; - - // Guard source UID before the IMAP move so reconcileDeletes cannot delete - // the DB row if an EXPUNGE arrives from the Snoozed folder while the move - // is in flight. - this._guardMoveUid(row.account_id, row.snoozed_folder, row.uid); - let newUid; - try { - // Move back to original folder - newUid = await this.moveMessageGetNewUid( - account, row.uid, row.snoozed_folder, row.original_folder - ); - - // Mark as unread so the user notices it - if (newUid) { - await this.setFlag(account, newUid, row.original_folder, '\\Seen', false); - } else if (row.message_id_header) { - // No UIDPLUS — server moved the message but returned no UID map. - // Search the destination folder by Message-ID to locate and unflag \Seen. - try { - await withFreshClient(account, async (client) => { - const lock = await client.getMailboxLock(row.original_folder); - try { - const uids = await client.search({ header: ['Message-ID', row.message_id_header] }, { uid: true }); - if (uids.length > 0) { - const r = await client.messageFlagsRemove(String(uids[0]), ['\\Seen'], { uid: true }); - if (r === false) console.warn(`Snooze wakeup: messageFlagsRemove returned false for ${row.original_folder}`); - } else { - console.warn(`Snooze wakeup: could not find message in ${row.original_folder} to mark unread (Message-ID: ${row.message_id_header})`); - } - } finally { - lock.release(); - } - }); - } catch (err) { - console.warn(`Snooze wakeup: could not mark message unread on server (no UIDPLUS): ${err.message}`); - } - } - - // Update DB: change folder, mark unread, and update UID if the move returned one. - if (newUid != null) { - await query( - 'UPDATE messages SET folder = $1, is_read = false, read_changed_at = NOW(), uid = $4 WHERE account_id = $2 AND message_id = $3 AND folder = $5', - [row.original_folder, row.account_id, row.message_id_header, newUid, row.snoozed_folder] - ); - } else { - // Non-UIDPLUS: DB holds the stale source UID at the destination. Guard it so - // reconcileDeletes does not treat it as an orphan before the next sync corrects it. - this._guardMoveUid(row.account_id, row.original_folder, row.uid); - await query( - 'UPDATE messages SET folder = $1, is_read = false, read_changed_at = NOW() WHERE account_id = $2 AND message_id = $3 AND folder = $4', - [row.original_folder, row.account_id, row.message_id_header, row.snoozed_folder] - ); - setTimeout(() => this._unguardMoveUid(row.account_id, row.original_folder, row.uid), 10_000); - } - } finally { - this._unguardMoveUid(row.account_id, row.snoozed_folder, row.uid); - } - + const restored = await restoreSnoozedRow(restoreImapManager, row, { + markUnread: true, + }); + if (!restored.restored) continue; // Remove snooze record await query('DELETE FROM snoozed_messages WHERE id = $1', [row.snooze_id]); @@ -4951,9 +4989,22 @@ export class ImapManager { 'SELECT DISTINCT folder FROM messages WHERE account_id = $1', [account.id] ); - if (!folderResult.rows.length) return; - - const folders = folderResult.rows.map(r => r.folder); + let delegatedFolder; + try { + const { folders: gtdFolders } = await getGtdConfig(account.id); + delegatedFolder = gtdFolders.delegated || null; + } catch (err) { + logger.debug(`Reconcile: delegated cleanup config unavailable for ${account.id}: ${err.message}`); + return; + } + // Always include the configured Delegated folder, even when it currently has no + // local rows. A successful empty server snapshot is the authority needed to clean + // metadata whose last local message row disappeared in an earlier failed pass. + const folders = [...new Set([ + ...folderResult.rows.map(r => r.folder), + ...(delegatedFolder ? [delegatedFolder] : []), + ])]; + if (!folders.length) return; // Phase 1 — fetch server UID sets for each folder (IMAP only, inside withFreshClient). const serverUidsByFolder = new Map(); // folder -> Set @@ -4984,14 +5035,17 @@ export class ImapManager { // Phase 2 — diff each folder's server UIDs against the DB and delete orphans. // Runs outside withFreshClient so DB errors never cause unnecessary pool eviction. let deletedCount = 0; + const delegatedThreadKeys = []; for (const [folder, serverUidSet] of serverUidsByFolder) { const dbResult = await query( - 'SELECT uid FROM messages WHERE account_id = $1 AND folder = $2 AND (synced_at IS NULL OR synced_at < $3)', + 'SELECT uid, thread_key FROM messages WHERE account_id = $1 AND folder = $2 AND (synced_at IS NULL OR synced_at < $3)', [account.id, folder, reconcileStartedAt] ); - const orphanUids = dbResult.rows - .map(r => Number(r.uid)) - .filter(uid => !serverUidSet.has(uid) && !this._isMoveUidGuarded(account.id, folder, uid)); + const orphanRows = dbResult.rows.filter(row => { + const uid = Number(row.uid); + return !serverUidSet.has(uid) && !this._isMoveUidGuarded(account.id, folder, uid); + }); + const orphanUids = orphanRows.map(row => Number(row.uid)); if (orphanUids.length === 0) continue; @@ -5013,6 +5067,39 @@ export class ImapManager { [account.id, folder] ); deletedCount += orphanUids.length; + if (folder === delegatedFolder) { + delegatedThreadKeys.push(...orphanRows.map(row => row.thread_key).filter(Boolean)); + } + } + + if (delegatedThreadKeys.length) { + await reconcileDelegatedRemovals({ + userId: account.user_id, + accountId: account.id, + delegatedFolder, + threadKeys: delegatedThreadKeys, + }); + } + if (delegatedFolder && serverUidsByFolder.has(delegatedFolder)) { + // Only sweep when the local folder is an exact mirror of the successful server + // snapshot. If the folder could not be opened, or the server has a UID that local + // ingest has not materialized yet (UIDVALIDITY rebuild / incomplete sync), local + // absence is not authoritative and valid delegation metadata must be retained. + const localDelegated = await query( + 'SELECT uid FROM messages WHERE account_id = $1 AND folder = $2 AND is_deleted = false', + [account.id, delegatedFolder] + ); + const serverUids = serverUidsByFolder.get(delegatedFolder); + const snapshotIsComplete = delegationSnapshotIsComplete(serverUids, localDelegated.rows); + if (snapshotIsComplete) { + // Retry cleanup even when the message row that originally triggered it was already + // removed by a prior pass whose metadata DELETE failed. + await sweepStaleDelegations({ + userId: account.user_id, + accountId: account.id, + delegatedFolder, + }); + } } if (deletedCount > 0) { diff --git a/backend/src/services/imapManager.test.js b/backend/src/services/imapManager.test.js index afd734a4..82be83af 100644 --- a/backend/src/services/imapManager.test.js +++ b/backend/src/services/imapManager.test.js @@ -12,11 +12,12 @@ vi.mock('../utils/redact.js', () => ({ redactEmail: vi.fn() })); vi.mock('./hostValidation.js', () => ({ resolveForConnection: vi.fn() })); vi.mock('./gtdTransitions.js', () => ({ runGtdTransitions: vi.fn(), threadKeysForMessageIds: vi.fn(), threadKeysInFolders: vi.fn() })); -import { ImapManager, providerProfile, makeClientCfg, gtdRelocateGuard, insertCopiedSibling, deleteMessageCopyRow, emitAfterDeferredCopySync, emitGtdSectionsRefreshOnDelete, emitGtdSectionsRefreshIfEnabled, selectGtdReevalIds, ensureMailbox, runGtdSyncTick, createKeyedSemaphore, isConnectionRefusal, connectCooldownMs, effectiveSyncIntervalMs, folderSyncDue, planModseqSync, connectStaggerFor, walkStructure } from './imapManager.js'; +import { ImapManager, providerProfile, makeClientCfg, gtdRelocateGuard, insertCopiedSibling, deleteMessageCopyRow, emitAfterDeferredCopySync, emitGtdSectionsRefreshOnDelete, emitGtdSectionsRefreshIfEnabled, selectGtdReevalIds, ensureMailbox, runGtdSyncTick, createKeyedSemaphore, isConnectionRefusal, connectCooldownMs, effectiveSyncIntervalMs, folderSyncDue, planModseqSync, connectStaggerFor, walkStructure, delegationSnapshotIsComplete } from './imapManager.js'; import { query } from './db.js'; import { invalidateGtdConfigCache } from './gtdConfig.js'; import { runGtdTransitions, threadKeysInFolders } from './gtdTransitions.js'; -import { parseMessage } from './messageParser.js'; +import { sanitizeEmail } from './emailSanitizer.js'; +import { parseMessage, snippetFromBody } from './messageParser.js'; const account = (imap_host, oauth_provider = null) => ({ imap_host, oauth_provider }); @@ -107,6 +108,22 @@ describe('providerProfile — host detection', () => { }); }); +// ── providerProfile — bodyBackfill allowlist ───────────────────────────────── + +describe('providerProfile — bodyBackfill allowlist', () => { + it('enables body backfill for well-behaved providers (apple, yahoo, generic)', () => { + expect(providerProfile(account('imap.icloud.com')).bodyBackfill).toBe(true); + expect(providerProfile(account('imap.mail.yahoo.com')).bodyBackfill).toBe(true); + expect(providerProfile(account('imap.fastmail.com')).bodyBackfill).toBe(true); + }); + + it('excludes throttle-hostile / unproven providers (google, purelymail, microsoft)', () => { + expect(providerProfile(account('imap.gmail.com')).bodyBackfill).toBe(false); + expect(providerProfile(account('imap.purelymail.com')).bodyBackfill).toBe(false); + expect(providerProfile(account('outlook.office365.com')).bodyBackfill).toBe(false); + }); +}); + // ── providerProfile — oauth_provider detection ──────────────────────────────── describe('providerProfile — oauth_provider fallback', () => { @@ -768,6 +785,34 @@ describe('runGtdSyncTick', () => { expect(runGtdTransitions).not.toHaveBeenCalled(); }); + it('publishes an awaitable in-flight sync for an immediate user-triggered retry', async () => { + const accountId = 'acct-tick-shared'; + invalidateGtdConfigCache(accountId); + query.mockResolvedValueOnce({ rows: [{ + gtd_enabled: true, + gtd_folders: { todo: 'Todo', watch: 'Todo', delegated: 'Todo', someday: 'Todo', reference: 'Todo' }, + }] }); + let releaseSync; + const pendingSync = new Promise(resolve => { releaseSync = resolve; }); + const mgr = mgrWithConnection(accountId, { + _gtdFolderFingerprint: vi.fn() + .mockResolvedValueOnce('before') + .mockResolvedValueOnce('after'), + _gtdSyncFolder: vi.fn().mockReturnValue(pendingSync), + onDemandSyncPromises: new Map(), + }); + const acct = { id: accountId, user_id: 'user-1' }; + + const tick = runGtdSyncTick(mgr, acct); + await vi.waitFor(() => expect(mgr._gtdSyncFolder).toHaveBeenCalledTimes(1)); + const userRetry = ImapManager.prototype.syncFolderOnDemand.call(mgr, acct, 'Todo'); + expect(mgr._gtdSyncFolder).toHaveBeenCalledTimes(1); + + releaseSync(); + await Promise.all([tick, userRetry]); + expect(mgr.onDemandSyncPromises.size).toBe(0); + }); + it('broadcasts gtd_sections_updated and re-runs transitions when a folder fingerprint changes', async () => { const allTodo = { todo: 'Todo', watch: 'Todo', delegated: 'Todo', someday: 'Todo', reference: 'Todo' }; query.mockResolvedValueOnce({ rows: [{ gtd_enabled: true, gtd_folders: allTodo }] }); @@ -1272,3 +1317,192 @@ describe('walkStructure attachment classification', () => { expect(results.attachments[0].filename).toBe('invoice.pdf'); }); }); + +describe('reconcileDeletes delegation cleanup durability', () => { + beforeEach(() => query.mockReset()); + + it('only treats local absence as authoritative when the successful server snapshot agrees', () => { + expect(delegationSnapshotIsComplete(new Set(), [])).toBe(true); + expect(delegationSnapshotIsComplete(new Set([41]), [])).toBe(false); + expect(delegationSnapshotIsComplete(new Set([41]), [{ uid: 41 }])).toBe(true); + expect(delegationSnapshotIsComplete(new Set([41]), [{ uid: 41 }, { uid: 42 }])).toBe(false); + }); + + it('aborts before any message deletion when delegation config cannot be loaded', async () => { + const accountId = 'acct-reconcile-config-fail'; + invalidateGtdConfigCache(accountId); + query + .mockResolvedValueOnce({ rows: [{ folder: 'Delegated' }] }) + .mockRejectedValueOnce(new Error('config unavailable')); + const mgr = new ImapManager(null); + + await expect(mgr.reconcileDeletes({ id: accountId, user_id: 'user-1' })) + .resolves.toBeUndefined(); + + expect(query).toHaveBeenCalledTimes(2); + expect(query.mock.calls.some(([sql]) => sql.startsWith('DELETE FROM messages'))).toBe(false); + }); +}); + +describe('ImapManager.fetchBodiesForMessages', () => { + // The constructor starts four setInterval timers; clear them so tests leave no open handles. + function makeManager() { + const mgr = new ImapManager({}); + clearInterval(mgr._healthCheckTimer); + clearInterval(mgr._snippetSchedulerTimer); + clearInterval(mgr._stalenessCheckTimer); + clearInterval(mgr._flagPushReconcilerTimer); + return mgr; + } + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns {fetched:0} without touching IMAP when given no ids', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn(); + const result = await mgr.fetchBodiesForMessages('acct-1', []); + expect(result).toEqual({ fetched: 0 }); + expect(mgr.fetchMessageBody).not.toHaveBeenCalled(); + }); + + it('fetches each still-empty message and writes its body via UPDATE', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockResolvedValue({ html: '

hi

', text: 'hi', attachments: [] }); + sanitizeEmail.mockReturnValue('

hi

'); + snippetFromBody.mockReturnValue('hi'); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1', email_address: 'x@y' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) { + return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }, { id: 'm2', uid: 6, folder: 'Sent' }] }); + } + return Promise.resolve({ rows: [] }); // UPDATE + }); + + const result = await mgr.fetchBodiesForMessages('acct-1', ['m1', 'm2']); + + expect(result).toEqual({ fetched: 2 }); + expect(mgr.fetchMessageBody).toHaveBeenCalledWith(expect.objectContaining({ id: 'acct-1' }), 5, 'INBOX'); + expect(mgr.fetchMessageBody).toHaveBeenCalledWith(expect.objectContaining({ id: 'acct-1' }), 6, 'Sent'); + const updateCalls = query.mock.calls.filter(([sql]) => /UPDATE messages/.test(sql)); + expect(updateCalls).toHaveLength(2); + }); + + it('skips a message whose body comes back empty (no html and no text)', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockResolvedValue({ html: null, text: null, attachments: [] }); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }] }); + return Promise.resolve({ rows: [] }); + }); + + const result = await mgr.fetchBodiesForMessages('acct-1', ['m1']); + + expect(result).toEqual({ fetched: 0 }); + const updateCalls = query.mock.calls.filter(([sql]) => /UPDATE messages/.test(sql)); + expect(updateCalls).toHaveLength(0); + }); + + it('propagates a fetchMessageBody failure so the drainer can back off', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockRejectedValue(new Error('Command failed')); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }] }); + return Promise.resolve({ rows: [] }); + }); + + await expect(mgr.fetchBodiesForMessages('acct-1', ['m1'])).rejects.toThrow('Command failed'); + }); + + it('skips a poison message that throws mid-batch and still fetches the rest (no wedge)', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn() + .mockRejectedValueOnce(new Error("Mailbox doesn't exist")) + .mockResolvedValueOnce({ html: '

hi

', text: 'hi', attachments: [] }); + sanitizeEmail.mockReturnValue('

hi

'); + snippetFromBody.mockReturnValue('hi'); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) { + return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'Stale' }, { id: 'm2', uid: 6, folder: 'INBOX' }] }); + } + return Promise.resolve({ rows: [] }); + }); + + const result = await mgr.fetchBodiesForMessages('acct-1', ['m1', 'm2']); + + // The poison message (m1) is skipped, not thrown — the loop keeps going so a batch + // containing it still makes forward progress instead of wedging the account forever. + expect(result).toEqual({ fetched: 1 }); + expect(mgr.fetchMessageBody).toHaveBeenCalledTimes(2); + const updateCalls = query.mock.calls.filter(([sql]) => /UPDATE messages/.test(sql)); + expect(updateCalls).toHaveLength(1); + }); + + it('rethrows when every message in the batch fails (genuine outage, not one bad message)', async () => { + const mgr = makeManager(); + mgr.fetchMessageBody = vi.fn().mockRejectedValue(new Error('Connection closed')); + query.mockImplementation((sql) => { + if (/FROM email_accounts/.test(sql)) return Promise.resolve({ rows: [{ id: 'acct-1' }] }); + if (/SELECT id, uid, folder FROM messages/.test(sql)) { + return Promise.resolve({ rows: [{ id: 'm1', uid: 5, folder: 'INBOX' }, { id: 'm2', uid: 6, folder: 'INBOX' }] }); + } + return Promise.resolve({ rows: [] }); + }); + + await expect(mgr.fetchBodiesForMessages('acct-1', ['m1', 'm2'])).rejects.toThrow('Connection closed'); + expect(mgr.fetchMessageBody).toHaveBeenCalledTimes(2); // both attempted, neither skipped silently + }); +}); + +describe('ImapManager.upsertDraftMessageRecord', () => { + function makeManager() { + const mgr = new ImapManager({}); + clearInterval(mgr._healthCheckTimer); + clearInterval(mgr._snippetSchedulerTimer); + clearInterval(mgr._stalenessCheckTimer); + clearInterval(mgr._flagPushReconcilerTimer); + return mgr; + } + + it('stores reply references without shifting later positional binds', async () => { + const mgr = makeManager(); + const date = new Date('2026-07-28T12:00:00.000Z'); + query.mockReset(); + query.mockResolvedValue({ rows: [] }); + + await mgr.upsertDraftMessageRecord( + { id: 'account-1' }, + 'Drafts', + 42, + { + messageId: '', + subject: 'Re: Thread', + fromName: 'Sender', + fromEmail: 'sender@example.com', + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + cc: [], + inReplyTo: '', + references: ' ', + snippet: 'Draft body', + bodyHtml: '

Draft body

', + bodyText: 'Draft body', + date, + }, + ); + + const [sql, binds] = query.mock.calls[0]; + expect(sql).toContain('in_reply_to, thread_references, date'); + expect(sql).toContain( + 'thread_references = COALESCE(EXCLUDED.thread_references, messages.thread_references)', + ); + expect(binds).toHaveLength(17); + expect(binds[9]).toBe(''); + expect(binds[10]).toBe(' '); + expect(binds[11]).toEqual(date); + expect(binds[16]).toBe(''); + }); +}); diff --git a/backend/src/services/inboxRules.js b/backend/src/services/inboxRules.js index 2335d5ad..9c4ad09e 100644 --- a/backend/src/services/inboxRules.js +++ b/backend/src/services/inboxRules.js @@ -132,7 +132,7 @@ function evaluateCondition(cond, msg) { } } -function evaluateRule(rule, msg) { +export function evaluateRule(rule, msg) { const conditions = Array.isArray(rule.conditions) ? rule.conditions : []; if (conditions.length === 0) return false; if (rule.condition_logic === 'OR') { @@ -141,6 +141,48 @@ function evaluateRule(rule, msg) { return conditions.every(c => evaluateCondition(c, msg)); } +export function matchingRules(rules, msg) { + return (Array.isArray(rules) ? rules : []) + .filter(rule => evaluateRule(rule, msg)) + .map(rule => ({ + id: rule.id, + name: rule.name, + would_match: true, + actions: Array.isArray(rule.actions) ? rule.actions : [], + })); +} + +export function toRuleMessage(row) { + let to = []; + try { + const raw = typeof row.to_addresses === 'string' + ? JSON.parse(row.to_addresses) + : row.to_addresses; + if (Array.isArray(raw)) { + to = raw.map(address => ({ + email: address.address || address.email || '', + name: address.name || '', + })); + } + } catch { + // Malformed to_addresses matches the previous /rules/run behavior: no recipients. + } + + return { + id: row.id, + uid: row.uid, + folder: row.folder, + fromEmail: row.from_email || '', + fromName: row.from_name || '', + to, + subject: row.subject || '', + hasAttachments: !!row.has_attachments, + isRead: !!row.is_read, + is_read: !!row.is_read, + parsedHeaders: {}, + }; +} + // Applies inbox rules to a batch of new INBOX messages. Returns { remaining, mutedIds }: // remaining — messages still in INBOX after rules ran (moved/archived/deleted excluded) // mutedIds — IDs of remaining messages that had mark_read applied by a rule; diff --git a/backend/src/services/mail/addresses.js b/backend/src/services/mail/addresses.js new file mode 100644 index 00000000..800406a6 --- /dev/null +++ b/backend/src/services/mail/addresses.js @@ -0,0 +1,51 @@ +// Extract name and email from an RFC 5322 address string. +// Handles "Name ", "Name", bare "", and bare "email" forms. +export function parseAddress(str) { + if (typeof str !== 'string') return { name: '', email: '' }; + const m = str.match(/^(.+?)\s*<([^>]+)>\s*$/); + if (m) return { name: m[1].trim().replace(/^"|"$/g, '').trim(), email: m[2].trim().toLowerCase() }; + const bare = str.match(/^\s*<([^>]+)>\s*$/); + if (bare) return { name: '', email: bare[1].trim().toLowerCase() }; + return { name: '', email: str.trim().toLowerCase() }; +} + +export function mapRecipientList(list) { + return (Array.isArray(list) ? list : []).filter(Boolean).map(addr => parseAddress(addr)); +} + +// Reject any recipient address that contains newlines, null bytes, or looks +// malformed — these are the classic email header-injection vectors. +export function normalizeRecipients(list, fieldName) { + if (!Array.isArray(list)) throw Object.assign(new Error(`${fieldName} must be an array`), { status: 400 }); + return list.map((addr, i) => { + if (typeof addr !== 'string' || !addr.trim()) { + throw Object.assign(new Error(`${fieldName}[${i}] is empty or not a string`), { status: 400 }); + } + const trimmed = addr.trim(); + if (/[\r\n\0]/.test(trimmed)) { + throw Object.assign(new Error(`${fieldName}[${i}] contains invalid characters`), { status: 400 }); + } + const at = trimmed.lastIndexOf('@'); + if (at < 1 || at === trimmed.length - 1) { + throw Object.assign(new Error(`${fieldName}[${i}] is not a valid email address`), { status: 400 }); + } + return trimmed; + }); +} + +// Strip header-injection characters from single-line header values. +export function sanitizeHeaderValue(value) { + if (typeof value !== 'string') return ''; + return value.replace(/[\r\n\0]/g, '').trim(); +} + +export function dedupePreferNamed(addrs) { + const deduped = new Map(); + for (const addr of addrs || []) { + const email = typeof addr?.email === 'string' ? addr.email.toLowerCase() : ''; + if (!email) continue; + const existing = deduped.get(email); + if (!existing || (!existing.name && addr.name)) deduped.set(email, addr); + } + return [...deduped.values()]; +} diff --git a/backend/src/services/mail/addresses.test.js b/backend/src/services/mail/addresses.test.js new file mode 100644 index 00000000..d29d1b12 --- /dev/null +++ b/backend/src/services/mail/addresses.test.js @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; +import { + dedupePreferNamed, + mapRecipientList, + normalizeRecipients, + parseAddress, + sanitizeHeaderValue, +} from './addresses.js'; + +describe('parseAddress', () => { + it.each([ + ['Name ', { name: 'Name', email: 'user@example.com' }], + ['"Quoted Name"', { name: 'Quoted Name', email: 'user@example.com' }], + ['', { name: '', email: 'user@example.com' }], + [' USER@Example.com ', { name: '', email: 'user@example.com' }], + ])('parses %j', (value, expected) => { + expect(parseAddress(value)).toEqual(expected); + }); + + it('keeps the guarded draft call-site behavior for non-string values', () => { + expect(parseAddress(null)).toEqual({ name: '', email: '' }); + expect(parseAddress(42)).toEqual({ name: '', email: '' }); + }); +}); + +describe('recipient helpers', () => { + it('maps recipient strings and tolerates absent lists', () => { + expect(mapRecipientList(['A ', 'b@example.com'])).toEqual([ + { name: 'A', email: 'a@example.com' }, + { name: '', email: 'b@example.com' }, + ]); + expect(mapRecipientList()).toEqual([]); + }); + + it('normalizes valid recipients without changing their display form', () => { + expect(normalizeRecipients([' Name '], 'to')).toEqual(['Name ']); + }); + + it.each([ + [undefined, 'to must be an array'], + [[''], 'to[0] is empty or not a string'], + [['a@example.com\nBcc: x@example.com'], 'to[0] contains invalid characters'], + [['missing-at'], 'to[0] is not a valid email address'], + [['missing@'], 'to[0] is not a valid email address'], + ])('rejects malformed recipients with status 400', (value, message) => { + expect(() => normalizeRecipients(value, 'to')).toThrowError( + expect.objectContaining({ message, status: 400 }), + ); + }); + + it('sanitizes single-line header values', () => { + expect(sanitizeHeaderValue(' hello\r\nBcc: x\0 ')).toBe('helloBcc: x'); + expect(sanitizeHeaderValue(null)).toBe(''); + }); +}); + +describe('dedupePreferNamed', () => { + it('deduplicates case-insensitively and replaces an unnamed entry with a named one', () => { + expect(dedupePreferNamed([ + { name: '', email: 'A@example.com' }, + { name: 'Alice', email: 'a@EXAMPLE.com' }, + { name: 'Bob', email: 'b@example.com' }, + { name: '', email: 'B@example.com' }, + ])).toEqual([ + { name: 'Alice', email: 'a@EXAMPLE.com' }, + { name: 'Bob', email: 'b@example.com' }, + ]); + }); +}); diff --git a/backend/src/services/mail/identity.js b/backend/src/services/mail/identity.js new file mode 100644 index 00000000..9d4b80f3 --- /dev/null +++ b/backend/src/services/mail/identity.js @@ -0,0 +1,52 @@ +export class AliasNotFoundError extends Error { + constructor(message = 'Alias not found') { + super(message); + this.name = 'AliasNotFoundError'; + this.status = 422; + this.code = 'alias_not_found'; + this.expose = true; + } +} + +/** + * Resolve a From identity for an account row already proven to belong to the caller. + * A supplied selector must resolve within that account and never silently falls back. + */ +export async function resolveFromIdentity(account, selector = {}, deps = {}) { + const defaultIdentity = { + fromName: account.sender_name || account.name, + fromEmail: account.email_address, + fromReplyTo: null, + signature: account.signature, + aliasId: null, + }; + + const aliasId = selector?.aliasId; + const aliasEmail = selector?.aliasEmail; + if (!aliasId && !aliasEmail) return defaultIdentity; + + let result; + if (aliasId) { + result = await deps.query( + 'SELECT * FROM account_aliases WHERE id = $1 AND account_id = $2', + [aliasId, account.id], + ); + } else { + result = await deps.query( + 'SELECT * FROM account_aliases WHERE account_id = $1 AND LOWER(email) = LOWER($2) LIMIT 1', + [account.id, aliasEmail], + ); + } + + const alias = result.rows[0]; + if (!alias) throw new AliasNotFoundError(); + + return { + fromName: alias.name, + fromEmail: alias.email, + fromReplyTo: alias.reply_to || null, + // null (DB default) means inherit from account; only override when alias has an explicit signature set + signature: alias.signature !== null ? alias.signature : defaultIdentity.signature, + aliasId: alias.id, + }; +} diff --git a/backend/src/services/mail/identity.test.js b/backend/src/services/mail/identity.test.js new file mode 100644 index 00000000..c81c475b --- /dev/null +++ b/backend/src/services/mail/identity.test.js @@ -0,0 +1,91 @@ +import { describe, expect, it, vi } from 'vitest'; +import { AliasNotFoundError, resolveFromIdentity } from './identity.js'; + +const account = { + id: 'account-1', + name: 'Account Name', + sender_name: 'Sender Name', + email_address: 'account@example.com', + signature: '

Account sig

', +}; + +describe('resolveFromIdentity', () => { + it('returns the already-scoped account identity when no selector is supplied', async () => { + const query = vi.fn(); + await expect(resolveFromIdentity(account, {}, { query })).resolves.toEqual({ + fromName: 'Sender Name', + fromEmail: 'account@example.com', + fromReplyTo: null, + signature: '

Account sig

', + aliasId: null, + }); + expect(query).not.toHaveBeenCalled(); + }); + + it('falls back from sender_name to account name', async () => { + const identity = await resolveFromIdentity({ ...account, sender_name: null }, null, { query: vi.fn() }); + expect(identity.fromName).toBe('Account Name'); + }); + + it('resolves an alias id within the scoped account and inherits a null signature', async () => { + const query = vi.fn().mockResolvedValue({ rows: [{ + id: 'alias-1', + name: 'Alias Name', + email: 'alias@example.com', + reply_to: 'reply@example.com', + signature: null, + }] }); + + await expect(resolveFromIdentity(account, { aliasId: 'alias-1' }, { query })).resolves.toEqual({ + fromName: 'Alias Name', + fromEmail: 'alias@example.com', + fromReplyTo: 'reply@example.com', + signature: '

Account sig

', + aliasId: 'alias-1', + }); + expect(query).toHaveBeenCalledWith( + 'SELECT * FROM account_aliases WHERE id = $1 AND account_id = $2', + ['alias-1', 'account-1'], + ); + }); + + it('resolves an alias email case-insensitively and honors an explicit empty signature', async () => { + const query = vi.fn().mockResolvedValue({ rows: [{ + id: 'alias-2', + name: 'Email Alias', + email: 'alias@example.com', + reply_to: null, + signature: '', + }] }); + + const identity = await resolveFromIdentity(account, { aliasEmail: 'ALIAS@example.com' }, { query }); + expect(identity).toEqual({ + fromName: 'Email Alias', + fromEmail: 'alias@example.com', + fromReplyTo: null, + signature: '', + aliasId: 'alias-2', + }); + expect(query).toHaveBeenCalledWith( + 'SELECT * FROM account_aliases WHERE account_id = $1 AND LOWER(email) = LOWER($2) LIMIT 1', + ['account-1', 'ALIAS@example.com'], + ); + }); + + it.each([ + [{ aliasId: 'missing' }], + [{ aliasEmail: 'missing@example.com' }], + ])('hard-errors rather than silently falling back for selector %j', async (selector) => { + const error = await resolveFromIdentity(account, selector, { + query: vi.fn().mockResolvedValue({ rows: [] }), + }).catch(err => err); + + expect(error).toBeInstanceOf(AliasNotFoundError); + expect(error).toMatchObject({ + message: 'Alias not found', + status: 422, + code: 'alias_not_found', + expose: true, + }); + }); +}); diff --git a/backend/src/services/mail/mimeBuilder.js b/backend/src/services/mail/mimeBuilder.js new file mode 100644 index 00000000..1e29858a --- /dev/null +++ b/backend/src/services/mail/mimeBuilder.js @@ -0,0 +1,84 @@ +import nodemailer from 'nodemailer'; +import sanitizeHtml from 'sanitize-html'; +import { sanitizeComposeBody } from '../emailSanitizer.js'; +import { sanitizeHeaderValue } from './addresses.js'; + +function escapeHtml(str) { + return str.replace(/&/g, '&').replace(//g, '>'); +} + +export function textToHtml(text) { + return '
' + + String(text || '').split('\n').map(l => `

${escapeHtml(l) || ' '}

`).join('') + + '
'; +} + +export function sigToPlainText(html) { + return sanitizeHtml(html || '', { allowedTags: [], allowedAttributes: {} }).trim(); +} + +export function bodyToPlain(body, isHtml) { + if (!isHtml) return body || ''; + return sanitizeHtml(body || '', { allowedTags: [], allowedAttributes: {} }); +} + +export function bodyToHtml(body, isHtml) { + if (!isHtml) return textToHtml(body || ''); + return sanitizeComposeBody(body || ''); +} + +function joinRecipients(recipients) { + const list = Array.isArray(recipients) ? recipients : [recipients]; + return list.filter(Boolean).join(', ') || undefined; +} + +export function buildMailOptions({ + messageId, + fromName, + fromEmail, + replyTo, + to, + cc, + bcc, + subject, + priority, + text, + html, + inReplyTo, + references, + attachments, +}) { + const mailOptions = { + messageId, + from: `${fromName} <${fromEmail}>`, + ...(replyTo ? { replyTo } : {}), + to: joinRecipients(to), + cc: joinRecipients(cc), + bcc: joinRecipients(bcc), + subject: sanitizeHeaderValue(subject || ''), + ...(priority && priority !== 'normal' ? { priority } : {}), + text, + ...(html !== undefined ? { html } : {}), + }; + + if (inReplyTo) { + mailOptions.inReplyTo = sanitizeHeaderValue(inReplyTo); + mailOptions.references = sanitizeHeaderValue(references || inReplyTo); + } else if (references) { + mailOptions.references = sanitizeHeaderValue(references); + } + if (attachments?.length) mailOptions.attachments = attachments; + return mailOptions; +} + +export async function renderRaw(mailOptions) { + const streamTransport = nodemailer.createTransport({ streamTransport: true, newline: 'unix' }); + const streamInfo = await streamTransport.sendMail(mailOptions); + const chunks = []; + await new Promise((resolve, reject) => { + streamInfo.message.on('data', chunk => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))); + streamInfo.message.on('end', resolve); + streamInfo.message.on('error', reject); + }); + return Buffer.concat(chunks); +} diff --git a/backend/src/services/mail/mimeBuilder.test.js b/backend/src/services/mail/mimeBuilder.test.js new file mode 100644 index 00000000..da2b921e --- /dev/null +++ b/backend/src/services/mail/mimeBuilder.test.js @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest'; +import { + bodyToHtml, + bodyToPlain, + buildMailOptions, + renderRaw, + sigToPlainText, + textToHtml, +} from './mimeBuilder.js'; + +describe('mail body helpers', () => { + it('escapes plain text and preserves line structure in HTML', () => { + expect(textToHtml('hello & \n')).toBe( + '
' + + '

hello & <world>

 

', + ); + }); + + it('turns signature and HTML bodies into plain text', () => { + expect(sigToPlainText('

Hello there

')).toBe('Hello there'); + expect(bodyToPlain('

Hello there

', true)).toBe('Hello there'); + expect(bodyToPlain('plain', false)).toBe('plain'); + }); + + it('sanitizes HTML compose bodies and converts plain text bodies', () => { + expect(bodyToHtml('

safe

', true)).toBe('

safe

'); + expect(bodyToHtml('plain', false)).toContain('

plain

'); + }); +}); + +describe('buildMailOptions', () => { + it('owns the complete shared mail header and content shape', () => { + const attachments = [{ filename: 'a.txt', content: Buffer.from('a') }]; + expect(buildMailOptions({ + messageId: '', + fromName: 'Sender', + fromEmail: 'sender@example.com', + replyTo: 'reply@example.com', + to: ['A '], + cc: ['c@example.com'], + bcc: ['b@example.com'], + subject: ' hello\r\n ', + priority: 'high', + text: 'plain body', + html: '

html body

', + inReplyTo: '\r\n', + references: ' ', + attachments, + })).toEqual({ + messageId: '', + from: 'Sender ', + replyTo: 'reply@example.com', + to: 'A ', + cc: 'c@example.com', + bcc: 'b@example.com', + subject: 'hello', + priority: 'high', + text: 'plain body', + html: '

html body

', + inReplyTo: '', + references: ' ', + attachments, + }); + }); + + it('omits empty optional headers and normal priority', () => { + expect(buildMailOptions({ + messageId: '', + fromName: 'Sender', + fromEmail: 'sender@example.com', + to: [], + subject: '', + priority: 'normal', + text: '', + })).toEqual({ + messageId: '', + from: 'Sender ', + to: undefined, + cc: undefined, + bcc: undefined, + subject: '', + text: '', + }); + }); +}); + +describe('renderRaw', () => { + it('renders a raw MIME buffer without opening a network listener', async () => { + const raw = await renderRaw(buildMailOptions({ + messageId: '', + fromName: 'Sender', + fromEmail: 'sender@example.com', + to: ['recipient@example.com'], + subject: 'Rendered', + text: 'hello', + html: '

hello

', + })); + + expect(Buffer.isBuffer(raw)).toBe(true); + expect(raw.toString()).toContain('Message-ID: '); + expect(raw.toString()).toContain('Subject: Rendered'); + }); +}); diff --git a/backend/src/services/mail/sentCopy.js b/backend/src/services/mail/sentCopy.js new file mode 100644 index 00000000..fb252d9e --- /dev/null +++ b/backend/src/services/mail/sentCopy.js @@ -0,0 +1,176 @@ +import { createHash, randomUUID as defaultRandomUUID } from 'crypto'; +import { runTransitionsForSentMessage as defaultRunTransitionsForSentMessage } from '../gtdTransitions.js'; +import { parseAddress } from './addresses.js'; +import { redactEmail } from '../../utils/redact.js'; +import { generateVCard as defaultGenerateVCard } from '../../utils/vcard.js'; + +const defaultSleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + +export async function resolveSentFolder(account, deps) { + const mapped = account.folder_mappings?.sent; + if (mapped) return mapped; + const result = await deps.query( + "SELECT path FROM folders WHERE account_id = $1 AND special_use = '\\Sent' LIMIT 1", + [account.id], + ); + return result.rows[0]?.path || null; +} + +export function scheduleSentMetadataUpsert(account, sentFolder, mailOptions, meta, deps) { + if (!sentFolder || !mailOptions.messageId) return; + const defer = deps.defer || setImmediate; + const sleep = deps.sleep || defaultSleep; + defer(async () => { + for (const delay of [3000, 10000, 20000]) { + await sleep(delay); + try { + const uid = await deps.imapManager.findUidByMessageId(account, sentFolder, mailOptions.messageId); + if (uid) { + await deps.imapManager.upsertSentMessageRecord(account, sentFolder, uid, meta); + return; + } + } catch (err) { + console.warn('Post-send sent metadata upsert failed:', err.message); + } + } + }); +} + +export async function persistSentCopy({ + account, + sentFolder, + rawMessage, + mailOptions, + meta, +}, deps) { + if (!sentFolder) return { sentCopySaved: null }; + const { imapManager } = deps; + const runTransitionsForSentMessage = deps.runTransitionsForSentMessage || defaultRunTransitionsForSentMessage; + + if (rawMessage) { + // Non-auto-saving account: APPEND the Sent copy ourselves — exactly ONCE. IMAP + // APPEND is NOT idempotent (unlike a \Seen flag), so we must not retry: a retry + // whose first attempt merely timed out (but still lands on the server) would store + // a SECOND copy. Bound the wait so a stalled connection can't hang the response; + // the abandoned append can at worst still save the single copy. On failure, warn + // the user and schedule a fallback sync in case the append landed late. Audit [2]. + let sentCopySaved = false; + try { + const { uid } = await Promise.race([ + imapManager.appendToSent(account, sentFolder, rawMessage), + new Promise((_, reject) => setTimeout(() => reject(new Error('Sent APPEND timed out')), 20000)), + ]); + sentCopySaved = true; + if (uid && meta) { + await imapManager.upsertSentMessageRecord(account, sentFolder, uid, meta) + .catch(err => console.warn('Sent metadata upsert failed:', err.message)); + } + setTimeout(() => { + imapManager.syncFolderOnDemand(account, sentFolder) + // Once the Sent copy is in the DB, re-run GTD transitions for its thread: a reply + // to a Todo/Someday thread means the owner acted, so that label should drop. The + // sent message reaches no other GTD hook (Sent isn't INBOX, and the tick watches + // only the state folders), so this is the only trigger. Swallow on failure — the + // next inbound sync / GTD tick self-heals. + .then(() => runTransitionsForSentMessage(imapManager, account, mailOptions.messageId) + .catch(error => console.warn(`Post-append GTD transition failed: ${error.message}`))) + .catch(error => console.error(`Post-append sync failed: ${error.message}`)); + }, 1000); + } catch (appendErr) { + console.error(`IMAP append to Sent failed for ${redactEmail(account.email_address)}/${sentFolder}: ${appendErr.message}`); + // The append may still have landed (or land shortly) — pull the folder so a + // late-completing append self-corrects the DB rather than staying invisible. + setTimeout(() => { + imapManager.syncFolderOnDemand(account, sentFolder) + .catch(error => console.error(`Post-append fallback sync failed: ${error.message}`)); + }, 8000); + } + return { sentCopySaved }; + } + + // Server auto-saves via SMTP; seed metadata once the Sent copy is searchable. + if (meta) scheduleSentMetadataUpsert(account, sentFolder, mailOptions, meta, deps); + // Server auto-saves via SMTP; just sync after a delay. Two attempts because the + // provider (e.g. Gmail) can be slow to expose the sent message; the 3s pass usually + // catches it, the 15s pass is the safety net. GTD transitions run after each: the 3s + // attempt may miss (Sent copy not yet visible → empty thread set → no-op) and the 15s + // attempt then catches it; if 3s already stripped, 15s is an idempotent no-op. + const syncAttempt = label => imapManager.syncFolderOnDemand(account, sentFolder) + .then(() => { + console.log(`Post-send ${label} sync done: ${redactEmail(account.email_address)}/${sentFolder}`); + return runTransitionsForSentMessage(imapManager, account, mailOptions.messageId) + .catch(error => console.warn(`Post-send ${label} GTD transition failed: ${error.message}`)); + }) + .catch(error => console.error(`Post-send ${label} sync failed: ${error.message}`)); + setTimeout(() => syncAttempt('3s'), 3000); + setTimeout(() => syncAttempt('15s'), 15000); + return { sentCopySaved: null }; +} + +export function learnSentRecipients({ userId, recipients }, deps) { + if (!recipients.length) return; + const defer = deps.defer || setImmediate; + const makeRandomUUID = deps.randomUUID || defaultRandomUUID; + const generateVCard = deps.generateVCard || defaultGenerateVCard; + const now = deps.now ? deps.now() : new Date(); + + defer(async () => { + try { + // Ensure the user's default address book exists + const abResult = await deps.query( + `INSERT INTO address_books (user_id, name) VALUES ($1, 'Personal') + ON CONFLICT (user_id, name) DO UPDATE SET updated_at = NOW() + RETURNING id`, + [userId], + ); + const addressBookId = abResult.rows[0].id; + + const results = await Promise.allSettled(recipients.map(addr => { + const { name, email } = parseAddress(addr); + if (!email) return Promise.resolve(); + const primaryEmail = email.toLowerCase(); + const displayName = name || primaryEmail; + const uid = makeRandomUUID(); + const emails = [{ value: primaryEmail, type: 'other', primary: true }]; + const vcard = generateVCard({ uid, displayName, emails }); + const etag = createHash('md5').update(vcard).digest('hex'); + // Upsert by (user_id, primary_email) — bump send_count and promote from is_auto. + // On conflict, preserve an existing vcard; only fill it in if the row had none. + return deps.query(` + INSERT INTO contacts ( + address_book_id, user_id, uid, vcard, etag, + display_name, primary_email, emails, is_auto, send_count, last_sent + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, false, 1, $9) + ON CONFLICT (address_book_id, primary_email) WHERE primary_email IS NOT NULL DO UPDATE + SET send_count = contacts.send_count + 1, + last_sent = $9, + is_auto = false, + display_name = CASE WHEN contacts.is_auto THEN $6 ELSE contacts.display_name END, + vcard = COALESCE(contacts.vcard, EXCLUDED.vcard), + etag = COALESCE(contacts.etag, EXCLUDED.etag), + updated_at = NOW() + RETURNING address_book_id + `, [addressBookId, userId, uid, vcard, etag, displayName, primaryEmail, JSON.stringify(emails), now]); + })); + + const failed = results.filter(result => result.status === 'rejected'); + if (failed.length) console.warn('Contact upsert errors:', failed.map(result => result.reason?.message)); + + // Collect distinct address books actually modified (contacts may live in non-default books). + const booksToSync = new Set(); + for (const result of results) { + if (result.status === 'fulfilled' && result.value?.rows?.[0]?.address_book_id) { + booksToSync.add(result.value.rows[0].address_book_id); + } + } + if (!booksToSync.size) booksToSync.add(addressBookId); + + await Promise.all([...booksToSync].map(bookId => + deps.query('UPDATE address_books SET sync_token = gen_random_uuid()::text, updated_at = NOW() WHERE id = $1', [bookId]) + )); + } catch (err) { + console.warn('Contact upsert setup error:', err.message); + } + }); +} diff --git a/backend/src/services/mail/sentCopy.test.js b/backend/src/services/mail/sentCopy.test.js new file mode 100644 index 00000000..1a3faf6e --- /dev/null +++ b/backend/src/services/mail/sentCopy.test.js @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + learnSentRecipients, + persistSentCopy, + resolveSentFolder, + scheduleSentMetadataUpsert, +} from './sentCopy.js'; + +const account = { + id: 'account-1', + email_address: 'sender@example.com', + folder_mappings: {}, +}; +const mailOptions = { messageId: '' }; +const meta = { messageId: '', subject: 'Subject' }; + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe('resolveSentFolder', () => { + it('prefers the account folder mapping', async () => { + const query = vi.fn(); + await expect(resolveSentFolder({ ...account, folder_mappings: { sent: 'Sent Items' } }, { query })) + .resolves.toBe('Sent Items'); + expect(query).not.toHaveBeenCalled(); + }); + + it('falls back to the detected special-use folder', async () => { + const query = vi.fn().mockResolvedValue({ rows: [{ path: 'Sent' }] }); + await expect(resolveSentFolder(account, { query })).resolves.toBe('Sent'); + expect(query).toHaveBeenCalledWith( + "SELECT path FROM folders WHERE account_id = $1 AND special_use = '\\Sent' LIMIT 1", + ['account-1'], + ); + }); +}); + +describe('persistSentCopy', () => { + it('APPENDs exactly once, upserts metadata, and schedules the post-append sync', async () => { + vi.useFakeTimers(); + const imapManager = { + appendToSent: vi.fn().mockResolvedValue({ uid: 42 }), + upsertSentMessageRecord: vi.fn().mockResolvedValue(undefined), + syncFolderOnDemand: vi.fn().mockResolvedValue(undefined), + }; + const runTransitionsForSentMessage = vi.fn().mockResolvedValue(undefined); + + await expect(persistSentCopy({ + account, + sentFolder: 'Sent', + rawMessage: Buffer.from('mime'), + mailOptions, + meta, + }, { imapManager, runTransitionsForSentMessage })).resolves.toEqual({ sentCopySaved: true }); + + expect(imapManager.appendToSent).toHaveBeenCalledTimes(1); + expect(imapManager.upsertSentMessageRecord).toHaveBeenCalledWith(account, 'Sent', 42, meta); + await vi.advanceTimersByTimeAsync(1000); + expect(imapManager.syncFolderOnDemand).toHaveBeenCalledWith(account, 'Sent'); + expect(runTransitionsForSentMessage).toHaveBeenCalledWith(imapManager, account, ''); + }); + + it('never retries a failed APPEND and schedules only the fallback sync', async () => { + vi.useFakeTimers(); + vi.spyOn(console, 'error').mockImplementation(() => {}); + const imapManager = { + appendToSent: vi.fn().mockRejectedValue(new Error('append failed')), + syncFolderOnDemand: vi.fn().mockResolvedValue(undefined), + }; + + await expect(persistSentCopy({ + account, + sentFolder: 'Sent', + rawMessage: Buffer.from('mime'), + mailOptions, + meta, + }, { imapManager, runTransitionsForSentMessage: vi.fn() })) + .resolves.toEqual({ sentCopySaved: false }); + + expect(imapManager.appendToSent).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(8000); + expect(imapManager.appendToSent).toHaveBeenCalledTimes(1); + expect(imapManager.syncFolderOnDemand).toHaveBeenCalledTimes(1); + }); + + it('returns not-applicable and schedules two syncs for provider auto-save', async () => { + vi.useFakeTimers(); + const imapManager = { + findUidByMessageId: vi.fn().mockResolvedValue(null), + syncFolderOnDemand: vi.fn().mockResolvedValue(undefined), + }; + const runTransitionsForSentMessage = vi.fn().mockResolvedValue(undefined); + + await expect(persistSentCopy({ + account, + sentFolder: 'Sent', + rawMessage: null, + mailOptions, + meta: null, + }, { imapManager, runTransitionsForSentMessage })).resolves.toEqual({ sentCopySaved: null }); + + await vi.advanceTimersByTimeAsync(15000); + expect(imapManager.syncFolderOnDemand).toHaveBeenCalledTimes(2); + expect(runTransitionsForSentMessage).toHaveBeenCalledTimes(2); + }); +}); + +describe('scheduleSentMetadataUpsert', () => { + it('searches until the sent message appears and then upserts once', async () => { + const imapManager = { + findUidByMessageId: vi.fn() + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(7), + upsertSentMessageRecord: vi.fn().mockResolvedValue(undefined), + }; + const sleep = vi.fn().mockResolvedValue(undefined); + let scheduled; + + scheduleSentMetadataUpsert(account, 'Sent', mailOptions, meta, { + imapManager, + defer: fn => { scheduled = fn; }, + sleep, + }); + await scheduled(); + + expect(sleep).toHaveBeenCalledTimes(2); + expect(imapManager.findUidByMessageId).toHaveBeenCalledTimes(2); + expect(imapManager.upsertSentMessageRecord).toHaveBeenCalledWith(account, 'Sent', 7, meta); + }); +}); + +describe('learnSentRecipients', () => { + it('schedules contact learning without making the send await database work', async () => { + const query = vi.fn() + .mockResolvedValueOnce({ rows: [{ id: 'book-1' }] }) + .mockResolvedValueOnce({ rows: [{ address_book_id: 'book-1' }] }) + .mockResolvedValueOnce({ rows: [] }); + let scheduled; + + const result = learnSentRecipients({ + userId: 'user-1', + recipients: ['Alice '], + }, { + query, + defer: fn => { scheduled = fn; }, + randomUUID: () => 'contact-1', + now: () => new Date('2026-07-28T00:00:00Z'), + }); + + expect(result).toBeUndefined(); + expect(query).not.toHaveBeenCalled(); + await scheduled(); + expect(query).toHaveBeenCalledTimes(3); + expect(query.mock.calls[1][1]).toContain('alice@example.com'); + }); +}); diff --git a/backend/src/services/mail/smtp.js b/backend/src/services/mail/smtp.js new file mode 100644 index 00000000..b2b611b2 --- /dev/null +++ b/backend/src/services/mail/smtp.js @@ -0,0 +1,90 @@ +import nodemailer from 'nodemailer'; +import { decrypt as defaultDecrypt } from '../encryption.js'; +import { resolveForConnection as defaultResolveForConnection } from '../hostValidation.js'; +import { getConnectionPolicy as defaultGetConnectionPolicy } from '../connectionPolicy.js'; + +function exposedError(message, status) { + return Object.assign(new Error(message), { status, expose: true }); +} + +export async function buildSmtpTransport(inputAccount, deps = {}) { + let account = inputAccount; + const decrypt = deps.decrypt || defaultDecrypt; + const getConnectionPolicy = deps.getConnectionPolicy || defaultGetConnectionPolicy; + const resolveForConnection = deps.resolveForConnection || defaultResolveForConnection; + const createTransport = deps.createTransport || nodemailer.createTransport.bind(nodemailer); + + if (account.oauth_provider === 'microsoft') { + // Only refresh when the token is near/at expiry (mirrors imapManager's + // ensureFreshToken). Refreshing on every send needlessly rotates the AAD + // refresh token and can invalidate it under concurrent sends. + const expiryMs = account.oauth_token_expiry ? new Date(account.oauth_token_expiry).getTime() : 0; + if (expiryMs - Date.now() < 5 * 60 * 1000) { + account = await deps.refreshMicrosoftToken(account); + } + } + + let smtpAuth; + if ((account.oauth_provider === 'microsoft' || account.oauth_provider === 'google') + && account.oauth_access_token) { + const accessToken = decrypt(account.oauth_access_token); + if (!accessToken) { + throw exposedError('OAuth access token is corrupted — please reconnect your account.', 502); + } + smtpAuth = { + type: 'OAuth2', + user: account.auth_user || account.email_address, + accessToken, + }; + } else { + const pass = decrypt(account.auth_pass); + if (!pass) { + throw exposedError('SMTP password is corrupted or missing — please re-enter your account password in Settings.', 502); + } + smtpAuth = { user: account.auth_user, pass }; + } + + const policy = await getConnectionPolicy(); + const smtpResolved = await resolveForConnection(account.smtp_host, { allowPrivate: policy.allowPrivateHosts }); + const smtpPlain = account.smtp_tls !== 'STARTTLS' && account.smtp_tls !== 'SSL'; + if (!policy.allowInsecureTls && smtpPlain) { + throw exposedError('Plain-text SMTP is not allowed: admin must enable "Allow insecure TLS"', 403); + } + const smtpTls = { rejectUnauthorized: !(policy.allowInsecureTls && account.imap_skip_tls_verify) }; + if (smtpResolved.servername) smtpTls.servername = smtpResolved.servername; + // For 'SSL': force direct TLS. For 'none': plain with no upgrade. + // For 'STARTTLS' (or any other/legacy value): fall back to port-based detection + // so existing accounts stored with the default 'STARTTLS' on port 465 keep working. + const smtpSecure = account.smtp_tls === 'SSL' || (account.smtp_tls !== 'none' && account.smtp_port === 465); + const transport = createTransport({ + host: smtpResolved.host, + port: account.smtp_port, + secure: smtpSecure, + ...(account.smtp_tls === 'none' ? { ignoreTLS: true } : {}), + auth: smtpAuth, + tls: smtpTls, + }); + + return { transport, account }; +} + +// Map SMTP/connection errors to user-friendly messages that don't expose server internals. +export function sanitizeSmtpError(err) { + const msg = err?.message || ''; + if (/ECONNREFUSED|ENOTFOUND|ETIMEDOUT|ECONNRESET|EHOSTUNREACH/i.test(msg)) { + return 'Could not connect to the mail server. Check your SMTP settings.'; + } + if (/535|534|530|invalid.?login|authentication.?fail|bad.*credentials|username.*password|password.*username/i.test(msg)) { + return 'Authentication failed. Check your email account credentials.'; + } + if (/throttl|rate.?limit|too many|4\.2\.|4\.7\.94/i.test(msg)) { + return 'The mail server is rate limiting sends. Please try again shortly.'; + } + if (/550|5\.[13]\.|reject|blacklist|spam|not.?accept/i.test(msg)) { + return 'Message was rejected by the mail server.'; + } + if (/TLS|SSL|certificate|handshake/i.test(msg)) { + return 'Secure connection to the mail server failed. Check your TLS settings.'; + } + return 'Failed to send message. Please try again.'; +} diff --git a/backend/src/services/mail/smtp.test.js b/backend/src/services/mail/smtp.test.js new file mode 100644 index 00000000..de03c4ca --- /dev/null +++ b/backend/src/services/mail/smtp.test.js @@ -0,0 +1,122 @@ +import { describe, expect, it, vi } from 'vitest'; +import { buildSmtpTransport, sanitizeSmtpError } from './smtp.js'; + +function baseAccount(overrides = {}) { + return { + id: 'account-1', + email_address: 'sender@example.com', + auth_user: 'smtp-user', + auth_pass: 'encrypted-password', + smtp_host: 'smtp.example.com', + smtp_port: 587, + smtp_tls: 'STARTTLS', + imap_skip_tls_verify: false, + ...overrides, + }; +} + +function baseDeps(overrides = {}) { + return { + decrypt: vi.fn(value => `decrypted:${value}`), + refreshMicrosoftToken: vi.fn(), + getConnectionPolicy: vi.fn().mockResolvedValue({ + allowPrivateHosts: false, + allowInsecureTls: false, + }), + resolveForConnection: vi.fn().mockResolvedValue({ + host: '203.0.113.10', + servername: 'smtp.example.com', + }), + createTransport: vi.fn(options => ({ options })), + ...overrides, + }; +} + +describe('buildSmtpTransport', () => { + it('refreshes a near-expiry Microsoft account and builds OAuth2 transport options', async () => { + const refreshed = baseAccount({ + oauth_provider: 'microsoft', + oauth_access_token: 'new-token', + oauth_token_expiry: new Date(Date.now() + 3_600_000).toISOString(), + smtp_port: 465, + }); + const deps = baseDeps({ + refreshMicrosoftToken: vi.fn().mockResolvedValue(refreshed), + }); + const stale = { ...refreshed, oauth_access_token: 'old-token', oauth_token_expiry: new Date(0).toISOString() }; + + const result = await buildSmtpTransport(stale, deps); + + expect(deps.refreshMicrosoftToken).toHaveBeenCalledWith(stale); + expect(result.account).toBe(refreshed); + expect(deps.createTransport).toHaveBeenCalledWith({ + host: '203.0.113.10', + port: 465, + secure: true, + auth: { + type: 'OAuth2', + user: 'smtp-user', + accessToken: 'decrypted:new-token', + }, + tls: { + rejectUnauthorized: true, + servername: 'smtp.example.com', + }, + }); + }); + + it('uses password auth and preserves explicit no-TLS behavior when policy permits it', async () => { + const deps = baseDeps({ + getConnectionPolicy: vi.fn().mockResolvedValue({ + allowPrivateHosts: true, + allowInsecureTls: true, + }), + resolveForConnection: vi.fn().mockResolvedValue({ host: '10.0.0.2' }), + }); + const account = baseAccount({ smtp_tls: 'none', imap_skip_tls_verify: true }); + + await buildSmtpTransport(account, deps); + + expect(deps.resolveForConnection).toHaveBeenCalledWith('smtp.example.com', { allowPrivate: true }); + expect(deps.createTransport).toHaveBeenCalledWith({ + host: '10.0.0.2', + port: 587, + secure: false, + ignoreTLS: true, + auth: { user: 'smtp-user', pass: 'decrypted:encrypted-password' }, + tls: { rejectUnauthorized: false }, + }); + }); + + it('refuses plain SMTP unless insecure TLS is enabled', async () => { + const error = await buildSmtpTransport(baseAccount({ smtp_tls: 'none' }), baseDeps()).catch(err => err); + expect(error).toMatchObject({ + message: 'Plain-text SMTP is not allowed: admin must enable "Allow insecure TLS"', + status: 403, + expose: true, + }); + }); + + it.each([ + [baseAccount({ oauth_provider: 'google', oauth_access_token: 'bad' }), 'OAuth access token is corrupted — please reconnect your account.'], + [baseAccount({ auth_pass: 'bad' }), 'SMTP password is corrupted or missing — please re-enter your account password in Settings.'], + ])('exposes credential corruption without attempting a connection', async (account, message) => { + const deps = baseDeps({ decrypt: vi.fn().mockReturnValue(null) }); + const error = await buildSmtpTransport(account, deps).catch(err => err); + expect(error).toMatchObject({ message, status: 502, expose: true }); + expect(deps.createTransport).not.toHaveBeenCalled(); + }); +}); + +describe('sanitizeSmtpError', () => { + it.each([ + ['connect ECONNREFUSED 127.0.0.1', 'Could not connect to the mail server. Check your SMTP settings.'], + ['535 invalid login', 'Authentication failed. Check your email account credentials.'], + ['server rate limit', 'The mail server is rate limiting sends. Please try again shortly.'], + ['550 rejected as spam', 'Message was rejected by the mail server.'], + ['TLS certificate handshake', 'Secure connection to the mail server failed. Check your TLS settings.'], + ['secret internal server detail', 'Failed to send message. Please try again.'], + ])('sanitizes %j', (message, expected) => { + expect(sanitizeSmtpError(new Error(message))).toBe(expected); + }); +}); diff --git a/backend/src/services/mailbox/archive.js b/backend/src/services/mailbox/archive.js new file mode 100644 index 00000000..f617c45f --- /dev/null +++ b/backend/src/services/mailbox/archive.js @@ -0,0 +1,207 @@ +import { query } from '../db.js'; +import { emitGtdIfRelevant } from '../gtdSections.js'; +import { + adjustFolderCounts, + isAllMailFolder, + resolveArchiveFolder, +} from '../../utils/mailUtils.js'; + +function emitGtdSectionsRefresh(imapManager, rows, userId) { + const byAccount = new Map(); + for (const m of rows) { + if (!m.message_id) continue; + if (!byAccount.has(m.account_id)) byAccount.set(m.account_id, { mids: new Set(), folders: new Set() }); + const entry = byAccount.get(m.account_id); + entry.mids.add(m.message_id); + if (m.folder) entry.folders.add(m.folder); + } + for (const [accountId, { mids, folders }] of byAccount) { + emitGtdIfRelevant(imapManager, accountId, userId, [...mids], [...folders]) + .catch(err => console.warn('GTD sections refresh emit failed:', err.message)); + } +} + +export async function bulkArchive(imapManager, { userId, accountIds, ids }) { + const moveGuards = []; + try { + const accountClause = accountIds == null ? '' : ' AND m.account_id = ANY($3::uuid[])'; + const params = accountIds == null ? [userId, ids] : [userId, ids, accountIds]; + const result = await query( + `SELECT m.*, a.user_id, a.folder_mappings FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1${accountClause}`, + params + ); + + const owned = result.rows; + const failedItems = []; + if (!owned.length) { + return { + ok: true, + archived: [], + archivedDetails: [], + failed: failedItems, + noArchiveFolder: [], + }; + } + + for (const m of owned) { + moveGuards.push({ accountId: m.account_id, folder: m.folder, uid: m.uid }); + imapManager._guardMoveUid(m.account_id, m.folder, m.uid); + } + + const byAccount = {}; + for (const msg of owned) { + (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); + } + + const archivedIds = []; + const noArchiveFolder = []; + const accountsById = {}; + const allMailDestFolders = new Set(); + + for (const [accountId, msgs] of Object.entries(byAccount)) { + const archiveFolder = await resolveArchiveFolder(accountId, msgs[0].folder_mappings); + if (!archiveFolder) { + noArchiveFolder.push(accountId); + continue; + } + if (await isAllMailFolder(accountId, archiveFolder)) { + allMailDestFolders.add(archiveFolder); + } + + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); + const account = accountResult.rows[0]; + accountsById[accountId] = account; + const byFolder = {}; + for (const msg of msgs) { + (byFolder[msg.folder] = byFolder[msg.folder] || []).push(msg); + } + for (const [srcFolder, folderMsgs] of Object.entries(byFolder)) { + const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); + const { uidMap, succeeded, failed } = await imapManager.bulkMoveMessages(account, folderMsgs.map(m => m.uid), srcFolder, archiveFolder); + for (const uid of succeeded) { + const msg = uidToMsg.get(String(uid)); + archivedIds.push({ id: msg.id, accountId, folder: archiveFolder, newUid: uidMap.get(Number(uid)) || null }); + } + for (const uid of failed) { + const msg = uidToMsg.get(String(uid)); + if (msg) failedItems.push({ id: msg.id, reason: 'IMAP move failed' }); + console.error(`bulk-archive IMAP uid ${uid}: IMAP move failed`); + } + } + } + + const byFolder = {}; + for (const { id, folder, newUid } of archivedIds) { + (byFolder[folder] = byFolder[folder] || []).push({ id, newUid }); + } + for (const [archiveFolder, entries] of Object.entries(byFolder)) { + const allIds = entries.map(e => e.id); + if (allMailDestFolders.has(archiveFolder)) { + await query('DELETE FROM messages WHERE id = ANY($1::uuid[])', [allIds]); + continue; + } + const withUid = entries.filter(e => e.newUid != null); + await query(` + WITH deleted AS ( + DELETE FROM messages WHERE id = ANY($1::uuid[]) RETURNING * + ), + uid_map(src_id, new_uid) AS ( + SELECT * FROM unnest($2::uuid[], $3::bigint[]) + ) + INSERT INTO messages ( + account_id, uid, folder, message_id, subject, + from_name, from_email, to_addresses, cc_addresses, + reply_to, in_reply_to, date, snippet, is_read, is_starred, + has_attachments, flags, body_html, body_text, attachments, + thread_references, thread_id, is_bulk, + read_changed_at, star_changed_at, spam_score_sa, spam_score_ml, + spam_verdict, spam_analyzed_at, spam_details, spam_user_override, + category, list_unsubscribe, list_unsubscribe_post, unsubscribed_at + ) + SELECT + d.account_id, u.new_uid, $4, d.message_id, d.subject, + d.from_name, d.from_email, d.to_addresses, d.cc_addresses, + d.reply_to, d.in_reply_to, d.date, d.snippet, d.is_read, d.is_starred, + d.has_attachments, d.flags, d.body_html, d.body_text, d.attachments, + d.thread_references, d.thread_id, d.is_bulk, + d.read_changed_at, d.star_changed_at, d.spam_score_sa, d.spam_score_ml, + d.spam_verdict, d.spam_analyzed_at, d.spam_details, d.spam_user_override, + d.category, d.list_unsubscribe, d.list_unsubscribe_post, d.unsubscribed_at + FROM deleted d + JOIN uid_map u ON d.id = u.src_id + ON CONFLICT (account_id, uid, folder) DO NOTHING + `, [allIds, withUid.map(e => e.id), withUid.map(e => e.newUid), archiveFolder]); + } + + const needResync = new Map(); + for (const e of archivedIds) { + if (e.newUid) continue; + if (allMailDestFolders.has(e.folder)) continue; + if (!needResync.has(e.accountId)) needResync.set(e.accountId, new Set()); + needResync.get(e.accountId).add(e.folder); + } + for (const [acctId, paths] of needResync) { + const acct = accountsById[acctId]; + if (!acct) continue; + for (const fp of paths) { + imapManager.syncFolderOnDemand(acct, fp) + .catch(err => console.warn('post-archive destination sync failed:', err.message)); + } + } + + if (archivedIds.length > 0) { + const idToArchiveDest = new Map(archivedIds.map(({ id, folder: dest }) => [id, dest])); + const folderDeltas = {}; + for (const msg of owned) { + const dest = idToArchiveDest.get(msg.id); + if (!dest) continue; + const wasUnread = !msg.is_read ? 1 : 0; + const srcKey = `${msg.account_id}:${msg.folder}`; + if (!folderDeltas[srcKey]) folderDeltas[srcKey] = { accountId: msg.account_id, path: msg.folder, totalDelta: 0, unreadDelta: 0 }; + folderDeltas[srcKey].totalDelta--; + folderDeltas[srcKey].unreadDelta -= wasUnread; + if (allMailDestFolders.has(dest)) continue; + const dstKey = `${msg.account_id}:${dest}`; + if (!folderDeltas[dstKey]) folderDeltas[dstKey] = { accountId: msg.account_id, path: dest, totalDelta: 0, unreadDelta: 0 }; + folderDeltas[dstKey].totalDelta++; + folderDeltas[dstKey].unreadDelta += wasUnread; + } + for (const { accountId, path, totalDelta, unreadDelta } of Object.values(folderDeltas)) { + adjustFolderCounts(accountId, path, totalDelta, unreadDelta); + } + const destFolders = [...new Set(archivedIds.map(a => a.folder))].filter(f => !allMailDestFolders.has(f)); + for (const dest of destFolders) { + const accountIdsForDest = [...new Set(archivedIds.filter(a => a.folder === dest).map(a => { + const msg = owned.find(m => m.id === a.id); + return msg?.account_id; + }).filter(Boolean))]; + for (const accountId of accountIdsForDest) { + imapManager.broadcast({ type: 'folder_updated', folder: dest, accountId }, userId); + } + } + } + + emitGtdSectionsRefresh(imapManager, owned, userId); + + return { + ok: true, + archived: archivedIds.map(a => a.id), + archivedDetails: archivedIds.map(a => ({ + id: a.id, + accountId: a.accountId, + folder: a.folder, + uid: a.newUid, + destinationUntracked: allMailDestFolders.has(a.folder), + })), + failed: failedItems, + noArchiveFolder, + }; + } catch (err) { + console.error('bulk-archive error:', err); + return { ok: false, status: 500, error: 'Failed to archive messages' }; + } finally { + for (const g of moveGuards) imapManager._unguardMoveUid(g.accountId, g.folder, g.uid); + } +} diff --git a/backend/src/services/mailbox/archive.test.js b/backend/src/services/mailbox/archive.test.js new file mode 100644 index 00000000..0a7134d6 --- /dev/null +++ b/backend/src/services/mailbox/archive.test.js @@ -0,0 +1,247 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + adjustFolderCounts: vi.fn(), + resolveArchiveFolder: vi.fn(), + isAllMailFolder: vi.fn(), + }; +}); +vi.mock('../gtdSections.js', () => ({ emitGtdIfRelevant: vi.fn().mockResolvedValue(undefined) })); + +import { query } from '../db.js'; +import { + adjustFolderCounts, + isAllMailFolder, + resolveArchiveFolder, +} from '../../utils/mailUtils.js'; +import { bulkArchive } from './archive.js'; + +const ID = '11111111-1111-4111-8111-111111111111'; +const message = { + id: ID, + account_id: 'a1', + uid: 10, + folder: 'INBOX', + message_id: '', + is_read: false, + folder_mappings: {}, +}; +const account = { id: 'a1' }; + +function manager(result = { + uidMap: new Map([[10, 110]]), + succeeded: [10], + failed: [], +}) { + return { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + bulkMoveMessages: vi.fn().mockResolvedValue(result), + syncFolderOnDemand: vi.fn().mockResolvedValue(undefined), + broadcast: vi.fn(), + }; +} + +function stubQueries() { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); +} + +beforeEach(() => { + query.mockReset(); + resolveArchiveFolder.mockReset(); + isAllMailFolder.mockReset(); + adjustFolderCounts.mockReset(); + resolveArchiveFolder.mockResolvedValue('Archive'); + isAllMailFolder.mockResolvedValue(false); +}); + +describe('bulkArchive', () => { + it('narrows ownership to accountIds before IMAP work', async () => { + query.mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await bulkArchive(imap, { + userId: 'u1', + accountIds: ['a1'], + ids: [ID], + }); + + expect(result).toEqual({ + ok: true, + archived: [], + archivedDetails: [], + failed: [], + noArchiveFolder: [], + }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($3::uuid[])'); + expect(params).toEqual(['u1', [ID], ['a1']]); + expect(imap.bulkMoveMessages).not.toHaveBeenCalled(); + }); + + it('preserves the UIDPLUS reinsert and signed source/destination deltas', async () => { + stubQueries(); + const imap = manager(); + + const result = await bulkArchive(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + }); + + expect(result).toEqual({ + ok: true, + archived: [ID], + archivedDetails: [{ + id: ID, + accountId: 'a1', + folder: 'Archive', + uid: 110, + destinationUntracked: false, + }], + failed: [], + noArchiveFolder: [], + }); + const cte = query.mock.calls.find(([sql]) => sql.includes('WITH deleted AS')); + expect(cte[1]).toEqual([[ID], [ID], [110], 'Archive']); + expect(adjustFolderCounts).toHaveBeenCalledWith('a1', 'INBOX', -1, -1); + expect(adjustFolderCounts).toHaveBeenCalledWith('a1', 'Archive', 1, 1); + expect(imap._unguardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + }); + + it('deletes the DB row instead of rehoming it for Gmail All Mail', async () => { + stubQueries(); + isAllMailFolder.mockResolvedValue(true); + resolveArchiveFolder.mockResolvedValue('[Gmail]/All Mail'); + const imap = manager(); + + const result = await bulkArchive(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + }); + + expect(result).toEqual({ + ok: true, + archived: [ID], + archivedDetails: [{ + id: ID, + accountId: 'a1', + folder: '[Gmail]/All Mail', + uid: 110, + destinationUntracked: true, + }], + failed: [], + noArchiveFolder: [], + }); + expect(query).toHaveBeenCalledWith( + 'DELETE FROM messages WHERE id = ANY($1::uuid[])', + [[ID]], + ); + expect(query.mock.calls.some(([sql]) => sql.includes('WITH deleted AS'))).toBe(false); + expect(adjustFolderCounts).toHaveBeenCalledTimes(1); + expect(adjustFolderCounts).toHaveBeenCalledWith('a1', 'INBOX', -1, -1); + }); + + it('keeps the non-UIDPLUS delete-only path and destination resync', async () => { + stubQueries(); + const imap = manager({ uidMap: new Map(), succeeded: [10], failed: [] }); + + const result = await bulkArchive(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + }); + + expect(result).toEqual({ + ok: true, + archived: [ID], + archivedDetails: [{ + id: ID, + accountId: 'a1', + folder: 'Archive', + uid: null, + destinationUntracked: false, + }], + failed: [], + noArchiveFolder: [], + }); + const cte = query.mock.calls.find(([sql]) => sql.includes('WITH deleted AS')); + expect(cte[1]).toEqual([[ID], [], [], 'Archive']); + expect(imap.syncFolderOnDemand).toHaveBeenCalledWith(account, 'Archive'); + }); + + it('releases the source guard when IMAP archive throws', async () => { + stubQueries(); + const imap = manager(); + imap.bulkMoveMessages.mockRejectedValue(new Error('archive failed')); + + const result = await bulkArchive(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + }); + + expect(result).toEqual({ ok: false, status: 500, error: 'Failed to archive messages' }); + expect(imap._guardMoveUid).toHaveBeenCalledTimes(1); + expect(imap._unguardMoveUid).toHaveBeenCalledTimes(1); + }); + + it('surfaces per-message IMAP failures', async () => { + const failedMessage = { + ...message, + id: '22222222-2222-4222-8222-222222222222', + uid: 11, + }; + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) { + return { rows: [message, failedMessage] }; + } + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = manager({ + uidMap: new Map([[10, 110]]), + succeeded: [10], + failed: [11], + }); + + const result = await bulkArchive(imap, { + userId: 'u1', + accountIds: null, + ids: [message.id, failedMessage.id], + }); + + expect(result.failed).toEqual([{ + id: failedMessage.id, + reason: 'IMAP move failed', + }]); + }); + + it('keeps accounts without archive folders in a named partition', async () => { + stubQueries(); + resolveArchiveFolder.mockResolvedValue(null); + + const result = await bulkArchive(manager(), { + userId: 'u1', + accountIds: null, + ids: [ID], + }); + + expect(result).toEqual({ + ok: true, + archived: [], + archivedDetails: [], + failed: [], + noArchiveFolder: ['a1'], + }); + }); +}); diff --git a/backend/src/services/mailbox/batch.js b/backend/src/services/mailbox/batch.js new file mode 100644 index 00000000..25bb76c3 --- /dev/null +++ b/backend/src/services/mailbox/batch.js @@ -0,0 +1,11 @@ +// Process IMAP operations in bounded batches so a 500-message bulk action +// does not spawn hundreds of parallel temporary IMAP connections. +export async function runInBatches(items, concurrency, fn) { + const results = []; + for (let i = 0; i < items.length; i += concurrency) { + const batch = items.slice(i, i + concurrency); + const batchResults = await Promise.allSettled(batch.map(fn)); + results.push(...batchResults); + } + return results; +} diff --git a/backend/src/services/mailbox/batch.test.js b/backend/src/services/mailbox/batch.test.js new file mode 100644 index 00000000..62d447fa --- /dev/null +++ b/backend/src/services/mailbox/batch.test.js @@ -0,0 +1,47 @@ +import { describe, expect, it, vi } from 'vitest'; +import { runInBatches } from './batch.js'; + +describe('runInBatches', () => { + it('does not start the next batch until the current batch settles', async () => { + const events = []; + const releases = []; + const fn = vi.fn((item) => new Promise((resolve) => { + events.push(`start:${item}`); + releases.push(() => { + events.push(`end:${item}`); + resolve(item * 2); + }); + })); + + const pending = runInBatches([1, 2, 3], 2, fn); + await Promise.resolve(); + expect(events).toEqual(['start:1', 'start:2']); + + releases.shift()(); + releases.shift()(); + await Promise.resolve(); + await Promise.resolve(); + expect(events).toEqual(['start:1', 'start:2', 'end:1', 'end:2', 'start:3']); + + releases.shift()(); + await expect(pending).resolves.toEqual([ + { status: 'fulfilled', value: 2 }, + { status: 'fulfilled', value: 4 }, + { status: 'fulfilled', value: 6 }, + ]); + }); + + it('returns rejected entries without aborting later batches', async () => { + const error = new Error('failed'); + const results = await runInBatches([1, 2, 3], 2, async item => { + if (item === 2) throw error; + return item; + }); + + expect(results).toEqual([ + { status: 'fulfilled', value: 1 }, + { status: 'rejected', reason: error }, + { status: 'fulfilled', value: 3 }, + ]); + }); +}); diff --git a/backend/src/services/mailbox/category.js b/backend/src/services/mailbox/category.js new file mode 100644 index 00000000..c6bb1d28 --- /dev/null +++ b/backend/src/services/mailbox/category.js @@ -0,0 +1,22 @@ +import { query } from '../db.js'; + +export async function setCategory(imapManager, { userId, accountIds, id, category }) { + void imapManager; + const accountClause = accountIds == null + ? '' + : ' AND messages.account_id = ANY($4::uuid[])'; + const params = accountIds == null + ? [category === 'primary' ? null : category, id, userId] + : [category === 'primary' ? null : category, id, userId, accountIds]; + const result = await query( + `UPDATE messages SET category = $1 + FROM email_accounts a + WHERE messages.id = $2 + AND messages.account_id = a.id + AND a.user_id = $3${accountClause} + RETURNING messages.id`, + params + ); + if (!result.rows.length) return { ok: false, status: 404, error: 'Message not found' }; + return { ok: true, category }; +} diff --git a/backend/src/services/mailbox/category.test.js b/backend/src/services/mailbox/category.test.js new file mode 100644 index 00000000..ac0395e4 --- /dev/null +++ b/backend/src/services/mailbox/category.test.js @@ -0,0 +1,42 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); + +import { query } from '../db.js'; +import { setCategory } from './category.js'; + +const ID = '11111111-1111-4111-8111-111111111111'; + +beforeEach(() => query.mockReset()); + +describe('setCategory', () => { + it('stores primary as null while returning the requested category', async () => { + query.mockResolvedValue({ rows: [{ id: ID }] }); + + const result = await setCategory(null, { + userId: 'u1', + accountIds: null, + id: ID, + category: 'primary', + }); + + expect(result).toEqual({ ok: true, category: 'primary' }); + expect(query.mock.calls[0][1]).toEqual([null, ID, 'u1']); + }); + + it('narrows the update to non-null accountIds', async () => { + query.mockResolvedValue({ rows: [] }); + + const result = await setCategory(null, { + userId: 'u1', + accountIds: ['a1'], + id: ID, + category: 'newsletter', + }); + + expect(result).toEqual({ ok: false, status: 404, error: 'Message not found' }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('messages.account_id = ANY($4::uuid[])'); + expect(params).toEqual(['newsletter', ID, 'u1', ['a1']]); + }); +}); diff --git a/backend/src/services/mailbox/flags.js b/backend/src/services/mailbox/flags.js new file mode 100644 index 00000000..55dd91b8 --- /dev/null +++ b/backend/src/services/mailbox/flags.js @@ -0,0 +1,183 @@ +import { query } from '../db.js'; +import { emitGtdIfRelevant } from '../gtdSections.js'; +import { + adjustFolderCounts, + fanOutBulkReadToSiblings, + fanOutReadToSiblings, + fanOutStarToSiblings, +} from '../../utils/mailUtils.js'; +import { runInBatches } from './batch.js'; + +function emitGtdSectionsRefresh(imapManager, rows, userId) { + const byAccount = new Map(); + for (const m of rows) { + if (!m.message_id) continue; + if (!byAccount.has(m.account_id)) byAccount.set(m.account_id, { mids: new Set(), folders: new Set() }); + const entry = byAccount.get(m.account_id); + entry.mids.add(m.message_id); + if (m.folder) entry.folders.add(m.folder); + } + for (const [accountId, { mids, folders }] of byAccount) { + emitGtdIfRelevant(imapManager, accountId, userId, [...mids], [...folders]) + .catch(err => console.warn('GTD sections refresh emit failed:', err.message)); + } +} + +export async function setRead(imapManager, { userId, accountIds, id, read }) { + const accountClause = accountIds == null ? '' : ' AND m.account_id = ANY($3::uuid[])'; + const params = accountIds == null ? [id, userId] : [id, userId, accountIds]; + const result = await query(` + SELECT m.*, a.user_id, + CASE WHEN m.message_id IS NULL THEN 1 + ELSE (SELECT COUNT(*) FROM messages s + WHERE s.account_id = m.account_id AND s.message_id = m.message_id) + END AS sibling_count + FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = $1 AND a.user_id = $2${accountClause} + `, params); + + if (!result.rows.length) return { ok: false, status: 404, error: 'Message not found' }; + const message = result.rows[0]; + + // Run DB update and account fetch concurrently — no dependency between them. + // read_changed_at tells the IMAP sync not to overwrite this change for 30 s, + // preventing a race where a concurrent sync fetch sees the old IMAP flag. + const [, accountResult] = await Promise.all([ + query('UPDATE messages SET is_read = $1, read_changed_at = NOW() WHERE id = $2', [read, id]), + query('SELECT * FROM email_accounts WHERE id = $1', [message.account_id]), + ]); + + // Keep the cached folder unread_count in sync so pagination totals stay accurate. + if (!!message.is_read !== !!read) { + adjustFolderCounts(message.account_id, message.folder, 0, read ? -1 : 1); + // Notify the user's OTHER sessions so a read/unread on one device reflects on the rest + // in place, without a full folder refetch (the originating device already applied it). + imapManager.broadcast({ type: 'message_flags', accountId: message.account_id, changes: [{ id, is_read: read }] }, userId); + } + + if (accountResult.rows[0]?.gtd_enabled && Number(message.sibling_count) > 1) { + await fanOutReadToSiblings(message.account_id, message.message_id, read); + } + + try { + await imapManager.setFlag(accountResult.rows[0], message.uid, message.folder, '\\Seen', read); + imapManager._resolveFlagPush(message.account_id, id, '\\Seen'); + } catch (err) { + console.error('IMAP flag update failed:', err.message); + imapManager._enqueueFlagPush(message.account_id, id, '\\Seen', read); + } + + emitGtdSectionsRefresh(imapManager, [message], userId); + + return { ok: true, is_read: read }; +} + +export async function setStarred(imapManager, { userId, accountIds, id, starred }) { + const accountClause = accountIds == null ? '' : ' AND m.account_id = ANY($3::uuid[])'; + const params = accountIds == null ? [id, userId] : [id, userId, accountIds]; + const result = await query(` + SELECT m.*, a.user_id, + CASE WHEN m.message_id IS NULL THEN 1 + ELSE (SELECT COUNT(*) FROM messages s + WHERE s.account_id = m.account_id AND s.message_id = m.message_id) + END AS sibling_count + FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = $1 AND a.user_id = $2${accountClause} + `, params); + + if (!result.rows.length) return { ok: false, status: 404, error: 'Message not found' }; + const message = result.rows[0]; + const updated = !!message.is_starred !== !!starred; + + const [, accountResult] = await Promise.all([ + query('UPDATE messages SET is_starred = $1, star_changed_at = NOW() WHERE id = $2', [starred, id]), + query('SELECT * FROM email_accounts WHERE id = $1', [message.account_id]), + ]); + + if (accountResult.rows[0]?.gtd_enabled && Number(message.sibling_count) > 1) { + await fanOutStarToSiblings(message.account_id, message.message_id, starred); + } + + try { + await imapManager.setFlag(accountResult.rows[0], message.uid, message.folder, '\\Flagged', starred); + imapManager._resolveFlagPush(message.account_id, id, '\\Flagged'); + } catch (err) { + console.error('IMAP star update failed:', err.message); + imapManager._enqueueFlagPush(message.account_id, id, '\\Flagged', starred); + } + + emitGtdSectionsRefresh(imapManager, [message], userId); + if (updated) { + imapManager.broadcast({ type: 'message_flags', accountId: message.account_id, changes: [{ id, is_starred: starred }] }, userId); + } + + return { ok: true, is_starred: starred, updated }; +} + +export async function bulkSetRead(imapManager, { userId, accountIds, ids, read }) { + try { + const accountClause = accountIds == null ? '' : ' AND m.account_id = ANY($3::uuid[])'; + const params = accountIds == null ? [userId, ids] : [userId, ids, accountIds]; + const result = await query( + `SELECT m.id, m.uid, m.folder, m.is_read, m.account_id, m.message_id, a.gtd_enabled FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1${accountClause}`, + params + ); + + const owned = result.rows; + if (!owned.length) return { ok: true, updated: [] }; + + const toUpdate = owned.filter(m => !!m.is_read !== !!read); + if (!toUpdate.length) return { ok: true, updated: [] }; + + await query( + 'UPDATE messages SET is_read = $1, read_changed_at = NOW() WHERE id = ANY($2::uuid[])', + [read, toUpdate.map(m => m.id)] + ); + + const folderDeltas = {}; + for (const msg of toUpdate) { + const key = `${msg.account_id}:${msg.folder}`; + if (!folderDeltas[key]) folderDeltas[key] = { accountId: msg.account_id, folder: msg.folder, delta: 0 }; + folderDeltas[key].delta += read ? -1 : 1; + } + for (const { accountId, folder, delta } of Object.values(folderDeltas)) { + adjustFolderCounts(accountId, folder, 0, delta); + } + + const gtdUpdatedIds = toUpdate.filter(m => m.gtd_enabled).map(m => m.id); + if (gtdUpdatedIds.length) await fanOutBulkReadToSiblings(gtdUpdatedIds, read); + imapManager.broadcast({ type: 'message_flags', changes: toUpdate.map(m => ({ id: m.id, is_read: read })) }, userId); + + const byAccount = {}; + for (const msg of toUpdate) { + (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); + } + for (const [accountId, msgs] of Object.entries(byAccount)) { + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); + const account = accountResult.rows[0]; + const results = await runInBatches( + msgs, 3, + msg => imapManager.setFlag(account, msg.uid, msg.folder, '\\Seen', read) + ); + results.forEach((r, i) => { + if (r.status === 'rejected') { + console.error(`bulk-read IMAP ${msgs[i].id}:`, r.reason.message); + imapManager._enqueueFlagPush(accountId, msgs[i].id, '\\Seen', read); + } else { + imapManager._resolveFlagPush(accountId, msgs[i].id, '\\Seen'); + } + }); + } + + emitGtdSectionsRefresh(imapManager, toUpdate, userId); + + return { ok: true, updated: toUpdate.map(m => m.id) }; + } catch (err) { + console.error('bulk-read error:', err); + return { ok: false, status: 500, error: 'Failed to update messages' }; + } +} diff --git a/backend/src/services/mailbox/flags.test.js b/backend/src/services/mailbox/flags.test.js new file mode 100644 index 00000000..189d5bd9 --- /dev/null +++ b/backend/src/services/mailbox/flags.test.js @@ -0,0 +1,157 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + adjustFolderCounts: vi.fn(), + fanOutReadToSiblings: vi.fn(), + fanOutStarToSiblings: vi.fn(), + fanOutBulkReadToSiblings: vi.fn(), + }; +}); +vi.mock('../gtdSections.js', () => ({ emitGtdIfRelevant: vi.fn().mockResolvedValue(undefined) })); + +import { query } from '../db.js'; +import { + fanOutBulkReadToSiblings, + fanOutReadToSiblings, + fanOutStarToSiblings, +} from '../../utils/mailUtils.js'; +import { bulkSetRead, setRead, setStarred } from './flags.js'; + +const ID = '11111111-1111-4111-8111-111111111111'; +const message = { + id: ID, + account_id: 'a1', + uid: 10, + folder: 'INBOX', + message_id: '', + is_read: false, + is_starred: false, + sibling_count: 2, + gtd_enabled: true, +}; +const account = { id: 'a1', gtd_enabled: true }; + +function manager() { + return { + setFlag: vi.fn().mockResolvedValue(undefined), + _resolveFlagPush: vi.fn(), + _enqueueFlagPush: vi.fn(), + broadcast: vi.fn(), + }; +} + +beforeEach(() => { + query.mockReset(); + fanOutReadToSiblings.mockReset(); + fanOutStarToSiblings.mockReset(); + fanOutBulkReadToSiblings.mockReset(); +}); + +describe('setRead', () => { + it('preserves the single-message read receipt and IMAP flag write', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = manager(); + + const result = await setRead(imap, { + userId: 'u1', + accountIds: null, + id: ID, + read: true, + }); + + expect(result).toEqual({ ok: true, is_read: true }); + expect(imap.setFlag).toHaveBeenCalledWith(account, 10, 'INBOX', '\\Seen', true); + expect(fanOutReadToSiblings).toHaveBeenCalledWith('a1', '', true); + }); + + it('narrows ownership to accountIds before DB mutation or IMAP work', async () => { + query.mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await setRead(imap, { + userId: 'u1', + accountIds: ['a1'], + id: ID, + read: true, + }); + + expect(result).toEqual({ ok: false, status: 404, error: 'Message not found' }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($3::uuid[])'); + expect(params).toEqual([ID, 'u1', ['a1']]); + expect(query).toHaveBeenCalledTimes(1); + expect(imap.setFlag).not.toHaveBeenCalled(); + }); +}); + +describe('setStarred', () => { + it('preserves the star receipt and GTD sibling fan-out', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = manager(); + + const result = await setStarred(imap, { + userId: 'u1', + accountIds: null, + id: ID, + starred: true, + }); + + expect(result).toEqual({ ok: true, is_starred: true, updated: true }); + expect(imap.setFlag).toHaveBeenCalledWith(account, 10, 'INBOX', '\\Flagged', true); + expect(fanOutStarToSiblings).toHaveBeenCalledWith('a1', '', true); + }); + + it('reports an already-targeted star state as a no-op', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id')) { + return { rows: [{ ...message, is_starred: true }] }; + } + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + + const result = await setStarred(manager(), { + userId: 'u1', + accountIds: null, + id: ID, + starred: true, + }); + + expect(result).toEqual({ ok: true, is_starred: true, updated: false }); + }); +}); + +describe('bulkSetRead', () => { + it('returns only ids whose read state changes', async () => { + const alreadyRead = { ...message, id: '22222222-2222-4222-8222-222222222222', is_read: true }; + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.id, m.uid')) return { rows: [message, alreadyRead] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = manager(); + + const result = await bulkSetRead(imap, { + userId: 'u1', + accountIds: null, + ids: [ID, alreadyRead.id], + read: true, + }); + + expect(result).toEqual({ ok: true, updated: [ID] }); + expect(imap.setFlag).toHaveBeenCalledTimes(1); + expect(fanOutBulkReadToSiblings).toHaveBeenCalledWith([ID], true); + }); +}); diff --git a/backend/src/services/mailbox/folders.js b/backend/src/services/mailbox/folders.js new file mode 100644 index 00000000..6453c489 --- /dev/null +++ b/backend/src/services/mailbox/folders.js @@ -0,0 +1,102 @@ +import { query } from '../db.js'; + +function accountScope(accountId, userId, accountIds, columns = '*') { + const accountClause = accountIds == null ? '' : ' AND id = ANY($3::uuid[])'; + const params = accountIds == null + ? [accountId, userId] + : [accountId, userId, accountIds]; + return query( + `SELECT ${columns} FROM email_accounts WHERE id = $1 AND user_id = $2${accountClause}`, + params, + ); +} + +export async function listFolders(imapManager, { userId, accountIds, accountId }) { + void imapManager; + const check = await accountScope(accountId, userId, accountIds, 'id'); + if (!check.rows.length) return { ok: false, status: 404, error: 'Account not found' }; + + const result = await query( + 'SELECT * FROM folders WHERE account_id = $1 ORDER BY path', + [accountId] + ); + return { ok: true, folders: result.rows }; +} + +export async function createFolder( + imapManager, + { userId, accountIds, accountId, name, parentPath }, +) { + const check = await accountScope(accountId, userId, accountIds); + if (!check.rows.length) return { ok: false, status: 404, error: 'Account not found' }; + + let path = name.trim(); + if (parentPath) { + const delimResult = await query('SELECT delimiter FROM folders WHERE account_id = $1 LIMIT 1', [accountId]); + const delim = delimResult.rows[0]?.delimiter || '/'; + path = `${parentPath}${delim}${name.trim()}`; + } + + try { + await imapManager.createFolder(check.rows[0], path); + await query( + `INSERT INTO folders (account_id, path, name) VALUES ($1, $2, $3) + ON CONFLICT (account_id, path) DO NOTHING`, + [accountId, path, name.trim()] + ); + return { ok: true, path }; + } catch (err) { + console.error('Create folder error:', err); + return { ok: false, status: 500, error: 'Failed to create folder' }; + } +} + +export async function deleteFolder(imapManager, { userId, accountIds, accountId, path }) { + const check = await accountScope(accountId, userId, accountIds); + if (!check.rows.length) return { ok: false, status: 404, error: 'Account not found' }; + + try { + await imapManager.deleteFolder(check.rows[0], path); + } catch (err) { + console.error(`IMAP deleteFolder failed for ${path}:`, err.message); + return { ok: false, status: 500, error: 'Failed to delete folder on server' }; + } + await query('DELETE FROM folders WHERE account_id = $1 AND path = $2', [accountId, path]); + await query('DELETE FROM messages WHERE account_id = $1 AND folder = $2', [accountId, path]); + return { ok: true }; +} + +export async function renameFolder( + imapManager, + { userId, accountIds, accountId, oldPath, newName }, +) { + const check = await accountScope(accountId, userId, accountIds); + if (!check.rows.length) return { ok: false, status: 404, error: 'Account not found' }; + + const delimResult = await query('SELECT delimiter FROM folders WHERE account_id = $1 AND path = $2', [accountId, oldPath]); + const delim = delimResult.rows[0]?.delimiter || '/'; + const parts = oldPath.split(delim); + parts[parts.length - 1] = newName.trim(); + const newPath = parts.join(delim); + + try { + await imapManager.renameFolder(check.rows[0], oldPath, newPath); + await query( + 'UPDATE folders SET path = $1, name = $2, updated_at = NOW() WHERE account_id = $3 AND path = $4', + [newPath, newName.trim(), accountId, oldPath] + ); + await query('UPDATE messages SET folder = $1 WHERE account_id = $2 AND folder = $3', [newPath, accountId, oldPath]); + return { ok: true, newPath }; + } catch (err) { + console.error('Rename folder error:', err); + return { ok: false, status: 500, error: 'Failed to rename folder' }; + } +} + +export async function countMessagesIn(accountId, path) { + const result = await query( + 'SELECT COUNT(*) AS count FROM messages WHERE account_id = $1 AND folder = $2', + [accountId, path], + ); + return Number(result.rows[0]?.count || 0); +} diff --git a/backend/src/services/mailbox/folders.test.js b/backend/src/services/mailbox/folders.test.js new file mode 100644 index 00000000..cee24cb7 --- /dev/null +++ b/backend/src/services/mailbox/folders.test.js @@ -0,0 +1,136 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); + +import { query } from '../db.js'; +import { + countMessagesIn, + createFolder, + deleteFolder, + listFolders, + renameFolder, +} from './folders.js'; + +const account = { id: 'a1', user_id: 'u1' }; + +function manager() { + return { + createFolder: vi.fn().mockResolvedValue(undefined), + deleteFolder: vi.fn().mockResolvedValue(undefined), + renameFolder: vi.fn().mockResolvedValue(undefined), + }; +} + +beforeEach(() => query.mockReset()); + +describe('listFolders', () => { + it('returns folders only after a scope-checked account lookup', async () => { + const folders = [{ account_id: 'a1', path: 'INBOX' }]; + query.mockResolvedValueOnce({ rows: [account] }).mockResolvedValueOnce({ rows: folders }); + + const result = await listFolders(null, { + userId: 'u1', + accountIds: ['a1'], + accountId: 'a1', + }); + + expect(result).toEqual({ ok: true, folders }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('id = ANY($3::uuid[])'); + expect(params).toEqual(['a1', 'u1', ['a1']]); + }); +}); + +describe('createFolder', () => { + it('prevents an out-of-scope account from reaching IMAP', async () => { + query.mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await createFolder(imap, { + userId: 'u1', + accountIds: ['a1'], + accountId: 'a2', + name: 'Projects', + parentPath: null, + }); + + expect(result).toEqual({ ok: false, status: 404, error: 'Account not found' }); + expect(query.mock.calls[0][0]).toContain('id = ANY($3::uuid[])'); + expect(imap.createFolder).not.toHaveBeenCalled(); + }); + + it('preserves delimiter-based child path creation', async () => { + query + .mockResolvedValueOnce({ rows: [account] }) + .mockResolvedValueOnce({ rows: [{ delimiter: '.' }] }) + .mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await createFolder(imap, { + userId: 'u1', + accountIds: null, + accountId: 'a1', + name: 'Child', + parentPath: 'Parent', + }); + + expect(result).toEqual({ ok: true, path: 'Parent.Child' }); + expect(imap.createFolder).toHaveBeenCalledWith(account, 'Parent.Child'); + }); +}); + +describe('renameFolder', () => { + it('renames only the last path component and updates folder/message rows', async () => { + query + .mockResolvedValueOnce({ rows: [account] }) + .mockResolvedValueOnce({ rows: [{ delimiter: '/' }] }) + .mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await renameFolder(imap, { + userId: 'u1', + accountIds: null, + accountId: 'a1', + oldPath: 'Parent/Old', + newName: 'New', + }); + + expect(result).toEqual({ ok: true, newPath: 'Parent/New' }); + expect(imap.renameFolder).toHaveBeenCalledWith(account, 'Parent/Old', 'Parent/New'); + }); +}); + +describe('deleteFolder', () => { + it('deletes the IMAP folder before local folder and message rows', async () => { + query.mockResolvedValueOnce({ rows: [account] }).mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await deleteFolder(imap, { + userId: 'u1', + accountIds: null, + accountId: 'a1', + path: 'Projects', + }); + + expect(result).toEqual({ ok: true }); + expect(imap.deleteFolder).toHaveBeenCalledWith(account, 'Projects'); + expect(query).toHaveBeenCalledWith( + 'DELETE FROM messages WHERE account_id = $1 AND folder = $2', + ['a1', 'Projects'], + ); + }); +}); + +describe('countMessagesIn', () => { + it('returns the live message count for an account and path', async () => { + query.mockResolvedValue({ rows: [{ count: '7' }] }); + + const count = await countMessagesIn('a1', 'Projects'); + + expect(count).toBe(7); + expect(query).toHaveBeenCalledWith( + expect.stringContaining('COUNT(*)'), + ['a1', 'Projects'], + ); + }); +}); diff --git a/backend/src/services/mailbox/move.js b/backend/src/services/mailbox/move.js new file mode 100644 index 00000000..82f3bdbf --- /dev/null +++ b/backend/src/services/mailbox/move.js @@ -0,0 +1,172 @@ +import { query } from '../db.js'; +import { emitGtdIfRelevant } from '../gtdSections.js'; +import { adjustFolderCounts } from '../../utils/mailUtils.js'; + +function emitGtdSectionsRefresh(imapManager, rows, userId) { + const byAccount = new Map(); + for (const m of rows) { + if (!m.message_id) continue; + if (!byAccount.has(m.account_id)) byAccount.set(m.account_id, { mids: new Set(), folders: new Set() }); + const entry = byAccount.get(m.account_id); + entry.mids.add(m.message_id); + if (m.folder) entry.folders.add(m.folder); + } + for (const [accountId, { mids, folders }] of byAccount) { + emitGtdIfRelevant(imapManager, accountId, userId, [...mids], [...folders]) + .catch(err => console.warn('GTD sections refresh emit failed:', err.message)); + } +} + +export async function resolveMovedIds(accountId, folder, uids) { + if (!uids.length) return []; + const result = await query( + `SELECT id, uid FROM messages + WHERE account_id = $1 AND folder = $2 AND uid = ANY($3::bigint[])`, + [accountId, folder, uids], + ); + return result.rows; +} + +export async function bulkMoveToFolder(imapManager, { userId, accountIds, ids, folder }) { + const moveGuards = []; + try { + const accountClause = accountIds == null ? '' : ' AND m.account_id = ANY($3::uuid[])'; + const params = accountIds == null ? [userId, ids] : [userId, ids, accountIds]; + const result = await query( + `SELECT m.*, a.user_id FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1${accountClause}`, + params + ); + + const owned = result.rows; + const movedDetails = []; + const failedItems = []; + const skippedAccounts = []; + if (!owned.length) { + return { ok: true, moved: [], movedDetails, failed: failedItems, skippedAccounts }; + } + + // Guard every source (account, folder, uid) for the whole bulk move. + for (const m of owned) { + moveGuards.push({ accountId: m.account_id, folder: m.folder, uid: m.uid }); + imapManager._guardMoveUid(m.account_id, m.folder, m.uid); + } + + const byAccount = {}; + for (const msg of owned) { + (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); + } + + const movedIds = []; + const uidUpdates = []; + const resyncAccounts = []; + for (const [accountId, msgs] of Object.entries(byAccount)) { + const folderCheck = await query( + 'SELECT 1 FROM folders WHERE account_id = $1 AND path = $2', + [accountId, folder] + ); + if (!folderCheck.rows.length) { + console.warn(`bulk-move: folder "${folder}" not found for account ${accountId}, skipping`); + skippedAccounts.push({ account_id: accountId, reason: 'folder_not_found' }); + continue; + } + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); + const account = accountResult.rows[0]; + const byFolder = {}; + for (const msg of msgs) { + (byFolder[msg.folder] = byFolder[msg.folder] || []).push(msg); + } + let accountMissingUid = false; + for (const [srcFolder, folderMsgs] of Object.entries(byFolder)) { + const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); + const { uidMap, succeeded, failed } = await imapManager.bulkMoveMessages(account, folderMsgs.map(m => m.uid), srcFolder, folder); + for (const uid of succeeded) { + const msg = uidToMsg.get(String(uid)); + movedIds.push(msg.id); + const newUid = uidMap.get(Number(uid)) || null; + movedDetails.push({ id: msg.id, accountId, uid: newUid }); + if (newUid) uidUpdates.push({ id: msg.id, newUid }); + else accountMissingUid = true; + } + for (const uid of failed) { + const msg = uidToMsg.get(String(uid)); + if (msg) failedItems.push({ id: msg.id, reason: 'IMAP move failed' }); + console.error(`bulk-move IMAP uid ${uid}: IMAP move failed`); + } + } + if (accountMissingUid) resyncAccounts.push(account); + } + + if (movedIds.length > 0) { + const uidUpdateMap = new Map(uidUpdates.map(u => [u.id, u.newUid])); + const withNewUid = movedIds.filter(id => uidUpdateMap.has(id)); + await query(` + WITH deleted AS ( + DELETE FROM messages WHERE id = ANY($1::uuid[]) RETURNING * + ), + uid_map(src_id, new_uid) AS ( + SELECT * FROM unnest($2::uuid[], $3::bigint[]) + ) + INSERT INTO messages ( + account_id, uid, folder, message_id, subject, + from_name, from_email, to_addresses, cc_addresses, + reply_to, in_reply_to, date, snippet, is_read, is_starred, + has_attachments, flags, body_html, body_text, attachments, + thread_references, thread_id, is_bulk, + read_changed_at, star_changed_at, spam_score_sa, spam_score_ml, + spam_verdict, spam_analyzed_at, spam_details, spam_user_override, + category, list_unsubscribe, list_unsubscribe_post, unsubscribed_at + ) + SELECT + d.account_id, u.new_uid, $4, d.message_id, d.subject, + d.from_name, d.from_email, d.to_addresses, d.cc_addresses, + d.reply_to, d.in_reply_to, d.date, d.snippet, d.is_read, d.is_starred, + d.has_attachments, d.flags, d.body_html, d.body_text, d.attachments, + d.thread_references, d.thread_id, d.is_bulk, + d.read_changed_at, d.star_changed_at, d.spam_score_sa, d.spam_score_ml, + d.spam_verdict, d.spam_analyzed_at, d.spam_details, d.spam_user_override, + d.category, d.list_unsubscribe, d.list_unsubscribe_post, d.unsubscribed_at + FROM deleted d + JOIN uid_map u ON d.id = u.src_id + ON CONFLICT (account_id, uid, folder) DO NOTHING + `, [movedIds, withNewUid, withNewUid.map(id => uidUpdateMap.get(id)), folder]); + for (const acct of resyncAccounts) { + imapManager.syncFolderOnDemand(acct, folder) + .catch(err => console.warn('post-move destination sync failed:', err.message)); + } + const movedSet = new Set(movedIds); + const srcTotals = {}; + for (const msg of owned) { + if (!movedSet.has(msg.id)) continue; + const key = `${msg.account_id}:${msg.folder}`; + if (!srcTotals[key]) srcTotals[key] = { accountId: msg.account_id, path: msg.folder, total: 0, unread: 0 }; + srcTotals[key].total++; + if (!msg.is_read) srcTotals[key].unread++; + } + for (const { accountId, path, total, unread } of Object.values(srcTotals)) { + adjustFolderCounts(accountId, path, -total, -unread); + adjustFolderCounts(accountId, folder, total, unread); + } + + for (const accountId of Object.keys(srcTotals).map(k => k.split(':')[0])) { + imapManager.broadcast({ type: 'folder_updated', folder, accountId }, userId); + } + } + + emitGtdSectionsRefresh(imapManager, owned, userId); + + return { + ok: true, + moved: movedIds, + movedDetails, + failed: failedItems, + skippedAccounts, + }; + } catch (err) { + console.error('bulk-move error:', err); + return { ok: false, status: 500, error: 'Failed to move messages' }; + } finally { + for (const g of moveGuards) imapManager._unguardMoveUid(g.accountId, g.folder, g.uid); + } +} diff --git a/backend/src/services/mailbox/move.test.js b/backend/src/services/mailbox/move.test.js new file mode 100644 index 00000000..c9ba0bff --- /dev/null +++ b/backend/src/services/mailbox/move.test.js @@ -0,0 +1,214 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, adjustFolderCounts: vi.fn() }; +}); +vi.mock('../gtdSections.js', () => ({ emitGtdIfRelevant: vi.fn().mockResolvedValue(undefined) })); + +import { query } from '../db.js'; +import { adjustFolderCounts } from '../../utils/mailUtils.js'; +import { bulkMoveToFolder, resolveMovedIds } from './move.js'; + +const ID = '11111111-1111-4111-8111-111111111111'; +const message = { + id: ID, + account_id: 'a1', + uid: 10, + folder: 'INBOX', + message_id: '', + is_read: false, +}; +const account = { id: 'a1' }; + +function manager(result = { + uidMap: new Map([[10, 110]]), + succeeded: [10], + failed: [], +}) { + return { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + bulkMoveMessages: vi.fn().mockResolvedValue(result), + syncFolderOnDemand: vi.fn().mockResolvedValue(undefined), + broadcast: vi.fn(), + }; +} + +function stubMoveQueries() { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id')) return { rows: [message] }; + if (sql.includes('SELECT 1 FROM folders')) return { rows: [{ exists: 1 }] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); +} + +beforeEach(() => { + query.mockReset(); + adjustFolderCounts.mockReset(); +}); + +describe('bulkMoveToFolder', () => { + it('narrows ownership to accountIds before IMAP work', async () => { + query.mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await bulkMoveToFolder(imap, { + userId: 'u1', + accountIds: ['a1'], + ids: [ID], + folder: 'Archive', + }); + + expect(result).toEqual({ + ok: true, + moved: [], + movedDetails: [], + failed: [], + skippedAccounts: [], + }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($3::uuid[])'); + expect(params).toEqual(['u1', [ID], ['a1']]); + expect(imap.bulkMoveMessages).not.toHaveBeenCalled(); + }); + + it('preserves the guarded UIDPLUS CTE path and signed count deltas', async () => { + stubMoveQueries(); + const imap = manager(); + + const result = await bulkMoveToFolder(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + folder: 'Archive', + }); + + expect(result).toEqual({ + ok: true, + moved: [ID], + movedDetails: [{ id: ID, accountId: 'a1', uid: 110 }], + failed: [], + skippedAccounts: [], + }); + expect(imap._guardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + expect(imap._unguardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + const cte = query.mock.calls.find(([sql]) => sql.includes('WITH deleted AS')); + expect(cte[1]).toEqual([[ID], [ID], [110], 'Archive']); + expect(adjustFolderCounts).toHaveBeenNthCalledWith(1, 'a1', 'INBOX', -1, -1); + expect(adjustFolderCounts).toHaveBeenNthCalledWith(2, 'a1', 'Archive', 1, 1); + }); + + it('keeps the non-UIDPLUS delete-only path and requests a destination resync', async () => { + stubMoveQueries(); + const imap = manager({ uidMap: new Map(), succeeded: [10], failed: [] }); + + const result = await bulkMoveToFolder(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + folder: 'Archive', + }); + + expect(result).toEqual({ + ok: true, + moved: [ID], + movedDetails: [{ id: ID, accountId: 'a1', uid: null }], + failed: [], + skippedAccounts: [], + }); + const cte = query.mock.calls.find(([sql]) => sql.includes('WITH deleted AS')); + expect(cte[1]).toEqual([[ID], [], [], 'Archive']); + expect(imap.syncFolderOnDemand).toHaveBeenCalledWith(account, 'Archive'); + }); + + it('releases every source guard when IMAP move throws', async () => { + stubMoveQueries(); + const imap = manager(); + imap.bulkMoveMessages.mockRejectedValue(new Error('move failed')); + + const result = await bulkMoveToFolder(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + folder: 'Archive', + }); + + expect(result).toEqual({ ok: false, status: 500, error: 'Failed to move messages' }); + expect(imap._guardMoveUid).toHaveBeenCalledTimes(1); + expect(imap._unguardMoveUid).toHaveBeenCalledTimes(1); + }); + + it('surfaces missing destination accounts and per-message IMAP failures', async () => { + const failedMessage = { + ...message, + id: '22222222-2222-4222-8222-222222222222', + uid: 11, + }; + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id')) return { rows: [message, failedMessage] }; + if (sql.includes('SELECT 1 FROM folders')) return { rows: [{ exists: 1 }] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = manager({ + uidMap: new Map([[10, 110]]), + succeeded: [10], + failed: [11], + }); + + const result = await bulkMoveToFolder(imap, { + userId: 'u1', + accountIds: null, + ids: [message.id, failedMessage.id], + folder: 'Archive', + }); + + expect(result.failed).toEqual([{ + id: failedMessage.id, + reason: 'IMAP move failed', + }]); + }); + + it('returns a skippedAccounts partition when a destination folder is absent', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id')) return { rows: [message] }; + if (sql.includes('SELECT 1 FROM folders')) return { rows: [] }; + return { rows: [] }; + }); + + const result = await bulkMoveToFolder(manager(), { + userId: 'u1', + accountIds: null, + ids: [ID], + folder: 'Missing', + }); + + expect(result).toEqual({ + ok: true, + moved: [], + movedDetails: [], + failed: [], + skippedAccounts: [{ + account_id: 'a1', + reason: 'folder_not_found', + }], + }); + }); +}); + +describe('resolveMovedIds', () => { + it('returns destination rows for the supplied account, folder, and UIDs', async () => { + query.mockResolvedValue({ rows: [{ id: 'new-id', uid: '110' }] }); + + const rows = await resolveMovedIds('a1', 'Archive', [110]); + + expect(rows).toEqual([{ id: 'new-id', uid: '110' }]); + expect(query).toHaveBeenCalledWith( + expect.stringContaining('account_id = $1'), + ['a1', 'Archive', [110]], + ); + }); +}); diff --git a/backend/src/services/mailbox/snooze.js b/backend/src/services/mailbox/snooze.js new file mode 100644 index 00000000..fdb27ecf --- /dev/null +++ b/backend/src/services/mailbox/snooze.js @@ -0,0 +1,298 @@ +import { query } from '../db.js'; +import { emitGtdIfRelevant } from '../gtdSections.js'; +import { adjustFolderCounts } from '../../utils/mailUtils.js'; + +function emitGtdSectionsRefresh(imapManager, rows, userId) { + const byAccount = new Map(); + for (const m of rows) { + if (!m.message_id) continue; + if (!byAccount.has(m.account_id)) byAccount.set(m.account_id, { mids: new Set(), folders: new Set() }); + const entry = byAccount.get(m.account_id); + entry.mids.add(m.message_id); + if (m.folder) entry.folders.add(m.folder); + } + for (const [accountId, { mids, folders }] of byAccount) { + emitGtdIfRelevant(imapManager, accountId, userId, [...mids], [...folders]) + .catch(err => console.warn('GTD sections refresh emit failed:', err.message)); + } +} + +async function connectedConversation(msg) { + if (!msg.thread_id) { + return { pool: [msg], seen: new Set([msg.message_id]) }; + } + const pool = (await query( + `SELECT id, uid, account_id, folder, message_id, in_reply_to, thread_references, is_read + FROM messages + WHERE account_id = $1 AND thread_id = $2 AND message_id IS NOT NULL`, + [msg.account_id, msg.thread_id] + )).rows; + + if (!pool.some(r => r.message_id === msg.message_id)) pool.push(msg); + + const refsOf = (r) => { + const ids = (r.thread_references || '').match(/<[^>]+>/g) || []; + if (r.in_reply_to) ids.push(r.in_reply_to); + return ids; + }; + + const adj = new Map(); + const node = (m) => { let s = adj.get(m); if (!s) { s = new Set(); adj.set(m, s); } return s; }; + for (const r of pool) node(r.message_id); + for (const r of pool) { + for (const ref of refsOf(r)) { + if (adj.has(ref)) { node(r.message_id).add(ref); node(ref).add(r.message_id); } + } + } + const seen = new Set([msg.message_id]); + const queue = [msg.message_id]; + while (queue.length) { + const cur = queue.shift(); + for (const nb of (adj.get(cur) || [])) if (!seen.has(nb)) { seen.add(nb); queue.push(nb); } + } + return { pool, seen }; +} + +// Gather the reply-chain conversation that should be snoozed alongside `msg`. +export async function gatherSnoozeConversation(msg) { + if (!msg.thread_id) return [msg]; + + const { pool, seen } = await connectedConversation(msg); + const already = new Set( + (await query( + 'SELECT message_id_header FROM snoozed_messages WHERE account_id = $1 AND message_id_header = ANY($2)', + [msg.account_id, [...seen]] + )).rows.map(r => r.message_id_header) + ); + const picked = new Map(); + for (const r of pool) { + if (seen.has(r.message_id) && r.folder === msg.folder && !already.has(r.message_id) && !picked.has(r.message_id)) { + picked.set(r.message_id, r); + } + } + const rest = [...picked.values()].filter(r => r.message_id !== msg.message_id); + const self = picked.get(msg.message_id) || msg; + return [self, ...rest]; +} + +export async function restoreSnoozedRow(imapManager, row, { markUnread }) { + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [row.account_id]); + if (!accountResult.rows.length) return { restored: false }; + const account = accountResult.rows[0]; + + // Guard source UID before the IMAP move so reconcileDeletes cannot delete + // the DB row if an EXPUNGE arrives from the Snoozed folder while the move + // is in flight. + imapManager._guardMoveUid(row.account_id, row.snoozed_folder, row.uid); + let newUid; + try { + newUid = await imapManager.moveMessageGetNewUid( + account, row.uid, row.snoozed_folder, row.original_folder + ); + + if (markUnread && newUid) { + await imapManager.setFlag(account, newUid, row.original_folder, '\\Seen', false); + } else if (markUnread && row.message_id_header) { + // No UIDPLUS — server moved the message but returned no UID map. + // Search the destination folder by Message-ID to locate and unflag \Seen. + try { + await imapManager._withFreshClient(account, async (client) => { + const lock = await client.getMailboxLock(row.original_folder); + try { + const uids = await client.search({ header: ['Message-ID', row.message_id_header] }, { uid: true }); + if (uids.length > 0) { + const r = await client.messageFlagsRemove(String(uids[0]), ['\\Seen'], { uid: true }); + if (r === false) console.warn(`Snooze wakeup: messageFlagsRemove returned false for ${row.original_folder}`); + } else { + console.warn(`Snooze wakeup: could not find message in ${row.original_folder} to mark unread (Message-ID: ${row.message_id_header})`); + } + } finally { + lock.release(); + } + }); + } catch (err) { + console.warn(`Snooze wakeup: could not mark message unread on server (no UIDPLUS): ${err.message}`); + } + } + + if (newUid != null) { + if (markUnread) { + await query( + 'UPDATE messages SET folder = $1, is_read = false, read_changed_at = NOW(), uid = $4 WHERE account_id = $2 AND message_id = $3 AND folder = $5', + [row.original_folder, row.account_id, row.message_id_header, newUid, row.snoozed_folder] + ); + } else { + await query( + 'UPDATE messages SET folder = $1, uid = $4 WHERE account_id = $2 AND message_id = $3 AND folder = $5', + [row.original_folder, row.account_id, row.message_id_header, newUid, row.snoozed_folder] + ); + } + } else { + // Non-UIDPLUS: DB holds the stale source UID at the destination. Guard it so + // reconcileDeletes does not treat it as an orphan before the next sync corrects it. + imapManager._guardMoveUid(row.account_id, row.original_folder, row.uid); + if (markUnread) { + await query( + 'UPDATE messages SET folder = $1, is_read = false, read_changed_at = NOW() WHERE account_id = $2 AND message_id = $3 AND folder = $4', + [row.original_folder, row.account_id, row.message_id_header, row.snoozed_folder] + ); + } else { + await query( + 'UPDATE messages SET folder = $1 WHERE account_id = $2 AND message_id = $3 AND folder = $4', + [row.original_folder, row.account_id, row.message_id_header, row.snoozed_folder] + ); + } + setTimeout(() => imapManager._unguardMoveUid(row.account_id, row.original_folder, row.uid), 10_000); + } + } finally { + imapManager._unguardMoveUid(row.account_id, row.snoozed_folder, row.uid); + } + + return { restored: true, folder: row.original_folder, newUid: newUid ?? null }; +} + +export async function snoozeConversation(imapManager, { userId, accountIds, id, until }) { + const accountClause = accountIds == null ? '' : ' AND m.account_id = ANY($3::uuid[])'; + const params = accountIds == null ? [id, userId] : [id, userId, accountIds]; + const msgResult = await query( + `SELECT m.*, a.user_id FROM messages m + JOIN email_accounts a ON a.id = m.account_id + WHERE m.id = $1 AND a.user_id = $2${accountClause}`, + params + ); + if (!msgResult.rows.length) return { ok: false, status: 404, error: 'Message not found' }; + const msg = msgResult.rows[0]; + + if (!msg.message_id) { + return { ok: false, status: 400, error: 'Message has no Message-ID header — cannot snooze' }; + } + + const snoozedFolder = 'Snoozed'; + + if (msg.folder === snoozedFolder) { + return { ok: false, status: 400, error: 'Message is already in Snoozed folder' }; + } + + const existing = await query( + 'SELECT id FROM snoozed_messages WHERE account_id = $1 AND message_id_header = $2', + [msg.account_id, msg.message_id] + ); + if (existing.rows.length) return { ok: false, status: 400, error: 'Message is already snoozed' }; + + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [msg.account_id]); + const account = accountResult.rows[0]; + + const convo = await gatherSnoozeConversation(msg); + + try { + await imapManager.ensureFolder(account, snoozedFolder); + } catch (err) { + console.error(`Snooze ensureFolder failed for message ${id}:`, err.message); + return { ok: false, status: 500, error: 'Failed to move message to Snoozed folder' }; + } + + const untilDate = until instanceof Date ? until : new Date(until); + const movedIds = []; + for (const tm of convo) { + imapManager._guardMoveUid(tm.account_id, tm.folder, tm.uid); + try { + let snoozedUid; + try { + snoozedUid = await imapManager.moveMessage(account, tm.uid, tm.folder, snoozedFolder); + } catch (err) { + console.error(`Snooze IMAP move failed for message ${tm.id}:`, err.message); + if (tm.id === msg.id) { + return { ok: false, status: 500, error: 'Failed to move message to Snoozed folder' }; + } + continue; + } + if (snoozedUid != null) { + await query('UPDATE messages SET folder = $1, uid = $2 WHERE id = $3', [snoozedFolder, snoozedUid, tm.id]); + } else { + imapManager._guardMoveUid(tm.account_id, snoozedFolder, tm.uid); + await query('UPDATE messages SET folder = $1 WHERE id = $2', [snoozedFolder, tm.id]); + setTimeout(() => imapManager._unguardMoveUid(tm.account_id, snoozedFolder, tm.uid), 10_000); + } + + await query( + `INSERT INTO snoozed_messages (user_id, account_id, message_id_header, original_folder, snooze_until, snoozed_folder) + VALUES ($1, $2, $3, $4, $5, $6)`, + [userId, tm.account_id, tm.message_id, tm.folder, untilDate.toISOString(), snoozedFolder] + ); + + adjustFolderCounts(tm.account_id, tm.folder, -1, tm.is_read ? 0 : -1); + adjustFolderCounts(tm.account_id, snoozedFolder, 1, tm.is_read ? 0 : 1); + movedIds.push(tm.id); + } finally { + imapManager._unguardMoveUid(tm.account_id, tm.folder, tm.uid); + } + } + + emitGtdSectionsRefresh(imapManager, convo, userId); + + return { + ok: true, + movedCount: movedIds.length, + movedIds, + folder: snoozedFolder, + }; +} + +export async function unsnoozeConversation( + imapManager, + { userId, accountIds, id, markUnread = false }, +) { + const accountClause = accountIds == null ? '' : ' AND m.account_id = ANY($3::uuid[])'; + const params = accountIds == null ? [id, userId] : [id, userId, accountIds]; + const msgResult = await query( + `SELECT m.*, a.user_id FROM messages m + JOIN email_accounts a ON a.id = m.account_id + WHERE m.id = $1 AND a.user_id = $2${accountClause}`, + params, + ); + if (!msgResult.rows.length) return { ok: false, status: 404, error: 'Message not found' }; + const msg = msgResult.rows[0]; + + const { seen } = await connectedConversation(msg); + const snoozed = await query( + `SELECT sm.id AS snooze_id, sm.user_id, sm.account_id, + sm.message_id_header, sm.original_folder, sm.snoozed_folder, + m.uid, m.is_read + FROM snoozed_messages sm + JOIN messages m ON m.account_id = sm.account_id + AND m.message_id = sm.message_id_header + AND m.folder = sm.snoozed_folder + AND m.is_deleted = false + WHERE sm.user_id = $1 + AND sm.account_id = $2 + AND sm.message_id_header = ANY($3)`, + [userId, msg.account_id, [...seen]], + ); + const acted = snoozed.rows.find(row => row.message_id_header === msg.message_id); + if (!acted) { + return { ok: false, status: 400, error: 'Message is not currently snoozed' }; + } + + const rows = [acted, ...snoozed.rows.filter(row => row !== acted)]; + let restored = 0; + for (const row of rows) { + const outcome = await restoreSnoozedRow(imapManager, row, { markUnread }); + if (!outcome.restored) continue; + await query('DELETE FROM snoozed_messages WHERE id = $1', [row.snooze_id]); + adjustFolderCounts(row.account_id, row.snoozed_folder, -1, row.is_read ? 0 : -1); + adjustFolderCounts( + row.account_id, + row.original_folder, + 1, + markUnread || !row.is_read ? 1 : 0, + ); + restored++; + } + + if (restored > 0) { + imapManager.broadcast?.({ type: 'snooze_wakeup', accountId: msg.account_id }, userId); + emitGtdSectionsRefresh(imapManager, [msg], userId); + } + + return { ok: true, restored, folder: acted.original_folder }; +} diff --git a/backend/src/services/mailbox/snooze.restore.test.js b/backend/src/services/mailbox/snooze.restore.test.js new file mode 100644 index 00000000..3fc854dc --- /dev/null +++ b/backend/src/services/mailbox/snooze.restore.test.js @@ -0,0 +1,122 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../../routes/oauth.js', () => ({ refreshMicrosoftToken: vi.fn() })); +vi.mock('../../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, adjustFolderCounts: vi.fn() }; +}); + +import { query } from '../db.js'; +import { adjustFolderCounts } from '../../utils/mailUtils.js'; +import { ImapManager } from '../imapManager.js'; +import { restoreSnoozedRow } from './snooze.js'; + +const row = { + snooze_id: 's1', + user_id: 'u1', + account_id: 'a1', + message_id_header: '', + original_folder: 'INBOX', + snoozed_folder: 'Snoozed', + uid: 10, + is_read: true, +}; +const account = { id: 'a1' }; + +beforeEach(() => { + query.mockReset(); + adjustFolderCounts.mockReset(); +}); + +describe('_runSnoozeWakeup watcher characterization', () => { + it('restores a due UIDPLUS row, marks it unread, removes its record, adjusts counts, and broadcasts', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT sm.id AS snooze_id')) return { rows: [row] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + moveMessageGetNewUid: vi.fn().mockResolvedValue(110), + setFlag: vi.fn().mockResolvedValue(undefined), + broadcast: vi.fn(), + }; + + await ImapManager.prototype._runSnoozeWakeup.call(imap); + + expect(imap._guardMoveUid).toHaveBeenCalledWith('a1', 'Snoozed', 10); + expect(imap.moveMessageGetNewUid).toHaveBeenCalledWith(account, 10, 'Snoozed', 'INBOX'); + expect(imap.setFlag).toHaveBeenCalledWith(account, 110, 'INBOX', '\\Seen', false); + expect(query).toHaveBeenCalledWith( + expect.stringContaining('UPDATE messages SET folder = $1, is_read = false'), + ['INBOX', 'a1', '', 110, 'Snoozed'], + ); + expect(query).toHaveBeenCalledWith('DELETE FROM snoozed_messages WHERE id = $1', ['s1']); + expect(adjustFolderCounts).toHaveBeenNthCalledWith(1, 'a1', 'Snoozed', -1, 0); + expect(adjustFolderCounts).toHaveBeenNthCalledWith(2, 'a1', 'INBOX', 1, 1); + expect(imap.broadcast).toHaveBeenCalledWith({ type: 'snooze_wakeup', accountId: 'a1' }, 'u1'); + expect(imap._unguardMoveUid).toHaveBeenCalledWith('a1', 'Snoozed', 10); + }); +}); + +describe('restoreSnoozedRow', () => { + it('owns the UIDPLUS move, unread flag, DB repoint, and guard protocol', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + moveMessageGetNewUid: vi.fn().mockResolvedValue(110), + setFlag: vi.fn().mockResolvedValue(undefined), + }; + + await restoreSnoozedRow(imap, row, { markUnread: true }); + + expect(imap._guardMoveUid).toHaveBeenCalledWith('a1', 'Snoozed', 10); + expect(imap.moveMessageGetNewUid).toHaveBeenCalledWith(account, 10, 'Snoozed', 'INBOX'); + expect(imap.setFlag).toHaveBeenCalledWith(account, 110, 'INBOX', '\\Seen', false); + expect(query).toHaveBeenCalledWith( + expect.stringContaining('is_read = false'), + ['INBOX', 'a1', '', 110, 'Snoozed'], + ); + expect(imap._unguardMoveUid).toHaveBeenCalledWith('a1', 'Snoozed', 10); + }); + + it('uses a Message-ID search to mark unread without UIDPLUS', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const lock = { release: vi.fn() }; + const client = { + getMailboxLock: vi.fn().mockResolvedValue(lock), + search: vi.fn().mockResolvedValue([210]), + messageFlagsRemove: vi.fn().mockResolvedValue(true), + }; + const imap = { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + moveMessageGetNewUid: vi.fn().mockResolvedValue(null), + _withFreshClient: vi.fn(async (_account, fn) => fn(client)), + }; + + await restoreSnoozedRow(imap, row, { markUnread: true }); + + expect(client.search).toHaveBeenCalledWith( + { header: ['Message-ID', ''] }, + { uid: true }, + ); + expect(client.messageFlagsRemove).toHaveBeenCalledWith('210', ['\\Seen'], { uid: true }); + expect(lock.release).toHaveBeenCalled(); + expect(query).toHaveBeenCalledWith( + expect.stringContaining('is_read = false'), + ['INBOX', 'a1', '', 'Snoozed'], + ); + expect(imap._guardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + expect(imap._unguardMoveUid).toHaveBeenCalledWith('a1', 'Snoozed', 10); + }); +}); diff --git a/backend/src/services/mailbox/snooze.test.js b/backend/src/services/mailbox/snooze.test.js new file mode 100644 index 00000000..56d97172 --- /dev/null +++ b/backend/src/services/mailbox/snooze.test.js @@ -0,0 +1,269 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, adjustFolderCounts: vi.fn() }; +}); +vi.mock('../gtdSections.js', () => ({ emitGtdIfRelevant: vi.fn().mockResolvedValue(undefined) })); + +import { query } from '../db.js'; +import { + gatherSnoozeConversation, + snoozeConversation, + unsnoozeConversation, +} from './snooze.js'; + +const ID = '11111111-1111-4111-8111-111111111111'; +const message = { + id: ID, + account_id: 'a1', + uid: 10, + folder: 'INBOX', + message_id: '', + thread_id: null, + is_read: false, +}; +const account = { id: 'a1' }; + +function manager() { + return { + ensureFolder: vi.fn().mockResolvedValue(undefined), + moveMessage: vi.fn().mockResolvedValue(110), + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + }; +} + +beforeEach(() => { + query.mockReset(); +}); + +describe('gatherSnoozeConversation', () => { + it('returns an unthreaded message without querying', async () => { + await expect(gatherSnoozeConversation(message)).resolves.toEqual([message]); + expect(query).not.toHaveBeenCalled(); + }); +}); + +describe('snoozeConversation', () => { + it('narrows ownership to accountIds before IMAP work', async () => { + query.mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await snoozeConversation(imap, { + userId: 'u1', + accountIds: ['a1'], + id: ID, + until: new Date(Date.now() + 60_000), + }); + + expect(result).toEqual({ ok: false, status: 404, error: 'Message not found' }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($3::uuid[])'); + expect(params).toEqual([ID, 'u1', ['a1']]); + expect(imap.moveMessage).not.toHaveBeenCalled(); + }); + + it('moves, records, and unguards the acted message', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) return { rows: [message] }; + if (sql.includes('SELECT id FROM snoozed_messages')) return { rows: [] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = manager(); + const until = new Date(Date.now() + 60_000); + + const result = await snoozeConversation(imap, { + userId: 'u1', + accountIds: null, + id: ID, + until, + }); + + expect(result).toEqual({ + ok: true, + movedCount: 1, + movedIds: [ID], + folder: 'Snoozed', + }); + expect(imap.ensureFolder).toHaveBeenCalledWith(account, 'Snoozed'); + expect(imap.moveMessage).toHaveBeenCalledWith(account, 10, 'INBOX', 'Snoozed'); + expect(imap._guardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + expect(imap._unguardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + expect(query).toHaveBeenCalledWith( + expect.stringContaining('INSERT INTO snoozed_messages'), + ['u1', 'a1', '', 'INBOX', until.toISOString(), 'Snoozed'], + ); + }); + + it('maps an acted-message IMAP failure while still releasing its guard', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) return { rows: [message] }; + if (sql.includes('SELECT id FROM snoozed_messages')) return { rows: [] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = manager(); + imap.moveMessage.mockRejectedValue(new Error('move failed')); + + const result = await snoozeConversation(imap, { + userId: 'u1', + accountIds: null, + id: ID, + until: new Date(Date.now() + 60_000), + }); + + expect(result).toEqual({ + ok: false, + status: 500, + error: 'Failed to move message to Snoozed folder', + }); + expect(imap._unguardMoveUid).toHaveBeenCalledTimes(1); + }); +}); + +describe('unsnoozeConversation', () => { + it('narrows ownership to accountIds before restore work', async () => { + query.mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await unsnoozeConversation(imap, { + userId: 'u1', + accountIds: ['a1'], + id: ID, + }); + + expect(result).toEqual({ ok: false, status: 404, error: 'Message not found' }); + expect(query.mock.calls[0][0]).toContain('m.account_id = ANY($3::uuid[])'); + expect(imap.moveMessage).not.toHaveBeenCalled(); + }); + + it('returns 400 when the acted message has no snooze record', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) return { rows: [message] }; + if (sql.includes('FROM snoozed_messages sm')) return { rows: [] }; + return { rows: [] }; + }); + + const result = await unsnoozeConversation(manager(), { + userId: 'u1', + accountIds: null, + id: ID, + }); + + expect(result).toEqual({ ok: false, status: 400, error: 'Message is not currently snoozed' }); + }); + + it('restores the whole reply chain without marking read messages unread', async () => { + const root = { ...message, thread_id: 't', folder: 'Snoozed', is_read: true }; + const reply = { + ...root, + id: '22222222-2222-4222-8222-222222222222', + uid: 11, + message_id: '', + in_reply_to: '', + thread_references: '', + is_read: false, + }; + const snoozedRows = [ + { + snooze_id: 's1', + user_id: 'u1', + account_id: 'a1', + message_id_header: root.message_id, + original_folder: 'INBOX', + snoozed_folder: 'Snoozed', + uid: 10, + is_read: true, + }, + { + snooze_id: 's2', + user_id: 'u1', + account_id: 'a1', + message_id_header: reply.message_id, + original_folder: 'INBOX', + snoozed_folder: 'Snoozed', + uid: 11, + is_read: false, + }, + ]; + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) return { rows: [root] }; + if (sql.includes('WHERE account_id = $1 AND thread_id = $2')) return { rows: [root, reply] }; + if (sql.includes('FROM snoozed_messages sm')) return { rows: snoozedRows }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + moveMessageGetNewUid: vi.fn() + .mockResolvedValueOnce(110) + .mockResolvedValueOnce(111), + setFlag: vi.fn(), + broadcast: vi.fn(), + }; + + const result = await unsnoozeConversation(imap, { + userId: 'u1', + accountIds: null, + id: ID, + }); + + expect(result).toEqual({ ok: true, restored: 2, folder: 'INBOX' }); + expect(imap.moveMessageGetNewUid).toHaveBeenCalledTimes(2); + expect(imap.setFlag).not.toHaveBeenCalled(); + expect(query.mock.calls.filter(([sql]) => sql === 'DELETE FROM snoozed_messages WHERE id = $1')).toHaveLength(2); + const repoints = query.mock.calls.filter(([sql]) => sql.includes('UPDATE messages SET folder')); + expect(repoints.every(([sql]) => !sql.includes('is_read = false'))).toBe(true); + }); + + it('marks restored messages unread when explicitly requested', async () => { + const root = { ...message, folder: 'Snoozed', is_read: true }; + const snoozedRow = { + snooze_id: 's1', + user_id: 'u1', + account_id: 'a1', + message_id_header: root.message_id, + original_folder: 'INBOX', + snoozed_folder: 'Snoozed', + uid: 10, + is_read: true, + }; + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id FROM messages')) return { rows: [root] }; + if (sql.includes('FROM snoozed_messages sm')) return { rows: [snoozedRow] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + const imap = { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + moveMessageGetNewUid: vi.fn().mockResolvedValue(110), + setFlag: vi.fn(), + broadcast: vi.fn(), + }; + + const result = await unsnoozeConversation(imap, { + userId: 'u1', + accountIds: null, + id: ID, + markUnread: true, + }); + + expect(result).toEqual({ ok: true, restored: 1, folder: 'INBOX' }); + expect(imap.setFlag).toHaveBeenCalledWith( + account, + 110, + 'INBOX', + '\\Seen', + false, + ); + expect(query).toHaveBeenCalledWith( + expect.stringContaining('is_read = false'), + ['INBOX', 'a1', '', 110, 'Snoozed'], + ); + }); +}); diff --git a/backend/src/services/mailbox/spamLabel.js b/backend/src/services/mailbox/spamLabel.js new file mode 100644 index 00000000..aef0ef98 --- /dev/null +++ b/backend/src/services/mailbox/spamLabel.js @@ -0,0 +1,177 @@ +import { query } from '../db.js'; +import { emitGtdIfRelevant } from '../gtdSections.js'; +import { + adjustFolderCounts, + resolveAllSpamPaths, + resolveSpamFolder, +} from '../../utils/mailUtils.js'; + +function scopedParams(id, userId, accountIds) { + return accountIds == null + ? { clause: '', params: [id, userId] } + : { clause: ' AND m.account_id = ANY($3::uuid[])', params: [id, userId, accountIds] }; +} + +function emitGtdSectionsRefresh(imapManager, rows, userId) { + const byAccount = new Map(); + for (const m of rows) { + if (!m.message_id) continue; + if (!byAccount.has(m.account_id)) byAccount.set(m.account_id, { mids: new Set(), folders: new Set() }); + const entry = byAccount.get(m.account_id); + entry.mids.add(m.message_id); + if (m.folder) entry.folders.add(m.folder); + } + for (const [accountId, { mids, folders }] of byAccount) { + emitGtdIfRelevant(imapManager, accountId, userId, [...mids], [...folders]) + .catch(err => console.warn('GTD sections refresh emit failed:', err.message)); + } +} + +// Move a single message to a destination folder, update DB, log to +// training_log, and broadcast folder_updated. Shared between spam and ham. +export async function moveForSpamLabel( + imapManager, + { userId, accountIds, messageId, destinationFolder, label }, +) { + const scope = scopedParams(messageId, userId, accountIds); + const result = await query(` + SELECT m.*, a.user_id, a.folder_mappings FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = $1 AND a.user_id = $2${scope.clause} + `, scope.params); + + if (!result.rows.length) return { ok: false, status: 404, error: 'Message not found' }; + const message = result.rows[0]; + + // No-op: message already in the destination folder. + if (message.folder === destinationFolder) { + // Still record the training label so the user's intent is captured + // (e.g. re-confirming a verdict), but skip the IMAP move. + await query( + `INSERT INTO spam_training_log + (user_id, account_id, message_id_header, message_uid, folder, label) + VALUES ($1, $2, $3, $4, $5, $6)`, + [userId, message.account_id, message.message_id, message.uid, message.folder, label] + ); + await query( + `UPDATE messages SET spam_user_override = $1, spam_verdict = $1, spam_analyzed_at = NOW() WHERE id = $2`, + [label, messageId] + ); + return { ok: true, status: 200, body: { ok: true, alreadyInFolder: true, folder: destinationFolder } }; + } + + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [message.account_id]); + const account = accountResult.rows[0]; + + // Guard the source UID before the IMAP move so reconcileDeletes cannot + // delete the DB row if an EXPUNGE arrives while the move is in flight. + imapManager._guardMoveUid(account.id, message.folder, message.uid); + let newUid; + try { + try { + newUid = await imapManager.moveMessage(account, message.uid, message.folder, destinationFolder); + } catch (err) { + console.error(`IMAP move for /${label} failed:`, err.message); + return { ok: false, status: 502, error: `IMAP move failed: ${err.message}` }; + } + if (newUid != null) { + await query('DELETE FROM messages WHERE account_id = $1 AND uid = $2 AND folder = $3 AND id != $4', + [account.id, newUid, destinationFolder, messageId]); + await query( + `UPDATE messages SET folder = $1, uid = $2, + spam_user_override = $3, spam_verdict = $3, spam_analyzed_at = NOW() + WHERE id = $4`, + [destinationFolder, newUid, label, messageId] + ); + } else { + // Non-UIDPLUS server: DB holds the stale source UID at the destination. + imapManager._guardMoveUid(account.id, destinationFolder, message.uid); + await query( + `UPDATE messages SET folder = $1, + spam_user_override = $2, spam_verdict = $2, spam_analyzed_at = NOW() + WHERE id = $3`, + [destinationFolder, label, messageId] + ); + setTimeout(() => imapManager._unguardMoveUid(account.id, destinationFolder, message.uid), 10_000); + } + } finally { + imapManager._unguardMoveUid(account.id, message.folder, message.uid); + } + + // Adjust cached folder counts. + const wasUnread = !message.is_read ? 1 : 0; + adjustFolderCounts(account.id, message.folder, -1, -wasUnread); + adjustFolderCounts(account.id, destinationFolder, 1, wasUnread); + + // Training log: capture the decision for future model training. + await query( + `INSERT INTO spam_training_log + (user_id, account_id, message_id_header, message_uid, folder, label, source) + VALUES ($1, $2, $3, $4, $5, $6, 'manual')`, + [userId, account.id, message.message_id, message.uid, destinationFolder, label] + ); + + // If folder_mappings.spam is not yet configured, learn from the discovered folder. + if (label === 'spam' && !account.folder_mappings?.spam) { + await query( + `UPDATE email_accounts SET folder_mappings = folder_mappings || jsonb_build_object('spam', $1::text) + WHERE id = $2 AND NOT (folder_mappings ? 'spam')`, + [destinationFolder, account.id] + ).catch(err => console.warn('Failed to auto-persist folder_mappings.spam:', err.message)); + } + + imapManager.broadcast( + { type: 'folder_updated', folder: destinationFolder, accountId: account.id }, + userId + ); + + // Refresh GTD section data if the (un)spammed message's thread carries a GTD label. + emitGtdSectionsRefresh(imapManager, [message], userId); + + return { ok: true, status: 200, body: { ok: true, folder: destinationFolder, newUid: newUid || null } }; +} + +export async function markSpam(imapManager, { userId, accountIds, id }) { + const scope = scopedParams(id, userId, accountIds); + const lookup = await query(` + SELECT m.account_id, a.folder_mappings FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = $1 AND a.user_id = $2${scope.clause} + `, scope.params); + + if (!lookup.rows.length) return { ok: false, status: 404, error: 'Message not found' }; + const spamFolder = await resolveSpamFolder(lookup.rows[0].account_id, lookup.rows[0].folder_mappings); + if (!spamFolder) return { ok: false, status: 422, error: 'No spam folder configured for this account' }; + + return moveForSpamLabel(imapManager, { + userId, + accountIds, + messageId: id, + destinationFolder: spamFolder, + label: 'spam', + }); +} + +export async function markNotSpam(imapManager, { userId, accountIds, id }) { + const scope = scopedParams(id, userId, accountIds); + const lookup = await query(` + SELECT m.account_id, m.folder, a.folder_mappings FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = $1 AND a.user_id = $2${scope.clause} + `, scope.params); + + if (!lookup.rows.length) return { ok: false, status: 404, error: 'Message not found' }; + const allSpam = await resolveAllSpamPaths(lookup.rows[0].account_id, lookup.rows[0].folder_mappings); + if (!allSpam.has(lookup.rows[0].folder)) { + return { ok: false, status: 400, error: 'Message is not in the spam folder' }; + } + + const inboxFolder = lookup.rows[0].folder_mappings?.inbox || 'INBOX'; + return moveForSpamLabel(imapManager, { + userId, + accountIds, + messageId: id, + destinationFolder: inboxFolder, + label: 'ham', + }); +} diff --git a/backend/src/services/mailbox/spamLabel.test.js b/backend/src/services/mailbox/spamLabel.test.js new file mode 100644 index 00000000..190990ec --- /dev/null +++ b/backend/src/services/mailbox/spamLabel.test.js @@ -0,0 +1,114 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + adjustFolderCounts: vi.fn(), + resolveSpamFolder: vi.fn(), + resolveAllSpamPaths: vi.fn(), + }; +}); +vi.mock('../gtdSections.js', () => ({ emitGtdIfRelevant: vi.fn().mockResolvedValue(undefined) })); + +import { query } from '../db.js'; +import { resolveAllSpamPaths, resolveSpamFolder } from '../../utils/mailUtils.js'; +import { markNotSpam, markSpam } from './spamLabel.js'; + +const ID = '11111111-1111-4111-8111-111111111111'; +const message = { + id: ID, + account_id: 'a1', + uid: 10, + folder: 'INBOX', + message_id: '', + is_read: false, + folder_mappings: {}, +}; +const account = { id: 'a1', user_id: 'u1', folder_mappings: {} }; + +function manager() { + return { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + moveMessage: vi.fn().mockResolvedValue(110), + broadcast: vi.fn(), + }; +} + +beforeEach(() => { + query.mockReset(); + resolveSpamFolder.mockReset(); + resolveAllSpamPaths.mockReset(); +}); + +describe('markSpam', () => { + it('preserves the REST path when accountIds is null', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.account_id, a.folder_mappings')) { + return { rows: [{ account_id: 'a1', folder_mappings: {} }] }; + } + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + resolveSpamFolder.mockResolvedValue('Junk'); + const imap = manager(); + + const result = await markSpam(imap, { userId: 'u1', accountIds: null, id: ID }); + + expect(result).toEqual({ ok: true, status: 200, body: { ok: true, folder: 'Junk', newUid: 110 } }); + expect(imap.moveMessage).toHaveBeenCalledWith(account, 10, 'INBOX', 'Junk'); + expect(imap._guardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + expect(imap._unguardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + }); + + it('narrows ownership to non-null accountIds before any IMAP work', async () => { + query.mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await markSpam(imap, { userId: 'u1', accountIds: ['a1'], id: ID }); + + expect(result).toEqual({ ok: false, status: 404, error: 'Message not found' }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($3::uuid[])'); + expect(params).toEqual([ID, 'u1', ['a1']]); + expect(imap.moveMessage).not.toHaveBeenCalled(); + }); + + it('releases the source UID guard when the IMAP move fails', async () => { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.account_id, a.folder_mappings')) { + return { rows: [{ account_id: 'a1', folder_mappings: {} }] }; + } + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [message] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); + resolveSpamFolder.mockResolvedValue('Junk'); + const imap = manager(); + imap.moveMessage.mockRejectedValue(new Error('move failed')); + + const result = await markSpam(imap, { userId: 'u1', accountIds: null, id: ID }); + + expect(result).toEqual({ ok: false, status: 502, error: 'IMAP move failed: move failed' }); + expect(imap._guardMoveUid).toHaveBeenCalledTimes(1); + expect(imap._unguardMoveUid).toHaveBeenCalledTimes(1); + }); +}); + +describe('markNotSpam', () => { + it('rejects a message outside all spam-like folders without moving it', async () => { + query.mockResolvedValue({ + rows: [{ account_id: 'a1', folder: 'INBOX', folder_mappings: {} }], + }); + resolveAllSpamPaths.mockResolvedValue(new Set(['Junk'])); + const imap = manager(); + + const result = await markNotSpam(imap, { userId: 'u1', accountIds: null, id: ID }); + + expect(result).toEqual({ ok: false, status: 400, error: 'Message is not in the spam folder' }); + expect(imap.moveMessage).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/services/mailbox/trash.js b/backend/src/services/mailbox/trash.js new file mode 100644 index 00000000..15676034 --- /dev/null +++ b/backend/src/services/mailbox/trash.js @@ -0,0 +1,280 @@ +import { query } from '../db.js'; +import { emitGtdIfRelevant } from '../gtdSections.js'; +import { + adjustFolderCounts, + resolveAllDraftsPaths, + resolveAllTrashPaths, + resolveTrashFolder, +} from '../../utils/mailUtils.js'; + +function emitGtdSectionsRefresh(imapManager, rows, userId) { + const byAccount = new Map(); + for (const m of rows) { + if (!m.message_id) continue; + if (!byAccount.has(m.account_id)) byAccount.set(m.account_id, { mids: new Set(), folders: new Set() }); + const entry = byAccount.get(m.account_id); + entry.mids.add(m.message_id); + if (m.folder) entry.folders.add(m.folder); + } + for (const [accountId, { mids, folders }] of byAccount) { + emitGtdIfRelevant(imapManager, accountId, userId, [...mids], [...folders]) + .catch(err => console.warn('GTD sections refresh emit failed:', err.message)); + } +} + +export async function bulkTrash( + imapManager, + { userId, accountIds, ids, allowPermanent = false }, +) { + const moveGuards = []; + try { + const accountClause = accountIds == null ? '' : ' AND m.account_id = ANY($3::uuid[])'; + const params = accountIds == null ? [userId, ids] : [userId, ids, accountIds]; + const result = await query( + `SELECT m.*, a.user_id, a.folder_mappings FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = ANY($2::uuid[]) AND a.user_id = $1${accountClause}`, + params + ); + + let owned = result.rows; + const failedItems = []; + if (!owned.length) { + return allowPermanent + ? { ok: true, deleted: [] } + : { + ok: true, + deleted: [], + trashedDetails: [], + failed: failedItems, + refused: [], + }; + } + + const refused = []; + if (!allowPermanent) { + const permitted = []; + const byAccountForRefusal = {}; + for (const msg of owned) { + (byAccountForRefusal[msg.account_id] = byAccountForRefusal[msg.account_id] || []).push(msg); + } + for (const [accountId, msgs] of Object.entries(byAccountForRefusal)) { + const allTrashPaths = await resolveAllTrashPaths(accountId, msgs[0].folder_mappings); + const allDraftsPaths = await resolveAllDraftsPaths(accountId, msgs[0].folder_mappings); + for (const msg of msgs) { + if (allTrashPaths.has(msg.folder)) { + refused.push({ + id: msg.id, + folder: msg.folder, + reason: 'already_in_trash_permanent_delete_required', + }); + } else if (allDraftsPaths.has(msg.folder)) { + refused.push({ + id: msg.id, + folder: msg.folder, + reason: 'draft_permanent_delete_required', + }); + } else { + permitted.push(msg); + } + } + } + owned = permitted; + if (!owned.length) { + return { + ok: true, + deleted: [], + trashedDetails: [], + failed: failedItems, + refused, + }; + } + } + + // Guard source UIDs for the whole operation so reconcileDeletes can't delete a + // trash-move source row between the IMAP move and the re-INSERT CTE. + for (const m of owned) { + moveGuards.push({ accountId: m.account_id, folder: m.folder, uid: m.uid }); + imapManager._guardMoveUid(m.account_id, m.folder, m.uid); + } + + const byAccount = {}; + for (const msg of owned) { + (byAccount[msg.account_id] = byAccount[msg.account_id] || []).push(msg); + } + + const expungeSucceeded = []; + const trashMoveSucceeded = []; + const accountsById = {}; + + for (const [accountId, msgs] of Object.entries(byAccount)) { + const accountResult = await query('SELECT * FROM email_accounts WHERE id = $1', [accountId]); + const account = accountResult.rows[0]; + accountsById[accountId] = account; + const trashPath = await resolveTrashFolder(accountId, msgs[0].folder_mappings); + const allTrashPaths = await resolveAllTrashPaths(accountId, msgs[0].folder_mappings); + const allDraftsPaths = await resolveAllDraftsPaths(accountId, msgs[0].folder_mappings); + + if (!trashPath) { + console.error(`bulk-delete: no Trash folder found for account ${accountId} — skipping ${msgs.length} messages`); + failedItems.push(...msgs.map(msg => ({ + id: msg.id, + reason: 'Trash folder not found', + }))); + continue; + } + + const toExpunge = msgs.filter(m => allTrashPaths.has(m.folder) || allDraftsPaths.has(m.folder)); + const toMove = msgs.filter(m => !allTrashPaths.has(m.folder) && !allDraftsPaths.has(m.folder)); + + if (toExpunge.length) { + const byExpungeFolder = {}; + for (const msg of toExpunge) { + (byExpungeFolder[msg.folder] = byExpungeFolder[msg.folder] || []).push(msg); + } + for (const [expungeFolder, folderMsgs] of Object.entries(byExpungeFolder)) { + const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); + const { succeeded, failed } = await imapManager.bulkPermanentDelete(account, folderMsgs.map(m => m.uid), expungeFolder); + for (const uid of succeeded) expungeSucceeded.push(uidToMsg.get(String(uid))); + for (const uid of failed) console.error(`bulk-delete IMAP expunge uid ${uid} from ${expungeFolder}: IMAP delete failed`); + } + } + + if (toMove.length) { + const byFolder = {}; + for (const msg of toMove) { + (byFolder[msg.folder] = byFolder[msg.folder] || []).push(msg); + } + for (const [srcFolder, folderMsgs] of Object.entries(byFolder)) { + const uidToMsg = new Map(folderMsgs.map(m => [String(m.uid), m])); + const { uidMap, succeeded, failed } = await imapManager.bulkMoveMessages(account, folderMsgs.map(m => m.uid), srcFolder, trashPath); + for (const uid of succeeded) { + trashMoveSucceeded.push({ msg: uidToMsg.get(String(uid)), trashPath, newUid: uidMap.get(Number(uid)) || null }); + } + for (const uid of failed) { + const msg = uidToMsg.get(String(uid)); + if (msg) failedItems.push({ id: msg.id, reason: 'IMAP move failed' }); + console.error(`bulk-delete IMAP move uid ${uid}: IMAP move failed`); + } + } + } + } + + if (expungeSucceeded.length) { + await query('DELETE FROM messages WHERE id = ANY($1::uuid[])', [expungeSucceeded.map(m => m.id)]); + } + + if (trashMoveSucceeded.length) { + const byTrashPath = {}; + for (const u of trashMoveSucceeded) { + (byTrashPath[u.trashPath] = byTrashPath[u.trashPath] || []).push(u); + } + for (const [trashPath, entries] of Object.entries(byTrashPath)) { + const allIds = entries.map(u => u.msg.id); + const withUid = entries.filter(u => u.newUid); + await query(` + WITH deleted AS ( + DELETE FROM messages WHERE id = ANY($1::uuid[]) RETURNING * + ), + uid_map(src_id, new_uid) AS ( + SELECT * FROM unnest($2::uuid[], $3::bigint[]) + ) + INSERT INTO messages ( + account_id, uid, folder, message_id, subject, + from_name, from_email, to_addresses, cc_addresses, + reply_to, in_reply_to, date, snippet, is_read, is_starred, + has_attachments, flags, body_html, body_text, attachments, + thread_references, thread_id, is_bulk, + read_changed_at, star_changed_at, spam_score_sa, spam_score_ml, + spam_verdict, spam_analyzed_at, spam_details, spam_user_override, + category, list_unsubscribe, list_unsubscribe_post, unsubscribed_at + ) + SELECT + d.account_id, u.new_uid, $4, d.message_id, d.subject, + d.from_name, d.from_email, d.to_addresses, d.cc_addresses, + d.reply_to, d.in_reply_to, d.date, d.snippet, d.is_read, d.is_starred, + d.has_attachments, d.flags, d.body_html, d.body_text, d.attachments, + d.thread_references, d.thread_id, d.is_bulk, + d.read_changed_at, d.star_changed_at, d.spam_score_sa, d.spam_score_ml, + d.spam_verdict, d.spam_analyzed_at, d.spam_details, d.spam_user_override, + d.category, d.list_unsubscribe, d.list_unsubscribe_post, d.unsubscribed_at + FROM deleted d + JOIN uid_map u ON d.id = u.src_id + ON CONFLICT (account_id, uid, folder) DO NOTHING + `, [allIds, withUid.map(u => u.msg.id), withUid.map(u => u.newUid), trashPath]); + } + const needResync = new Map(); + for (const u of trashMoveSucceeded) { + if (u.newUid) continue; + if (!needResync.has(u.msg.account_id)) needResync.set(u.msg.account_id, new Set()); + needResync.get(u.msg.account_id).add(u.trashPath); + } + for (const [acctId, paths] of needResync) { + const acct = accountsById[acctId]; + if (!acct) continue; + for (const tp of paths) { + imapManager.syncFolderOnDemand(acct, tp) + .catch(err => console.warn('post-trash destination sync failed:', err.message)); + } + } + } + + const allSucceeded = [ + ...expungeSucceeded.map(m => m.id), + ...trashMoveSucceeded.map(u => u.msg.id), + ]; + if (allSucceeded.length) { + const srcDeltas = {}; + for (const msg of expungeSucceeded) { + const key = `${msg.account_id}:${msg.folder}`; + if (!srcDeltas[key]) srcDeltas[key] = { accountId: msg.account_id, path: msg.folder, total: 0, unread: 0 }; + srcDeltas[key].total++; + if (!msg.is_read) srcDeltas[key].unread++; + } + for (const { msg } of trashMoveSucceeded) { + const key = `${msg.account_id}:${msg.folder}`; + if (!srcDeltas[key]) srcDeltas[key] = { accountId: msg.account_id, path: msg.folder, total: 0, unread: 0 }; + srcDeltas[key].total++; + if (!msg.is_read) srcDeltas[key].unread++; + } + for (const { accountId, path, total, unread } of Object.values(srcDeltas)) { + adjustFolderCounts(accountId, path, -total, -unread); + } + const dstDeltas = {}; + for (const { msg, trashPath } of trashMoveSucceeded) { + const key = `${msg.account_id}:${trashPath}`; + if (!dstDeltas[key]) dstDeltas[key] = { accountId: msg.account_id, path: trashPath, total: 0, unread: 0 }; + dstDeltas[key].total++; + if (!msg.is_read) dstDeltas[key].unread++; + } + for (const { accountId, path, total, unread } of Object.values(dstDeltas)) { + adjustFolderCounts(accountId, path, total, unread); + } + for (const { accountId, path } of Object.values(dstDeltas)) { + imapManager.broadcast({ type: 'folder_updated', folder: path, accountId }, userId); + } + } + + emitGtdSectionsRefresh(imapManager, owned, userId); + + return allowPermanent + ? { ok: true, deleted: allSucceeded } + : { + ok: true, + deleted: allSucceeded, + trashedDetails: trashMoveSucceeded.map(({ msg, trashPath, newUid }) => ({ + id: msg.id, + accountId: msg.account_id, + folder: trashPath, + uid: newUid, + })), + failed: failedItems, + refused, + }; + } catch (err) { + console.error('bulk-delete error:', err); + return { ok: false, status: 500, error: 'Failed to delete messages' }; + } finally { + for (const g of moveGuards) imapManager._unguardMoveUid(g.accountId, g.folder, g.uid); + } +} diff --git a/backend/src/services/mailbox/trash.test.js b/backend/src/services/mailbox/trash.test.js new file mode 100644 index 00000000..b858bd6e --- /dev/null +++ b/backend/src/services/mailbox/trash.test.js @@ -0,0 +1,230 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../../utils/mailUtils.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + adjustFolderCounts: vi.fn(), + resolveTrashFolder: vi.fn(), + resolveAllTrashPaths: vi.fn(), + resolveAllDraftsPaths: vi.fn(), + }; +}); +vi.mock('../gtdSections.js', () => ({ emitGtdIfRelevant: vi.fn().mockResolvedValue(undefined) })); + +import { query } from '../db.js'; +import { + resolveAllDraftsPaths, + resolveAllTrashPaths, + resolveTrashFolder, +} from '../../utils/mailUtils.js'; +import { bulkTrash } from './trash.js'; + +const ID = '11111111-1111-4111-8111-111111111111'; +const message = { + id: ID, + account_id: 'a1', + uid: 10, + folder: 'INBOX', + message_id: '', + is_read: false, + folder_mappings: {}, +}; +const account = { id: 'a1' }; + +function manager() { + return { + _guardMoveUid: vi.fn(), + _unguardMoveUid: vi.fn(), + bulkMoveMessages: vi.fn().mockResolvedValue({ + uidMap: new Map([[10, 110]]), + succeeded: [10], + failed: [], + }), + bulkPermanentDelete: vi.fn().mockResolvedValue({ succeeded: [10], failed: [] }), + syncFolderOnDemand: vi.fn().mockResolvedValue(undefined), + broadcast: vi.fn(), + }; +} + +function stubQueries(row = message) { + query.mockImplementation(async (sql) => { + if (sql.includes('SELECT m.*, a.user_id, a.folder_mappings')) return { rows: [row] }; + if (sql.includes('SELECT * FROM email_accounts')) return { rows: [account] }; + return { rows: [] }; + }); +} + +beforeEach(() => { + query.mockReset(); + resolveTrashFolder.mockReset(); + resolveAllTrashPaths.mockReset(); + resolveAllDraftsPaths.mockReset(); + resolveTrashFolder.mockResolvedValue('Trash'); + resolveAllTrashPaths.mockResolvedValue(new Set(['Trash'])); + resolveAllDraftsPaths.mockResolvedValue(new Set(['Drafts'])); +}); + +describe('bulkTrash', () => { + it('narrows ownership to accountIds before IMAP work', async () => { + query.mockResolvedValue({ rows: [] }); + const imap = manager(); + + const result = await bulkTrash(imap, { + userId: 'u1', + accountIds: ['a1'], + ids: [ID], + allowPermanent: true, + }); + + expect(result).toEqual({ ok: true, deleted: [] }); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('m.account_id = ANY($3::uuid[])'); + expect(params).toEqual(['u1', [ID], ['a1']]); + expect(imap.bulkMoveMessages).not.toHaveBeenCalled(); + expect(imap.bulkPermanentDelete).not.toHaveBeenCalled(); + }); + + it('refuses an already-trash message when permanent deletion is not allowed', async () => { + stubQueries({ ...message, folder: 'Trash' }); + const imap = manager(); + + const result = await bulkTrash(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + allowPermanent: false, + }); + + expect(result).toEqual({ + ok: true, + deleted: [], + trashedDetails: [], + failed: [], + refused: [{ + id: ID, + folder: 'Trash', + reason: 'already_in_trash_permanent_delete_required', + }], + }); + expect(imap.bulkPermanentDelete).not.toHaveBeenCalled(); + expect(imap.bulkMoveMessages).not.toHaveBeenCalled(); + expect(imap._guardMoveUid).not.toHaveBeenCalled(); + }); + + it('returns destination metadata for the non-permanent tool path', async () => { + stubQueries(); + + const result = await bulkTrash(manager(), { + userId: 'u1', + accountIds: null, + ids: [ID], + allowPermanent: false, + }); + + expect(result).toEqual({ + ok: true, + deleted: [ID], + trashedDetails: [{ + id: ID, + accountId: 'a1', + folder: 'Trash', + uid: 110, + }], + failed: [], + refused: [], + }); + }); + + it('surfaces per-message move failures for the non-permanent tool path', async () => { + stubQueries(); + const imap = manager(); + imap.bulkMoveMessages.mockResolvedValue({ + uidMap: new Map(), + succeeded: [], + failed: [10], + }); + + const result = await bulkTrash(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + allowPermanent: false, + }); + + expect(result.failed).toEqual([{ id: ID, reason: 'IMAP move failed' }]); + }); + + it('preserves permanent expunge for the REST-compatible allowPermanent path', async () => { + stubQueries({ ...message, folder: 'Trash' }); + const imap = manager(); + + const result = await bulkTrash(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + allowPermanent: true, + }); + + expect(result).toEqual({ ok: true, deleted: [ID] }); + expect(imap.bulkPermanentDelete).toHaveBeenCalledWith(account, [10], 'Trash'); + expect(query).toHaveBeenCalledWith('DELETE FROM messages WHERE id = ANY($1::uuid[])', [[ID]]); + }); + + it('preserves the guarded move-to-trash UIDPLUS path', async () => { + stubQueries(); + const imap = manager(); + + const result = await bulkTrash(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + allowPermanent: true, + }); + + expect(result).toEqual({ ok: true, deleted: [ID] }); + expect(imap.bulkMoveMessages).toHaveBeenCalledWith(account, [10], 'INBOX', 'Trash'); + expect(imap._guardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + expect(imap._unguardMoveUid).toHaveBeenCalledWith('a1', 'INBOX', 10); + }); + + it('keeps the non-UIDPLUS delete-only path and trash-folder resync', async () => { + stubQueries(); + const imap = manager(); + imap.bulkMoveMessages.mockResolvedValue({ + uidMap: new Map(), + succeeded: [10], + failed: [], + }); + + const result = await bulkTrash(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + allowPermanent: true, + }); + + expect(result).toEqual({ ok: true, deleted: [ID] }); + const cte = query.mock.calls.find(([sql]) => sql.includes('WITH deleted AS')); + expect(cte[1]).toEqual([[ID], [], [], 'Trash']); + expect(imap.syncFolderOnDemand).toHaveBeenCalledWith(account, 'Trash'); + }); + + it('releases the source guard when a trash move throws', async () => { + stubQueries(); + const imap = manager(); + imap.bulkMoveMessages.mockRejectedValue(new Error('trash failed')); + + const result = await bulkTrash(imap, { + userId: 'u1', + accountIds: null, + ids: [ID], + allowPermanent: true, + }); + + expect(result).toEqual({ ok: false, status: 500, error: 'Failed to delete messages' }); + expect(imap._guardMoveUid).toHaveBeenCalledTimes(1); + expect(imap._unguardMoveUid).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/services/messageService.js b/backend/src/services/messageService.js index 7102ee36..00ff8a18 100644 --- a/backend/src/services/messageService.js +++ b/backend/src/services/messageService.js @@ -1,9 +1,21 @@ import { query } from './db.js'; import { resolveAccountScope } from './unifiedInbox.js'; - -export async function listMessages({ userId, accountId, folder = 'INBOX', limit = 50, offset = 0, unreadOnly, threaded, category }) { +import { DELEGATION_SELECT_SQL, delegationJoinSql, mapDelegationRow } from './gtdDelegations.js'; + +export async function listMessages({ + userId, + accountId, + folder = 'INBOX', + limit = 50, + offset = 0, + unreadOnly, + threaded, + category, + excludeClaimedSourceDrafts = false, +}) { const accountsResult = await query( - 'SELECT id, include_in_unified_inbox FROM email_accounts WHERE user_id = $1 AND enabled = true', + `SELECT id, include_in_unified_inbox, folder_mappings + FROM email_accounts WHERE user_id = $1 AND enabled = true`, [userId] ); const { @@ -41,20 +53,21 @@ export async function listMessages({ userId, accountId, folder = 'INBOX', limit whereConditions.push(`(m.category IS NULL OR m.category = 'primary')`); } - const where = whereConditions.join(' AND '); - const safeLimit = Math.min(Math.max(parseInt(limit) || 50, 1), 500); const safeOffset = Math.max(parseInt(offset) || 0, 0); let total = 0; + let folderRow = null; try { if (isSpecificAccount) { const r = await query( - 'SELECT total_count, unread_count FROM folders WHERE account_id = $1 AND path = $2', + `SELECT total_count, unread_count, special_use, name + FROM folders WHERE account_id = $1 AND path = $2`, [accountId, folder] ); if (r.rows.length) { - total = isUnreadOnly ? (r.rows[0].unread_count ?? 0) : (r.rows[0].total_count ?? 0); + [folderRow] = r.rows; + total = isUnreadOnly ? (folderRow.unread_count ?? 0) : (folderRow.total_count ?? 0); } } else { const r = isUnreadOnly @@ -72,7 +85,41 @@ export async function listMessages({ userId, accountId, folder = 'INBOX', limit total = 0; } - if (threaded === 'true' || threaded === true) { + const selectedAccount = isSpecificAccount + ? accountsResult.rows.find(account => account.id === resolvedAccountId) + : null; + const mappedDraftsFolder = selectedAccount?.folder_mappings?.drafts; + const isDraftsFolder = isSpecificAccount && ( + mappedDraftsFolder === folder + || folderRow?.special_use === '\\Drafts' + || folderRow?.name?.toLowerCase().includes('draft') + ); + const filterClaimedSources = excludeClaimedSourceDrafts && isDraftsFolder; + if (filterClaimedSources) { + whereConditions.push(`NOT EXISTS ( + SELECT 1 FROM compose_sessions cs + WHERE cs.user_id = $${p++} + AND cs.source_draft_account_id = m.account_id + AND cs.source_draft_folder = m.folder + AND cs.source_draft_uid = m.uid + )`); + values.push(userId); + } + + const where = whereConditions.join(' AND '); + const isThreaded = threaded === 'true' || threaded === true; + + if (filterClaimedSources && !isThreaded) { + const countResult = await query( + `SELECT COUNT(*)::int AS total + FROM messages m + WHERE ${where}`, + values, + ); + total = countResult.rows[0]?.total ?? 0; + } + + if (isThreaded) { const filterValues = [...values]; const threadAccountParam = isSpecificAccount ? [resolvedAccountId] : scopedAccountIds; // For INBOX-specific views the thread badge must match the expansion, so scope @@ -100,6 +147,7 @@ export async function listMessages({ userId, accountId, folder = 'INBOX', limit m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, m.category, m.list_unsubscribe, m.list_unsubscribe_post, m.delivery_addresses, + ${DELEGATION_SELECT_SQL}, a.name AS account_name, a.email_address AS account_email, a.color AS account_color, @@ -109,6 +157,7 @@ export async function listMessages({ userId, accountId, folder = 'INBOX', limit LEFT JOIN contacts co ON co.user_id = a.user_id AND co.primary_email = lower(m.from_email) AND co.photo_data IS NOT NULL + ${delegationJoinSql('m', 'a')} WHERE ${where} AND m.thread_key IN (SELECT thread_id FROM paged_threads) ORDER BY m.account_id, @@ -147,7 +196,7 @@ export async function listMessages({ userId, accountId, folder = 'INBOX', limit account_name, account_email, account_color, category, list_unsubscribe, list_unsubscribe_post, delivery_addresses, message_count, unread_count, - thread_has_contact_photo AS has_contact_photo + thread_has_contact_photo AS has_contact_photo, delegation FROM ranked WHERE rn = 1 ORDER BY date DESC @@ -160,7 +209,7 @@ export async function listMessages({ userId, accountId, folder = 'INBOX', limit `, filterValues); return { - messages: threadResult.rows, + messages: threadResult.rows.map(row => ({ ...row, delegation: mapDelegationRow(row) })), total: threadCountResult.rows[0]?.total ?? 0, threaded: true, resolvedAccountId, @@ -178,19 +227,21 @@ export async function listMessages({ userId, accountId, folder = 'INBOX', limit m.has_attachments, m.account_id, m.category, m.list_unsubscribe, m.list_unsubscribe_post, m.delivery_addresses, a.name as account_name, a.email_address as account_email, a.color as account_color, - (co.id IS NOT NULL) AS has_contact_photo + (co.id IS NOT NULL) AS has_contact_photo, + ${DELEGATION_SELECT_SQL} FROM messages m JOIN email_accounts a ON m.account_id = a.id LEFT JOIN contacts co ON co.user_id = a.user_id AND co.primary_email = lower(m.from_email) AND co.photo_data IS NOT NULL + ${delegationJoinSql('m', 'a')} WHERE ${where} ORDER BY m.date DESC LIMIT $${limitParam} OFFSET $${offsetParam} `, values); return { - messages: result.rows, + messages: result.rows.map(row => ({ ...row, delegation: mapDelegationRow(row) })), total, resolvedAccountId, }; diff --git a/backend/src/services/messageService.test.js b/backend/src/services/messageService.test.js index e696f04f..6fb11bc7 100644 --- a/backend/src/services/messageService.test.js +++ b/backend/src/services/messageService.test.js @@ -186,6 +186,27 @@ describe('listMessages — threaded mode', () => { }); describe('listMessages — message shape', () => { + it('projects delegation metadata in flat and threaded results', async () => { + const delegation = JSON.stringify({ contact_id: null, display_name: 'Casey' }); + query + .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }) + .mockResolvedValueOnce({ rows: [{ total_count: 1, unread_count: 0 }] }) + .mockResolvedValueOnce({ rows: [{ id: 'msg-1', delegation }] }); + const flat = await listMessages({ userId: 'user-1', accountId: 'acc-1' }); + expect(flat.messages[0].delegation).toEqual({ contact_id: null, display_name: 'Casey' }); + expect(query.mock.calls[2][0]).toContain('gtd_delegations'); + + query.mockReset(); + query + .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }) + .mockResolvedValueOnce({ rows: [{ total_count: 1, unread_count: 0 }] }) + .mockResolvedValueOnce({ rows: [{ id: 'msg-1', delegation }] }) + .mockResolvedValueOnce({ rows: [{ total: 1 }] }); + const threaded = await listMessages({ userId: 'user-1', accountId: 'acc-1', threaded: true }); + expect(threaded.messages[0].delegation.display_name).toBe('Casey'); + expect(query.mock.calls[2][0]).toContain('gtd_delegations'); + }); + it('selects delivery_addresses in the flat query', async () => { query .mockResolvedValueOnce({ rows: [{ id: 'acc-1' }] }) diff --git a/backend/src/services/migrations.collation.test.js b/backend/src/services/migrations.collation.test.js new file mode 100644 index 00000000..d0690543 --- /dev/null +++ b/backend/src/services/migrations.collation.test.js @@ -0,0 +1,58 @@ +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('./db.js', () => ({ pool: { query: vi.fn(), connect: vi.fn() } })); +import { warnOnCollationMismatch } from './migrations.js'; + +const row = (over = {}) => ({ + rows: [{ db: 'mailflow', recorded: '2.36', actual: '2.36', ...over }], +}); + +describe('warnOnCollationMismatch', () => { + it('warns loudly, naming REINDEX DATABASE, when versions diverge', async () => { + const warn = vi.fn(); + const query = vi.fn().mockResolvedValue(row({ recorded: '1.2.38', actual: '2.36' })); + const hit = await warnOnCollationMismatch({ query, warn }); + expect(hit).toBe(true); + expect(warn).toHaveBeenCalledTimes(1); + const msg = warn.mock.calls[0][0]; + expect(msg).toContain('collation version mismatch'); + expect(msg).toContain('1.2.38'); + expect(msg).toContain('2.36'); + expect(msg).toContain('REINDEX DATABASE "mailflow";'); + expect(msg).toContain('REFRESH COLLATION VERSION'); + }); + + it('stays silent when the recorded and actual versions match', async () => { + const warn = vi.fn(); + const hit = await warnOnCollationMismatch({ query: vi.fn().mockResolvedValue(row()), warn }); + expect(hit).toBe(false); + expect(warn).not.toHaveBeenCalled(); + }); + + it('warns on the alpine→glibc signature: NULL recorded version, versioned current libc', async () => { + // Field-verified: musl (postgres:16-alpine) records NO datcollversion, so after + // the swap to pgvector/pgvector:pg16 the row reads recorded=NULL, actual=2.36 — + // and Postgres itself stays silent. This is the primary case to catch. + const warn = vi.fn(); + const query = vi.fn().mockResolvedValue(row({ recorded: null, actual: '2.36' })); + expect(await warnOnCollationMismatch({ query, warn })).toBe(true); + const msg = warn.mock.calls[0][0]; + expect(msg).toContain('reported no collation'); + expect(msg).toContain('REINDEX DATABASE "mailflow";'); + }); + + it('stays silent for versionless locales (NULL actual, e.g. C/POSIX or still on musl)', async () => { + const warn = vi.fn(); + for (const over of [{ actual: null }, { recorded: null, actual: null }]) { + expect(await warnOnCollationMismatch({ query: vi.fn().mockResolvedValue(row(over)), warn })).toBe(false); + } + expect(warn).not.toHaveBeenCalled(); + }); + + it('never throws — a failing query (e.g. PG < 15) only skips the check', async () => { + const warn = vi.fn(); + const query = vi.fn().mockRejectedValue(new Error('column "datcollversion" does not exist')); + await expect(warnOnCollationMismatch({ query, warn })).resolves.toBe(false); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/src/services/migrations.gtdDelegations.test.js b/backend/src/services/migrations.gtdDelegations.test.js new file mode 100644 index 00000000..190211f4 --- /dev/null +++ b/backend/src/services/migrations.gtdDelegations.test.js @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { readdirSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +const migrationsDir = fileURLToPath(new URL('../../migrations/', import.meta.url)); + +describe('gtd delegations migration', () => { + it('defines thread-stable ownership, snapshots, and deletion behavior', () => { + const files = readdirSync(migrationsDir).filter(name => /^\d{4}_gtd_delegations\.sql$/.test(name)); + expect(files).toHaveLength(1); + const sql = readFileSync(`${migrationsDir}/${files[0]}`, 'utf8'); + expect(sql).toMatch(/PRIMARY KEY \(user_id, account_id, thread_key\)/i); + expect(sql).toMatch(/contact_id UUID REFERENCES contacts\(id\) ON DELETE SET NULL/i); + expect(sql).toMatch(/contact_display_name_snapshot TEXT NOT NULL/i); + expect(sql).toMatch(/contact_primary_email_snapshot TEXT/i); + expect(sql).toMatch(/delegated_at TIMESTAMPTZ NOT NULL DEFAULT NOW\(\)/i); + expect(sql).toMatch(/updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW\(\)/i); + }); +}); diff --git a/backend/src/services/migrations.js b/backend/src/services/migrations.js index 38fc9a20..6345fc9d 100644 --- a/backend/src/services/migrations.js +++ b/backend/src/services/migrations.js @@ -94,3 +94,70 @@ export async function runMigrations() { client.release(); } } + +// --- Collation-version drift check (best-effort, never aborts boot) --------------- +// The compose move from postgres:16-alpine (musl libc) to pgvector/pgvector:pg16 +// (Debian, glibc) silently changes how the OS collates text. Postgres records the +// collation version a database was created under (pg_database.datcollversion, PG15+); +// pg_database_collation_actual_version(oid) reports what the running server's libc/ICU +// provides NOW. Two shapes of drift matter here (field-verified on both images): +// - recorded != actual: a versioned libc changed underneath (e.g. a glibc bump). +// Postgres emits its own per-connection WARNING for this, easily lost in logs. +// - recorded IS NULL while actual is not: the database was created under a libc +// that reports NO collation version — exactly what musl/alpine does — and now +// runs under glibc. This is the actual alpine → pgvector upgrade signature, and +// Postgres itself stays completely SILENT about it. +// (actual is NULL for versionless locales like C/POSIX — byte-order collation is +// immune to libc swaps, so there is nothing to warn about.) +// Either way, text indexes built under the old ordering may silently return wrong or +// missing rows until reindexed, so surface it loudly at boot with the remedy. The +// REFRESH COLLATION VERSION step records the current version and silences this +// warning on subsequent boots. Returns whether a mismatch was reported +// (observability only; never throws). +export async function warnOnCollationMismatch(deps = {}) { + const q = deps.query || ((text) => pool.query(text)); + const warn = deps.warn || console.warn; + try { + const { rows } = await q(` + SELECT current_database() AS db, + datcollversion AS recorded, + pg_database_collation_actual_version(oid) AS actual + FROM pg_database + WHERE datname = current_database() + `); + const r = rows[0]; + if (!r || !r.actual || r.recorded === r.actual) return false; + const origin = r.recorded + ? [ + ` Database "${r.db}" was created under collation version ${r.recorded}, but`, + ` the operating system now provides ${r.actual}.`, + ] + : [ + ` Database "${r.db}" was created under a C library that reported no collation`, + ` version (e.g. postgres:16-alpine/musl); the current one provides ${r.actual}.`, + ]; + warn([ + '='.repeat(76), + 'WARNING: database collation version mismatch detected', + '', + ...origin, + ' This happens when the Postgres image\'s C library changes — e.g. the', + ' docker-compose switch from postgres:16-alpine (musl) to', + ' pgvector/pgvector:pg16 (glibc).', + '', + ' Text indexes built under the old collation can silently return wrong or', + ' missing rows. Reindex once, then record the new version:', + '', + ` docker compose exec postgres psql -U mailflow -d ${r.db} \\`, + ` -c 'REINDEX DATABASE "${r.db}";' \\`, + ` -c 'ALTER DATABASE "${r.db}" REFRESH COLLATION VERSION;'`, + '='.repeat(76), + ].join('\n')); + return true; + } catch (err) { + // Postgres < 15 has no datcollversion; a restricted role may not read pg_database. + // The check is observability only — never block or fail the boot on it. + console.log(`Collation version check skipped: ${err.message}`); + return false; + } +} diff --git a/backend/src/services/outboxService.js b/backend/src/services/outboxService.js new file mode 100644 index 00000000..d6f2b848 --- /dev/null +++ b/backend/src/services/outboxService.js @@ -0,0 +1,227 @@ +import { withTransaction as defaultWithTransaction } from './db.js'; +import { deleteDraft as defaultDeleteDraft } from './draftService.js'; +import { sendMessage as defaultSendMessage } from './sendService.js'; +import { sanitizeSmtpError } from './mail/smtp.js'; + +export const MAX_UNDO_SECONDS = 120; +export const UNDO_CHOICES = [0, 10, 30, 60, 120]; + +export function normalizeUndoWindow(requested, preference) { + const raw = requested ?? preference ?? 0; + const seconds = Number(raw); + if (!Number.isFinite(seconds) || seconds <= 0) return 0; + return Math.min(Math.trunc(seconds), MAX_UNDO_SECONDS); +} + +export async function enqueue({ + userId, + accountId, + payload, + undoSeconds, + idempotencyKey, + subject, + toPreview, + messageId, +}, deps) { + const storedPayload = { ...payload, account_id: accountId }; + const result = await deps.query( + `INSERT INTO outbox_messages + (user_id, account_id, payload, send_at, subject, to_preview, message_id, idempotency_key) + VALUES ($1, $2, $3, NOW() + ($4 * INTERVAL '1 second'), $5, $6, $7, $8) + ON CONFLICT (user_id, idempotency_key) WHERE idempotency_key IS NOT NULL + DO UPDATE SET idempotency_key = EXCLUDED.idempotency_key + RETURNING id, send_at`, + [ + userId, + accountId, + storedPayload, + undoSeconds, + subject, + toPreview, + messageId, + idempotencyKey || null, + ], + ); + return { + outbox_id: result.rows[0].id, + send_at: result.rows[0].send_at, + undo_seconds: undoSeconds, + }; +} + +export async function cancel({ id, userId }, deps = {}) { + const withTransaction = deps.withTransaction || defaultWithTransaction; + return withTransaction(async (client) => { + const cancelled = await client.query( + `UPDATE outbox_messages + SET status='cancelled', payload='{}'::jsonb, updated_at=NOW() + WHERE id=$1 AND user_id=$2 AND status='pending' + RETURNING id`, + [id, userId], + ); + if (cancelled.rows.length) return { cancelled: true }; + + const existing = await client.query( + 'SELECT status FROM outbox_messages WHERE id=$1 AND user_id=$2', + [id, userId], + ); + if (!existing.rows.length) return { cancelled: false, reason: 'not_found' }; + if (existing.rows[0].status === 'cancelled') { + return { cancelled: false, reason: 'cancelled' }; + } + if (existing.rows[0].status === 'sent' || existing.rows[0].status === 'claimed') { + return { cancelled: false, reason: 'already_sent' }; + } + return { cancelled: false, reason: 'not_found' }; + }); +} + +export async function listPending({ userId }, deps) { + const result = await deps.query( + `SELECT id, subject, to_preview, send_at + FROM outbox_messages + WHERE user_id=$1 AND status='pending' + ORDER BY send_at`, + [userId], + ); + return result.rows; +} + +export async function claimDue({ limit = 50 } = {}, deps = {}) { + const withTransaction = deps.withTransaction || defaultWithTransaction; + return withTransaction(async (client) => { + const result = await client.query( + `UPDATE outbox_messages o + SET status='claimed', claimed_at=NOW(), attempts=attempts+1, updated_at=NOW() + FROM (SELECT id FROM outbox_messages + WHERE status='pending' AND send_at <= NOW() + ORDER BY send_at + FOR UPDATE SKIP LOCKED + LIMIT $1) d + WHERE o.id = d.id + RETURNING o.*`, + [limit], + ); + return result.rows; + }); +} + +export async function markSent(id, sentMessageId, deps) { + const result = await deps.query( + `UPDATE outbox_messages + SET status='sent', payload='{}'::jsonb, sent_message_id=$2, + error=NULL, updated_at=NOW() + WHERE id=$1 AND status='claimed'`, + [id, sentMessageId], + ); + return result.rowCount; +} + +export async function markFailed(id, error, deps) { + const result = await deps.query( + `UPDATE outbox_messages + SET status='failed', payload='{}'::jsonb, error=$2, updated_at=NOW() + WHERE id=$1 AND status='claimed'`, + [id, error], + ); + return result.rowCount; +} + +export async function sweepStaleClaims(deps) { + const result = await deps.query( + `UPDATE outbox_messages + SET status='failed', payload='{}'::jsonb, + error='delivery interrupted — the send was not retried; check your Sent folder', + updated_at=NOW() + WHERE status='claimed' AND claimed_at < NOW() - INTERVAL '5 minutes'`, + ); + return result.rowCount; +} + +export async function purgeTerminalRows(deps) { + const result = await deps.query( + `DELETE FROM outbox_messages + WHERE status IN ('sent','cancelled','failed') + AND updated_at < NOW() - INTERVAL '7 days'`, + ); + return result.rowCount; +} + +export function startOutboxWorker(deps, { tickMs = 5000 } = {}) { + let running = false; + let tickCount = 0; + let stopped = false; + const deliver = deps.sendMessage || defaultSendMessage; + + sweepStaleClaims(deps) + .catch(err => console.error('Outbox stale-claim sweep error:', err.message)); + + const timer = setInterval(() => { + if (running || stopped) return; + running = true; + tickCount += 1; + + (async () => { + if (tickCount % 12 === 0) await sweepStaleClaims(deps); + if (tickCount % 720 === 0) await purgeTerminalRows(deps); + + const rows = await claimDue({ limit: 50 }, deps); + for (const row of rows) { + try { + const accountResult = await deps.query( + 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', + [row.account_id, row.user_id], + ); + if (!accountResult.rows.length) throw new Error('Account not found'); + const account = accountResult.rows[0]; + + const result = await deliver({ + ...row.payload, + userId: row.user_id, + account, + account_id: row.account_id, + messageId: row.message_id, + }, deps); + await markSent(row.id, result.messageId || row.message_id, deps); + const draft = row.payload?.deleteDraftOnSend; + if (draft) { + try { + let draftAccount = account; + if (draft.accountId && draft.accountId !== account.id) { + const sourceAccountResult = await deps.query( + 'SELECT * FROM email_accounts WHERE id = $1 AND user_id = $2', + [draft.accountId, row.user_id], + ); + if (!sourceAccountResult.rows.length) { + throw new Error('Source draft account not found'); + } + draftAccount = sourceAccountResult.rows[0]; + } + const deleteDraft = deps.draftService?.deleteDraft || defaultDeleteDraft; + await deleteDraft({ + account: draftAccount, + uid: draft.uid, + folder: draft.folder, + }, deps); + } catch (err) { + console.error('Outbox draft cleanup error:', err.message); + } + } + } catch (err) { + await markFailed(row.id, sanitizeSmtpError(err), deps); + } + } + })() + .catch(err => console.error('Outbox worker error:', err.message)) + .finally(() => { running = false; }); + }, tickMs); + timer.unref?.(); + + return { + stop() { + if (stopped) return; + stopped = true; + clearInterval(timer); + }, + }; +} diff --git a/backend/src/services/outboxService.test.js b/backend/src/services/outboxService.test.js new file mode 100644 index 00000000..1eef5861 --- /dev/null +++ b/backend/src/services/outboxService.test.js @@ -0,0 +1,295 @@ +import { describe, expect, it, vi } from 'vitest'; + +describe('normalizeUndoWindow', () => { + it.each([ + [undefined, undefined, 0], + [undefined, 30, 30], + [200, undefined, 120], + [-5, undefined, 0], + ['30', undefined, 30], + ['not-a-number', undefined, 0], + ])('normalizes requested=%j preference=%j to %d', async (requested, preference, expected) => { + const service = await import('./outboxService.js').catch(() => ({})); + + expect(service.normalizeUndoWindow).toBeTypeOf('function'); + expect(service.normalizeUndoWindow(requested, preference)).toBe(expected); + }); +}); + +describe('enqueue', () => { + it('stores a fully resolved payload with denormalized receipt fields', async () => { + const service = await import('./outboxService.js'); + const sendAt = new Date('2026-07-28T12:00:30.000Z'); + const query = vi.fn().mockResolvedValue({ + rows: [{ id: 'outbox-1', send_at: sendAt }], + }); + + expect(service.enqueue).toBeTypeOf('function'); + const result = await service.enqueue({ + userId: 'user-1', + accountId: 'account-1', + payload: { + userId: 'user-1', + to: ['Recipient '], + body: 'Hello', + }, + undoSeconds: 30, + idempotencyKey: 'compose-1', + subject: 'Subject', + toPreview: ['Recipient '], + messageId: '', + }, { query }); + + expect(query).toHaveBeenCalledTimes(1); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain('INSERT INTO outbox_messages'); + expect(sql).toContain('ON CONFLICT (user_id, idempotency_key)'); + expect(params).toEqual([ + 'user-1', + 'account-1', + { + userId: 'user-1', + to: ['Recipient '], + body: 'Hello', + account_id: 'account-1', + }, + 30, + 'Subject', + ['Recipient '], + '', + 'compose-1', + ]); + expect(result).toEqual({ + outbox_id: 'outbox-1', + send_at: sendAt, + undo_seconds: 30, + }); + }); +}); + +describe('cancel', () => { + function depsFor(query) { + return { + withTransaction: async (fn) => fn({ query }), + }; + } + + it('atomically cancels a pending row and wipes its payload', async () => { + const service = await import('./outboxService.js'); + const query = vi.fn().mockResolvedValue({ rows: [{ id: 'outbox-1' }] }); + + expect(service.cancel).toBeTypeOf('function'); + await expect(service.cancel( + { id: 'outbox-1', userId: 'user-1' }, + depsFor(query), + )).resolves.toEqual({ cancelled: true }); + + expect(query).toHaveBeenCalledTimes(1); + expect(query.mock.calls[0][0]).toContain("status='cancelled'"); + expect(query.mock.calls[0][0]).toContain("payload='{}'::jsonb"); + expect(query.mock.calls[0][0]).toContain("status='pending'"); + expect(query.mock.calls[0][1]).toEqual(['outbox-1', 'user-1']); + }); + + it.each(['sent', 'claimed'])('reports %s rows as already sent', async (status) => { + const service = await import('./outboxService.js'); + const query = vi.fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [{ status }] }); + + expect(service.cancel).toBeTypeOf('function'); + await expect(service.cancel( + { id: 'outbox-1', userId: 'user-1' }, + depsFor(query), + )).resolves.toEqual({ cancelled: false, reason: 'already_sent' }); + }); + + it('does not reveal a row owned by another user', async () => { + const service = await import('./outboxService.js'); + const query = vi.fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }); + + expect(service.cancel).toBeTypeOf('function'); + await expect(service.cancel( + { id: 'outbox-1', userId: 'user-2' }, + depsFor(query), + )).resolves.toEqual({ cancelled: false, reason: 'not_found' }); + expect(query.mock.calls[1][0]).toContain('WHERE id=$1 AND user_id=$2'); + }); + + it('treats an already-cancelled row as an idempotent no-op', async () => { + const service = await import('./outboxService.js'); + const query = vi.fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [{ status: 'cancelled' }] }); + + expect(service.cancel).toBeTypeOf('function'); + await expect(service.cancel( + { id: 'outbox-1', userId: 'user-1' }, + depsFor(query), + )).resolves.toEqual({ cancelled: false, reason: 'cancelled' }); + }); +}); + +describe('listPending', () => { + it('returns only pending receipt fields for the scoped user', async () => { + const service = await import('./outboxService.js'); + const rows = [{ + id: 'outbox-1', + subject: 'Subject', + to_preview: ['recipient@example.com'], + send_at: new Date('2026-07-28T12:00:30.000Z'), + }]; + const query = vi.fn().mockResolvedValue({ rows }); + + expect(service.listPending).toBeTypeOf('function'); + await expect(service.listPending( + { userId: 'user-1' }, + { query }, + )).resolves.toEqual(rows); + expect(query).toHaveBeenCalledTimes(1); + expect(query.mock.calls[0][0]).toContain("user_id=$1 AND status='pending'"); + expect(query.mock.calls[0][0]).toContain('ORDER BY send_at'); + expect(query.mock.calls[0][1]).toEqual(['user-1']); + }); +}); + +describe('claimDue', () => { + it('claims due rows with the plan SQL under a transaction', async () => { + const service = await import('./outboxService.js'); + const rows = [{ id: 'outbox-1', status: 'claimed', attempts: 1 }]; + const query = vi.fn().mockResolvedValue({ rows }); + const withTransaction = vi.fn(async (fn) => fn({ query })); + + expect(service.claimDue).toBeTypeOf('function'); + await expect(service.claimDue( + { limit: 25, now: new Date('2026-07-28T12:00:00.000Z') }, + { withTransaction }, + )).resolves.toEqual(rows); + + expect(withTransaction).toHaveBeenCalledTimes(1); + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain("SET status='claimed', claimed_at=NOW(), attempts=attempts+1"); + expect(sql).toContain("WHERE status='pending' AND send_at <= NOW()"); + expect(sql).toContain('FOR UPDATE SKIP LOCKED'); + expect(sql).toContain('LIMIT $1'); + expect(params).toEqual([25]); + }); + + it('allows exactly one winner when cancel and claim race', async () => { + const service = await import('./outboxService.js'); + let status = 'pending'; + let arrivals = 0; + let release; + const barrier = new Promise(resolve => { release = resolve; }); + const query = vi.fn(async (sql) => { + if (sql.includes("status='pending'")) { + arrivals += 1; + if (arrivals === 2) release(); + await barrier; + } + if (sql.includes("SET status='claimed'")) { + if (status !== 'pending') return { rows: [] }; + status = 'claimed'; + return { rows: [{ id: 'outbox-1', status }] }; + } + if (sql.includes("SET status='cancelled'")) { + if (status !== 'pending') return { rows: [] }; + status = 'cancelled'; + return { rows: [{ id: 'outbox-1' }] }; + } + if (sql.includes('SELECT status')) return { rows: [{ status }] }; + return { rows: [] }; + }); + const withTransaction = vi.fn(async (fn) => fn({ query })); + const deps = { withTransaction }; + + expect(service.claimDue).toBeTypeOf('function'); + const [claimed, cancelled] = await Promise.all([ + service.claimDue({ limit: 1 }, deps), + service.cancel({ id: 'outbox-1', userId: 'user-1' }, deps), + ]); + + const claimWon = claimed.length === 1; + const cancelWon = cancelled.cancelled === true; + expect(Number(claimWon) + Number(cancelWon)).toBe(1); + expect(withTransaction).toHaveBeenCalledTimes(2); + }); +}); + +describe('markSent', () => { + it('marks only a claimed row sent and wipes the payload atomically', async () => { + const service = await import('./outboxService.js'); + const query = vi.fn().mockResolvedValue({ rowCount: 1, rows: [] }); + + expect(service.markSent).toBeTypeOf('function'); + await expect(service.markSent( + 'outbox-1', + '', + { query }, + )).resolves.toBe(1); + + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain("status='sent'"); + expect(sql).toContain("payload='{}'::jsonb"); + expect(sql).toContain("WHERE id=$1 AND status='claimed'"); + expect(params).toEqual(['outbox-1', '']); + }); +}); + +describe('markFailed', () => { + it('marks only a claimed row failed with an error and wipes the payload', async () => { + const service = await import('./outboxService.js'); + const query = vi.fn().mockResolvedValue({ rowCount: 1, rows: [] }); + + expect(service.markFailed).toBeTypeOf('function'); + await expect(service.markFailed( + 'outbox-1', + 'Failed to send message.', + { query }, + )).resolves.toBe(1); + + const [sql, params] = query.mock.calls[0]; + expect(sql).toContain("status='failed'"); + expect(sql).toContain("payload='{}'::jsonb"); + expect(sql).toContain("WHERE id=$1 AND status='claimed'"); + expect(params).toEqual(['outbox-1', 'Failed to send message.']); + }); +}); + +describe('sweepStaleClaims', () => { + it('fails stale claims loudly instead of returning them to pending', async () => { + const service = await import('./outboxService.js'); + const query = vi.fn().mockResolvedValue({ rowCount: 2, rows: [] }); + + expect(service.sweepStaleClaims).toBeTypeOf('function'); + await expect(service.sweepStaleClaims({ query })).resolves.toBe(2); + + const [sql] = query.mock.calls[0]; + expect(sql).toContain("SET status='failed', payload='{}'::jsonb"); + expect(sql).toContain( + "error='delivery interrupted — the send was not retried; check your Sent folder'", + ); + expect(sql).toContain("status='claimed'"); + expect(sql).toContain("claimed_at < NOW() - INTERVAL '5 minutes'"); + expect(sql).not.toContain("status='pending'"); + }); +}); + +describe('purgeTerminalRows', () => { + it('deletes only terminal rows whose last update is older than seven days', async () => { + const service = await import('./outboxService.js'); + const query = vi.fn().mockResolvedValue({ rowCount: 3, rows: [] }); + + expect(service.purgeTerminalRows).toBeTypeOf('function'); + await expect(service.purgeTerminalRows({ query })).resolves.toBe(3); + + const [sql] = query.mock.calls[0]; + expect(sql).toContain('DELETE FROM outbox_messages'); + expect(sql).toContain("status IN ('sent','cancelled','failed')"); + expect(sql).toContain("updated_at < NOW() - INTERVAL '7 days'"); + expect(sql).not.toContain("'pending'"); + expect(sql).not.toContain("'claimed'"); + }); +}); diff --git a/backend/src/services/outboxWorker.test.js b/backend/src/services/outboxWorker.test.js new file mode 100644 index 00000000..64d557b0 --- /dev/null +++ b/backend/src/services/outboxWorker.test.js @@ -0,0 +1,392 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { readFileSync } from 'node:fs'; + +const NOW = new Date('2026-07-28T12:00:00.000Z'); + +function deferred() { + let resolve; + const promise = new Promise(r => { resolve = r; }); + return { promise, resolve }; +} + +function workerDeps({ rows = [], sendMessage, missingAccountIds = [] } = {}) { + const state = rows.map(row => ({ status: 'pending', ...row })); + let claimQueries = 0; + const query = vi.fn(async (sql, params = []) => { + if (sql.includes('SELECT * FROM email_accounts')) { + if (missingAccountIds.includes(params[0])) return { rows: [] }; + return { + rows: [{ + id: params[0], + user_id: params[1], + email_address: 'sender@example.com', + }], + }; + } + if (sql.includes("SET status='sent'")) { + const row = state.find(candidate => candidate.id === params[0]); + if (row?.status !== 'claimed') return { rowCount: 0, rows: [] }; + row.status = 'sent'; + return { rowCount: 1, rows: [] }; + } + if (sql.includes("SET status='failed'") && sql.includes('WHERE id=$1')) { + const row = state.find(candidate => candidate.id === params[0]); + if (row?.status !== 'claimed') return { rowCount: 0, rows: [] }; + row.status = 'failed'; + row.error = params[1]; + return { rowCount: 1, rows: [] }; + } + if (sql.includes("claimed_at < NOW() - INTERVAL '5 minutes'")) { + return { rowCount: 0, rows: [] }; + } + if (sql.includes('DELETE FROM outbox_messages')) { + return { rowCount: 0, rows: [] }; + } + return { rowCount: 0, rows: [] }; + }); + const withTransaction = vi.fn(async (fn) => fn({ + query: vi.fn(async (sql) => { + if (!sql.includes('FOR UPDATE SKIP LOCKED')) return { rows: [] }; + claimQueries += 1; + const due = state.filter(row => ( + row.status === 'pending' && new Date(row.send_at).getTime() <= Date.now() + )); + for (const row of due) row.status = 'claimed'; + return { rows: due }; + }), + })); + return { + deps: { + query, + withTransaction, + imapManager: {}, + redisClient: {}, + refreshMicrosoftToken: vi.fn(), + sendMessage: sendMessage || vi.fn().mockResolvedValue({ messageId: '' }), + }, + query, + state, + claimCount: () => claimQueries, + }; +} + +describe('startOutboxWorker', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it('claims and delivers only rows whose send time is due', async () => { + const service = await import('./outboxService.js'); + const due = { + id: 'due', + user_id: 'user-1', + account_id: 'account-1', + send_at: new Date(NOW.getTime() - 1), + message_id: '', + payload: { to: ['due@example.com'], body: 'Due' }, + }; + const future = { + id: 'future', + user_id: 'user-1', + account_id: 'account-1', + send_at: new Date(NOW.getTime() + 60_000), + message_id: '', + payload: { to: ['future@example.com'], body: 'Future' }, + }; + const { deps, state } = workerDeps({ rows: [due, future] }); + + expect(service.startOutboxWorker).toBeTypeOf('function'); + const worker = service.startOutboxWorker(deps, { tickMs: 1_000 }); + await vi.advanceTimersByTimeAsync(1_000); + + expect(deps.sendMessage).toHaveBeenCalledTimes(1); + expect(deps.sendMessage).toHaveBeenCalledWith( + expect.objectContaining({ + account: expect.objectContaining({ id: 'account-1', user_id: 'user-1' }), + account_id: 'account-1', + messageId: '', + userId: 'user-1', + }), + deps, + ); + expect(state.find(row => row.id === 'due').status).toBe('sent'); + expect(state.find(row => row.id === 'future').status).toBe('pending'); + worker.stop(); + }); + + it('deletes a queued draft only after delivery succeeds and the row is marked sent', async () => { + const service = await import('./outboxService.js'); + const row = { + id: 'draft-send', + user_id: 'user-1', + account_id: 'account-1', + send_at: NOW, + message_id: '', + payload: { + to: ['recipient@example.com'], + body: 'Draft body', + deleteDraftOnSend: { uid: 7, folder: 'Drafts' }, + }, + }; + const { deps, state } = workerDeps({ rows: [row] }); + deps.draftService = { + deleteDraft: vi.fn().mockResolvedValue({ ok: true }), + }; + + const worker = service.startOutboxWorker(deps, { tickMs: 1_000 }); + await vi.advanceTimersByTimeAsync(1_000); + + expect(state[0].status).toBe('sent'); + expect(deps.draftService.deleteDraft).toHaveBeenCalledWith({ + account: expect.objectContaining({ id: 'account-1', user_id: 'user-1' }), + uid: 7, + folder: 'Drafts', + }, deps); + const markSentCall = deps.query.mock.invocationCallOrder.find((_, index) => ( + deps.query.mock.calls[index][0].includes("SET status='sent'") + )); + expect(markSentCall).toBeLessThan( + deps.draftService.deleteDraft.mock.invocationCallOrder[0], + ); + worker.stop(); + }); + + it('owner-resolves and deletes a queued source draft through its cross-account source', async () => { + const service = await import('./outboxService.js'); + const row = { + id: 'cross-account-draft-send', + user_id: 'user-1', + account_id: 'destination-account', + send_at: NOW, + message_id: '', + payload: { + to: ['recipient@example.com'], + body: 'Draft body', + deleteDraftOnSend: { + accountId: 'source-account', + uid: 7, + folder: 'Drafts', + }, + }, + }; + const { deps, state } = workerDeps({ rows: [row] }); + deps.draftService = { + deleteDraft: vi.fn().mockResolvedValue({ ok: true }), + }; + + const worker = service.startOutboxWorker(deps, { tickMs: 1_000 }); + await vi.advanceTimersByTimeAsync(1_000); + + expect(state[0].status).toBe('sent'); + expect(deps.query).toHaveBeenCalledWith( + expect.stringMatching(/FROM email_accounts[\s\S]+id\s*=\s*\$1[\s\S]+user_id\s*=\s*\$2/), + ['source-account', 'user-1'], + ); + expect(deps.draftService.deleteDraft).toHaveBeenCalledWith({ + account: expect.objectContaining({ id: 'source-account', user_id: 'user-1' }), + uid: 7, + folder: 'Drafts', + }, deps); + worker.stop(); + }); + + it('never deletes a queued source through the destination when the owned source is missing', async () => { + const service = await import('./outboxService.js'); + const row = { + id: 'missing-source-draft-send', + user_id: 'user-1', + account_id: 'destination-account', + send_at: NOW, + message_id: '', + payload: { + to: ['recipient@example.com'], + body: 'Draft body', + deleteDraftOnSend: { + accountId: 'missing-source-account', + uid: 7, + folder: 'Drafts', + }, + }, + }; + const { deps, state } = workerDeps({ + rows: [row], + missingAccountIds: ['missing-source-account'], + }); + deps.draftService = { deleteDraft: vi.fn() }; + vi.spyOn(console, 'error').mockImplementation(() => {}); + + const worker = service.startOutboxWorker(deps, { tickMs: 1_000 }); + await vi.advanceTimersByTimeAsync(1_000); + + expect(state[0].status).toBe('sent'); + expect(deps.draftService.deleteDraft).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith( + 'Outbox draft cleanup error:', + 'Source draft account not found', + ); + worker.stop(); + }); + + it('keeps a delivered row sent when deferred draft cleanup fails', async () => { + const service = await import('./outboxService.js'); + const row = { + id: 'draft-cleanup-failure', + user_id: 'user-1', + account_id: 'account-1', + send_at: NOW, + message_id: '', + payload: { + to: ['recipient@example.com'], + body: 'Draft body', + deleteDraftOnSend: { uid: 7, folder: 'Drafts' }, + }, + }; + const { deps, query, state } = workerDeps({ rows: [row] }); + deps.draftService = { + deleteDraft: vi.fn().mockRejectedValue(new Error('IMAP delete failed')), + }; + vi.spyOn(console, 'error').mockImplementation(() => {}); + + const worker = service.startOutboxWorker(deps, { tickMs: 1_000 }); + await vi.advanceTimersByTimeAsync(1_000); + + expect(state[0].status).toBe('sent'); + expect(query.mock.calls.some(([sql]) => ( + sql.includes("SET status='failed'") && sql.includes('WHERE id=$1') + ))).toBe(false); + expect(console.error).toHaveBeenCalledWith( + 'Outbox draft cleanup error:', + 'IMAP delete failed', + ); + worker.stop(); + }); + + it('does not stack ticks while delivery is still running', async () => { + const service = await import('./outboxService.js'); + const slow = deferred(); + const row = { + id: 'slow', + user_id: 'user-1', + account_id: 'account-1', + send_at: NOW, + message_id: '', + payload: { to: ['slow@example.com'], body: 'Slow' }, + }; + const { deps, claimCount } = workerDeps({ + rows: [row], + sendMessage: vi.fn().mockReturnValue(slow.promise), + }); + + expect(service.startOutboxWorker).toBeTypeOf('function'); + const worker = service.startOutboxWorker(deps, { tickMs: 1_000 }); + await vi.advanceTimersByTimeAsync(3_000); + expect(claimCount()).toBe(1); + + slow.resolve({ messageId: '' }); + await vi.runAllTicks(); + await vi.advanceTimersByTimeAsync(1_000); + expect(claimCount()).toBe(2); + worker.stop(); + }); + + it('marks a delivery failure with a sanitized error and never retries it', async () => { + const service = await import('./outboxService.js'); + vi.spyOn(console, 'error').mockImplementation(() => {}); + const row = { + id: 'failure', + user_id: 'user-1', + account_id: 'account-1', + send_at: NOW, + message_id: '', + payload: { to: ['failure@example.com'], body: 'Failure' }, + }; + const sendMessage = vi.fn().mockRejectedValue( + new Error('ECONNREFUSED smtp.secret.internal:2525'), + ); + const { deps, state } = workerDeps({ rows: [row], sendMessage }); + + expect(service.startOutboxWorker).toBeTypeOf('function'); + const worker = service.startOutboxWorker(deps, { tickMs: 1_000 }); + await vi.advanceTimersByTimeAsync(5_000); + + expect(sendMessage).toHaveBeenCalledTimes(1); + expect(state[0]).toMatchObject({ + status: 'failed', + error: 'Could not connect to the mail server. Check your SMTP settings.', + }); + worker.stop(); + }); + + it('stop clears the interval', async () => { + const service = await import('./outboxService.js'); + const { deps, claimCount } = workerDeps(); + const clearIntervalSpy = vi.spyOn(globalThis, 'clearInterval'); + + expect(service.startOutboxWorker).toBeTypeOf('function'); + const worker = service.startOutboxWorker(deps, { tickMs: 1_000 }); + worker.stop(); + await vi.advanceTimersByTimeAsync(3_000); + + expect(clearIntervalSpy).toHaveBeenCalledTimes(1); + expect(claimCount()).toBe(0); + }); + + it('unrefs the interval timer so it cannot keep the process alive', async () => { + const service = await import('./outboxService.js'); + const { deps } = workerDeps(); + const timer = { unref: vi.fn() }; + vi.spyOn(globalThis, 'setInterval').mockReturnValue(timer); + + expect(service.startOutboxWorker).toBeTypeOf('function'); + const worker = service.startOutboxWorker(deps, { tickMs: 1_000 }); + + expect(timer.unref).toHaveBeenCalledTimes(1); + worker.stop(); + }); + + it('sweeps on startup and every 12th tick, and purges every 720th tick', async () => { + const service = await import('./outboxService.js'); + const { deps, query } = workerDeps(); + + expect(service.startOutboxWorker).toBeTypeOf('function'); + const worker = service.startOutboxWorker(deps, { tickMs: 1 }); + await vi.runAllTicks(); + for (let tick = 0; tick < 720; tick += 1) { + await vi.advanceTimersByTimeAsync(1); + } + + const sweepCalls = query.mock.calls.filter(([sql]) => ( + sql.includes("claimed_at < NOW() - INTERVAL '5 minutes'") + )); + const purgeCalls = query.mock.calls.filter(([sql]) => ( + sql.includes('DELETE FROM outbox_messages') + )); + expect(sweepCalls).toHaveLength(61); + expect(purgeCalls).toHaveLength(1); + worker.stop(); + }); +}); + +describe('outbox worker startup wiring', () => { + it('arms beside the snooze watcher and exposes the service to MCP', () => { + const source = readFileSync(new URL('../index.js', import.meta.url), 'utf8'); + + expect(source).toContain("import * as outboxService from './services/outboxService.js';"); + expect(source.indexOf('outboxService.startOutboxWorker(')).toBeGreaterThan( + source.indexOf('imapManager.startSnoozeWatcher();'), + ); + expect(source).toContain(`outboxService.startOutboxWorker( + { imapManager, refreshMicrosoftToken, redisClient, query }, + { tickMs: 5000 }, +);`); + expect(source).toMatch( + /mountMcp\(app,\s*\{[\s\S]*?sendService,\s*outboxService,\s*draftService,[\s\S]*?\}\);/, + ); + }); +}); diff --git a/backend/src/services/replyService.js b/backend/src/services/replyService.js new file mode 100644 index 00000000..03e2b0a7 --- /dev/null +++ b/backend/src/services/replyService.js @@ -0,0 +1,396 @@ +import { dedupePreferNamed, parseAddress } from './mail/addresses.js'; +import { AliasNotFoundError } from './mail/identity.js'; + +function addressObjects(value) { + if (Array.isArray(value)) return value; + try { + const parsed = JSON.parse(value || '[]'); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } +} + +export function selfAddressSet(account, aliases) { + return new Set([ + account?.email_address, + ...(aliases || []).flatMap(alias => [alias?.email, alias?.reply_to]), + ].filter(Boolean).map(address => address.toLowerCase())); +} + +export function pickReplyTarget(msg) { + const replyTo = addressObjects(msg?.reply_to); + if (replyTo[0]?.email) return replyTo[0]; + return { name: msg?.from_name || '', email: msg?.from_email || '' }; +} + +function parseRecipientInput(value) { + return (Array.isArray(value) ? value : []).map(recipient => ( + typeof recipient === 'string' ? parseAddress(recipient) : recipient + )); +} + +function withoutEmails(recipients, emails) { + return recipients.filter(recipient => !emails.has(recipient?.email?.toLowerCase())); +} + +function serviceError(message, code) { + return Object.assign(new Error(message), { + status: 400, + code, + expose: true, + }); +} + +export function computeReplyRecipients(msg, { + account, + aliases = [], + replyAll = false, + to: toOverride, + cc: ccOverride, + bcc: bccOverride, + toAdd, + ccAdd, + bccAdd, + remove, +}) { + for (const [field, override, additive] of [ + ['to', toOverride, toAdd], + ['cc', ccOverride, ccAdd], + ['bcc', bccOverride, bccAdd], + ]) { + if (override !== undefined && additive !== undefined) { + throw serviceError(`${field} and ${field}Add are mutually exclusive`, 'invalid_arguments'); + } + } + + const replyTarget = pickReplyTarget(msg); + let to = replyTarget.email ? [replyTarget] : []; + let cc = []; + let bcc = []; + + if (replyAll) { + const self = selfAddressSet(account, aliases); + const targetEmail = replyTarget.email?.toLowerCase(); + cc = dedupePreferNamed([ + ...addressObjects(msg?.to_addresses), + ...addressObjects(msg?.cc_addresses), + ].filter(recipient => { + const email = recipient?.email?.toLowerCase(); + return email && !self.has(email) && email !== targetEmail; + })); + } + + if (toOverride !== undefined) { + const explicitTo = parseRecipientInput(toOverride); + const explicitToEmails = new Set(explicitTo.map(recipient => recipient?.email?.toLowerCase())); + cc = withoutEmails(dedupePreferNamed([ + ...to, + ...cc, + ...parseRecipientInput(ccOverride), + ]), explicitToEmails); + to = explicitTo; + } else if (ccOverride !== undefined) { + cc = parseRecipientInput(ccOverride); + } + if (bccOverride !== undefined) bcc = parseRecipientInput(bccOverride); + + to.push(...parseRecipientInput(toAdd)); + cc.push(...parseRecipientInput(ccAdd)); + bcc.push(...parseRecipientInput(bccAdd)); + + const removed = new Set(parseRecipientInput(remove).map(recipient => recipient?.email?.toLowerCase())); + to = withoutEmails(to, removed); + cc = withoutEmails(cc, removed); + bcc = withoutEmails(bcc, removed); + + const explicitlyNamed = new Set([ + ...parseRecipientInput(toOverride), + ...parseRecipientInput(ccOverride), + ...parseRecipientInput(bccOverride), + ...parseRecipientInput(toAdd), + ...parseRecipientInput(ccAdd), + ...parseRecipientInput(bccAdd), + ].map(recipient => recipient?.email?.toLowerCase()).filter(Boolean)); + const self = selfAddressSet(account, aliases); + const finalPass = recipients => recipients.filter(recipient => { + const email = recipient?.email?.toLowerCase(); + return email && (!self.has(email) || explicitlyNamed.has(email)); + }); + to = finalPass(to); + cc = finalPass(cc); + bcc = finalPass(bcc); + + if (to.length + cc.length + bcc.length > 100) { + throw serviceError('Too many recipients (max 100)', 'too_many_recipients'); + } + return { to, cc, bcc }; +} + +const RE_PREFIX = /^\s*re\s*:/i; +const FWD_PREFIX = /^\s*(fwd?|fw)\s*:/i; +const MAX_ATTACHMENT_BYTES = 26_214_400; + +export function replySubject(subject) { + const value = typeof subject === 'string' ? subject.trim() : ''; + if (!value) return 'Re:'; + return RE_PREFIX.test(value) ? value : `Re: ${value}`; +} + +export function forwardSubject(subject) { + const value = typeof subject === 'string' ? subject.trim() : ''; + if (!value) return 'Fwd:'; + return FWD_PREFIX.test(value) ? value : `Fwd: ${value}`; +} + +export function buildReferences(msg) { + const inReplyTo = msg?.message_id || null; + const ids = `${msg?.thread_references || ''} ${msg?.message_id || ''}` + .split(/\s+/) + .filter(id => id.startsWith('<') && id.endsWith('>')); + const seen = new Set(); + const chain = ids.filter(id => !seen.has(id) && seen.add(id)); + const bounded = chain.length > 21 ? [chain[0], ...chain.slice(-20)] : chain; + return { + inReplyTo, + references: bounded.join(' ') || inReplyTo, + }; +} + +export function autoSelectAlias(msg, aliases) { + if (!aliases?.length) return null; + const originalRecipients = [ + ...addressObjects(msg?.to_addresses), + ...addressObjects(msg?.cc_addresses), + ].map(recipient => recipient?.email?.toLowerCase()).filter(Boolean); + const fromEmail = (msg?.from_email || '').toLowerCase(); + const match = aliases.find(alias => { + const aliasEmail = alias?.email?.toLowerCase(); + return aliasEmail && ( + originalRecipients.includes(aliasEmail) || + fromEmail === aliasEmail + ); + }); + return match?.id || null; +} + +function quoteAuthor(msg) { + const safeName = (msg?.from_name || '').replace(/[\r\n]+/g, ' '); + return safeName + ? `${safeName} <${msg?.from_email || ''}>` + : msg?.from_email || ''; +} + +function normalizeCid(value) { + return String(value || '').replace(/^cid:/i, '').replace(/^<|>$/g, '').toLowerCase(); +} + +function rewriteCidImages(html, attachments) { + const byCid = new Map(addressObjects(attachments).map(attachment => [ + normalizeCid(attachment?.cid || attachment?.content_id || attachment?.contentId), + attachment, + ]).filter(([cid]) => cid)); + return html.replace( + /]*\bsrc\s*=\s*(?:"cid:([^"]+)"|'cid:([^']+)'|cid:([^\s>]+))[^>]*>/gi, + (_tag, doubleQuoted, singleQuoted, unquoted) => { + const cid = normalizeCid(doubleQuoted || singleQuoted || unquoted); + const attachment = byCid.get(cid); + return `[inline image: ${attachment?.filename || attachment?.name || 'attachment'}]`; + }, + ); +} + +function referencedCids(html) { + const matches = String(html || '').matchAll( + /]*\bsrc\s*=\s*(?:"cid:([^"]+)"|'cid:([^']+)'|cid:([^\s>]+))[^>]*>/gi, + ); + return [...new Set([...matches].map(match => ( + normalizeCid(match[1] || match[2] || match[3]) + )).filter(Boolean))]; +} + +async function fetchInlineAttachments(msg, account, deps) { + const attachments = addressObjects(msg?.attachments); + const byCid = new Map(attachments.map(attachment => [ + normalizeCid(attachment?.cid || attachment?.content_id || attachment?.contentId), + attachment, + ]).filter(([cid]) => cid)); + const resolved = []; + let totalBytes = 0; + for (const cid of referencedCids(msg?.body_html)) { + const attachment = byCid.get(cid); + if (!attachment?.part) continue; + const content = await deps.imapManager.fetchAttachment( + account, + msg.uid, + msg.folder, + attachment.part, + ); + const buffer = Buffer.isBuffer(content) ? content : Buffer.from(content || ''); + totalBytes += buffer.length; + if (totalBytes > MAX_ATTACHMENT_BYTES) { + throw serviceError('Total attachment size exceeds 25 MB', 'attachment_too_large'); + } + resolved.push({ + filename: attachment.filename || attachment.name || 'attachment', + content: buffer.toString('base64'), + contentType: attachment.type || attachment.contentType || 'application/octet-stream', + cid, + }); + } + return resolved; +} + +function wrapQuoteHtml(header, html) { + return `

${header}

${html}
`; +} + +export function buildQuote(msg, { includeInlineImages = false } = {}) { + const localeDate = msg?.date ? new Date(msg.date).toLocaleString() : ''; + const author = quoteAuthor(msg); + const quotedBody = msg?.body_text + ? `\n\n---\nOn ${localeDate}, ${author} wrote:\n${msg.body_text.split('\n').map(line => `> ${line}`).join('\n')}` + : ''; + const html = includeInlineImages + ? msg?.body_html + : rewriteCidImages(msg?.body_html || '', msg?.attachments); + const quotedBodyHtml = msg?.body_html + ? wrapQuoteHtml(`On ${localeDate}, ${author} wrote:`, html) + : null; + return { quotedBody, quotedBodyHtml }; +} + +function formatAddressField(value) { + return addressObjects(value).map(recipient => ( + recipient?.name + ? `${recipient.name} <${recipient.email}>` + : recipient?.email + )).filter(Boolean).join(', '); +} + +export function buildForwardQuote(msg) { + const localeDate = msg?.date ? new Date(msg.date).toLocaleString() : ''; + const author = quoteAuthor(msg); + const subject = (msg?.subject || '').replace(/[\r\n]+/g, ' '); + const to = formatAddressField(msg?.to_addresses); + const cc = formatAddressField(msg?.cc_addresses); + const headers = `From: ${author}\nDate: ${localeDate}\nSubject: ${subject}${to ? `\nTo: ${to}` : ''}${cc ? `\nCc: ${cc}` : ''}`; + const quotedBody = `\n\n---------- Forwarded message ----------\n${headers}\n\n${msg?.body_text || ''}`; + const html = rewriteCidImages(msg?.body_html || '', msg?.attachments); + const htmlHeaders = `---------- Forwarded message ----------
From: ${author}
Date: ${localeDate}
Subject: ${subject}${to ? `
To: ${to}` : ''}${cc ? `
Cc: ${cc}` : ''}`; + const quotedBodyHtml = msg?.body_html + ? wrapQuoteHtml(htmlHeaders, html) + : null; + return { quotedBody, quotedBodyHtml }; +} + +function formatRecipient(recipient) { + return recipient.name + ? `${recipient.name} <${recipient.email}>` + : recipient.email; +} + +async function selectAliasId(message, account, aliases, alias, deps) { + if (alias === undefined || alias === null) { + return autoSelectAlias(message, aliases); + } + const resolved = deps?.resolveAlias + ? await deps.resolveAlias(account.id, alias) + : aliases.find(candidate => candidate?.email?.toLowerCase() === alias.toLowerCase()); + if (!resolved) throw new AliasNotFoundError(); + return resolved.id; +} + +export async function buildReply({ + message, + account, + aliases = [], + replyAll = false, + body = '', + bodyIsHtml = false, + to, + cc, + bcc, + toAdd, + ccAdd, + bccAdd, + remove, + noQuote = false, + includeInlineImages = false, + alias, +}, deps = {}) { + const recipients = computeReplyRecipients(message, { + account, + aliases, + replyAll, + to, + cc, + bcc, + toAdd, + ccAdd, + bccAdd, + remove, + }); + const aliasId = await selectAliasId(message, account, aliases, alias, deps); + const threading = buildReferences(message); + const quote = noQuote + ? { quotedBody: '', quotedBodyHtml: null } + : buildQuote(message, { includeInlineImages }); + const inlineAttachments = includeInlineImages && !noQuote + ? await fetchInlineAttachments(message, account, deps) + : []; + + return { + account, + aliasId, + userId: account.user_id, + to: recipients.to.map(formatRecipient), + cc: recipients.cc.map(formatRecipient), + bcc: recipients.bcc.map(formatRecipient), + subject: replySubject(message.subject), + body, + bodyIsHtml, + quotedBody: quote.quotedBody, + quotedBodyHtml: quote.quotedBodyHtml, + inReplyTo: threading.inReplyTo, + references: threading.references, + ...(inlineAttachments.length ? { attachments: inlineAttachments } : {}), + }; +} + +export async function buildForward({ + message, + account, + aliases = [], + to = [], + note = '', + skipAttachments = false, + alias, +}, deps = {}) { + if (to.length > 100) { + throw serviceError('Too many recipients (max 100)', 'too_many_recipients'); + } + const aliasId = await selectAliasId(message, account, aliases, alias, deps); + const quote = buildForwardQuote(message); + const forwardedAttachments = addressObjects(message?.attachments) + .filter(attachment => attachment?.part) + .map(attachment => ({ + messageId: message.id, + part: attachment.part, + })); + + return { + account, + aliasId, + userId: account.user_id, + to, + cc: [], + bcc: [], + subject: forwardSubject(message.subject), + body: note, + bodyIsHtml: false, + quotedBody: quote.quotedBody, + quotedBodyHtml: quote.quotedBodyHtml, + ...(!skipAttachments ? { forwardedAttachments } : {}), + }; +} diff --git a/backend/src/services/replyService.test.js b/backend/src/services/replyService.test.js new file mode 100644 index 00000000..8cfbc810 --- /dev/null +++ b/backend/src/services/replyService.test.js @@ -0,0 +1,760 @@ +import { describe, expect, it, vi } from 'vitest'; +import { openReplyFromMessage } from '../../../frontend/src/utils/composeFromMessage.js'; +import * as replyService from './replyService.js'; + +const { selfAddressSet } = replyService; + +describe('selfAddressSet', () => { + it('includes the account address plus every alias email and reply-to address', () => { + expect(selfAddressSet( + { email_address: 'Me@Example.com' }, + [ + { email: 'Team@Example.com', reply_to: 'Forward@Example.com' }, + { email: 'Other@Example.com', reply_to: null }, + ], + )).toEqual(new Set([ + 'me@example.com', + 'team@example.com', + 'forward@example.com', + 'other@example.com', + ])); + }); +}); + +describe('pickReplyTarget', () => { + it('uses the first reply-to entry when it has an email', () => { + expect(replyService.pickReplyTarget({ + reply_to: JSON.stringify([ + { name: 'Reply Desk', email: 'reply@example.com' }, + { name: 'Other', email: 'other@example.com' }, + ]), + from_name: 'Sender', + from_email: 'sender@example.com', + })).toEqual({ name: 'Reply Desk', email: 'reply@example.com' }); + }); + + it('falls back to the From address when reply-to is empty', () => { + expect(replyService.pickReplyTarget({ + reply_to: [], + from_name: 'Sender', + from_email: 'sender@example.com', + })).toEqual({ name: 'Sender', email: 'sender@example.com' }); + }); +}); + +describe('computeReplyRecipients', () => { + const message = { + reply_to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + from_name: 'Sender', + from_email: 'sender@example.com', + to_addresses: [ + { name: '', email: 'person@example.com' }, + { name: 'Me', email: 'me@example.com' }, + { name: 'Forwarding Self', email: 'forward@example.com' }, + ], + cc_addresses: [ + { name: 'Person', email: 'PERSON@example.com' }, + { name: 'Reply Desk', email: 'reply@example.com' }, + { name: 'Colleague', email: 'colleague@example.com' }, + ], + }; + const account = { email_address: 'me@example.com' }; + const aliases = [{ email: 'alias@example.com', reply_to: 'forward@example.com' }]; + + it('computes a plain reply with only the reply target in To', () => { + expect(replyService.computeReplyRecipients(message, { + account, + aliases, + replyAll: false, + })).toEqual({ + to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + cc: [], + bcc: [], + }); + }); + + it('computes reply-all without self or target addresses and dedupes preferring names', () => { + expect(replyService.computeReplyRecipients(message, { + account, + aliases, + replyAll: true, + })).toEqual({ + to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + cc: [ + { name: 'Person', email: 'PERSON@example.com' }, + { name: 'Colleague', email: 'colleague@example.com' }, + ], + bcc: [], + }); + }); + + it('moves computed To and Cc recipients into Cc when explicit To replaces them', () => { + expect(replyService.computeReplyRecipients(message, { + account, + aliases, + replyAll: true, + to: ['Redirect '], + cc: ['Extra '], + })).toEqual({ + to: [{ name: 'Redirect', email: 'redirect@example.com' }], + cc: [ + { name: 'Reply Desk', email: 'reply@example.com' }, + { name: 'Person', email: 'PERSON@example.com' }, + { name: 'Colleague', email: 'colleague@example.com' }, + { name: 'Extra', email: 'extra@example.com' }, + ], + bcc: [], + }); + }); + + it('replaces computed Cc without moving recipients when only explicit Cc is supplied', () => { + expect(replyService.computeReplyRecipients(message, { + account, + aliases, + replyAll: true, + cc: ['Only '], + })).toEqual({ + to: [{ name: 'Reply Desk', email: 'reply@example.com' }], + cc: [{ name: 'Only', email: 'only@example.com' }], + bcc: [], + }); + }); + + it('appends additive recipients and applies removal after all recipient computation', () => { + expect(replyService.computeReplyRecipients(message, { + account, + aliases, + replyAll: true, + toAdd: ['New To '], + ccAdd: ['New Cc '], + bccAdd: ['New Bcc '], + remove: ['reply@example.com', 'NEW-CC@example.com'], + })).toEqual({ + to: [{ name: 'New To', email: 'new-to@example.com' }], + cc: [ + { name: 'Person', email: 'PERSON@example.com' }, + { name: 'Colleague', email: 'colleague@example.com' }, + ], + bcc: [{ name: 'New Bcc', email: 'new-bcc@example.com' }], + }); + }); + + it.each([ + [{ to: ['a@example.com'], toAdd: ['b@example.com'] }, 'to and toAdd'], + [{ cc: ['a@example.com'], ccAdd: ['b@example.com'] }, 'cc and ccAdd'], + [{ bcc: ['a@example.com'], bccAdd: ['b@example.com'] }, 'bcc and bccAdd'], + ])('rejects mutually exclusive override and additive forms', (overrides, fields) => { + expect(() => replyService.computeReplyRecipients(message, { + account, + aliases, + replyAll: true, + ...overrides, + })).toThrowError(expect.objectContaining({ + message: expect.stringContaining(fields), + status: 400, + code: 'invalid_arguments', + })); + }); + + it('drops self addresses in the final pass unless the caller explicitly names them', () => { + const selfSender = { + ...message, + reply_to: [], + from_name: 'Me', + from_email: 'me@example.com', + }; + + expect(replyService.computeReplyRecipients(selfSender, { + account, + aliases, + replyAll: false, + }).to).toEqual([]); + expect(replyService.computeReplyRecipients(selfSender, { + account, + aliases, + replyAll: false, + ccAdd: ['Forwarding Self '], + }).cc).toEqual([ + { name: 'Forwarding Self', email: 'forward@example.com' }, + ]); + }); + + it('drops blank recipient entries in the final pass', () => { + expect(replyService.computeReplyRecipients(message, { + account, + aliases, + toAdd: ['', null], + }).to).toEqual([ + { name: 'Reply Desk', email: 'reply@example.com' }, + ]); + }); + + it('refuses more than 100 final recipients', () => { + const recipients = Array.from( + { length: 100 }, + (_, index) => `person-${index}@example.com`, + ); + expect(() => replyService.computeReplyRecipients(message, { + account, + aliases, + toAdd: recipients, + })).toThrowError(expect.objectContaining({ + status: 400, + code: 'too_many_recipients', + })); + }); +}); + +describe('subject helpers', () => { + it.each([ + ['Topic', 'Re: Topic'], + [' Re: Topic ', 'Re: Topic'], + ['RE: Topic', 'RE: Topic'], + ['re : Topic', 're : Topic'], + ['', 'Re:'], + [null, 'Re:'], + ])('builds an idempotent reply subject from %j', (subject, expected) => { + expect(replyService.replySubject(subject)).toBe(expected); + }); + + it.each([ + ['Topic', 'Fwd: Topic'], + [' Fwd: Topic ', 'Fwd: Topic'], + ['FW: Topic', 'FW: Topic'], + ['fwd : Topic', 'fwd : Topic'], + ['', 'Fwd:'], + [null, 'Fwd:'], + ])('builds an idempotent forward subject from %j', (subject, expected) => { + expect(replyService.forwardSubject(subject)).toBe(expected); + }); +}); + +describe('buildReferences', () => { + it('uses the full ancestor chain plus message id and removes duplicates', () => { + expect(replyService.buildReferences({ + thread_references: ' ', + message_id: '', + in_reply_to: '', + })).toEqual({ + inReplyTo: '', + references: ' ', + }); + }); + + it('keeps the root and last 20 ids when a chain exceeds 21 entries', () => { + const ids = Array.from({ length: 25 }, (_, index) => ``); + const result = replyService.buildReferences({ + thread_references: ids.join(' '), + message_id: '', + }); + const bounded = result.references.split(' '); + + expect(bounded).toHaveLength(21); + expect(bounded[0]).toBe(ids[0]); + expect(bounded.slice(1)).toEqual([...ids.slice(-19), '']); + }); + + it('falls back to the message id when there is no stored ancestor chain', () => { + expect(replyService.buildReferences({ + thread_references: null, + message_id: '', + })).toEqual({ + inReplyTo: '', + references: '', + }); + }); +}); + +describe('autoSelectAlias', () => { + const aliases = [ + { id: 'alias-1', email: 'first@example.com' }, + { id: 'alias-2', email: 'second@example.com' }, + ]; + + it('selects the first alias addressed in the original To or Cc list', () => { + expect(replyService.autoSelectAlias({ + to_addresses: JSON.stringify([{ name: '', email: 'SECOND@example.com' }]), + cc_addresses: [{ name: '', email: 'first@example.com' }], + from_email: 'sender@example.com', + }, aliases)).toBe('alias-1'); + }); + + it('selects an alias that sent the original message', () => { + expect(replyService.autoSelectAlias({ + to_addresses: [], + cc_addresses: [], + from_email: 'SECOND@example.com', + }, aliases)).toBe('alias-2'); + }); + + it('returns null when no alias matches', () => { + expect(replyService.autoSelectAlias({ + to_addresses: [], + cc_addresses: [], + from_email: 'sender@example.com', + }, aliases)).toBeNull(); + }); +}); + +describe('buildReply', () => { + const account = { + id: 'account-1', + user_id: 'user-1', + email_address: 'me@example.com', + }; + const aliases = [{ id: 'alias-1', email: 'team@example.com', reply_to: null }]; + const message = { + id: 'message-row-1', + account_id: 'account-1', + uid: 42, + folder: 'INBOX', + message_id: '', + thread_references: '', + subject: 'Topic', + from_name: 'Sender', + from_email: 'sender@example.com', + reply_to: [], + to_addresses: [{ name: 'Team', email: 'team@example.com' }], + cc_addresses: [{ name: 'Colleague', email: 'colleague@example.com' }], + body_text: 'Original', + body_html: '

Original

', + attachments: [], + }; + + it('returns a sendMessage compose input and hard-validates an explicit alias', async () => { + const result = await replyService.buildReply({ + message, + account, + aliases, + replyAll: true, + body: 'Response', + bodyIsHtml: false, + noQuote: true, + alias: 'TEAM@example.com', + }, { + resolveAlias: async (accountId, aliasEmail) => ( + accountId === account.id && aliasEmail === 'TEAM@example.com' + ? aliases[0] + : null + ), + }); + + expect(result).toEqual({ + account, + aliasId: 'alias-1', + userId: 'user-1', + to: ['Sender '], + cc: ['Colleague '], + bcc: [], + subject: 'Re: Topic', + body: 'Response', + bodyIsHtml: false, + quotedBody: '', + quotedBodyHtml: null, + inReplyTo: '', + references: ' ', + }); + }); + + it('throws alias_not_found when an explicit selector does not resolve', async () => { + await expect(replyService.buildReply({ + message, + account, + aliases, + body: 'Response', + noQuote: true, + alias: 'missing@example.com', + }, { + resolveAlias: async () => null, + })).rejects.toMatchObject({ + name: 'AliasNotFoundError', + status: 422, + code: 'alias_not_found', + }); + }); + + it('re-fetches referenced cid parts and returns base64 attachments with matching cids', async () => { + const fetchAttachment = vi.fn(async (_account, _uid, _folder, part) => ( + Buffer.from(`content-${part}`) + )); + const inlineMessage = { + ...message, + body_html: '

Original

', + attachments: [ + { part: '2', cid: 'image-1', filename: 'one.png', type: 'image/png' }, + { part: '3', content_id: '', filename: 'two.jpg', type: 'image/jpeg' }, + ], + }; + + const result = await replyService.buildReply({ + message: inlineMessage, + account, + aliases, + body: 'Response', + includeInlineImages: true, + }, { + imapManager: { fetchAttachment }, + }); + + expect(result.quotedBodyHtml).toContain(''); + expect(result.attachments).toEqual([ + { + filename: 'one.png', + content: Buffer.from('content-2').toString('base64'), + contentType: 'image/png', + cid: 'image-1', + }, + { + filename: 'two.jpg', + content: Buffer.from('content-3').toString('base64'), + contentType: 'image/jpeg', + cid: 'image-2', + }, + ]); + expect(fetchAttachment).toHaveBeenCalledTimes(2); + expect(fetchAttachment).toHaveBeenNthCalledWith(1, account, 42, 'INBOX', '2'); + expect(fetchAttachment).toHaveBeenNthCalledWith(2, account, 42, 'INBOX', '3'); + }); + + it('does not fetch inline parts when quoting is disabled', async () => { + const fetchAttachment = vi.fn(); + const result = await replyService.buildReply({ + message: { + ...message, + body_html: '', + attachments: [{ part: '2', cid: 'image-1' }], + }, + account, + aliases, + body: 'Response', + noQuote: true, + includeInlineImages: true, + }, { + imapManager: { fetchAttachment }, + }); + + expect(fetchAttachment).not.toHaveBeenCalled(); + expect(result).not.toHaveProperty('attachments'); + }); + + it('enforces the 25 MB attachment budget for re-fetched inline images', async () => { + await expect(replyService.buildReply({ + message: { + ...message, + body_html: '', + attachments: [{ part: '2', cid: 'image-1' }], + }, + account, + aliases, + body: 'Response', + includeInlineImages: true, + }, { + imapManager: { + fetchAttachment: async () => Buffer.alloc(26_214_401), + }, + })).rejects.toMatchObject({ + status: 400, + code: 'attachment_too_large', + }); + }); +}); + +describe('buildQuote', () => { + const date = '2026-07-28T12:34:56.000Z'; + const localeDate = new Date(date).toLocaleString(); + + it('builds the frontend-compatible text quote and adds gmail_quote to the HTML wrapper', () => { + expect(replyService.buildQuote({ + date, + from_name: 'Sender\r\nInjected', + from_email: 'sender@example.com', + body_text: 'First line\nSecond line', + body_html: '

Original HTML

', + attachments: [], + }, {})).toEqual({ + quotedBody: `\n\n---\nOn ${localeDate}, Sender Injected wrote:\n> First line\n> Second line`, + quotedBodyHtml: `

On ${localeDate}, Sender Injected wrote:

Original HTML

`, + }); + }); + + it('uses empty quote values when the corresponding source body is absent', () => { + expect(replyService.buildQuote({ + from_name: '', + from_email: 'sender@example.com', + body_text: '', + body_html: '', + }, {})).toEqual({ + quotedBody: '', + quotedBodyHtml: null, + }); + }); + + it('replaces cid image tags with filename placeholders by default', () => { + expect(replyService.buildQuote({ + from_name: 'Sender', + from_email: 'sender@example.com', + body_html: '

Before

photo', + attachments: [ + { part: '2', cid: 'image-1', filename: 'photo.png', type: 'image/png' }, + ], + }, {}).quotedBodyHtml).toContain( + '

Before

[inline image: photo.png][inline image: attachment]', + ); + }); + + it('keeps cid image tags when inline images are requested', () => { + const html = '

Before

'; + expect(replyService.buildQuote({ + from_name: 'Sender', + from_email: 'sender@example.com', + body_html: html, + attachments: [{ part: '2', content_id: '', filename: 'photo.png' }], + }, { includeInlineImages: true }).quotedBodyHtml).toContain(html); + }); +}); + +describe('buildForwardQuote', () => { + it('builds frontend-compatible forwarded headers and body with gmail_quote HTML', () => { + const date = '2026-07-28T12:34:56.000Z'; + const localeDate = new Date(date).toLocaleString(); + const result = replyService.buildForwardQuote({ + date, + from_name: 'Sender\r\nInjected', + from_email: 'sender@example.com', + subject: 'Topic\r\nBcc: hidden@example.com', + to_addresses: JSON.stringify([ + { name: 'One', email: 'one@example.com' }, + { name: '', email: 'two@example.com' }, + ]), + cc_addresses: [{ name: 'Three', email: 'three@example.com' }], + body_text: 'Original text', + body_html: '

Original HTML

', + attachments: [{ cid: 'image-1', filename: 'photo.png' }], + }); + + expect(result).toEqual({ + quotedBody: `\n\n---------- Forwarded message ----------\nFrom: Sender Injected \nDate: ${localeDate}\nSubject: Topic Bcc: hidden@example.com\nTo: One , two@example.com\nCc: Three \n\nOriginal text`, + quotedBodyHtml: `

---------- Forwarded message ----------
From: Sender Injected
Date: ${localeDate}
Subject: Topic Bcc: hidden@example.com
To: One , two@example.com
Cc: Three

Original HTML

[inline image: photo.png]
`, + }); + }); +}); + +describe('buildForward', () => { + const account = { + id: 'account-1', + user_id: 'user-1', + email_address: 'me@example.com', + }; + const aliases = [{ id: 'alias-1', email: 'team@example.com' }]; + const message = { + id: '11111111-1111-4111-8111-111111111111', + uid: 42, + folder: 'INBOX', + subject: 'Topic', + from_name: 'Sender', + from_email: 'sender@example.com', + to_addresses: [{ name: 'Team', email: 'team@example.com' }], + cc_addresses: [], + body_text: 'Original text', + body_html: '

Original HTML

', + attachments: [ + { part: '2', filename: 'deck.pdf', type: 'application/pdf', size: 100 }, + { part: '', filename: 'missing-part.txt', type: 'text/plain', size: 10 }, + ], + }; + + it('returns a sendMessage compose input and forwards stored attachment parts by reference', async () => { + const result = await replyService.buildForward({ + message, + account, + aliases, + to: ['Recipient '], + note: 'Please review.', + }, {}); + + expect(result).toMatchObject({ + account, + aliasId: 'alias-1', + userId: 'user-1', + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Fwd: Topic', + body: 'Please review.', + bodyIsHtml: false, + quotedBody: expect.stringContaining('---------- Forwarded message ----------'), + quotedBodyHtml: expect.stringContaining('class="gmail_quote"'), + forwardedAttachments: [{ + messageId: '11111111-1111-4111-8111-111111111111', + part: '2', + }], + }); + }); + + it('omits forwardedAttachments when skipAttachments is true', async () => { + const result = await replyService.buildForward({ + message, + account, + aliases, + to: ['recipient@example.com'], + skipAttachments: true, + }, {}); + + expect(result).not.toHaveProperty('forwardedAttachments'); + }); + + it('hard-validates an explicit alias selector', async () => { + await expect(replyService.buildForward({ + message, + account, + aliases, + to: ['recipient@example.com'], + alias: 'missing@example.com', + }, { + resolveAlias: async () => null, + })).rejects.toMatchObject({ + status: 422, + code: 'alias_not_found', + }); + }); +}); + +describe('composeFromMessage port parity', () => { + const account = { + id: 'account-1', + email_address: 'me@example.com', + aliases: [{ + id: 'alias-1', + email: 'team@example.com', + reply_to: 'forwarding-self@example.com', + }], + }; + const baseMessage = { + id: 'message-row-1', + account_id: 'account-1', + message_id: '', + in_reply_to: '', + thread_references: '', + subject: 'Topic', + from_name: 'Sender', + from_email: 'sender@example.com', + reply_to: [], + to_addresses: [ + { name: 'Me', email: 'me@example.com' }, + { name: 'Person', email: 'person@example.com' }, + ], + cc_addresses: [{ name: 'Colleague', email: 'colleague@example.com' }], + date: '2026-07-28T12:34:56.000Z', + body_text: 'First line\nSecond line', + body_html: '

Original HTML

', + attachments: [], + }; + + async function runFrontend(message, replyAll = true) { + let compose; + await openReplyFromMessage(message, { + accounts: [account], + replyAll, + openCompose: value => { compose = value; }, + getMessageBody: async () => ({ + text: message.body_text, + html: message.body_html, + attachments: message.attachments, + }), + }); + return compose; + } + + it('matches frontend recipients, subject, text quote, wrapper style, and short references', async () => { + const frontend = await runFrontend(baseMessage); + const recipients = replyService.computeReplyRecipients(baseMessage, { + account, + aliases: account.aliases, + replyAll: true, + }); + const quote = replyService.buildQuote(baseMessage, {}); + const threading = replyService.buildReferences(baseMessage); + + expect(recipients.to).toEqual(frontend.to); + expect(recipients.cc).toEqual(frontend.cc); + expect(replyService.replySubject(baseMessage.subject)).toBe(frontend.subject); + expect(quote.quotedBody).toBe(frontend.quotedBody); + expect(quote.quotedBodyHtml?.replace(' class="gmail_quote"', '')).toBe(frontend.quotedBodyHtml); + expect(quote.quotedBodyHtml).toContain('class="gmail_quote"'); + expect(threading.inReplyTo).toBe(frontend.inReplyTo); + expect(threading.references).toBe(frontend.references); + }); + + it('intentionally differs by deduping reply-all Cc case-insensitively and preferring names', async () => { + const message = { + ...baseMessage, + to_addresses: [{ name: '', email: 'duplicate@example.com' }], + cc_addresses: [{ name: 'Duplicate', email: 'DUPLICATE@example.com' }], + }; + const frontend = await runFrontend(message); + const backend = replyService.computeReplyRecipients(message, { + account, + aliases: account.aliases, + replyAll: true, + }); + + expect(frontend.cc).toEqual([ + { name: '', email: 'duplicate@example.com' }, + { name: 'Duplicate', email: 'DUPLICATE@example.com' }, + ]); + expect(backend.cc).toEqual([ + { name: 'Duplicate', email: 'DUPLICATE@example.com' }, + ]); + expect(backend.cc).not.toEqual(frontend.cc); + }); + + it('intentionally differs by recognizing an uppercase RE prefix', async () => { + const message = { ...baseMessage, subject: 'RE: Topic' }; + const frontend = await runFrontend(message, false); + const backend = replyService.replySubject(message.subject); + + expect(frontend.subject).toBe('Re: RE: Topic'); + expect(backend).toBe('RE: Topic'); + expect(backend).not.toBe(frontend.subject); + }); + + it('intentionally differs by retaining the full thread_references ancestor chain', async () => { + const message = { + ...baseMessage, + in_reply_to: '', + thread_references: ' ', + }; + const frontend = await runFrontend(message, false); + const backend = replyService.buildReferences(message); + + expect(frontend.references).toBe(' '); + expect(backend.references).toBe( + ' ', + ); + expect(backend.references).not.toBe(frontend.references); + }); + + it('intentionally differs by treating alias reply-to addresses as self', async () => { + const message = { + ...baseMessage, + to_addresses: [{ name: 'Me', email: 'me@example.com' }], + cc_addresses: [{ + name: 'Forwarding Self', + email: 'forwarding-self@example.com', + }], + }; + const frontend = await runFrontend(message); + const backend = replyService.computeReplyRecipients(message, { + account, + aliases: account.aliases, + replyAll: true, + }); + + expect(frontend.cc).toEqual([{ + name: 'Forwarding Self', + email: 'forwarding-self@example.com', + }]); + expect(backend.cc).toEqual([]); + expect(backend.cc).not.toEqual(frontend.cc); + }); +}); diff --git a/backend/src/services/search/ftsBackfill.js b/backend/src/services/search/ftsBackfill.js new file mode 100644 index 00000000..bacef25b --- /dev/null +++ b/backend/src/services/search/ftsBackfill.js @@ -0,0 +1,108 @@ +import { query } from '../db.js'; +import { searchFtsExpr, FTS_VERSION } from './lexicalRepo.js'; +import { upsertJob } from '../backgroundJobs.js'; + +const KIND = 'fts_backfill'; +const BATCH = 3000; // 2–5k rows/batch +const IDLE_MS = 5 * 60 * 1000; +const defaultSleep = () => new Promise((r) => setTimeout(r, 250)); + +let running = false; + +// FTS_VERSION is inlined as a literal (not a bind param) so the predicate +// matches the partial index idx_messages_fts_stale_v1 exactly and the batch +// scan is index-served rather than a growing seq scan as the tail drains. +const STALE_PRED = `fts_version IS DISTINCT FROM ${FTS_VERSION}`; + +const batchSql = ` + WITH batch AS ( + SELECT id FROM messages + WHERE ${STALE_PRED} + ORDER BY date DESC NULLS LAST + LIMIT $1 + ) + UPDATE messages m + SET search_fts = ${searchFtsExpr('m')}, + fts_version = ${FTS_VERSION} + FROM batch + WHERE m.id = batch.id +`; + +async function processBatch() { + try { + const res = await query(batchSql, [BATCH]); + return res.rowCount; + } catch (err) { + if (err.code !== '54000') throw err; // program_limit_exceeded + return processBatchRowByRow(); + } +} + +// A single pathological body tripped SQLSTATE 54000 for the whole batch. +// Re-process row by row so one bad row can't wedge the drainer. +async function processBatchRowByRow() { + const { rows } = await query( + `SELECT id FROM messages WHERE ${STALE_PRED} ORDER BY date DESC NULLS LAST LIMIT $1`, + [BATCH] + ); + let done = 0; + for (const { id } of rows) { + try { + await query( + `UPDATE messages m SET search_fts = ${searchFtsExpr('m')}, fts_version = ${FTS_VERSION} WHERE m.id = $1`, + [id] + ); + } catch (err) { + if (err.code !== '54000') throw err; + // Stamp the version so it is never retried; leave search_fts NULL (still + // served by the ILIKE fallback in lexicalRepo). + await query(`UPDATE messages SET fts_version = ${FTS_VERSION} WHERE id = $1 AND search_fts IS NULL`, [id]); + console.warn(`FTS backfill: skipped oversized message ${id} (tsvector too large)`); + } + done++; + } + return done; +} + +export async function runFtsBackfill({ sleep = defaultSleep } = {}) { + if (running) return; // single-flight + running = true; + try { + const { rows: [{ remaining }] } = await query( + `SELECT count(*)::int AS remaining FROM messages WHERE ${STALE_PRED}` + ); + if (remaining === 0) { + await upsertJob({ kind: KIND, state: 'done', processed: 0, total: 0 }); + return; + } + const total = remaining; + let processed = 0; + await upsertJob({ kind: KIND, state: 'running', processed, total }); + + while (true) { + let n; + try { + n = await processBatch(); + } catch (err) { + await upsertJob({ kind: KIND, state: 'error', processed, total, lastError: err.message }); + throw err; + } + if (n === 0) break; + processed += n; + await upsertJob({ kind: KIND, state: 'running', processed: Math.min(processed, total), total }); + await sleep(); + } + await upsertJob({ kind: KIND, state: 'done', processed: total, total }); + } finally { + running = false; + } +} + +export function scheduleFtsBackfill() { + runFtsBackfill().catch((err) => console.error('FTS backfill error:', err.message)); + const timer = setInterval(() => { + runFtsBackfill().catch((err) => console.error('FTS backfill error:', err.message)); + }, IDLE_MS); + timer.unref?.(); + return timer; +} diff --git a/backend/src/services/search/ftsBackfill.test.js b/backend/src/services/search/ftsBackfill.test.js new file mode 100644 index 00000000..3539739a --- /dev/null +++ b/backend/src/services/search/ftsBackfill.test.js @@ -0,0 +1,45 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +// Mock inline + import the mocked bindings (avoids the vi.mock hoisting TDZ trap). +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('../backgroundJobs.js', () => ({ upsertJob: vi.fn() })); +import { query } from '../db.js'; +import { upsertJob } from '../backgroundJobs.js'; +import { runFtsBackfill } from './ftsBackfill.js'; + +const noSleep = () => Promise.resolve(); +beforeEach(() => { query.mockReset(); upsertJob.mockReset(); }); + +describe('runFtsBackfill', () => { + it('marks the job done and does no work when nothing is stale', async () => { + query.mockResolvedValueOnce({ rows: [{ remaining: 0 }] }); // count + await runFtsBackfill({ sleep: noSleep }); + expect(query).toHaveBeenCalledTimes(1); // count only, no batch UPDATE + expect(upsertJob).toHaveBeenCalledWith(expect.objectContaining({ kind: 'fts_backfill', state: 'done' })); + }); + + it('drains in batches until an UPDATE affects zero rows, reporting progress', async () => { + query + .mockResolvedValueOnce({ rows: [{ remaining: 5000 }] }) // count + .mockResolvedValueOnce({ rowCount: 3000 }) // batch 1 + .mockResolvedValueOnce({ rowCount: 2000 }) // batch 2 + .mockResolvedValueOnce({ rowCount: 0 }); // batch 3 → stop + await runFtsBackfill({ sleep: noSleep }); + const batchSql = query.mock.calls[1][0]; + expect(batchSql).toContain('fts_version IS DISTINCT FROM 1'); + expect(batchSql).toContain("setweight(to_tsvector('english', coalesce(m.subject,'')), 'A')"); + expect(upsertJob).toHaveBeenLastCalledWith( + expect.objectContaining({ state: 'done', processed: 5000, total: 5000 }) + ); + }); + + it('is single-flight: a second concurrent call returns immediately', async () => { + let release; + const gate = new Promise((r) => (release = r)); + query.mockImplementationOnce(async () => { await gate; return { rows: [{ remaining: 0 }] }; }); + const first = runFtsBackfill({ sleep: noSleep }); + await runFtsBackfill({ sleep: noSleep }); // returns immediately (guarded) + release(); + await first; + expect(query).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/services/search/lexicalRepo.js b/backend/src/services/search/lexicalRepo.js new file mode 100644 index 00000000..ea10ea99 --- /dev/null +++ b/backend/src/services/search/lexicalRepo.js @@ -0,0 +1,348 @@ +import { shouldExcludeTrashFromSearch } from './queryParser.js'; +import { DELEGATION_SELECT_SQL, delegationJoinSql, mapDelegationRow } from '../gtdDelegations.js'; + +// Postgres refuses to build a tsvector larger than ~1MB of packed lexemes +// (SQLSTATE 54000). Cap the text fed to to_tsvector at 600k chars — matching +// msgvault's maxFTSBodyChars (internal/store/dialect_pg.go). Reused by the +// slice-02 search_fts trigger and backfill via searchFtsExpr(). +export const FTS_BODY_CHAR_CAP = 600000; + +// Single source of truth for the stored-tsvector layout version. The trigger +// and backfill both stamp fts_version = FTS_VERSION; a layout/dictionary +// change bumps this and re-runs the backfill. +export const FTS_VERSION = 1; + +// Wraps a positive condition so that when negated it also matches rows where +// the underlying columns are NULL. +export function negateCond(sql) { + return `NOT COALESCE((${sql}), false)`; +} + +export function trashFolderExclusionCondition() { + return `NOT EXISTS ( + SELECT 1 + FROM folders f + WHERE f.account_id = m.account_id + AND f.path = m.folder + AND (f.special_use = '\\Trash' + OR lower(f.name) LIKE '%trash%' + OR lower(f.name) LIKE '%deleted%') + )`; +} + +// One free-text term matches if it appears in the sender, subject, the stored +// search_vector, or the length-capped body. Extracted so the body cap is a +// single, testable source of truth (the ranked, search_fts-first variant takes +// over once rows are backfilled). +export function freeTextTermCondition(likeIdx, ftsIdx) { + return `( + m.from_name ILIKE $${likeIdx} + OR m.from_email ILIKE $${likeIdx} + OR m.subject ILIKE $${likeIdx} + OR m.search_vector @@ plainto_tsquery('english', $${ftsIdx}) + OR to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP})) @@ plainto_tsquery('english', $${ftsIdx}) + )`; +} + +// BM25-style lexical rank (ts_rank_cd; class weights D,C,B,A → 10:4:1 +// subject:from:rest; length normalization 32). Port of pgvector/fused.go:31-36. +// Exported so the fused query reuses it as the lexical leg. +export const LEXICAL_RANK_SQL = (vectorExpr, queryExpr) => + `ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], ${vectorExpr}, ${queryExpr}, 32)`; + +// A term counts as searchable text only if it has at least one letter or +// digit; msgvault's hasFTSToken drops punctuation-only tokens ("!!!", "***") +// the same way, since they'd normalize to zero lexemes and can't usefully +// match or rank anything. Exported so every caller building a free-text +// query (searchLexical, searchService's semantic branch, vectorStore's fused +// BM25 leg) applies the identical hygiene rather than re-deriving it. +export function hasSearchableToken(term) { + return /[\p{L}\p{N}]/u.test(term); +} + +// A bare free-text word is a single search token; a term containing whitespace +// only arises from a quoted multi-word phrase (queryParser doesn't emit these +// today, but the builders below stay correct if it ever does). Phrases keep +// ordinary non-prefix matching — prefix-expanding a multi-word phrase isn't a +// single lexeme operation and would change its meaning. +function isPhraseTerm(term) { + return /\s/.test(term); +} + +// msgvault's BuildFTSArg prefix-matches every bare word (typing "amaz" still +// finds "amazon") — plainto_tsquery only matches whole stemmed words, which +// regressed recall vs the old ILIKE substring behavior once a row is +// backfilled onto search_fts. quote_literal($N) is a plain SQL bind param +// reference (no JS-side string building), so the raw term travels through as +// an ordinary parameter; quote_literal() at query time safely wraps it as a +// single tsquery lexeme literal — neutralizing any &, |, !, (, ), ', or : +// the term might contain — before ':*' marks it for prefix matching. +// +// Exported (as the raw tsquery expression, not the whole `vectorExpr @@ ...` +// predicate) so `vectorStore.fusedSearch`'s BM25 leg builds its `@@` match +// AND its ts_rank_cd query-arg from this SAME per-term construction, rather +// than forking a second, non-prefix `plainto_tsquery` builder — a review +// caught exactly that fork (msgvault's fused.go reuses BuildFTSTerm for the +// identical reason: one construction, every caller). +export function ftsTermQueryArg(ftsIdx, term) { + return isPhraseTerm(term) + ? `plainto_tsquery('english', $${ftsIdx})` + : `to_tsquery('english', quote_literal($${ftsIdx}) || ':*')`; +} + +function ftsMatchExpr(vectorExpr, ftsIdx, term) { + return `${vectorExpr} @@ ${ftsTermQueryArg(ftsIdx, term)}`; +} + +// Ranked variant used once rows carry search_fts: backfilled rows match via the +// GIN-indexed tsvector; rows not yet backfilled (search_fts IS NULL) fall back +// to the legacy ILIKE/search_vector/body branch so recall never regresses +// mid-backfill. The fallback matches nothing once search_fts is populated, +// letting the planner use the GIN index exclusively. +export function freeTextTermConditionRanked(likeIdx, ftsIdx, term) { + return `( + ${ftsMatchExpr('m.search_fts', ftsIdx, term)} + OR (m.search_fts IS NULL AND ${freeTextTermCondition(likeIdx, ftsIdx)}) + )`; +} + +// English stopwords ("the", "for", "you", …) normalize to an EMPTY tsquery +// (numnode = 0), and `tsvector @@ ` is FALSE for every row — so one +// stopword in an AND'd term chain silently nuked ALL results once rows were +// backfilled onto search_fts ("waiting for invoice" → 0 hits, because of +// "for"; the ILIKE fallback arm is dead once search_fts is populated). +// A term whose tsquery normalizes empty must contribute NOTHING instead: +// this wraps the term's already-polarity-applied condition so it is +// vacuously TRUE — for positive AND negated terms alike — whenever the +// term's tsquery is empty. The numnode() probe runs on the SAME bind (and +// the SAME prefix-or-phrase construction, via ftsTermQueryArg) the match +// uses, so guard and match can never disagree; a query of ONLY stopwords +// degrades to a filter-only, date-ordered search. The relevance rank needs +// no guard: `&&` drops an empty tsquery operand and ts_rank_cd over a fully +// empty tsquery is 0 — both verified against pgvector/pg16 (2026-07-17). +export function stopwordSafeCondition(ftsIdx, term, condition) { + return `(numnode(${ftsTermQueryArg(ftsIdx, term)}) = 0 OR ${condition})`; +} + +// The one owner of a free-text term's FULL metadata-scope predicate — ranked +// FTS match, un-backfilled ILIKE fallback, polarity, stopword vacuity — shared +// by searchLexical, the fused query's NOT-conditions (negatedFreeTextClause), +// and MCP stage_deletion (engineAdapter), so a search preview and a staged +// deletion set can never disagree on what a term matches. `bind(value) → '$n'` +// pushes onto the caller's params; the raw ordinals freeTextTermConditionRanked +// expects are recovered from the placeholders so callers don't juggle them. +export function freeTextTermClause(term, negate, bind) { + const likeIdx = Number(bind(`%${term}%`).slice(1)); + const ftsIdx = Number(bind(term).slice(1)); + const cond = freeTextTermConditionRanked(likeIdx, ftsIdx, term); + return stopwordSafeCondition(ftsIdx, term, negate ? negateCond(cond) : cond); +} + +// Negated free-text NOT-condition for the fused (vector/hybrid) path, built from +// the SAME per-term construction (prefix/phrase match + the un-backfilled ILIKE +// fallback + stopword vacuity) that searchLexical applies, so `invoice -draft` +// excludes drafts identically in every mode. Prefix-vs-phrase semantics follow +// the term, via ftsTermQueryArg, exactly as the positive lexical predicate does. +export function negatedFreeTextClause(term, bind) { + return freeTextTermClause(term, true, bind); +} + +// Body-only free-text match for scope:'body' (MCP search_message_bodies leg): +// matches ONLY the message body, length-capped at the SAME FTS_BODY_CHAR_CAP as +// the stored FTS body cap and the body_text return cap (frozen contract — never +// a second cap, so every FTS body match is locatable in the returned text). +// Query-time (not GIN-served); bodies are sparse and this pool is bounded (D3). +export function bodyTermCondition(ftsIdx, term) { + return ftsMatchExpr(`to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}))`, ftsIdx, term); +} + +// The weighted tsvector written into messages.search_fts. `ref` is the row +// reference: 'm' for the backfill UPDATE, 'NEW' for the BEFORE trigger. The +// class weights map to ts_rank_cd's D,C,B,A array (10:4:1 subject:from:rest). +// MUST stay identical to migration 0041's trigger body. +export function searchFtsExpr(ref) { + return `setweight(to_tsvector('english', coalesce(${ref}.subject,'')), 'A') || + setweight(to_tsvector('english', coalesce(${ref}.from_name,'') || ' ' || coalesce(${ref}.from_email,'')), 'B') || + setweight(to_tsvector('english', coalesce(${ref}.to_addresses::text,'') || ' ' || coalesce(${ref}.cc_addresses::text,'')), 'C') || + setweight(to_tsvector('english', LEFT(coalesce(${ref}.body_text,''), ${FTS_BODY_CHAR_CAP})), 'D')`; +} + +// Structured operator predicates (from/to/cc/subject/has/is/after/before), +// negation-aware. `bind(value) → '$n'` pushes value onto the caller's param +// list and returns its placeholder — the caller owns `params`/the running +// index, so this stays reusable for both searchLexical (one shared params +// array) and the fused query (its own per-leg bind closure, README "one +// search seam" — lexicalRepo remains the single owner of these predicates). +// Excludes free-text terms and folder scope (`in:`), which are not +// structured row predicates. Extracted from searchLexical byte-identically — +// same branches, same ILIKE-arm reuse, same skip rules. +export function buildOperatorClauses(filters, bind) { + const conditions = []; + for (const f of filters) { + if (f.key === 'in') continue; // controls scope, not a row condition + let cond = null; + + if (f.key === 'from') { + const idx = bind(`%${f.value}%`); + cond = `(m.from_email ILIKE ${idx} OR m.from_name ILIKE ${idx})`; + } else if (f.key === 'subject') { + cond = `m.subject ILIKE ${bind(`%${f.value}%`)}`; + } else if (f.key === 'to') { + const idx = bind(`%${f.value}%`); + cond = `(m.to_addresses::text ILIKE ${idx} OR m.cc_addresses::text ILIKE ${idx})`; + } else if (f.key === 'cc') { + cond = `m.cc_addresses::text ILIKE ${bind(`%${f.value}%`)}`; + } else if (f.key === 'has') { + if (f.value === 'attachment' || f.value === 'attachments') cond = `m.has_attachments = true`; + } else if (f.key === 'is') { + if (f.value === 'unread') cond = `m.is_read = false`; + else if (f.value === 'read') cond = `m.is_read = true`; + else if (f.value === 'starred') cond = `m.is_starred = true`; + } else if (f.key === 'after') { + const d = new Date(f.value); + if (!isNaN(d)) cond = `m.date >= ${bind(d.toISOString())}`; + } else if (f.key === 'before') { + const d = new Date(f.value); + if (!isNaN(d)) cond = `m.date < ${bind(d.toISOString())}`; + } + + if (cond) conditions.push(f.negate ? negateCond(cond) : cond); + } + return conditions; +} + +// Folder-scope predicate(s) for the messages table, shared by the lexical path +// and the fused (vector/hybrid) path so every mode scopes to the SAME folder — +// semantic search must not leak Sent/Archive/Trash into an Inbox search, and an +// explicit in:sent must apply. `folderScope`/`folderFuzzy` come from +// resolveSearchFolderScope: a truthy fuzzy scope (in:) matches the bare +// name OR any .../ path; an exact scope (REST ?folder=) matches the full +// path; a null scope carries no explicit folder and excludes trash-like folders +// (the default search scope). `bind(value) → '$n'` — the caller owns params. +export function buildFolderScopeClauses(folderScope, folderFuzzy, bind) { + const conditions = []; + if (folderScope) { + if (folderFuzzy) { + const name = bind(folderScope); + const path = bind(`%/${folderScope}`); + conditions.push(`(m.folder ILIKE ${name} OR m.folder ILIKE ${path})`); + } else { + conditions.push(`m.folder = ${bind(folderScope)}`); + } + } else if (shouldExcludeTrashFromSearch(folderScope)) { + conditions.push(trashFolderExclusionCondition()); + } + return conditions; +} + +export async function searchLexical(client, { parsed, accountIds, folderScope, folderFuzzy, ordering, scope = 'metadata', limit, offset }) { + const { filters, terms } = parsed; + const params = [accountIds]; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + const conditions = buildOperatorClauses(filters, bind); + + for (const term of terms.slice(0, 10)) { + // Single-char terms are too broad/expensive; a punctuation-only term + // (e.g. "!!!") has no letters/digits to search on — msgvault's + // hasFTSToken drops it the same way rather than handing Postgres a + // token that would normalize to zero lexemes. + if (term.value.length < 2 || !hasSearchableToken(term.value)) continue; + if (scope === 'body') { + params.push(term.value); + const ftsIdx = p++; + const cond = bodyTermCondition(ftsIdx, term.value); + conditions.push(stopwordSafeCondition(ftsIdx, term.value, term.negate ? negateCond(cond) : cond)); + } else { + conditions.push(freeTextTermClause(term.value, term.negate, bind)); + } + } + + // A bare in:inbox (or lone folder param) must never dump a whole folder. (No total + // here: MCP search_metadata pre-checks free text, so a no-condition search — a + // filter-only/empty query — only reaches this path via REST, which ignores total.) + if (!conditions.length) return { rows: [], hasCondition: false }; + + for (const cond of buildFolderScopeClauses(folderScope, folderFuzzy, bind)) conditions.push(cond); + + // Snapshot the predicate binds (accountIds + operator/term/folder) BEFORE the + // rank/LIMIT/OFFSET binds are appended, so the metadata COUNT reuses the exact same + // WHERE with the exact same param ordinals (MCP search_metadata needs a real total). + const countParams = params.slice(); + + // D5: rank a free-text search by ts_rank_cd over the combined positive terms + // (date/id tiebreak); a filter-only search stays date-ordered. The rank arg + // must be bound before LIMIT/OFFSET. + let orderBy = 'ORDER BY m.date DESC'; + const positiveTerms = terms.slice(0, 10) + .filter(t => !t.negate && t.value.length >= 2 && hasSearchableToken(t.value)) + .map(t => t.value); + if (ordering === 'relevance' && positiveTerms.length) { + // Rank with the SAME prefix-aware tsquery the MATCH predicate uses + // (freeTextTermConditionRanked → ftsTermQueryArg): one bind per term, + // combined with && (ts_rank_cd takes a single tsquery), mirroring the fused + // query's BM25 leg. Reusing the one term→tsquery-arg builder is what makes + // predicate and rank impossible to diverge — a prefix-only hit ("invo" + // matching "invoice") then ranks by ts_rank_cd instead of getting rank 0 + // (which COALESCE(...,0) would collapse to date order). Phrases keep + // non-prefix matching via ftsTermQueryArg, exactly as the predicate does. + const rankArgs = positiveTerms.map((term) => { params.push(term); return p++; }); + const rankQuery = positiveTerms.map((term, i) => ftsTermQueryArg(rankArgs[i], term)).join(' && '); + const rankExpr = LEXICAL_RANK_SQL('m.search_fts', rankQuery); + // ts_rank_cd(..., NULL::tsvector, ...) returns NULL for un-backfilled rows + // (search_fts IS NULL), and Postgres sorts NULLs FIRST in DESC order — so + // without COALESCE every un-backfilled row would outrank every properly + // ranked hit during the backfill window. COALESCE(...,0) keeps them + // sorting by the date/id tiebreak instead, below any real rank score. + orderBy = `ORDER BY COALESCE(${rankExpr}, 0) DESC, m.date DESC, m.id DESC`; + } + + // scope:'body' returns the body so MCP can compute keyword excerpts without a + // second query. The return cap is FTS_BODY_CHAR_CAP — the SAME constant as the + // FTS body cap (frozen contract: never a second cap), so every FTS body match + // is locatable in the returned text. Metadata scope keeps the historical + // column list (REST response byte-identical). + const bodyCol = scope === 'body' + ? `,\n LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}) AS body_text` + : ''; + + params.push(limit); + params.push(offset); + + const result = await client(` + SELECT + m.id, m.uid, m.folder, m.subject, m.from_name, m.from_email, + m.date, m.snippet, m.is_read, m.is_starred, m.has_attachments, m.account_id, + a.name as account_name, a.email_address as account_email, a.color as account_color, + ${DELEGATION_SELECT_SQL}${bodyCol} + FROM messages m + JOIN email_accounts a ON m.account_id = a.id + ${delegationJoinSql('m', 'a')} + WHERE m.account_id = ANY($1) + AND m.is_deleted = false + AND ${conditions.join('\n AND ')} + ${orderBy} + LIMIT $${p} OFFSET $${p + 1} + `, params); + + // Bounded metadata total: a COUNT(*) over the SAME predicate (no LIMIT/OFFSET), for + // scope:'metadata' only. body/semantic stay total:-1 upstream. + let total; + if (scope === 'metadata') { + const countResult = await client(` + SELECT COUNT(*) AS total + FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.account_id = ANY($1) + AND m.is_deleted = false + AND ${conditions.join('\n AND ')} + `, countParams); + total = Number(countResult.rows[0]?.total ?? 0); // COUNT(*) always returns one row in prod + } + + return { + rows: result.rows.map(row => ({ ...row, delegation: mapDelegationRow(row) })), + hasCondition: true, + ...(total !== undefined ? { total } : {}), + }; +} diff --git a/backend/src/services/search/lexicalRepo.test.js b/backend/src/services/search/lexicalRepo.test.js new file mode 100644 index 00000000..8386b259 --- /dev/null +++ b/backend/src/services/search/lexicalRepo.test.js @@ -0,0 +1,439 @@ +import { describe, it, expect, vi } from 'vitest'; +import { + FTS_BODY_CHAR_CAP, + FTS_VERSION, + negateCond, + trashFolderExclusionCondition, + freeTextTermCondition, + bodyTermCondition, + searchFtsExpr, + searchLexical, + LEXICAL_RANK_SQL, + freeTextTermConditionRanked, + freeTextTermClause, + negatedFreeTextClause, + stopwordSafeCondition, + buildOperatorClauses, + buildFolderScopeClauses, +} from './lexicalRepo.js'; + +describe('SQL fragment builders', () => { + it('pins the body cap at 600000 and FTS version at 1 (msgvault parity)', () => { + expect(FTS_BODY_CHAR_CAP).toBe(600000); + expect(FTS_VERSION).toBe(1); + }); + + it('caps the body tsvector inside the free-text condition', () => { + const cond = freeTextTermCondition(3, 4); + expect(cond).toContain( + `to_tsvector('english', LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP}))` + ); + expect(cond).toContain('m.from_name ILIKE $3'); + expect(cond).toContain("m.search_vector @@ plainto_tsquery('english', $4)"); + expect(cond).not.toContain("to_tsvector('english', coalesce(m.body_text,'')) @@"); + }); + + it('bodyTermCondition prefix-matches a single-word term (msgvault BuildFTSArg parity)', () => { + const cond = bodyTermCondition(4, 'invoice'); + expect(cond).toBe( + "to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@ to_tsquery('english', quote_literal($4) || ':*')" + ); + expect(cond).not.toContain('ILIKE'); + expect(cond).not.toContain('search_vector'); + expect(cond).not.toContain('plainto_tsquery'); + }); + + it('bodyTermCondition keeps non-prefix phrase matching for a quoted multi-word term', () => { + const cond = bodyTermCondition(4, 'weekly report'); + expect(cond).toBe( + "to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@ plainto_tsquery('english', $4)" + ); + expect(cond).not.toContain('quote_literal'); + }); + + it('negateCond wraps a positive condition to also match NULL columns', () => { + expect(negateCond('m.is_read = true')).toBe('NOT COALESCE((m.is_read = true), false)'); + }); + + it('trashFolderExclusionCondition targets trash-like folders', () => { + const sql = trashFolderExclusionCondition(); + expect(sql).toContain('NOT EXISTS'); + expect(sql).toContain('%trash%'); + expect(sql).toContain('%deleted%'); + }); + + it('searchFtsExpr builds a weighted A/B/C/D tsvector for the given row alias', () => { + const expr = searchFtsExpr('m'); + expect(expr).toContain("setweight(to_tsvector('english', coalesce(m.subject,'')), 'A')"); + expect(expr).toContain("coalesce(m.from_name,'') || ' ' || coalesce(m.from_email,'')"); + expect(expr).toContain("LEFT(coalesce(m.body_text,''), 600000)), 'D')"); + // NEW.-form for the trigger must be produced by the same builder. + expect(searchFtsExpr('NEW')).toContain("coalesce(NEW.subject,'')"); + }); +}); + +describe('searchLexical', () => { + function mockClient() { + const calls = []; + const fn = vi.fn(async (text, params) => { calls.push({ text, params }); return { rows: [{ id: 'x' }] }; }); + return { fn, calls }; + } + + it('returns hasCondition:false and never queries when there is no real condition', async () => { + const { fn } = mockClient(); + const res = await searchLexical(fn, { + parsed: { filters: [{ key: 'in', value: 'inbox', negate: false }], terms: [] }, + accountIds: ['a1'], folderScope: 'inbox', folderFuzzy: true, ordering: 'date', limit: 50, offset: 0, + }); + expect(res).toEqual({ rows: [], hasCondition: false }); + expect(fn).not.toHaveBeenCalled(); + }); + + it('emits the pre-existing SQL shape and bind params for a from:+term all-folder query', async () => { + const { fn, calls } = mockClient(); + const res = await searchLexical(fn, { + parsed: { + filters: [{ key: 'from', value: 'amazon', negate: false }], + terms: [{ value: 'invoice', negate: false }], + }, + accountIds: ['a1', 'a2'], folderScope: null, folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + expect(res.hasCondition).toBe(true); + expect(res.rows).toEqual([{ id: 'x', delegation: null }]); + const { text, params } = calls[0]; + // Params: [accountIds, from-like, term-like, term-fts, limit, offset] + expect(params).toEqual([['a1', 'a2'], '%amazon%', '%invoice%', 'invoice', 50, 0]); + // Load-bearing SQL shape (byte-identical gate: the human-runnable old-vs-new diff). + expect(text).toContain('WHERE m.account_id = ANY($1)'); + expect(text).toContain('AND m.is_deleted = false'); + expect(text).toContain('(m.from_email ILIKE $2 OR m.from_name ILIKE $2)'); + expect(text).toContain('NOT EXISTS'); // trash exclusion on all-folder search + expect(text).toContain('ORDER BY m.date DESC'); + expect(text).toContain('LIMIT $5 OFFSET $6'); + }); + + it('adds a cc_addresses predicate for the new cc: operator', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [{ key: 'cc', value: 'boss', negate: false }], terms: [] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + expect(calls[0].text).toContain('m.cc_addresses::text ILIKE $2'); + expect(calls[0].params).toEqual([['a1'], '%boss%', 'INBOX', 50, 0]); + }); + + it('scope:body matches only the capped body branch and returns body_text at the SAME cap', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', scope: 'body', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // Body-only free-text leg (no sender/subject ILIKE, no search_vector), + // prefix-matched (msgvault BuildFTSArg parity — "invoic" still finds "invoice"). + expect(text).toContain("to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@ to_tsquery('english', quote_literal($2) || ':*')"); + expect(text).not.toContain('ILIKE $2'); + // body_text returned at FTS_BODY_CHAR_CAP (600000) — same cap as the FTS body match (frozen contract). + expect(text).toContain("LEFT(coalesce(m.body_text,''), 600000) AS body_text"); + // Body scope pushes only the fts param per term (no %like%): [accountIds, term, folder, limit, offset]. + expect(params).toEqual([['a1'], 'invoice', 'INBOX', 50, 0]); + }); +}); + +describe('ranked lexical query (slice 02)', () => { + function mockClient() { + const calls = []; + const fn = vi.fn(async (text, params) => { calls.push({ text, params }); return { rows: [] }; }); + return { fn, calls }; + } + + it('LEXICAL_RANK_SQL emits ts_rank_cd with the D,C,B,A weight array and normalization 32', () => { + const expr = LEXICAL_RANK_SQL('m.search_fts', "plainto_tsquery('english', $7)"); + expect(expr).toBe("ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, plainto_tsquery('english', $7), 32)"); + }); + + it('freeTextTermConditionRanked prefix-matches a single-word term against search_fts, falling back only when it IS NULL', () => { + const cond = freeTextTermConditionRanked(3, 4, 'invoice'); + expect(cond).toContain("m.search_fts @@ to_tsquery('english', quote_literal($4) || ':*')"); + expect(cond).not.toContain("m.search_fts @@ plainto_tsquery"); + expect(cond).toContain('m.search_fts IS NULL AND'); + // The IS-NULL fallback branch is untouched legacy SQL (freeTextTermCondition): + // still plainto_tsquery + ILIKE, since it only serves un-backfilled rows. + expect(cond).toContain(`LEFT(coalesce(m.body_text,''), ${FTS_BODY_CHAR_CAP})`); + expect(cond).toContain("m.search_vector @@ plainto_tsquery('english', $4)"); + }); + + it('freeTextTermConditionRanked keeps non-prefix phrase matching for a quoted multi-word term', () => { + const cond = freeTextTermConditionRanked(3, 4, 'weekly report'); + expect(cond).toContain("m.search_fts @@ plainto_tsquery('english', $4)"); + expect(cond).not.toContain("m.search_fts @@ to_tsquery"); + }); + + it('prefix-matching passes the raw term through as an ordinary bind param — quoting/escaping happens in SQL via quote_literal, never in JS', async () => { + const { fn, calls } = mockClient(); + const weird = `d'Angelo:*(evil)`; + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: weird, negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // The term is bound VERBATIM (no JS-side escaping) — quote_literal() at + // query time is what neutralizes '/:/* /()/etc. so it can't act as tsquery + // syntax or break out of the quoted lexeme. + expect(params).toContain(weird); + expect(text).toContain("to_tsquery('english', quote_literal($3) || ':*')"); + }); + + it('drops a punctuation-only term (msgvault hasFTSToken parity) instead of emitting an invalid tsquery', async () => { + const { fn } = mockClient(); + const res = await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: '!!!', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + // A punctuation-only term contributes no condition at all — same + // treatment as an under-length term — so a bare "!!!" search never + // dumps the whole folder and never reaches Postgres with a term that + // would normalize to zero lexemes. + expect(res).toEqual({ rows: [], hasCondition: false }); + expect(fn).not.toHaveBeenCalled(); + }); + + it('drops a punctuation-only term while keeping a real term alongside it', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: '***', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + // Only the real term's params were pushed: [accountIds, like, fts, folder, limit, offset]. + expect(calls[0].params).toEqual([['a1'], '%invoice%', 'invoice', 'INBOX', 50, 0]); + }); + + it('orders by ts_rank_cd then date/id for a relevance query, binding the rank args before LIMIT/OFFSET', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: 'urgent', negate: false }] }, + accountIds: ['a1'], folderScope: null, folderFuzzy: false, ordering: 'relevance', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // NULL-safe: ts_rank_cd(..., NULL::tsvector, ...) returns NULL, and Postgres + // sorts NULLs FIRST in DESC order — without COALESCE, every un-backfilled + // (search_fts IS NULL) row would outrank every properly ranked hit during + // the backfill window. COALESCE(..., 0) keeps un-backfilled rows sorting + // by date/id like today, below any real (non-negative) rank score. + expect(text).toContain('ORDER BY COALESCE(ts_rank_cd'); + expect(text).toContain('32), 0) DESC, m.date DESC, m.id DESC'); + // Fix 5: rank reuses the SAME prefix-aware per-term tsquery as the MATCH + // predicate (ftsTermQueryArg), one bind per term combined with && — NOT a + // single plainto_tsquery over the joined string — so a prefix-only hit ranks + // by ts_rank_cd instead of collapsing to COALESCE(0) date order. + expect(text).toContain( + "ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, to_tsquery('english', quote_literal($6) || ':*') && to_tsquery('english', quote_literal($7) || ':*'), 32)" + ); + // Params: [accountIds, t1-like, t1-fts, t2-like, t2-fts, t1-rank, t2-rank, limit, offset] + expect(params).toEqual([['a1'], '%invoice%', 'invoice', '%urgent%', 'urgent', 'invoice', 'urgent', 50, 0]); + }); + + it('ranks a prefix-only term with the SAME builder output as the MATCH predicate (Fix 5 — never rank 0)', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invo', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'relevance', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // The MATCH predicate prefix-matches via ftsTermQueryArg; the rank arg is the + // SAME per-term construction (different bind ordinal only), so predicate and + // rank can never diverge on prefix-vs-plainto. + expect(text).toContain("m.search_fts @@ to_tsquery('english', quote_literal($3) || ':*')"); // predicate + expect(text).toContain( + "ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, to_tsquery('english', quote_literal($5) || ':*'), 32)" // rank + ); + // Params: [accountIds, term-like, term-fts(predicate), folder, term-fts(rank), limit, offset] + expect(params).toEqual([['a1'], '%invo%', 'invo', 'INBOX', 'invo', 50, 0]); + }); + + it('keeps non-prefix phrase matching in the rank for a quoted multi-word positive term (mirrors the predicate)', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'weekly report', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'relevance', limit: 50, offset: 0, + }); + const { text } = calls[0]; + // A phrase term ranks (and matches) via plainto_tsquery — no prefix ':*'. + expect(text).toContain("ts_rank_cd(ARRAY[0.1, 0.1, 0.4, 1.0]::real[], m.search_fts, plainto_tsquery('english', $5), 32)"); + expect(text).not.toContain("quote_literal($5) || ':*'"); + }); + + it('keeps date ordering (no ts_rank_cd) for a filter-only query', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [{ key: 'is', value: 'unread', negate: false }], terms: [] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + expect(calls[0].text).toContain('ORDER BY m.date DESC'); + expect(calls[0].text).not.toContain('ts_rank_cd'); + }); +}); + +describe('stopword-safe free-text predicates (Wave D Fix 1)', () => { + // Root cause (verified against pgvector/pg16, 2026-07-17): + // to_tsquery('english', quote_literal('for') || ':*') normalizes to an + // EMPTY tsquery (numnode = 0), and `tsvector @@ ` is FALSE for + // every row — so once rows were backfilled onto search_fts, one english + // stopword in an AND'd term chain ("waiting for invoice") nuked ALL + // results. The guard makes such a term vacuously TRUE instead. + function mockClient() { + const calls = []; + const fn = vi.fn(async (text, params) => { calls.push({ text, params }); return { rows: [] }; }); + return { fn, calls }; + } + + it('stopwordSafeCondition wraps a clause with a numnode()=0 escape on the SAME bind as the match', () => { + expect(stopwordSafeCondition(4, 'invoice', 'X')).toBe( + "(numnode(to_tsquery('english', quote_literal($4) || ':*')) = 0 OR X)" + ); + // Phrase terms probe emptiness through the SAME plainto construction the + // match uses ("the on" normalizes empty exactly like a stopword word). + expect(stopwordSafeCondition(4, 'weekly report', 'X')).toBe( + "(numnode(plainto_tsquery('english', $4)) = 0 OR X)" + ); + }); + + it('wraps every positive free-text term condition, binding NO extra params', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [ + { value: 'waiting', negate: false }, + { value: 'for', negate: false }, + { value: 'invoice', negate: false }, + ] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + const { text, params } = calls[0]; + // One guard per term, reusing that term's fts bind ordinal ($3, $5, $7). + for (const ftsIdx of [3, 5, 7]) { + expect(text).toContain(`(numnode(to_tsquery('english', quote_literal($${ftsIdx}) || ':*')) = 0 OR (`); + } + // Guard adds no binds: [accountIds, like1, fts1, like2, fts2, like3, fts3, folder, limit, offset] + expect(params).toEqual([['a1'], '%waiting%', 'waiting', '%for%', 'for', '%invoice%', 'invoice', 'INBOX', 50, 0]); + }); + + it('applies the guard OUTSIDE the negation so a negated stopword also contributes nothing', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'the', negate: true }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', limit: 50, offset: 0, + }); + const { text } = calls[0]; + // (empty OR NOT COALESCE(match)) — vacuously TRUE for a stopword in BOTH + // polarities; a plain NOT-wrap of the guarded condition would instead be + // FALSE and exclude everything. + expect(text).toContain("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR NOT COALESCE("); + }); + + it('guards the body-scope term condition with the same construction', async () => { + const { fn, calls } = mockClient(); + await searchLexical(fn, { + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + accountIds: ['a1'], folderScope: 'INBOX', folderFuzzy: false, ordering: 'date', scope: 'body', limit: 50, offset: 0, + }); + expect(calls[0].text).toContain( + "(numnode(to_tsquery('english', quote_literal($2) || ':*')) = 0 OR to_tsvector('english', LEFT(coalesce(m.body_text,''), 600000)) @@" + ); + }); + + it('freeTextTermClause is the one owner searchLexical and the staging path share', () => { + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + const clause = freeTextTermClause('invoice', false, bind); + expect(params).toEqual(['%invoice%', 'invoice']); + expect(clause.startsWith("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR (")).toBe(true); + expect(clause).toContain("m.search_fts @@ to_tsquery('english', quote_literal($3) || ':*')"); + expect(clause).toContain('m.search_fts IS NULL AND'); + }); + + it('negatedFreeTextClause carries the guard outside the NOT (fused NOT-conditions)', () => { + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + const clause = negatedFreeTextClause('the', bind); + expect(clause.startsWith("(numnode(to_tsquery('english', quote_literal($3) || ':*')) = 0 OR NOT COALESCE(")).toBe(true); + }); +}); + +describe('buildOperatorClauses (Phase 4 Task 2a — extracted from searchLexical)', () => { + function bindHarness(start = 2) { + const params = []; + let p = start; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { bind, params }; + } + + it('builds a from: predicate reusing one bind for both ILIKE arms', () => { + const { bind, params } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'from', value: 'amazon', negate: false }], bind); + expect(conds).toEqual(['(m.from_email ILIKE $2 OR m.from_name ILIKE $2)']); + expect(params).toEqual(['%amazon%']); + }); + + it('builds a cc: predicate', () => { + const { bind, params } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'cc', value: 'boss', negate: false }], bind); + expect(conds).toEqual(['m.cc_addresses::text ILIKE $2']); + expect(params).toEqual(['%boss%']); + }); + + it('negates a structured operator via negateCond', () => { + const { bind } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'subject', value: 'invoice', negate: true }], bind); + expect(conds).toEqual(['NOT COALESCE((m.subject ILIKE $2), false)']); + }); + + it('skips in: (scope control, not a row condition) and malformed after/before', () => { + const { bind, params } = bindHarness(); + const conds = buildOperatorClauses([ + { key: 'in', value: 'inbox', negate: false }, + { key: 'after', value: 'not-a-date', negate: false }, + ], bind); + expect(conds).toEqual([]); + expect(params).toEqual([]); + }); + + it('does not bind free-text terms or folder scope (structured operators only)', () => { + const { bind } = bindHarness(); + const conds = buildOperatorClauses([{ key: 'is', value: 'unread', negate: false }], bind); + expect(conds).toEqual(['m.is_read = false']); + }); +}); + +describe('buildFolderScopeClauses (folder scope — shared by lexical + fused)', () => { + function bindHarness(start = 2) { + const params = []; + let p = start; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { bind, params }; + } + + it('fuzzy in: matches the bare name or any .../ path', () => { + const { bind, params } = bindHarness(); + const conds = buildFolderScopeClauses('sent', true, bind); + expect(conds).toEqual(['(m.folder ILIKE $2 OR m.folder ILIKE $3)']); + expect(params).toEqual(['sent', '%/sent']); + }); + + it('an exact folderScope (REST ?folder=) matches the full path', () => { + const { bind, params } = bindHarness(); + const conds = buildFolderScopeClauses('INBOX', false, bind); + expect(conds).toEqual(['m.folder = $2']); + expect(params).toEqual(['INBOX']); + }); + + it('a null folderScope excludes trash-like folders (default search scope), binding nothing', () => { + const { bind, params } = bindHarness(); + const conds = buildFolderScopeClauses(null, false, bind); + expect(conds).toHaveLength(1); + expect(conds[0]).toContain('NOT EXISTS'); + expect(conds[0]).toContain('%trash%'); + expect(params).toEqual([]); + }); +}); diff --git a/backend/src/services/search/lexicalRepo.total.test.js b/backend/src/services/search/lexicalRepo.total.test.js new file mode 100644 index 00000000..065201a8 --- /dev/null +++ b/backend/src/services/search/lexicalRepo.total.test.js @@ -0,0 +1,29 @@ +import { describe, it, expect, vi } from 'vitest'; +vi.mock('../db.js', () => ({ query: vi.fn() })); +import { searchLexical } from './lexicalRepo.js'; + +// NOTE: the real searchLexical takes `client` as a FUNCTION (client(sql, params)), +// not an object with a .query method (the plan's Consumes was written earlier — +// real code wins). We adapt the fake accordingly. +const parsed = { filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }; + +describe('searchLexical metadata total', () => { + it('returns a real total via a bounded COUNT over the same predicate (no LIMIT/OFFSET)', async () => { + const client = vi.fn() + .mockResolvedValueOnce({ rows: [{ id: 'm1' }] }) // page query + .mockResolvedValueOnce({ rows: [{ total: '42' }] }); // count query + const out = await searchLexical(client, { parsed, accountIds: ['acc-1'], scope: 'metadata', limit: 20, offset: 0 }); + expect(out.total).toBe(42); + const countSql = client.mock.calls[1][0]; + expect(countSql).toMatch(/COUNT\(\*\)/i); + expect(countSql).not.toMatch(/\bLIMIT\b/i); + expect(countSql).not.toMatch(/\bOFFSET\b/i); + }); + + it("does not COUNT for scope:'body' (no total)", async () => { + const client = vi.fn().mockResolvedValueOnce({ rows: [] }); // page query only + const out = await searchLexical(client, { parsed, accountIds: ['acc-1'], scope: 'body', limit: 20, offset: 0 }); + expect(out.total).toBeUndefined(); + expect(client.mock.calls.every((c) => !/COUNT\(\*\)/i.test(c[0]))).toBe(true); + }); +}); diff --git a/backend/src/services/search/migrations.test.js b/backend/src/services/search/migrations.test.js new file mode 100644 index 00000000..58b777a1 --- /dev/null +++ b/backend/src/services/search/migrations.test.js @@ -0,0 +1,89 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { searchFtsExpr, FTS_VERSION } from './lexicalRepo.js'; + +const MIGRATIONS = join(dirname(fileURLToPath(import.meta.url)), '../../../migrations'); +const read = (f) => readFileSync(join(MIGRATIONS, f), 'utf8'); +const norm = (s) => s.replace(/\s+/g, ' ').trim(); + +describe('0041_search_fts.sql', () => { + const sql = read('0041_search_fts.sql'); + + it('is transactional (no no-transaction header — the $$ body cannot survive the ; splitter)', () => { + expect(/^--\s*no-transaction/im.test(sql)).toBe(false); + }); + + it('adds nullable search_fts + fts_version with IF NOT EXISTS (fast metadata DDL, D1)', () => { + expect(norm(sql)).toContain('ADD COLUMN IF NOT EXISTS search_fts tsvector'); + expect(norm(sql)).toContain('ADD COLUMN IF NOT EXISTS fts_version int'); + expect(sql).not.toContain('GENERATED ALWAYS AS'); // D1: not a generated column + }); + + it('installs a BEFORE INSERT OR UPDATE trigger whose body equals searchFtsExpr(NEW)', () => { + expect(norm(sql)).toContain('BEFORE INSERT OR UPDATE ON messages'); + expect(norm(sql)).toContain(norm(searchFtsExpr('NEW'))); + expect(norm(sql)).toContain(`NEW.fts_version := ${FTS_VERSION}`); + }); + + it('handles the oversized-tsvector case gracefully (never fails the row write)', () => { + expect(sql).toContain('EXCEPTION WHEN program_limit_exceeded'); + }); + + it('skips recompute only on columns that actually feed searchFtsExpr — snippet is not one of them', () => { + // snippet isn't part of the weighted A/B/C/D expression (subject, from, + // to/cc, body_text), so guarding on it made a snippet-only UPDATE (e.g. + // read/star-adjacent metadata writes that also touch snippet) recompute + // an identical search_fts for nothing. + expect(sql).toContain('NEW.subject IS NOT DISTINCT FROM OLD.subject'); + expect(sql).toContain('NEW.from_name IS NOT DISTINCT FROM OLD.from_name'); + expect(sql).toContain('NEW.from_email IS NOT DISTINCT FROM OLD.from_email'); + expect(sql).toContain('NEW.to_addresses IS NOT DISTINCT FROM OLD.to_addresses'); + expect(sql).toContain('NEW.cc_addresses IS NOT DISTINCT FROM OLD.cc_addresses'); + expect(sql).toContain('NEW.body_text IS NOT DISTINCT FROM OLD.body_text'); + expect(sql).not.toContain('OLD.snippet'); + expect(sql).not.toContain('NEW.snippet'); + }); +}); + +describe('0043_search_fts_index.sql', () => { + const sql = read('0043_search_fts_index.sql'); + + it('runs outside a transaction and builds both indexes CONCURRENTLY', () => { + expect(/^--\s*no-transaction/im.test(sql)).toBe(true); + expect(sql).toContain('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_search_fts'); + expect(sql).toContain('USING GIN (search_fts)'); + expect(sql).toContain('idx_messages_fts_stale_v1'); + expect(sql).toContain(`WHERE fts_version IS DISTINCT FROM ${FTS_VERSION}`); + expect(sql).not.toContain('$$'); // no function bodies — safe for the ; splitter + }); + + // A cancelled/crashed CREATE INDEX CONCURRENTLY leaves an INVALID index under the + // target name; retrying with IF NOT EXISTS silently skips it, so the scan stays + // unindexed forever. Drop-if-exists before each create makes the retry crash-idempotent. + it('drops each index CONCURRENTLY before creating it (invalid-index retry hazard)', () => { + for (const name of ['idx_messages_search_fts', 'idx_messages_fts_stale_v1']) { + expect(sql).toContain(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`); + expect(sql.indexOf(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`)) + .toBeLessThan(sql.indexOf(`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${name}`)); + } + }); +}); + +describe('0045_embed_pending_index.sql', () => { + const sql = read('0045_embed_pending_index.sql'); + + it('runs outside a transaction and builds the partial index CONCURRENTLY', () => { + expect(/^--\s*no-transaction/im.test(sql)).toBe(true); + expect(sql).toContain('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_embed_pending'); + expect(sql).toContain('WHERE embed_gen IS NULL'); + expect(sql).not.toContain('$$'); // no function bodies — safe for the ; splitter + }); + + it('drops the index CONCURRENTLY before creating it (invalid-index retry hazard)', () => { + expect(sql).toContain('DROP INDEX CONCURRENTLY IF EXISTS idx_messages_embed_pending'); + expect(sql.indexOf('DROP INDEX CONCURRENTLY IF EXISTS idx_messages_embed_pending')) + .toBeLessThan(sql.indexOf('CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_messages_embed_pending')); + }); +}); diff --git a/backend/src/services/search/queryParser.js b/backend/src/services/search/queryParser.js new file mode 100644 index 00000000..bdba1e36 --- /dev/null +++ b/backend/src/services/search/queryParser.js @@ -0,0 +1,185 @@ +// Pure query grammar for lexical search. Ports msgvault's operator set +// (internal/search/parser.go); operators Mailflow's schema cannot serve +// (larger:/smaller: — no size column, bcc: — not stored, label:/l: — no +// labels) are RECORDED as unsupported, never silently widened into a match. + +// Multi-char keys precede single-char `l` so the alternation resolves +// `label:` before `l:`. The leading (-?) captures optional negation; \b sits +// between an optional '-' and the key so both `from:` and `-from:` match. +const OP_KEYS = 'from|to|cc|bcc|subject|has|is|after|before|in|older_than|newer_than|larger|smaller|label|l'; +const OP_PATTERN = new RegExp(`(-?)\\b(${OP_KEYS}):("([^"]*)"|([\\S]+))`, 'gi'); + +// Bare quoted phrases (msgvault tokenize parity, parser.go:395-464): a double- +// OR single-quoted span that STARTS a token (preceded by start/whitespace, +// optionally negated with '-') becomes ONE phrase term. A quote glued to the +// tail of a token — from:"John Smith", d'Angelo — never starts a phrase: +// operator values belong to OP_PATTERN and mid-word apostrophes are text. +// Backslash escapes keep a quote char from terminating the span. +const PHRASE_PATTERN = /(^|\s)(-?)(["'])((?:\\.|(?!\3)[^\\])*)\3/g; + +// Port of msgvault's unescapeQuotedValue: `\\` and an escaped quote collapse +// to the bare char; any other `\x` keeps its backslash literally. +function unescapePhrase(s) { + let out = ''; + let escaped = false; + for (const ch of s) { + if (escaped) { + out += (ch === '\\' || ch === '"' || ch === "'") ? ch : '\\' + ch; + escaped = false; + } else if (ch === '\\') { + escaped = true; + } else { + out += ch; + } + } + if (escaped) out += '\\'; + return out; +} + +// Sizes: 5M / 100K / 1G → bytes. Longer suffixes checked first. Returns null +// on anything unparseable (port of parser.go parseSize). +function parseSize(value) { + const v = value.trim().toUpperCase(); + const mult = { KB: 1024, MB: 1048576, GB: 1073741824, K: 1024, M: 1048576, G: 1073741824 }; + for (const suffix of ['KB', 'MB', 'GB', 'K', 'M', 'G']) { + if (v.endsWith(suffix)) { + const num = parseFloat(v.slice(0, -suffix.length)); + if (Number.isNaN(num)) return null; + return Math.floor(num * mult[suffix]); + } + } + return /^\d+$/.test(v) ? parseInt(v, 10) : null; +} + +// Relative ages: 7d / 2w / 1m / 1y → an absolute ISO timestamp relative to +// `now` (port of parser.go parseRelativeDate). Returns null if unparseable. +function relativeAgeToISO(value, now) { + const m = /^(\d+)([dwmy])$/.exec(value.trim().toLowerCase()); + if (!m) return null; + const n = parseInt(m[1], 10); + const d = new Date(now.getTime()); + switch (m[2]) { + case 'd': d.setUTCDate(d.getUTCDate() - n); break; + case 'w': d.setUTCDate(d.getUTCDate() - n * 7); break; + case 'm': d.setUTCMonth(d.getUTCMonth() - n); break; + case 'y': d.setUTCFullYear(d.getUTCFullYear() - n); break; + default: return null; + } + return d.toISOString(); +} + +function applyOperator(key, value, negate, ctx) { + const { filters, unsupported, errors, now } = ctx; + switch (key) { + case 'from': case 'to': case 'cc': + case 'subject': case 'has': case 'is': + case 'in': + if (value) filters.push({ key, value, negate }); + return; + case 'after': + case 'before': { + if (!value) return; + // A bad date must be RECORDED, not silently dropped downstream + // (buildOperatorClauses skips unparseable dates via isNaN, which would + // silently WIDEN the results). Same uniform message parser.go's + // operatorValueError emits for a bad before:/after: value. + if (isNaN(new Date(value))) { + errors.push(`invalid value "${value}" for ${key}: — expected a date like YYYY-MM-DD`); + return; + } + filters.push({ key, value, negate }); + return; + } + case 'newer_than': + case 'older_than': { + const iso = relativeAgeToISO(value, now); + if (!iso) { + errors.push(`invalid value "${value}" for ${key}: — expected a relative age like 7d, 2w, 1m, or 1y`); + return; + } + filters.push({ key: key === 'newer_than' ? 'after' : 'before', value: iso, negate }); + return; + } + case 'larger': + case 'smaller': { + if (parseSize(value) === null) { + errors.push(`invalid value "${value}" for ${key}: — expected a size like 5M, 100K, or 1G`); + return; + } + // Recognized and well-formed, but messages has no size column. + unsupported.push({ key, token: `${key}:${value}` }); + return; + } + case 'bcc': + case 'label': + case 'l': + // No bcc column / no labels concept in Mailflow. + if (value) unsupported.push({ key: key === 'l' ? 'label' : key, token: `${key}:${value}` }); + return; + default: + return; + } +} + +export function parseQuery(raw, { now = new Date() } = {}) { + const filters = []; + const terms = []; + const unsupported = []; + const errors = []; + const ctx = { filters, unsupported, errors, now }; + + // Phase 1: lift bare quoted phrases out BEFORE the operator grammar runs, so + // a colon inside a phrase ("subject:not an operator") can't be parsed as an + // operator. Each phrase leaves a U+E000-delimited placeholder in the string + // (a Private Use Area char) — restored in term order below — so phrases and + // bare words keep their relative positions. U+E000 is stripped from the raw + // input first, so user text can never forge a placeholder. + const phrases = []; + const withPhrases = (raw || '').replace(/\uE000/g, '').replace(PHRASE_PATTERN, (_, pre, neg, _q, body) => { + const value = unescapePhrase(body); + if (!value.trim()) return pre; // empty phrase ("") contributes nothing + phrases.push({ value, negate: neg === '-' }); + return `${pre}\uE000${phrases.length - 1}\uE000`; + }); + + const remaining = withPhrases.replace(OP_PATTERN, (_, neg, key, _v, quoted, unquoted) => { + const k = key.toLowerCase(); + const value = (quoted !== undefined ? quoted : (unquoted || '')).toLowerCase().trim(); + applyOperator(k, value, neg === '-', ctx); + return ' '; + }).trim(); + + for (const word of remaining.split(/\s+/)) { + let w = word.trim(); + if (!w || w === '-') continue; // skip blanks and a lone '-' (nothing to negate) + const ph = /^\uE000(\d+)\uE000$/.exec(w); + if (ph) { terms.push(phrases[Number(ph[1])]); continue; } + let negate = false; + if (w[0] === '-' && w.length > 1) { negate = true; w = w.slice(1); } + terms.push({ value: w, negate }); + } + + return { filters, terms, unsupported, errors }; +} + +export function resolveSearchFolderScope(filters, folderParam = '') { + let folderScope; + let folderFuzzy = false; // in: matches loosely; the folder param is exact + + for (const f of filters) { + if (f.key !== 'in') continue; + if (f.value === 'all') { folderScope = null; } + else { folderScope = f.value; folderFuzzy = true; } + } + + if (folderScope === undefined) { + folderScope = (folderParam || '').trim() || null; + folderFuzzy = false; + } + + return { folderScope, folderFuzzy }; +} + +export function shouldExcludeTrashFromSearch(folderScope) { + return folderScope === null; +} diff --git a/backend/src/services/search/queryParser.test.js b/backend/src/services/search/queryParser.test.js new file mode 100644 index 00000000..58485aa4 --- /dev/null +++ b/backend/src/services/search/queryParser.test.js @@ -0,0 +1,225 @@ +import { describe, it, expect } from 'vitest'; +import { + parseQuery, + resolveSearchFolderScope, + shouldExcludeTrashFromSearch, +} from './queryParser.js'; + +const NOW = new Date('2026-07-16T00:00:00.000Z'); + +describe('parseQuery — preserved grammar', () => { + it('treats bare words as free-text terms', () => { + const { filters, terms } = parseQuery('hello world'); + expect(filters).toEqual([]); + expect(terms).toEqual([ + { value: 'hello', negate: false }, + { value: 'world', negate: false }, + ]); + }); + + it('extracts positive operators and lowercases their values', () => { + const { filters, terms } = parseQuery('from:Amazon subject:Invoice hello'); + expect(filters).toEqual([ + { key: 'from', value: 'amazon', negate: false }, + { key: 'subject', value: 'invoice', negate: false }, + ]); + expect(terms).toEqual([{ value: 'hello', negate: false }]); + }); + + it('supports quoted operator values with spaces', () => { + const { filters } = parseQuery('from:"John Smith" report'); + expect(filters).toEqual([{ key: 'from', value: 'john smith', negate: false }]); + }); + + it('negates an operator when prefixed with -', () => { + const { filters } = parseQuery('-from:Smith report'); + expect(filters).toEqual([{ key: 'from', value: 'smith', negate: true }]); + }); + + it('negates a free-text term when prefixed with -', () => { + const { terms } = parseQuery('report -invoice'); + expect(terms).toEqual([ + { value: 'report', negate: false }, + { value: 'invoice', negate: true }, + ]); + }); + + it('preserves repeated and mixed positive/negative operators', () => { + const { filters } = parseQuery('from:alice -from:bob is:unread'); + expect(filters).toEqual([ + { key: 'from', value: 'alice', negate: false }, + { key: 'from', value: 'bob', negate: true }, + { key: 'is', value: 'unread', negate: false }, + ]); + }); + + it('ignores a lone - so it is not treated as a negated empty term', () => { + const { terms } = parseQuery('report - draft'); + expect(terms).toEqual([ + { value: 'report', negate: false }, + { value: 'draft', negate: false }, + ]); + }); + + it('returns empty structures for blank input', () => { + expect(parseQuery('')).toEqual({ filters: [], terms: [], unsupported: [], errors: [] }); + expect(parseQuery(' ')).toEqual({ filters: [], terms: [], unsupported: [], errors: [] }); + }); + + it('parses in:all and named folders', () => { + expect(parseQuery('in:all invoice').filters).toEqual([ + { key: 'in', value: 'all', negate: false }, + ]); + expect(parseQuery('in:Sent proposal').filters).toEqual([ + { key: 'in', value: 'sent', negate: false }, + ]); + }); +}); + +describe('parseQuery — new msgvault operators', () => { + it('adds cc: as a real filter', () => { + const { filters } = parseQuery('cc:boss@corp.com report'); + expect(filters).toEqual([{ key: 'cc', value: 'boss@corp.com', negate: false }]); + }); + + it('maps newer_than: to an after: date and older_than: to a before: date', () => { + const newer = parseQuery('newer_than:2w', { now: NOW }).filters; + expect(newer[0].key).toBe('after'); + expect(newer[0].value.startsWith('2026-07-02')).toBe(true); + + const older = parseQuery('older_than:7d', { now: NOW }).filters; + expect(older[0].key).toBe('before'); + expect(older[0].value.startsWith('2026-07-09')).toBe(true); + }); + + it('records larger:/smaller:/bcc:/label:/l: as recognized-but-unsupported (never widened)', () => { + const { filters, unsupported } = parseQuery('larger:5M bcc:x@y.com label:work l:home smaller:100K'); + expect(filters).toEqual([]); // none of these silently become predicates + expect(unsupported).toEqual([ + { key: 'larger', token: 'larger:5m' }, + { key: 'bcc', token: 'bcc:x@y.com' }, + { key: 'label', token: 'label:work' }, + { key: 'label', token: 'l:home' }, + { key: 'smaller', token: 'smaller:100k' }, + ]); + }); + + it('records a malformed typed value as an error, not a filter', () => { + const { errors, unsupported } = parseQuery('larger:5X older_than:3q'); + expect(unsupported).toEqual([]); + expect(errors).toHaveLength(2); + expect(errors[0]).toContain('larger'); + expect(errors[1]).toContain('older_than'); + }); +}); + +describe('parseQuery — bare quoted phrases (Wave D Fix 4, msgvault tokenize parity)', () => { + it('treats a double-quoted phrase as ONE term without the quote chars (port "quoted phrase")', () => { + expect(parseQuery('"hello world"').terms).toEqual([{ value: 'hello world', negate: false }]); + }); + + it('mixes phrases with operators and bare words in order (port "mixed operators and text")', () => { + const { filters, terms } = parseQuery('from:alice@example.com "meeting notes" urgent'); + expect(filters).toEqual([{ key: 'from', value: 'alice@example.com', negate: false }]); + expect(terms).toEqual([ + { value: 'meeting notes', negate: false }, + { value: 'urgent', negate: false }, + ]); + }); + + it('keeps colons inside a phrase from becoming operators (port QuotedPhrasesWithColons)', () => { + expect(parseQuery('"foo:bar"').terms).toEqual([{ value: 'foo:bar', negate: false }]); + expect(parseQuery('"meeting at 10:30"').terms).toEqual([{ value: 'meeting at 10:30', negate: false }]); + expect(parseQuery('"check http://example.com"').terms).toEqual([{ value: 'check http://example.com', negate: false }]); + expect(parseQuery('"a:b:c:d"').terms).toEqual([{ value: 'a:b:c:d', negate: false }]); + }); + + it('parses a colon phrase alongside a real operator (port "quoted colon phrase mixed with real operator")', () => { + const { filters, terms } = parseQuery('from:alice@example.com "subject:not an operator"'); + expect(filters).toEqual([{ key: 'from', value: 'alice@example.com', negate: false }]); + expect(terms).toEqual([{ value: 'subject:not an operator', negate: false }]); + }); + + it('parses a leading phrase before an operator (port "operator followed by quoted colon phrase")', () => { + const { filters, terms } = parseQuery('"re: meeting notes" from:bob@example.com'); + expect(filters).toEqual([{ key: 'from', value: 'bob@example.com', negate: false }]); + expect(terms).toEqual([{ value: 're: meeting notes', negate: false }]); + }); + + it('accepts single-quoted phrases (msgvault tokenize takes both quote chars)', () => { + expect(parseQuery("'hello world'").terms).toEqual([{ value: 'hello world', negate: false }]); + }); + + it('never starts a phrase at an apostrophe INSIDE a word', () => { + expect(parseQuery("d'Angelo report").terms).toEqual([ + { value: "d'Angelo", negate: false }, + { value: 'report', negate: false }, + ]); + }); + + it('unescapes backslash-escaped quotes inside a phrase (msgvault unescapeQuotedValue)', () => { + expect(parseQuery('"say \\"hi\\" now"').terms).toEqual([{ value: 'say "hi" now', negate: false }]); + expect(parseQuery('"a\\\\b"').terms).toEqual([{ value: 'a\\b', negate: false }]); + }); + + it('negates a phrase with a leading -', () => { + expect(parseQuery('report -"weekly digest"').terms).toEqual([ + { value: 'report', negate: false }, + { value: 'weekly digest', negate: true }, + ]); + }); + + it('drops an empty phrase and leaves op:"value" quoting to the operator grammar', () => { + expect(parseQuery('""').terms).toEqual([]); + expect(parseQuery('from:"John Smith"').filters).toEqual([{ key: 'from', value: 'john smith', negate: false }]); + expect(parseQuery('from:"John Smith"').terms).toEqual([]); + }); +}); + +describe('parseQuery — before:/after: validation (Wave D Fix 5)', () => { + it('records an invalid before:/after: value as an error, never a filter (no silent widening)', () => { + const { filters, errors } = parseQuery('before:notadate after:2025-99-99 invoice'); + expect(filters).toEqual([]); + expect(errors).toEqual([ + 'invalid value "notadate" for before: — expected a date like YYYY-MM-DD', + 'invalid value "2025-99-99" for after: — expected a date like YYYY-MM-DD', + ]); + }); + + it('keeps valid dates as filters (msgvault parseDate accepts several forms)', () => { + const { filters, errors } = parseQuery('after:2025-01-02 before:2025/03/04'); + expect(errors).toEqual([]); + expect(filters.map((f) => f.key)).toEqual(['after', 'before']); + }); +}); + +describe('resolveSearchFolderScope / shouldExcludeTrashFromSearch', () => { + it('scopes to the client folder param when no in: operator is present', () => { + const { filters } = parseQuery('subject:newsletter'); + expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ + folderScope: 'INBOX', + folderFuzzy: false, + }); + }); + + it('lets in: override the client folder param', () => { + const { filters } = parseQuery('in:trash subject:newsletter'); + expect(resolveSearchFolderScope(filters, 'INBOX')).toEqual({ + folderScope: 'trash', + folderFuzzy: true, + }); + }); + + it('flags all-folder searches for trash exclusion', () => { + const { filters } = parseQuery('subject:newsletter'); + const { folderScope } = resolveSearchFolderScope(filters); + expect(folderScope).toBeNull(); + expect(shouldExcludeTrashFromSearch(folderScope)).toBe(true); + }); + + it('keeps explicit folder searches eligible to find trash', () => { + const { filters } = parseQuery('in:trash subject:newsletter'); + const { folderScope } = resolveSearchFolderScope(filters); + expect(shouldExcludeTrashFromSearch(folderScope)).toBe(false); + }); +}); diff --git a/backend/src/services/search/searchService.js b/backend/src/services/search/searchService.js new file mode 100644 index 00000000..847b67fa --- /dev/null +++ b/backend/src/services/search/searchService.js @@ -0,0 +1,172 @@ +import { query } from '../db.js'; +import { resolveAccountScope } from '../unifiedInbox.js'; +import { searchLexical, buildOperatorClauses, hasSearchableToken, buildFolderScopeClauses, negatedFreeTextClause } from './lexicalRepo.js'; +import { resolveSearchFolderScope } from './queryParser.js'; +import { hybridSearch, isLexicalFallback, MissingFreeTextError } from '../embeddings/hybrid.js'; + +function clampLimit(limit) { + return Math.max(1, Math.min(parseInt(limit) || 50, 200)); +} + +// MCP-facing envelope helpers, applied only when the caller opts in +// (REST never sets `explain`/`scope:'body'`, so its response is unaffected). +function withExplainScores(hits) { + for (const h of hits) { + h.score = { + rrf: h.rrf_score, + ...(h.bm25_score != null ? { bm25: h.bm25_score } : {}), + ...(h.vector_score != null ? { vector: h.vector_score } : {}), + subject_boosted: !!h.subject_boosted, + }; + } +} +function attachBestChunk(hits) { + for (const h of hits) { + h.best_chunk = h.best_char_start == null ? null + : { chunk_index: h.best_chunk_index, char_start: h.best_char_start, char_end: h.best_char_end, score: h.vector_score }; + } +} + +// The vector/hybrid SQL projects the message id as `message_id` +// (vectorStore.fusedSearch DISPLAY_COLS), but every hit consumer — REST +// serialization and MCP hydration alike — keys on `id`. Additively alias `id` +// onto each vector/hybrid hit so the shape is mode-invariant with lexical, leaving +// `message_id` untouched (the REST response is a superset — additive, never +// renamed). searchService.search() is the one seam both modes flow through, so +// this is the single place the alias is applied. +function aliasIdFromMessageId(hits) { + for (const h of hits) h.id = h.message_id; +} + +// A caller that already resolved its scope (e.g. the MCP handler, from the +// bearer-token owner's enabled accounts) passes accountIds directly and it is +// trusted as-is; otherwise derive it from the session userId. The REST route +// never forwards a raw accountIds, so a browser client cannot widen its scope. +// Shared by both the lexical and semantic branches. +async function resolveAccountIds(request) { + const { userId, accountIds: providedScope, accountId } = request; + if (providedScope) { + if (!providedScope.length) return []; + return accountId && providedScope.includes(accountId) ? [accountId] : providedScope; + } + const accountsResult = await query( + 'SELECT id, include_in_unified_inbox FROM email_accounts WHERE user_id = $1 AND enabled = true', + [userId] + ); + return resolveAccountScope(accountsResult.rows, accountId).accountIds; +} + +// Phase 1's original lexical search, extracted verbatim so the mode dispatch +// below can reuse it both as the default path and as the fallback target when +// a semantic search degrades. Returns the pre-Phase-4 shape (no `mode` field — +// the caller adds that uniformly). +async function runLexical(request) { + const { parsed, folderParam = '', limit = 50, offset = 0, scope } = request; + + // Frozen contract: scope ∈ 'metadata' | 'body' (default metadata). Anything + // else coerces to metadata (today's behavior); phase 5's body tool passes 'body'. + const searchScope = scope === 'body' ? 'body' : 'metadata'; + + const cap = clampLimit(limit); + const off = Math.max(0, parseInt(offset) || 0); + const emptyPage = { offset: off, limit: cap, hasMore: false }; + + const accountIds = await resolveAccountIds(request); + if (!accountIds.length) return { messages: [], page: emptyPage }; + + const { folderScope, folderFuzzy } = resolveSearchFolderScope(parsed.filters, folderParam); + + // D5: a free-text search (≥1 positive, non-trivial term) ranks by relevance; + // a filter-only search stays date-ordered. + const hasPositiveText = parsed.terms.some(t => !t.negate && t.value.length >= 2); + const ordering = hasPositiveText ? 'relevance' : 'date'; + + const { rows, total, hasCondition } = await searchLexical(query, { + parsed, accountIds, folderScope, folderFuzzy, ordering, scope: searchScope, limit: cap, offset: off, + }); + if (!hasCondition) return { messages: [], page: emptyPage }; + + return { + messages: rows, + ...(total !== undefined ? { total } : {}), + page: { offset: off, limit: cap, hasMore: rows.length === cap }, + }; +} + +// The single search entry point shared by REST and MCP. Owns account scoping, +// mode dispatch (lexical/vector/hybrid), the D5 ordering decision (lexical), +// and result shaping. No HTTP framing here. +export async function search(request) { + const mode = request.mode === 'vector' || request.mode === 'hybrid' ? request.mode : 'lexical'; + if (mode === 'lexical') { + return { ...(await runLexical(request)), mode: 'lexical' }; + } + + const limit = clampLimit(request.limit); + const offset = Math.max(0, parseInt(request.offset) || 0); + // Resolve once, up front, and thread it into a lexical fallback below — a + // fallback must not re-resolve accounts via a second DB round-trip. + const accountIds = await resolveAccountIds(request); + const scopedRequest = { ...request, accountIds }; + try { + if (accountIds.length === 0) { + return { messages: [], mode, pool_saturated: false, generation: null, + page: { offset, limit, hasMore: false } }; + } + const { parsed } = request; // already parsed by the caller — never re-parse + const { folderScope, folderFuzzy } = resolveSearchFolderScope(parsed.filters, request.folderParam || ''); + // One buildFilters owner threads the SAME predicates the lexical path applies + // into BOTH fused legs (fusedSearch applies it to the FTS pool and the ANN + // EXISTS alike, on the joined messages table): structured operators, the + // folder scope (so semantic search can't leak Sent/Archive/Trash into an + // Inbox search and an explicit in:sent applies), and negated free-text terms + // as NOT-conditions (so `invoice -draft` excludes drafts in semantic mode + // just as FTS exclusion does in lexical mode). Same term hygiene throughout: + // drop sub-2-char / punctuation-only tokens. + const buildFilters = (bind) => [ + ...buildOperatorClauses(parsed.filters, bind), + ...buildFolderScopeClauses(folderScope, folderFuzzy, bind), + ...parsed.terms + .filter(t => t.negate && t.value.length >= 2 && hasSearchableToken(t.value)) + .map(t => negatedFreeTextClause(t.value, bind)), + ]; + // Only non-negated terms drive the embedding + BM25 leg; negated terms are + // enforced via buildFilters above, never fed to the embedder. + const freeText = parsed.terms + .filter(t => !t.negate && t.value.length >= 2 && hasSearchableToken(t.value)) + .map(t => t.value).join(' '); + + // Ranked pools are bounded (D3); fetch one past the page window so a full + // page can observe whether more hits exist, then slice the page. + const window = offset + limit; + const { hits, poolSaturated, generation } = await hybridSearch({ + mode, freeText, accountIds, buildFilters, limit: window + 1, + }); + const page = hits.slice(offset, offset + limit); + aliasIdFromMessageId(page); // mode-invariant `id` alias (seam contract) + if (request.explain) withExplainScores(page); + if (request.scope === 'body') attachBestChunk(page); + return { + messages: page, + mode, + pool_saturated: poolSaturated, + generation: generation || null, // rich {id,model,dimension,fingerprint,state} object + page: { offset, limit, hasMore: hits.length > window }, + }; + } catch (err) { + if (isLexicalFallback(err)) { + if (request.strictVector) { + if (err instanceof MissingFreeTextError) throw err; // invalid input, not unavailability + throw err; // VectorUnavailableError already carries .reason + } + const lexical = { ...(await runLexical(scopedRequest)), mode: 'lexical' }; + // A filter-only query in semantic mode (MissingFreeTextError) is not a + // degradation — there is nothing to embed and the lexical result IS the + // answer, so no fellBack (the UI keys its amber "index building" hint on + // it). Real unavailability (unconfigured/building/stale/ + // embedding_timeout) keeps the flag. + return err instanceof MissingFreeTextError ? lexical : { ...lexical, fellBack: true }; + } + throw err; + } +} diff --git a/backend/src/services/search/searchService.test.js b/backend/src/services/search/searchService.test.js new file mode 100644 index 00000000..eaacb0c8 --- /dev/null +++ b/backend/src/services/search/searchService.test.js @@ -0,0 +1,364 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Mock inline and import the mocked bindings — referencing an outer `const fn` +// from a vi.mock factory hits Vitest's hoisting temporal-dead-zone error. +vi.mock('../db.js', () => ({ query: vi.fn() })); +vi.mock('./lexicalRepo.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, searchLexical: vi.fn() }; +}); +vi.mock('../embeddings/hybrid.js', () => ({ + hybridSearch: vi.fn(), + isLexicalFallback: vi.fn(() => true), + MissingFreeTextError: class extends Error {}, +})); + +import { query } from '../db.js'; +import { searchLexical } from './lexicalRepo.js'; +import * as hybrid from '../embeddings/hybrid.js'; +import { search } from './searchService.js'; +import { VectorUnavailableError } from '../embeddings/vectorErrors.js'; + +beforeEach(() => { + query.mockReset(); searchLexical.mockReset(); + hybrid.hybridSearch.mockReset(); + hybrid.isLexicalFallback.mockReset(); + hybrid.isLexicalFallback.mockReturnValue(true); +}); + +function withAccounts(ids) { + query.mockResolvedValueOnce({ rows: ids.map(id => ({ id })) }); +} + +describe('searchService.search', () => { + it('returns an empty shaped result when the user has no enabled accounts', async () => { + withAccounts([]); + const res = await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'hi', negate: false }] } }); + expect(res).toEqual({ messages: [], mode: 'lexical', page: { offset: 0, limit: 50, hasMore: false } }); + expect(searchLexical).not.toHaveBeenCalled(); + }); + + it('scopes to a single account only when it belongs to the user', async () => { + withAccounts(['a1', 'a2']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', accountId: 'a2', parsed: { filters: [], terms: [{ value: 'hi', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].accountIds).toEqual(['a2']); + }); + + it('trusts a caller-provided accountIds scope and skips the DB account lookup (MCP path)', async () => { + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ accountIds: ['a1', 'a2'], parsed: { filters: [], terms: [{ value: 'hi', negate: false }] } }); + expect(query).not.toHaveBeenCalled(); // no user_id → email_accounts lookup + expect(searchLexical.mock.calls[0][1].accountIds).toEqual(['a1', 'a2']); + }); + + it('passes through an optional total when searchLexical returns one', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [{ id: 'm' }], total: 137, hasCondition: true }); + const res = await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(res.total).toBe(137); + }); + + it('passes scope through to searchLexical, defaulting to metadata and coercing unknown values', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].scope).toBe('metadata'); + + query.mockReset(); searchLexical.mockReset(); + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', scope: 'body', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].scope).toBe('body'); + + query.mockReset(); searchLexical.mockReset(); + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', scope: 'nonsense', parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].scope).toBe('metadata'); + }); + + it('chooses relevance ordering for free text and date ordering for filter-only queries', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] } }); + expect(searchLexical.mock.calls[0][1].ordering).toBe('relevance'); + + query.mockReset(); searchLexical.mockReset(); + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + await search({ userId: 'u1', parsed: { filters: [{ key: 'is', value: 'unread', negate: false }], terms: [] } }); + expect(searchLexical.mock.calls[0][1].ordering).toBe('date'); + }); + + it('shapes rows into messages + page, clamping the limit to 200 and flagging hasMore on a full page', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: new Array(200).fill({ id: 'm' }), hasCondition: true }); + const res = await search({ userId: 'u1', limit: 9999, offset: 40, parsed: { filters: [], terms: [{ value: 'x', negate: false }] } }); + expect(res.messages).toHaveLength(200); + expect(res.mode).toBe('lexical'); + expect(res.page).toEqual({ offset: 40, limit: 200, hasMore: true }); + expect(searchLexical.mock.calls[0][1].limit).toBe(200); + }); + + it('returns empty when there is no real search condition', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: false }); + const res = await search({ userId: 'u1', parsed: { filters: [{ key: 'in', value: 'inbox', negate: false }], terms: [] } }); + expect(res.messages).toEqual([]); + expect(res.page.hasMore).toBe(false); + }); +}); + +describe('searchService mode dispatch (Phase 4 Task 5)', () => { + const hybridReq = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'quarterly', negate: false }, { value: 'revenue', negate: false }] }, + ...extra, + }); + + it('defaults to lexical and marks the mode', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [], hasCondition: true }); + const res = await search({ userId: 'u1', parsed: { filters: [], terms: [{ value: 'hello', negate: false }] } }); + expect(res.mode).toBe('lexical'); + expect(res.fellBack).toBeUndefined(); + expect(hybrid.hybridSearch).not.toHaveBeenCalled(); + }); + + it('hybrid success returns score-ordered messages with pool_saturated + the rich generation object, no total', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 0.04, subject: 's' }], + poolSaturated: true, generation: { id: 7 }, + }); + const res = await search(hybridReq()); + expect(res.mode).toBe('hybrid'); + expect(res.pool_saturated).toBe(true); + // Task 5b supersedes Task 5's bare generation.id with the rich object + // fusedSearch/hybridSearch echoes back. + expect(res.generation).toEqual({ id: 7 }); + expect(res.total).toBeUndefined(); + expect(res.messages[0].message_id).toBe('a'); + }); + + it('additively aliases id onto message_id for BOTH vector and hybrid hits (mode-invariant seam contract), leaving message_id intact', async () => { + // The vector/hybrid SQL (fusedSearch DISPLAY_COLS) projects the message id + // as `message_id`, NOT `id` — the real seam shape the live MCP round-trip + // exposed. The lexical path and every hit consumer (REST serialization, MCP + // hydration via getMessageSummariesByIDs) key on `id`, so the seam must add + // it for both modes. Regression guard for `semantic_search_messages` + // returning 0. + for (const mode of ['vector', 'hybrid']) { + query.mockReset(); hybrid.hybridSearch.mockReset(); + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'real-uuid', uid: 7, folder: 'INBOX', subject: 'Run failed: CD deploy', rrf_score: 0.04 }], + poolSaturated: false, generation: { id: 1 }, + }); + const res = await search({ + userId: 'u1', mode, limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'deploy', negate: false }] }, + }); + expect(res.messages[0].message_id).toBe('real-uuid'); // unchanged — REST reads it + expect(res.messages[0].id).toBe('real-uuid'); // additive alias — MCP hydration + lexical parity + } + }); + + it('drops sub-2-char and punctuation-only terms from the semantic freeText, matching the lexical path\'s hygiene (review MINOR 3)', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ hits: [], poolSaturated: false, generation: null }); + await search(hybridReq({ + parsed: { + filters: [], + terms: [ + { value: 'quarterly', negate: false }, + { value: 'a', negate: false }, // 1-char — dropped + { value: '!!!', negate: false }, // punctuation-only — dropped + { value: 'revenue', negate: false }, + ], + }, + })); + expect(hybrid.hybridSearch.mock.calls[0][0].freeText).toBe('quarterly revenue'); + }); + + it('falls back to lexical (fellBack:true) when hybrid degrades', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [{ id: 'lex1', subject: 'kw' }], hasCondition: true }); + hybrid.hybridSearch.mockRejectedValue(Object.assign(new Error('building'), { code: 'INDEX_BUILDING' })); + const res = await search(hybridReq()); + expect(res.mode).toBe('lexical'); + expect(res.fellBack).toBe(true); + expect(res.messages[0].id).toBe('lex1'); + }); + + it('serves a filter-only semantic query lexically WITHOUT fellBack — not an "index building" event (Wave D Fix 6)', async () => { + withAccounts(['a1']); + searchLexical.mockResolvedValue({ rows: [{ id: 'lex1' }], hasCondition: true }); + hybrid.hybridSearch.mockRejectedValue(new hybrid.MissingFreeTextError()); + const res = await search(hybridReq({ parsed: { filters: [{ key: 'is', value: 'unread', negate: false }], terms: [] } })); + expect(res.mode).toBe('lexical'); + expect(res.fellBack).toBeUndefined(); // the UI keys an amber degradation hint on fellBack + expect(res.messages[0].id).toBe('lex1'); + }); + + it('propagates an unexpected (non-degradation) error instead of falling back', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockRejectedValue(new Error('boom')); + hybrid.isLexicalFallback.mockReturnValue(false); + await expect(search(hybridReq())).rejects.toThrow('boom'); + }); +}); + +describe('searchService fused folder scope + negation (Fix 2 + Fix 4)', () => { + const semReq = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + ...extra, + }); + // Run the buildFilters closure the semantic branch hands hybridSearch, so we + // can inspect the SQL predicates + params it applies to BOTH fused legs. + function runBuildFilters() { + const buildFilters = hybrid.hybridSearch.mock.calls[0][0].buildFilters; + const params = []; + let p = 2; + const bind = (v) => { params.push(v); return `$${p++}`; }; + return { clauses: buildFilters(bind), params }; + } + + beforeEach(() => { + hybrid.hybridSearch.mockResolvedValue({ hits: [], poolSaturated: false, generation: null }); + }); + + it('scopes the fused query to an in: for BOTH vector and hybrid (never leaks other folders)', async () => { + for (const mode of ['vector', 'hybrid']) { + query.mockReset(); hybrid.hybridSearch.mockReset(); + hybrid.hybridSearch.mockResolvedValue({ hits: [], poolSaturated: false, generation: null }); + withAccounts(['a1']); + await search(semReq({ mode, parsed: { filters: [{ key: 'in', value: 'sent', negate: false }], terms: [{ value: 'invoice', negate: false }] } })); + const { clauses, params } = runBuildFilters(); + expect(clauses.some((c) => /m\.folder ILIKE .+ OR m\.folder ILIKE/.test(c))).toBe(true); + expect(params).toContain('sent'); + expect(params).toContain('%/sent'); + } + }); + + it('honors an exact REST folderParam on the fused query', async () => { + withAccounts(['a1']); + await search(semReq({ folderParam: 'INBOX' })); + const { clauses, params } = runBuildFilters(); + expect(clauses.some((c) => c === 'm.folder = $2')).toBe(true); + expect(params).toContain('INBOX'); + }); + + it('defaults the fused query to the trash-excluding scope when no folder is specified', async () => { + withAccounts(['a1']); + await search(semReq()); + const { clauses } = runBuildFilters(); + expect(clauses.some((c) => /NOT EXISTS/.test(c) && /%trash%/.test(c))).toBe(true); + }); + + it('enforces a negated free-text term as a NOT-condition on the fused query (invoice -draft excludes drafts)', async () => { + withAccounts(['a1']); + await search(semReq({ parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: 'draft', negate: true }] } })); + const { clauses, params } = runBuildFilters(); + // Positive term drives freeText (embedded + BM25 leg); the negated term becomes + // a stopword-guarded NOT COALESCE(...) filter using the SAME prefix-aware + // FTS builder as lexical (guard OUTSIDE the NOT, so a negated stopword + // contributes nothing instead of excluding everything — Fix 1). + expect(hybrid.hybridSearch.mock.calls[0][0].freeText).toBe('invoice'); + const notClause = clauses.find((c) => /OR NOT COALESCE/.test(c)); + expect(notClause).toBeDefined(); + expect(notClause).toMatch(/^\(numnode\(to_tsquery\('english', quote_literal\(\$\d+\) \|\| ':\*'\)\) = 0 OR NOT COALESCE/); + expect(notClause).toContain("m.search_fts @@ to_tsquery('english', quote_literal($"); + expect(params).toContain('draft'); + }); + + it('drops a sub-2-char / punctuation-only negated term from the fused NOT-conditions (lexical hygiene parity)', async () => { + withAccounts(['a1']); + await search(semReq({ parsed: { filters: [], terms: [{ value: 'invoice', negate: false }, { value: 'x', negate: true }, { value: '!!!', negate: true }] } })); + const { clauses } = runBuildFilters(); + expect(clauses.some((c) => /NOT COALESCE/.test(c))).toBe(false); + }); +}); + +describe('searchService semantic pagination hasMore (Fix 3)', () => { + const semReq = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 2, offset: 0, + parsed: { filters: [], terms: [{ value: 'invoice', negate: false }] }, + ...extra, + }); + + it('fetches window+1 and flags hasMore when a sentinel hit is present', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 3 }, { message_id: 'b', rrf_score: 2 }, { message_id: 'c', rrf_score: 1 }], + poolSaturated: false, generation: null, + }); + const res = await search(semReq()); + expect(hybrid.hybridSearch.mock.calls[0][0].limit).toBe(3); // window(2) + 1 + expect(res.messages).toHaveLength(2); // page sliced to limit + expect(res.page.hasMore).toBe(true); + }); + + it('flags hasMore false when exactly the window is returned', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 2 }, { message_id: 'b', rrf_score: 1 }], + poolSaturated: false, generation: null, + }); + const res = await search(semReq()); + expect(res.messages).toHaveLength(2); + expect(res.page.hasMore).toBe(false); + }); + + it('respects offset: window+1 = offset+limit+1 and slices the page', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a' }, { message_id: 'b' }, { message_id: 'c' }, { message_id: 'd' }, { message_id: 'e' }], + poolSaturated: false, generation: null, + }); + const res = await search(semReq({ offset: 2, limit: 2 })); + expect(hybrid.hybridSearch.mock.calls[0][0].limit).toBe(5); // offset(2)+limit(2)+1 + expect(res.messages.map((m) => m.message_id)).toEqual(['c', 'd']); + expect(res.page.hasMore).toBe(true); // 5 hits > window(4) + }); +}); + +describe('searchService strictVector + envelope (Task 5b)', () => { + const hybridReq2 = (extra = {}) => ({ + userId: 'u1', mode: 'hybrid', limit: 50, offset: 0, + parsed: { filters: [], terms: [{ value: 'q', negate: false }] }, + ...extra, + }); + + it('rethrows the VectorUnavailableError (no fallback) under strictVector', async () => { + withAccounts(['a1']); + const unavail = new VectorUnavailableError('index_building'); + hybrid.hybridSearch.mockRejectedValue(unavail); + const err = await search(hybridReq2({ strictVector: true })).catch(e => e); + expect(err).toBe(unavail); + expect(err.reason).toBe('index_building'); + }); + + it('returns the rich generation object and per-hit score under explain', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 0.04, bm25_score: 0.01, vector_score: 0.9, subject_boosted: true, best_char_start: null }], + poolSaturated: false, generation: { id: 7, model: 'm', dimension: 4, fingerprint: 'm:4:x', state: 'active' }, + }); + const res = await search(hybridReq2({ explain: true })); + expect(res.generation).toEqual({ id: 7, model: 'm', dimension: 4, fingerprint: 'm:4:x', state: 'active' }); + expect(res.messages[0].score).toEqual({ rrf: 0.04, bm25: 0.01, vector: 0.9, subject_boosted: true }); + }); + + it('attaches best_chunk metadata under scope=body', async () => { + withAccounts(['a1']); + hybrid.hybridSearch.mockResolvedValue({ + hits: [{ message_id: 'a', rrf_score: 0.04, vector_score: 0.9, best_chunk_index: 2, best_char_start: 6, best_char_end: 40 }], + poolSaturated: false, generation: { id: 1, model: 'm', dimension: 4, fingerprint: 'f', state: 'active' }, + }); + const res = await search(hybridReq2({ scope: 'body' })); + expect(res.messages[0].best_chunk).toEqual({ chunk_index: 2, char_start: 6, char_end: 40, score: 0.9 }); + }); +}); diff --git a/backend/src/services/search/searchService.total.test.js b/backend/src/services/search/searchService.total.test.js new file mode 100644 index 00000000..f7c5bf8b --- /dev/null +++ b/backend/src/services/search/searchService.total.test.js @@ -0,0 +1,12 @@ +import { it, expect, vi } from 'vitest'; +vi.mock('./lexicalRepo.js', () => ({ searchLexical: vi.fn(), buildOperatorClauses: vi.fn(), hasSearchableToken: vi.fn() })); +import { searchLexical } from './lexicalRepo.js'; +import { search } from './searchService.js'; + +it('passes the metadata total through to the service result', async () => { + searchLexical.mockResolvedValue({ rows: [{ id: 'm1' }], total: 42, hasCondition: true }); + const parsed = { filters: [], terms: [{ value: 'budget', negate: false }], unsupported: [] }; + const r = await search({ mode: 'lexical', scope: 'metadata', parsed, accountIds: ['acc-1'], limit: 20, offset: 0 }); + expect(r.total).toBe(42); + expect(searchLexical.mock.calls[0][1]).toMatchObject({ scope: 'metadata', accountIds: ['acc-1'] }); +}); diff --git a/backend/src/services/sendService.js b/backend/src/services/sendService.js new file mode 100644 index 00000000..88c00f53 --- /dev/null +++ b/backend/src/services/sendService.js @@ -0,0 +1,404 @@ +import { randomBytes as defaultRandomBytes } from 'crypto'; +import { sanitizeSignature } from './emailSanitizer.js'; +import { embedInlineDataImages as defaultEmbedInlineDataImages } from '../utils/inlineImages.js'; +import { redactEmail } from '../utils/redact.js'; +import { + mapRecipientList, + normalizeRecipients, + sanitizeHeaderValue, +} from './mail/addresses.js'; +import { resolveFromIdentity as defaultResolveFromIdentity } from './mail/identity.js'; +import { + bodyToHtml, + bodyToPlain, + buildMailOptions as defaultBuildMailOptions, + renderRaw as defaultRenderRaw, + sigToPlainText, + textToHtml, +} from './mail/mimeBuilder.js'; +import { buildSmtpTransport as defaultBuildSmtpTransport } from './mail/smtp.js'; +import { + learnSentRecipients as defaultLearnSentRecipients, + persistSentCopy as defaultPersistSentCopy, + resolveSentFolder as defaultResolveSentFolder, +} from './mail/sentCopy.js'; + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const MAX_ATTACHMENT_BYTES = 26_214_400; +const VALID_PRIORITIES = new Set(['high', 'normal', 'low']); + +function serviceError(message, status, extra = {}) { + return Object.assign(new Error(message), { status, expose: true, ...extra }); +} + +function estimatedBase64Bytes(content) { + return typeof content === 'string' ? Math.ceil(content.length * 0.75) : 0; +} + +function validateCompose(input) { + if (input.attachments !== undefined) { + if (!Array.isArray(input.attachments)) throw serviceError('attachments must be an array', 400); + if (input.attachments.length > 100) throw serviceError('Too many attachments (max 100)', 400); + const totalBytes = input.attachments.reduce((sum, attachment) => sum + estimatedBase64Bytes(attachment.content), 0); + if (totalBytes > MAX_ATTACHMENT_BYTES) throw serviceError('Total attachment size exceeds 25 MB', 400); + for (const [i, attachment] of input.attachments.entries()) { + if (typeof attachment.filename !== 'string' || !attachment.filename.trim()) { + throw serviceError(`attachments[${i}].filename is required`, 400); + } + if (typeof attachment.content !== 'string') { + throw serviceError(`attachments[${i}].content must be a base64 string`, 400); + } + } + } + + if (input.forwardedAttachments !== undefined) { + if (!Array.isArray(input.forwardedAttachments)) { + throw serviceError('forwardedAttachments must be an array', 400); + } + for (const [i, attachment] of input.forwardedAttachments.entries()) { + if (typeof attachment.messageId !== 'string' || !UUID_RE.test(attachment.messageId)) { + throw serviceError(`forwardedAttachments[${i}].messageId is invalid`, 400); + } + if (typeof attachment.part !== 'string' || !attachment.part.trim()) { + throw serviceError(`forwardedAttachments[${i}].part is required`, 400); + } + } + } + + try { + return { + to: normalizeRecipients(input.to, 'to'), + cc: normalizeRecipients(input.cc || [], 'cc'), + bcc: normalizeRecipients(input.bcc || [], 'bcc'), + subject: sanitizeHeaderValue(input.subject || ''), + priority: VALID_PRIORITIES.has(input.priority) ? input.priority : 'normal', + }; + } catch (err) { + err.expose = true; + throw err; + } +} + +async function resolveForwardedAttachments(input, deps) { + if (!input.forwardedAttachments?.length) return []; + try { + const resolved = await Promise.all(input.forwardedAttachments.map(async (forwarded) => { + const msgResult = await deps.query( + `SELECT m.uid, m.folder, m.attachments, to_jsonb(a) AS account FROM messages m + JOIN email_accounts a ON m.account_id = a.id + WHERE m.id = $1 AND a.user_id = $2`, + [forwarded.messageId, input.userId], + ); + if (!msgResult.rows.length) throw serviceError('Forwarded message not found', 404); + const message = msgResult.rows[0]; + + const storedAttachments = typeof message.attachments === 'string' + ? JSON.parse(message.attachments || '[]') + : (message.attachments || []); + const attachment = storedAttachments.find(candidate => candidate.part === forwarded.part); + if (!attachment) throw serviceError('Attachment not found in message', 404); + + const buffer = await deps.imapManager.fetchAttachment( + message.account, + message.uid, + message.folder, + forwarded.part, + ); + if (!buffer) throw serviceError(`Could not fetch attachment: ${attachment.filename}`, 502); + + return { + filename: sanitizeHeaderValue(attachment.filename || 'attachment'), + content: buffer, + contentType: attachment.type || 'application/octet-stream', + }; + })); + + const uploadedBytes = (input.attachments || []).reduce( + (sum, attachment) => sum + estimatedBase64Bytes(attachment.content), + 0, + ); + const forwardedBytes = resolved.reduce((sum, attachment) => sum + (attachment.content?.length || 0), 0); + if (uploadedBytes + forwardedBytes > MAX_ATTACHMENT_BYTES) { + throw serviceError('Total attachment size exceeds 25 MB', 400); + } + return resolved; + } catch (err) { + if (err.expose) throw err; + throw serviceError(err.message || 'Failed to fetch forwarded attachments', err.status || 500); + } +} + +function attachmentSize(attachment) { + if (Buffer.isBuffer(attachment.content)) return attachment.content.length; + return estimatedBase64Bytes(attachment.content); +} + +export function buildReceipt({ + from, + to, + cc, + bcc, + subject, + attachments, + messageId, + sentCopySaved, + folder, +}) { + return { + from, + to: mapRecipientList(to), + cc: mapRecipientList(cc), + bcc: mapRecipientList(bcc), + subject, + attachments: (attachments || []).map(attachment => ({ + filename: attachment.filename, + size: attachmentSize(attachment), + })), + messageId, + sentCopySaved, + folder, + }; +} + +export async function sendMessage(input, deps) { + const normalized = validateCompose(input); + if (!input.account) throw serviceError('Account not found', 404, { code: 'account_not_found' }); + + const resolveFromIdentity = deps.resolveFromIdentity || defaultResolveFromIdentity; + const buildSmtpTransport = deps.buildSmtpTransport || defaultBuildSmtpTransport; + const buildMailOptions = deps.buildMailOptions || defaultBuildMailOptions; + const renderRaw = deps.renderRaw || defaultRenderRaw; + const embedInlineDataImages = deps.embedInlineDataImages || defaultEmbedInlineDataImages; + const learnSentRecipients = deps.learnSentRecipients || defaultLearnSentRecipients; + const resolveSentFolder = deps.resolveSentFolder || defaultResolveSentFolder; + const persistSentCopy = deps.persistSentCopy || defaultPersistSentCopy; + const makeRandomBytes = deps.randomBytes || defaultRandomBytes; + + const identity = await resolveFromIdentity( + input.account, + { aliasId: input.aliasId, aliasEmail: input.aliasEmail }, + deps, + ); + + // Allow the caller to override the signature per-send (undefined means use DB value). + // Sanitize client-supplied HTML to prevent scripts or tracking pixels in sent mail. + const effectiveSignature = input.editedSignature !== undefined + ? (input.editedSignature ? sanitizeSignature(input.editedSignature) : null) + : identity.signature; + + // Fetch forwarded content before SMTP setup so these failures stay descriptive. + const forwardedAttachments = await resolveForwardedAttachments(input, deps); + const smtp = await buildSmtpTransport(input.account, deps); + const account = smtp.account; + + const domain = identity.fromEmail.split('@')[1] || 'mailflow.local'; + const messageId = input.messageId || `<${makeRandomBytes(16).toString('hex')}@${domain}>`; + const text = effectiveSignature + ? bodyToPlain(input.body, input.bodyIsHtml) + '\n\n-- \n' + + sigToPlainText(effectiveSignature) + (input.quotedBody || '') + : bodyToPlain(input.body, input.bodyIsHtml) + (input.quotedBody || ''); + + let html; + let inlineImageAttachments = []; + if (!input.plaintextEmail) { + const rawHtml = bodyToHtml(input.body, input.bodyIsHtml) + + (effectiveSignature + ? '
' + effectiveSignature + '
' + : '') + + (input.quotedBodyHtml || (input.quotedBody ? textToHtml(input.quotedBody) : '')); + const embedded = embedInlineDataImages(rawHtml); + html = embedded.html; + inlineImageAttachments = embedded.attachments; + } + + const uploadedAttachments = input.attachments?.length + ? input.attachments.map(attachment => ({ + filename: sanitizeHeaderValue(attachment.filename), + content: Buffer.from(attachment.content, 'base64'), + contentType: typeof attachment.contentType === 'string' + ? attachment.contentType + : 'application/octet-stream', + // replyService re-fetches quoted inline images with a cid so the reply's + // keeps resolving; dropping it breaks those images. + ...(typeof attachment.cid === 'string' && attachment.cid + ? { cid: sanitizeHeaderValue(attachment.cid) } + : {}), + })) + : []; + const allAttachments = [ + ...inlineImageAttachments, + ...uploadedAttachments, + ...forwardedAttachments, + ]; + const mailOptions = buildMailOptions({ + messageId, + fromName: identity.fromName, + fromEmail: identity.fromEmail, + replyTo: identity.fromReplyTo, + to: normalized.to, + cc: normalized.cc, + bcc: normalized.bcc, + subject: normalized.subject, + priority: normalized.priority, + text, + ...(html !== undefined ? { html } : {}), + inReplyTo: input.inReplyTo, + references: input.references, + attachments: allAttachments, + }); + + // OAuth providers save Sent automatically; other accounts need this exact raw MIME. + const serverAutoSaves = !!account.oauth_provider; + const rawMessage = serverAutoSaves ? null : await renderRaw(mailOptions); + + // Idempotent callers inject the Redis reservation here: after MIME rendering and + // immediately before the delivery boundary. + if (deps.beforeDelivery) await deps.beforeDelivery(); + await smtp.transport.sendMail(mailOptions); + if (deps.afterDelivery) deps.afterDelivery(); + + const recipients = [...normalized.to, ...normalized.cc, ...normalized.bcc]; + learnSentRecipients({ userId: input.userId, recipients }, deps); + const sentFolder = await resolveSentFolder(account, deps); + console.log(`Post-send: ${redactEmail(account.email_address)} sentFolder=${sentFolder} autoSaves=${serverAutoSaves}`); + + const sentMeta = sentFolder ? { + messageId, + subject: normalized.subject, + fromName: identity.fromName, + fromEmail: identity.fromEmail, + to: mapRecipientList(normalized.to), + cc: mapRecipientList(normalized.cc), + snippet: bodyToPlain(input.body, input.bodyIsHtml).replace(/\s+/g, ' ').trim().substring(0, 200), + date: new Date(), + } : null; + const { sentCopySaved } = await persistSentCopy({ + account, + sentFolder, + rawMessage, + mailOptions, + meta: sentMeta, + }, deps); + + const receipt = buildReceipt({ + from: { name: identity.fromName, email: identity.fromEmail }, + to: normalized.to, + cc: normalized.cc, + bcc: normalized.bcc, + subject: normalized.subject, + attachments: allAttachments, + messageId, + sentCopySaved, + folder: sentFolder, + }); + return { ok: true, messageId, sentCopySaved, receipt }; +} + +function conflict(message) { + return serviceError(message, 409, { code: 'idempotency_conflict' }); +} + +export async function sendMessageIdempotent({ idempotencyKey, ...input }, deps) { + const boundedKey = typeof idempotencyKey === 'string' ? idempotencyKey.slice(0, 128) : null; + const key = boundedKey ? `send_idem:${input.userId}:${boundedKey}` : null; + if (!key) return sendMessage(input, deps); + + const cached = await deps.redisClient.get(key).catch(() => null); + if (cached === '__inflight__') throw conflict('This message is already being sent.'); + if (cached) return JSON.parse(cached); + + let delivered = false; + let reservationConflict = false; + try { + const result = await sendMessage(input, { + ...deps, + beforeDelivery: async () => { + const reserved = await deps.redisClient + .set(key, '__inflight__', { NX: true, EX: 300 }) + .catch(() => 'OK'); + if (reserved === null) { + reservationConflict = true; + throw conflict('This message is already being sent.'); + } + }, + afterDelivery: () => { + delivered = true; + }, + }); + deps.redisClient.set(key, JSON.stringify(result), { EX: 86400 }).catch(() => {}); + return result; + } catch (err) { + if (reservationConflict) throw err; + if (delivered) { + deps.redisClient.set(key, JSON.stringify({ ok: true }), { EX: 86400 }).catch(() => {}); + } else { + deps.redisClient.del(key).catch(() => {}); + } + throw err; + } +} + +export async function sendOrEnqueue(input, deps) { + if (!input.undoSeconds) { + const immediateInput = { ...input }; + delete immediateInput.messageId; + return sendMessageIdempotent(immediateInput, deps); + } + if (!input.account) throw serviceError('Account not found', 404, { code: 'account_not_found' }); + + const normalized = validateCompose(input); + const makeRandomBytes = deps.randomBytes || defaultRandomBytes; + const resolveFromIdentity = deps.resolveFromIdentity || defaultResolveFromIdentity; + const identity = await resolveFromIdentity( + input.account, + { aliasId: input.aliasId, aliasEmail: input.aliasEmail }, + deps, + ); + const domain = identity.fromEmail.split('@')[1] || 'mailflow.local'; + const messageId = `<${makeRandomBytes(16).toString('hex')}@${domain}>`; + const payload = { + userId: input.userId, + account_id: input.account.id, + to: normalized.to, + cc: normalized.cc, + bcc: normalized.bcc, + subject: normalized.subject, + priority: normalized.priority, + body: input.body, + bodyIsHtml: input.bodyIsHtml, + attachments: input.attachments, + forwardedAttachments: input.forwardedAttachments, + aliasId: input.aliasId, + aliasEmail: input.aliasEmail, + editedSignature: input.editedSignature, + quotedBody: input.quotedBody, + quotedBodyHtml: input.quotedBodyHtml, + plaintextEmail: input.plaintextEmail, + inReplyTo: input.inReplyTo, + references: input.references, + ...(input.deleteDraftOnSend + ? { deleteDraftOnSend: input.deleteDraftOnSend } + : {}), + ...(input.composeSessionRestore + ? { composeSessionRestore: input.composeSessionRestore } + : {}), + messageId, + }; + + const queued = await deps.outboxService.enqueue({ + userId: input.userId, + accountId: input.account.id, + payload, + undoSeconds: input.undoSeconds, + idempotencyKey: input.idempotencyKey, + subject: normalized.subject, + toPreview: normalized.to, + messageId, + }, deps); + return { + queued: true, + outboxId: queued.outbox_id, + sendAt: queued.send_at, + undoSeconds: queued.undo_seconds, + }; +} diff --git a/backend/src/services/sendService.test.js b/backend/src/services/sendService.test.js new file mode 100644 index 00000000..ec45c416 --- /dev/null +++ b/backend/src/services/sendService.test.js @@ -0,0 +1,477 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + buildReceipt, + sendMessage, + sendMessageIdempotent, +} from './sendService.js'; + +const account = { + id: 'account-1', + email_address: 'sender@example.com', + name: 'Sender', + signature: null, + oauth_provider: null, +}; + +function input(overrides = {}) { + return { + userId: 'user-1', + account, + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Subject', + body: 'Body', + bodyIsHtml: false, + ...overrides, + }; +} + +function serviceDeps(overrides = {}) { + const transport = { sendMail: vi.fn().mockResolvedValue({}) }; + return { + query: vi.fn(), + imapManager: {}, + resolveFromIdentity: vi.fn().mockResolvedValue({ + fromName: 'Sender', + fromEmail: 'sender@example.com', + fromReplyTo: null, + signature: null, + aliasId: null, + }), + buildSmtpTransport: vi.fn().mockResolvedValue({ transport, account }), + buildMailOptions: vi.fn(options => ({ + messageId: options.messageId, + from: `${options.fromName} <${options.fromEmail}>`, + to: options.to.join(', '), + subject: options.subject, + text: options.text, + ...(options.html !== undefined ? { html: options.html } : {}), + ...(options.attachments?.length ? { attachments: options.attachments } : {}), + })), + renderRaw: vi.fn().mockResolvedValue(Buffer.from('raw mime')), + embedInlineDataImages: vi.fn(html => ({ html, attachments: [] })), + learnSentRecipients: vi.fn(), + resolveSentFolder: vi.fn().mockResolvedValue('Sent'), + persistSentCopy: vi.fn().mockResolvedValue({ sentCopySaved: true }), + randomBytes: vi.fn(() => Buffer.alloc(16, 1)), + ...overrides, + transport, + }; +} + +describe('sendMessage', () => { + it('validates, renders, delivers, and persists a non-OAuth message in order', async () => { + const events = []; + const deps = serviceDeps({ + renderRaw: vi.fn(async () => { + events.push('render'); + return Buffer.from('raw mime'); + }), + beforeDelivery: vi.fn(async () => { events.push('reserve'); }), + afterDelivery: vi.fn(() => { events.push('delivered'); }), + persistSentCopy: vi.fn(async () => { + events.push('persist'); + return { sentCopySaved: false }; + }), + }); + deps.transport.sendMail.mockImplementation(async () => { events.push('send'); }); + + const result = await sendMessage(input({ + attachments: [{ + filename: 'note.txt', + content: Buffer.from('hello').toString('base64'), + contentType: 'text/plain', + }], + priority: 'high', + }), deps); + + expect(events).toEqual(['render', 'reserve', 'send', 'delivered', 'persist']); + expect(deps.resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasId: undefined, aliasEmail: undefined }, + deps, + ); + expect(deps.buildMailOptions).toHaveBeenCalledWith(expect.objectContaining({ + to: ['Recipient '], + priority: 'high', + attachments: [expect.objectContaining({ + filename: 'note.txt', + content: Buffer.from('hello'), + contentType: 'text/plain', + })], + })); + expect(result).toMatchObject({ + ok: true, + sentCopySaved: false, + receipt: { + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'Recipient', email: 'recipient@example.com' }], + subject: 'Subject', + folder: 'Sent', + sentCopySaved: false, + }, + }); + expect(result.messageId).toMatch(/^<01010101.+@example\.com>$/); + }); + + it('preserves cid on inline attachments and omits it otherwise', async () => { + const deps = serviceDeps(); + + await sendMessage(input({ + attachments: [ + { + filename: 'inline.png', + content: Buffer.from('img').toString('base64'), + contentType: 'image/png', + cid: 'part1.abc@example.com', + }, + { + filename: 'plain.txt', + content: Buffer.from('txt').toString('base64'), + contentType: 'text/plain', + }, + ], + }), deps); + + const { attachments } = deps.buildMailOptions.mock.calls[0][0]; + expect(attachments[0]).toMatchObject({ filename: 'inline.png', cid: 'part1.abc@example.com' }); + expect(attachments[1]).not.toHaveProperty('cid'); + }); + + it('skips raw rendering for OAuth providers', async () => { + const oauthAccount = { ...account, oauth_provider: 'google' }; + const deps = serviceDeps({ + buildSmtpTransport: vi.fn().mockImplementation(async () => ({ + transport: deps.transport, + account: oauthAccount, + })), + }); + + await sendMessage(input({ account: oauthAccount }), deps); + expect(deps.renderRaw).not.toHaveBeenCalled(); + expect(deps.persistSentCopy).toHaveBeenCalledWith(expect.objectContaining({ rawMessage: null }), deps); + }); + + it('reuses a message ID generated when an outbox row was enqueued', async () => { + const deps = serviceDeps(); + + const result = await sendMessage(input({ + messageId: '', + }), deps); + + expect(deps.buildMailOptions).toHaveBeenCalledWith(expect.objectContaining({ + messageId: '', + })); + expect(result.messageId).toBe(''); + }); + + it.each([ + [{ attachments: 'bad' }, 'attachments must be an array'], + [{ attachments: Array.from({ length: 101 }, () => ({})) }, 'Too many attachments (max 100)'], + [{ attachments: [{ filename: '', content: '' }] }, 'attachments[0].filename is required'], + [{ forwardedAttachments: [{ messageId: 'bad', part: '1' }] }, 'forwardedAttachments[0].messageId is invalid'], + [{ to: ['bad-address'] }, 'to[0] is not a valid email address'], + ])('rejects invalid compose input before building SMTP: %j', async (overrides, message) => { + const deps = serviceDeps(); + const error = await sendMessage(input(overrides), deps).catch(err => err); + expect(error).toMatchObject({ message, status: 400, expose: true }); + expect(deps.buildSmtpTransport).not.toHaveBeenCalled(); + }); + + it('fetches forwarded attachment content through a user-scoped account query', async () => { + const referencedAccount = { ...account, id: 'account-2' }; + const deps = serviceDeps({ + query: vi.fn().mockResolvedValueOnce({ rows: [{ + uid: 9, + folder: 'Inbox', + account: referencedAccount, + attachments: [{ part: '2', filename: 'forwarded.pdf', type: 'application/pdf' }], + }] }), + imapManager: { + fetchAttachment: vi.fn().mockResolvedValue(Buffer.from('pdf')), + }, + }); + + await sendMessage(input({ + forwardedAttachments: [{ + messageId: '11111111-1111-4111-8111-111111111111', + part: '2', + }], + }), deps); + + expect(deps.query).toHaveBeenCalledTimes(1); + expect(deps.query.mock.calls[0][0]).toContain('to_jsonb(a) AS account'); + expect(deps.query.mock.calls[0][1]).toEqual([ + '11111111-1111-4111-8111-111111111111', + 'user-1', + ]); + expect(deps.imapManager.fetchAttachment).toHaveBeenCalledWith(referencedAccount, 9, 'Inbox', '2'); + }); +}); + +describe('sendMessageIdempotent', () => { + it('uses the unchanged Redis namespace and reserves after rendering immediately before sendMail', async () => { + const events = []; + const redisClient = { + get: vi.fn().mockResolvedValue(null), + set: vi.fn().mockImplementation(async (_key, value) => { + events.push(value === '__inflight__' ? 'reserve' : 'cache'); + return 'OK'; + }), + del: vi.fn(), + }; + const deps = serviceDeps({ + redisClient, + renderRaw: vi.fn(async () => { + events.push('render'); + return Buffer.from('raw'); + }), + }); + deps.transport.sendMail.mockImplementation(async () => { events.push('send'); }); + + await sendMessageIdempotent(input({ idempotencyKey: 'key-1' }), deps); + + expect(events.slice(0, 3)).toEqual(['render', 'reserve', 'send']); + expect(redisClient.get).toHaveBeenCalledWith('send_idem:user-1:key-1'); + expect(redisClient.set.mock.calls[0]).toEqual([ + 'send_idem:user-1:key-1', + '__inflight__', + { NX: true, EX: 300 }, + ]); + }); + + it('returns cached results and conflicts on in-flight results without sending', async () => { + const cachedDeps = serviceDeps({ + redisClient: { + get: vi.fn().mockResolvedValue(JSON.stringify({ ok: true })), + set: vi.fn(), + del: vi.fn(), + }, + }); + await expect(sendMessageIdempotent(input({ idempotencyKey: 'cached' }), cachedDeps)) + .resolves.toEqual({ ok: true }); + expect(cachedDeps.transport.sendMail).not.toHaveBeenCalled(); + + const inflightDeps = serviceDeps({ + redisClient: { + get: vi.fn().mockResolvedValue('__inflight__'), + set: vi.fn(), + del: vi.fn(), + }, + }); + await expect(sendMessageIdempotent(input({ idempotencyKey: 'busy' }), inflightDeps)) + .rejects.toMatchObject({ status: 409, expose: true }); + expect(inflightDeps.transport.sendMail).not.toHaveBeenCalled(); + }); + + it('does not delete somebody else’s reservation when NX reports a conflict', async () => { + const redisClient = { + get: vi.fn().mockResolvedValue(null), + set: vi.fn().mockResolvedValue(null), + del: vi.fn(), + }; + const deps = serviceDeps({ redisClient }); + + await expect(sendMessageIdempotent(input({ idempotencyKey: 'race' }), deps)) + .rejects.toMatchObject({ status: 409 }); + expect(redisClient.del).not.toHaveBeenCalled(); + expect(deps.transport.sendMail).not.toHaveBeenCalled(); + }); + + it('releases the reservation on pre-delivery failure and stores durable success after delivery', async () => { + const preRedis = { + get: vi.fn().mockResolvedValue(null), + set: vi.fn().mockResolvedValue('OK'), + del: vi.fn().mockResolvedValue(1), + }; + const preDeps = serviceDeps({ redisClient: preRedis }); + preDeps.transport.sendMail.mockRejectedValueOnce(new Error('SMTP failed')); + await expect(sendMessageIdempotent(input({ idempotencyKey: 'pre' }), preDeps)).rejects.toThrow('SMTP failed'); + expect(preRedis.del).toHaveBeenCalledWith('send_idem:user-1:pre'); + + const postRedis = { + get: vi.fn().mockResolvedValue(null), + set: vi.fn().mockResolvedValue('OK'), + del: vi.fn(), + }; + const postDeps = serviceDeps({ + redisClient: postRedis, + resolveSentFolder: vi.fn().mockRejectedValue(new Error('DB failed after delivery')), + }); + await expect(sendMessageIdempotent(input({ idempotencyKey: 'post' }), postDeps)) + .rejects.toThrow('DB failed after delivery'); + expect(postRedis.set).toHaveBeenLastCalledWith( + 'send_idem:user-1:post', + JSON.stringify({ ok: true }), + { EX: 86400 }, + ); + expect(postRedis.del).not.toHaveBeenCalled(); + }); +}); + +describe('sendOrEnqueue', () => { + it('uses the existing idempotent immediate-send path when undo is off', async () => { + const service = await import('./sendService.js'); + const redisClient = { + get: vi.fn().mockResolvedValue(null), + set: vi.fn().mockResolvedValue('OK'), + del: vi.fn(), + }; + const deps = serviceDeps({ redisClient }); + + expect(service.sendOrEnqueue).toBeTypeOf('function'); + const result = await service.sendOrEnqueue(input({ + undoSeconds: 0, + idempotencyKey: 'immediate-1', + messageId: '', + }), deps); + + expect(result.ok).toBe(true); + expect(result.messageId).toBe('<01010101010101010101010101010101@example.com>'); + expect(redisClient.get).toHaveBeenCalledWith('send_idem:user-1:immediate-1'); + expect(deps.transport.sendMail).toHaveBeenCalledTimes(1); + }); + + it('enqueues a validated credential-free payload with the same idempotency key', async () => { + const service = await import('./sendService.js'); + const sendAt = new Date('2026-07-28T12:00:30.000Z'); + const outboxService = { + enqueue: vi.fn().mockResolvedValue({ + outbox_id: 'outbox-1', + send_at: sendAt, + undo_seconds: 30, + }), + }; + const deps = serviceDeps({ outboxService }); + + expect(service.sendOrEnqueue).toBeTypeOf('function'); + const result = await service.sendOrEnqueue(input({ + accountId: 'account-1', + undoSendSeconds: 30, + undoSeconds: 30, + idempotencyKey: 'queued-1', + priority: 'unexpected', + auth_pass: 'must-never-be-stored', + deleteDraftOnSend: { + accountId: 'source-account-1', + uid: 7, + folder: 'Drafts', + }, + composeSessionRestore: { + originalSessionId: '11111111-1111-4111-8111-111111111111', + preferredSlot: 2, + }, + }), deps); + + expect(deps.transport.sendMail).not.toHaveBeenCalled(); + expect(outboxService.enqueue).toHaveBeenCalledTimes(1); + const [queued, enqueueDeps] = outboxService.enqueue.mock.calls[0]; + expect(enqueueDeps).toBe(deps); + expect(queued).toMatchObject({ + userId: 'user-1', + accountId: 'account-1', + undoSeconds: 30, + idempotencyKey: 'queued-1', + subject: 'Subject', + toPreview: ['Recipient '], + messageId: '<01010101010101010101010101010101@example.com>', + }); + expect(queued.payload).toMatchObject({ + userId: 'user-1', + account_id: 'account-1', + to: ['Recipient '], + cc: [], + bcc: [], + subject: 'Subject', + priority: 'normal', + body: 'Body', + messageId: '<01010101010101010101010101010101@example.com>', + deleteDraftOnSend: { + accountId: 'source-account-1', + uid: 7, + folder: 'Drafts', + }, + composeSessionRestore: { + originalSessionId: '11111111-1111-4111-8111-111111111111', + preferredSlot: 2, + }, + }); + expect(queued.payload).not.toHaveProperty('account'); + expect(queued.payload).not.toHaveProperty('auth_pass'); + expect(queued.payload).not.toHaveProperty('undoSeconds'); + expect(queued.payload).not.toHaveProperty('undoSendSeconds'); + expect(queued.payload).not.toHaveProperty('idempotencyKey'); + expect(result).toEqual({ + queued: true, + outboxId: 'outbox-1', + sendAt, + undoSeconds: 30, + }); + }); + + it('generates the queued message ID from the resolved alias identity', async () => { + const service = await import('./sendService.js'); + const outboxService = { + enqueue: vi.fn().mockResolvedValue({ + outbox_id: 'outbox-1', + send_at: new Date('2026-07-28T12:00:30.000Z'), + undo_seconds: 30, + }), + }; + const deps = serviceDeps({ + outboxService, + resolveFromIdentity: vi.fn().mockResolvedValue({ + fromName: 'Alias', + fromEmail: 'sender@alias.example', + fromReplyTo: null, + signature: null, + aliasId: 'alias-1', + }), + }); + + expect(service.sendOrEnqueue).toBeTypeOf('function'); + await service.sendOrEnqueue(input({ + undoSeconds: 30, + aliasId: 'alias-1', + }), deps); + + expect(deps.resolveFromIdentity).toHaveBeenCalledWith( + account, + { aliasId: 'alias-1', aliasEmail: undefined }, + deps, + ); + expect(outboxService.enqueue).toHaveBeenCalledWith( + expect.objectContaining({ + messageId: '<01010101010101010101010101010101@alias.example>', + }), + deps, + ); + }); +}); + +describe('buildReceipt', () => { + it('returns normalized recipient and attachment metadata without content', () => { + expect(buildReceipt({ + from: { name: 'Sender', email: 'sender@example.com' }, + to: ['A '], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'a.txt', content: Buffer.from('abc') }], + messageId: '', + sentCopySaved: true, + folder: 'Sent', + })).toEqual({ + from: { name: 'Sender', email: 'sender@example.com' }, + to: [{ name: 'A', email: 'a@example.com' }], + cc: [], + bcc: [], + subject: 'Subject', + attachments: [{ filename: 'a.txt', size: 3 }], + messageId: '', + sentCopySaved: true, + folder: 'Sent', + }); + }); +}); diff --git a/backend/src/testSupport/mockSurface.js b/backend/src/testSupport/mockSurface.js new file mode 100644 index 00000000..aa3fc448 --- /dev/null +++ b/backend/src/testSupport/mockSurface.js @@ -0,0 +1,24 @@ +import { vi } from 'vitest'; + +// Mock-drift guard. Given a mocked module namespace (`import * as ns` from a +// vi.mock'd module, or a hand-built `{ fnA, fnB }` of its named imports) and the +// real module (`await vi.importActual(path)`), return the names that are mocked +// as functions but do NOT exist as functions on the real module. A non-empty +// result means a suite invented a seam the production module never implemented — +// e.g. `generations.chunkCount`, mocked as a vi.fn() in two suites while the real +// module never exported it, which made live `collectStats` throw and silently drop +// get_stats' vector_search block. +// +// Scope note: this catches the missing/renamed EXPORT drift class only, not +// value-shape drift (a mock whose fn returns the wrong row shape still passes — +// the earlier message_id/id bug would not have been caught here). Pair it with +// real-shape fixtures where the return shape is load-bearing. +export function mockSurfaceDrift(mockedNs, realNs) { + const drift = []; + for (const [name, value] of Object.entries(mockedNs)) { + if (vi.isMockFunction(value) && typeof realNs[name] !== 'function') { + drift.push(name); + } + } + return drift; +} diff --git a/backend/src/utils/textExcerpt.js b/backend/src/utils/textExcerpt.js new file mode 100644 index 00000000..7c9f92ca --- /dev/null +++ b/backend/src/utils/textExcerpt.js @@ -0,0 +1,21 @@ +// Low-level UTF-8 / line primitives shared by the excerpt builders on both search +// paths: mcp/bodyMatch.js (keyword matches) and services/embeddings/chunkmatch.js +// (vector chunk matches). All offsets are UTF-8 BYTES (msgvault wire contract). The +// higher-level keyword-vs-chunk assembly stays in each owner; only these primitives +// live here so they cannot drift. + +// Default snippet width in bytes for a match excerpt. +export const SNIPPET_BYTES = 300; + +// True when `byte` is a UTF-8 leading byte (not a 10xxxxxx continuation byte), so a +// slice boundary at it does not split a multi-byte rune. +export const isRuneStart = (byte) => (byte & 0xc0) !== 0x80; + +// 1-based line number at a byte offset: one plus the count of '\n' bytes before it. +export function lineNumberAt(buf, byteOffset) { + if (byteOffset <= 0) return 1; + const o = Math.min(byteOffset, buf.length); + let n = 1; + for (let i = 0; i < o; i++) if (buf[i] === 0x0a) n++; + return n; +} diff --git a/backend/src/utils/textExcerpt.test.js b/backend/src/utils/textExcerpt.test.js new file mode 100644 index 00000000..036046bf --- /dev/null +++ b/backend/src/utils/textExcerpt.test.js @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest'; +import { SNIPPET_BYTES, isRuneStart, lineNumberAt } from './textExcerpt.js'; + +describe('SNIPPET_BYTES', () => { + it('is the shared 300-byte excerpt width', () => { + expect(SNIPPET_BYTES).toBe(300); + }); +}); + +describe('isRuneStart', () => { + it('flags UTF-8 leading bytes, not continuation bytes', () => { + const buf = Buffer.from('é', 'utf8'); // 0xC3 0xA9 + expect(isRuneStart(buf[0])).toBe(true); // 0xC3 lead byte + expect(isRuneStart(buf[1])).toBe(false); // 0xA9 continuation byte + expect(isRuneStart(0x61)).toBe(true); // ASCII 'a' + }); +}); + +describe('lineNumberAt', () => { + it('counts newlines before the byte offset (1-based)', () => { + const buf = Buffer.from('a\nb\nc', 'utf8'); + expect(lineNumberAt(buf, 0)).toBe(1); + expect(lineNumberAt(buf, 2)).toBe(2); + expect(lineNumberAt(buf, 4)).toBe(3); + }); + it('clamps a negative or over-long offset', () => { + const buf = Buffer.from('a\nb', 'utf8'); + expect(lineNumberAt(buf, -1)).toBe(1); + expect(lineNumberAt(buf, 999)).toBe(2); + }); +}); diff --git a/backend/src/utils/validation.js b/backend/src/utils/validation.js new file mode 100644 index 00000000..33555278 --- /dev/null +++ b/backend/src/utils/validation.js @@ -0,0 +1,11 @@ +// Validate a folder name / path component: no control chars, max 255 chars. +export function isValidFolderName(name) { + // eslint-disable-next-line no-control-regex -- intentionally rejecting control characters + return typeof name === 'string' && name.length > 0 && name.length <= 255 && !/[\x00-\x1f\x7f]/.test(name); +} + +export const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export function areValidUUIDs(ids) { + return ids.every(id => typeof id === 'string' && UUID_RE.test(id)); +} diff --git a/backend/src/utils/validation.test.js b/backend/src/utils/validation.test.js new file mode 100644 index 00000000..a6487f70 --- /dev/null +++ b/backend/src/utils/validation.test.js @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest'; +import { UUID_RE, areValidUUIDs, isValidFolderName } from './validation.js'; + +describe('mail validation helpers', () => { + it('accepts UUID-shaped message ids case-insensitively', () => { + expect(UUID_RE.test('AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA')).toBe(true); + expect(areValidUUIDs([ + '11111111-1111-4111-8111-111111111111', + 'AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA', + ])).toBe(true); + }); + + it('rejects non-string or malformed ids in an array', () => { + expect(areValidUUIDs(['not-a-uuid'])).toBe(false); + expect(areValidUUIDs([null])).toBe(false); + }); + + it('accepts folder paths up to 255 characters without control characters', () => { + expect(isValidFolderName('Projects/2026')).toBe(true); + expect(isValidFolderName('x'.repeat(255))).toBe(true); + }); + + it('rejects empty, oversized, non-string, and control-character folder names', () => { + expect(isValidFolderName('')).toBe(false); + expect(isValidFolderName('x'.repeat(256))).toBe(false); + expect(isValidFolderName(null)).toBe(false); + expect(isValidFolderName('Inbox\nArchive')).toBe(false); + }); +}); diff --git a/docker-compose.ghcr.yml b/docker-compose.ghcr.yml index 3c51af41..aaeb02e7 100644 --- a/docker-compose.ghcr.yml +++ b/docker-compose.ghcr.yml @@ -71,7 +71,7 @@ services: # ── PostgreSQL ─────────────────────────────────────────────────────────────── postgres: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 container_name: mailflow-postgres restart: unless-stopped environment: diff --git a/docker-compose.yml b/docker-compose.yml index 398c1426..38ceeba4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: # ── PostgreSQL ─────────────────────────────────────────────────────────────── postgres: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 container_name: mailflow-postgres restart: unless-stopped # PUID/PGID default to 0:0 (root), identical to the image's normal behaviour. diff --git a/frontend/nginx.conf b/frontend/nginx.conf index c4ca0c89..5d486404 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -87,6 +87,19 @@ server { proxy_read_timeout 60s; } + # MCP endpoint (Streamable HTTP — long-lived SSE responses, no buffering) + location /mcp { + proxy_pass http://backend:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_read_timeout 300s; + proxy_buffering off; + proxy_cache off; + } + # WebSocket proxy location /ws { proxy_pass http://backend:3000; @@ -215,6 +228,19 @@ server { proxy_read_timeout 60s; } + # MCP endpoint (Streamable HTTP — long-lived SSE responses, no buffering) + location /mcp { + proxy_pass http://backend:3000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; + proxy_read_timeout 300s; + proxy_buffering off; + proxy_cache off; + } + # WebSocket proxy location /ws { proxy_pass http://backend:3000; diff --git a/frontend/src/commands/CommandRuntimeContext.jsx b/frontend/src/commands/CommandRuntimeContext.jsx new file mode 100644 index 00000000..2e7a63cb --- /dev/null +++ b/frontend/src/commands/CommandRuntimeContext.jsx @@ -0,0 +1,13 @@ +import { createContext, useContext } from 'react'; + +const CommandRuntimeContext = createContext(null); + +export function CommandRuntimeProvider({ runtime, children }) { + return {children}; +} + +export function useCommandRuntimeContext() { + const runtime = useContext(CommandRuntimeContext); + if (!runtime) throw new Error('useCommandRuntimeContext must be used inside CommandRuntimeProvider'); + return runtime; +} diff --git a/frontend/src/commands/CommandRuntimeContext.test.js b/frontend/src/commands/CommandRuntimeContext.test.js new file mode 100644 index 00000000..60a50e35 --- /dev/null +++ b/frontend/src/commands/CommandRuntimeContext.test.js @@ -0,0 +1,12 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +describe('CommandRuntimeContext source contract', () => { + it('exports one provider and one strict consumer hook', () => { + const source = fs.readFileSync(new URL('./CommandRuntimeContext.jsx', import.meta.url), 'utf8'); + assert.match(source, /export function CommandRuntimeProvider/); + assert.match(source, /export function useCommandRuntimeContext/); + assert.match(source, /must be used inside CommandRuntimeProvider/); + }); +}); diff --git a/frontend/src/commands/appCommands.js b/frontend/src/commands/appCommands.js new file mode 100644 index 00000000..783e0ce5 --- /dev/null +++ b/frontend/src/commands/appCommands.js @@ -0,0 +1,119 @@ +import { mailCommandDefinitions } from './mailActions.js'; +import { shortcutCommandDefinitions } from './shortcutCommands.js'; +import { + composeSessionCommandDefinitions, + createComposeSessionCommandExecutors, +} from './composeSessionCommands.js'; + +const globalCommand = (id, titleKey, icon, group, executorId, params = {}, overrides = {}) => ({ + id, titleKey, aliasKeys: [], icon, group, + defaultKeys: { primary: null, secondary: [] }, + rank: { base: 50 }, isAvailable: () => true, targetMode: 'global', executorId, params, + ...overrides, +}); + +const settingsTabs = ['accounts', 'notifications', 'rules', 'categories', 'appearance', 'shortcuts', + 'security', 'integrations', 'ai-actions', 'about']; +const adminTabs = ['users', 'sso', 'ai']; +const gtdSections = ['todo', 'watch', 'delegated', 'reference', 'someday']; + +export function createAppCommandDefinitions({ accounts = [], folders = {}, themes = {}, user = {} }) { + const definitions = [ + globalCommand('navigation.search', 'commands.navigation.search.title', 'search', 'navigation', 'navigation.search', {}, { + defaultKeys: { primary: '/', secondary: [] }, rank: { base: 90 }, + isAvailable: context => context.surface !== 'compose' && context.surface !== 'settings', + }), + globalCommand('navigation.unified-inbox', 'commands.navigation.unifiedInbox.title', 'inbox', 'navigation', 'navigation.inbox', { + accountId: null, folder: 'INBOX', + }), + ]; + + for (const account of accounts) { + definitions.push(globalCommand( + `navigation.account-inbox.${account.id}`, + 'commands.navigation.accountInbox.title', + 'inbox', 'navigation', 'navigation.inbox', + { accountId: account.id, folder: 'INBOX', name: account.name || account.email_address }, + )); + for (const folder of folders[account.id] || []) { + if (folder.path === 'INBOX') continue; + definitions.push(globalCommand( + `navigation.folder.${account.id}.${folder.path}`, + 'commands.navigation.folder.title', + 'folder', 'navigation', 'navigation.folder', + { accountId: account.id, folder: folder.path, name: folder.name || folder.path }, + )); + } + } + + for (const section of gtdSections) { + definitions.push(globalCommand( + `navigation.gtd.${section}`, + `commands.navigation.gtd.${section}.title`, + 'gtd', 'navigation', 'navigation.gtd', { section }, + { isAvailable: context => context.gtdAvailable }, + )); + } + for (const [theme, value] of Object.entries(themes)) { + definitions.push(globalCommand( + `appearance.theme.${theme}`, + 'commands.appearance.theme.title', + 'appearance', 'appearance', 'appearance.theme', { theme, name: value.label || theme }, + )); + } + for (const tab of [...settingsTabs, ...(user.isAdmin ? adminTabs : [])]) { + definitions.push(globalCommand( + `settings.${tab}`, + `commands.settings.${tab}.title`, + 'settings', 'settings', 'settings.open', { tab }, + )); + } + definitions.push(...composeSessionCommandDefinitions); + definitions.push(...mailCommandDefinitions); + definitions.push(...shortcutCommandDefinitions.filter(command => command.id !== 'compose.send')); + return definitions; +} + +export function createAppCommandExecutors({ getState, emitShortcut }) { + return Object.freeze({ + ...createComposeSessionCommandExecutors({ + getController: () => getState().composeWorkspaceController, + openCompose: changes => getState().openCompose(changes), + }), + 'navigation.search': () => { + emitShortcut('focusSearch'); + return { status: 'success' }; + }, + 'navigation.contacts': () => { + const state = getState(); + state.setSelectedMessage(null); + state.clearSelectedMessageIds(); + state.setShowContacts(true); + return { status: 'success' }; + }, + 'navigation.inbox': ({ command }) => { + getState().setSelectedAccount(command.params.accountId, 'INBOX'); + return { status: 'success' }; + }, + 'navigation.folder': ({ command }) => { + getState().setSelectedAccount(command.params.accountId, command.params.folder); + return { status: 'success' }; + }, + 'navigation.gtd': ({ command }) => { + const state = getState(); + state.setSelectedAccount(state.selectedAccountId, 'INBOX'); + state.setActiveGtdTab(command.params.section); + return { status: 'success' }; + }, + 'appearance.theme': ({ command }) => { + getState().setTheme(command.params.theme); + return { status: 'success' }; + }, + 'settings.open': ({ command }) => { + const state = getState(); + state.setAdminTab(command.params.tab); + state.setShowAdmin(true); + return { status: 'success' }; + }, + }); +} diff --git a/frontend/src/commands/appCommands.test.js b/frontend/src/commands/appCommands.test.js new file mode 100644 index 00000000..d4185578 --- /dev/null +++ b/frontend/src/commands/appCommands.test.js @@ -0,0 +1,83 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createCommandContext } from './contracts.js'; +import { createCommandRegistry } from './registry.js'; +import { createAppCommandDefinitions, createAppCommandExecutors } from './appCommands.js'; + +const snapshot = { + accounts: [{ id: 'acct-1', name: 'Work', gtd_enabled: true }], + folders: { 'acct-1': [{ path: 'INBOX', name: 'Inbox' }, { path: 'Projects', name: 'Projects' }] }, + themes: { system: { label: 'System' }, midnight: { label: 'Midnight' } }, + user: { isAdmin: false }, +}; +const context = createCommandContext({ + surface: 'list', activeConversationId: null, selectedConversationIds: [], conversations: [], + accountId: 'acct-1', folder: 'INBOX', draft: null, gtdAvailable: true, + cardDavConnected: false, modal: null, editing: false, platform: 'mac', shortcutOverrides: {}, + translate: (key, values) => values?.name || key, +}); + +describe('application commands', () => { + it('builds application destinations and the shared mail command set', () => { + const ids = createAppCommandDefinitions(snapshot).map(command => command.id); + for (const id of [ + 'compose.new', 'navigation.search', 'navigation.contacts', 'navigation.unified-inbox', + 'navigation.account-inbox.acct-1', 'navigation.folder.acct-1.Projects', + 'navigation.gtd.todo', 'appearance.theme.system', 'settings.accounts', 'settings.shortcuts', + 'mail.archive', 'mail.move', 'mail.replyAll', 'mail.toggleRead', 'gtd.todo', + ]) assert.ok(ids.includes(id), `missing ${id}`); + }); + + it('keeps administrator-only settings absent for non-admin users', () => { + const userIds = createAppCommandDefinitions(snapshot).map(command => command.id); + assert.equal(userIds.includes('settings.users'), false); + const adminIds = createAppCommandDefinitions({ ...snapshot, user: { isAdmin: true } }).map(command => command.id); + assert.equal(adminIds.includes('settings.users'), true); + assert.equal(adminIds.includes('settings.sso'), true); + assert.equal(adminIds.includes('settings.ai'), true); + }); + + it('searches account/folder labels through localized dynamic keys', () => { + const registry = createCommandRegistry(createAppCommandDefinitions(snapshot)); + assert.equal(registry.search('Projects', context)[0].command.id, 'navigation.folder.acct-1.Projects'); + }); + + it('routes execution through injected state services', async () => { + const calls = []; + const state = { + openCompose: value => calls.push(['compose', value]), + setSelectedAccount: (accountId, folder) => calls.push(['navigate', accountId, folder]), + setShowContacts: value => calls.push(['contacts', value]), + setActiveGtdTab: value => calls.push(['gtd', value]), + setAdminTab: value => calls.push(['tab', value]), + setShowAdmin: value => calls.push(['admin', value]), + setTheme: value => calls.push(['theme', value]), + }; + const executors = createAppCommandExecutors({ getState: () => state, emitShortcut: id => calls.push(['shortcut', id]) }); + await executors['navigation.folder']({ command: { params: { accountId: 'acct-1', folder: 'Projects' } } }); + await executors['settings.open']({ command: { params: { tab: 'appearance' } } }); + await executors['navigation.search']({ command: { params: {} } }); + assert.deepEqual(calls, [ + ['navigate', 'acct-1', 'Projects'], ['tab', 'appearance'], ['admin', true], ['shortcut', 'focusSearch'], + ]); + }); + + it('selects a GTD destination after navigation clears the previous tab', async () => { + const state = { + selectedAccountId: 'acct-1', + activeGtdTab: 'watch', + setSelectedAccount(accountId, folder) { + this.selectedAccountId = accountId; + this.selectedFolder = folder; + this.activeGtdTab = null; + }, + setActiveGtdTab(section) { + this.activeGtdTab = section; + }, + }; + const executors = createAppCommandExecutors({ getState: () => state, emitShortcut() {} }); + await executors['navigation.gtd']({ command: { params: { section: 'delegated' } } }); + assert.equal(state.selectedFolder, 'INBOX'); + assert.equal(state.activeGtdTab, 'delegated'); + }); +}); diff --git a/frontend/src/commands/appContext.js b/frontend/src/commands/appContext.js new file mode 100644 index 00000000..9808f821 --- /dev/null +++ b/frontend/src/commands/appContext.js @@ -0,0 +1,119 @@ +import { createCommandContext, stableConversationId } from './contracts.js'; +import { normalizeLegacyShortcutOverrides } from '../utils/defaultShortcuts.js'; + +export function detectCommandPlatform(navigatorLike = {}) { + const value = navigatorLike.userAgentData?.platform || navigatorLike.platform || ''; + if (/mac|iphone|ipad|ipod/i.test(value)) return 'mac'; + if (/win/i.test(value)) return 'windows'; + return 'linux'; +} + +function allConversations(state) { + const gtd = Object.values(state.gtdSections || {}).flatMap(section => section?.threads || []); + return [ + ...(state.messages || []), + ...(state.searchResults || []), + ...Object.values(state.threadMessages || {}).flat(), + ...gtd, + ].filter((message, index, all) => { + const id = stableConversationId(message); + return id && all.findIndex(candidate => stableConversationId(candidate) === id) === index; + }); +} + +export function buildAppCommandContext(state, { + translate, + platform = detectCommandPlatform(globalThis.navigator), + editing = false, + modal = null, +} = {}) { + const conversations = allConversations(state); + const visibleMessages = state.searchQuery?.trim() + ? (state.searchResults || []) + : state.activeGtdTab + ? (state.gtdSections?.[state.activeGtdTab]?.threads || []) + : (state.messages || []); + const selectedRows = new Set(state.selectedMessageIds || []); + const selectedConversationIds = conversations + .filter(message => selectedRows.has(message.id)) + .map(stableConversationId); + const selected = conversations.find(message => message.id === state.selectedMessageId); + const listCursor = visibleMessages.find(message => message.id === state.lastViewedMessageId); + const active = state.showContacts ? null : (selected || listCursor); + const targetedMessages = selectedConversationIds.length + ? conversations.filter(message => selectedConversationIds.includes(stableConversationId(message))) + : active ? [active] : []; + const targetedAccountIds = [...new Set(targetedMessages.map(message => message.account_id).filter(Boolean))]; + const accounts = state.accounts || []; + const gtdAvailable = targetedAccountIds.length + ? targetedAccountIds.every(id => accounts.find(account => account.id === id)?.gtd_enabled) + : state.selectedAccountId + ? Boolean(accounts.find(account => account.id === state.selectedAccountId)?.gtd_enabled) + : accounts.some(account => account.gtd_enabled); + const composeSessions = Array.isArray(state.composeWorkspaceController?.sessions) + ? state.composeWorkspaceController.sessions + : state.composeWorkspaceController?.getSnapshot?.()?.sessions || []; + const controllerFocus = state.composeWorkspaceController?.focusedSessionId; + const focusedComposeSessionId = controllerFocus === undefined + ? state.focusedComposeSessionId + : controllerFocus; + const focusedComposeSession = focusedComposeSessionId + ? composeSessions.find(session => session.id === focusedComposeSessionId) + : null; + const visibleComposeSessionIds = new Set( + state.composeWorkspaceController?.getSnapshot?.()?.visibleSessions?.map(session => session.id) || [], + ); + const surface = modal ? 'picker' + : state.showAdmin || state.showContacts ? 'settings' + : focusedComposeSession ? 'compose' + : state.selectedMessageId ? 'conversation' : 'list'; + const shortcutOverrides = normalizeLegacyShortcutOverrides(state.shortcuts || {}); + + return createCommandContext({ + surface, + activeConversationId: stableConversationId(active), + activeMessage: active || null, + selectedConversationIds, + visibleConversationIds: state.showContacts ? [] : visibleMessages.map(stableConversationId).filter(Boolean), + conversations, + accountId: state.selectedAccountId, + folder: state.selectedFolder, + draft: focusedComposeSession ? { + id: focusedComposeSession.id, + slot: focusedComposeSession.slot, + revision: focusedComposeSession.baseRevision ?? focusedComposeSession.revision, + } : null, + composeSlots: composeSessions.map(session => ({ + id: session.id, + slot: session.slot, + presentationState: session.presentationState, + createdAt: session.createdAt, + lastFocusedAt: session.lastFocusedAt, + status: session.status, + terminalPending: session.terminalPending || null, + visible: visibleComposeSessionIds.size + ? visibleComposeSessionIds.has(session.id) + : session.presentationState !== 'minimized', + })), + gtdAvailable, + cardDavConnected: Boolean(state.carddavStatus?.connected), + carddavStatus: state.carddavStatus, + carddavStatusLoaded: state.carddavStatusLoaded, + modal, + editing: editing || Boolean(focusedComposeSession) || Boolean(state.showAdmin) || Boolean(state.showContacts), + undoAvailable: (state.notifications || []).some(notification => typeof notification.onUndo === 'function'), + platform, + shortcutOverrides, + translate, + }); +} + +export function commandTargetLabel(context) { + if (context.selectedConversationIds.length > 1) { + return { key: 'commandPalette.target.selected', values: { count: context.selectedConversationIds.length } }; + } + if (context.selectedConversationIds.length === 1 || context.activeConversationId) { + return { key: 'commandPalette.target.conversation', values: {} }; + } + return { key: 'commandPalette.target.application', values: {} }; +} diff --git a/frontend/src/commands/appContext.test.js b/frontend/src/commands/appContext.test.js new file mode 100644 index 00000000..097114c8 --- /dev/null +++ b/frontend/src/commands/appContext.test.js @@ -0,0 +1,124 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { buildAppCommandContext, commandTargetLabel, detectCommandPlatform } from './appContext.js'; + +const state = overrides => ({ + messages: [ + { id: 'row-a', message_id: '', account_id: 'acct-1' }, + { id: 'row-b', message_id: '', account_id: 'acct-1' }, + ], + searchResults: [], searchQuery: '', threadMessages: {}, gtdSections: null, + selectedMessageId: null, selectedMessageIds: new Set(), selectedAccountId: 'acct-1', + selectedFolder: 'INBOX', composeWorkspaceController: null, + focusedComposeSessionId: null, showAdmin: false, + accounts: [{ id: 'acct-1', gtd_enabled: true }], + carddavStatus: { connected: true }, carddavStatusLoaded: true, + notifications: [], activeGtdTab: null, + shortcuts: {}, ...overrides, +}); +const translate = (key, values) => values?.count == null ? key : `${key}:${values.count}`; + +describe('buildAppCommandContext', () => { + it('maps list, conversation, compose, and settings surfaces', () => { + assert.equal(buildAppCommandContext(state(), { translate, platform: 'mac' }).surface, 'list'); + assert.equal(buildAppCommandContext(state({ selectedMessageId: 'row-a' }), { translate, platform: 'mac' }).surface, 'conversation'); + const composeWorkspaceController = { + sessions: [{ id: 'draft-1', slot: 2, revision: 7 }], + }; + const composeContext = buildAppCommandContext(state({ + composeWorkspaceController, + focusedComposeSessionId: 'draft-1', + }), { translate, platform: 'mac' }); + assert.equal(composeContext.surface, 'compose'); + assert.deepEqual(composeContext.draft, { id: 'draft-1', slot: 2, revision: 7 }); + assert.deepEqual(composeContext.composeSlots, [{ + id: 'draft-1', slot: 2, presentationState: undefined, + createdAt: undefined, lastFocusedAt: undefined, status: undefined, + terminalPending: null, visible: true, + }]); + assert.equal(buildAppCommandContext(state({ showAdmin: true }), { translate, platform: 'mac' }).surface, 'settings'); + }); + + it('requires the focused compose ID to resolve to an actual controller session', () => { + const composeWorkspaceController = { + sessions: [{ id: 'draft-1', slot: 2, revision: 7 }], + }; + const listContext = buildAppCommandContext(state({ + composeWorkspaceController, + focusedComposeSessionId: 'missing-draft', + }), { translate, platform: 'mac' }); + assert.equal(listContext.surface, 'list'); + assert.equal(listContext.draft, null); + + const unfocusedContext = buildAppCommandContext(state({ + composeWorkspaceController, + focusedComposeSessionId: null, + }), { translate, platform: 'mac' }); + assert.equal(unfocusedContext.surface, 'list'); + assert.equal(unfocusedContext.draft, null); + }); + + it('converts selected row IDs to account-scoped RFC identities and exposes integrations', () => { + const context = buildAppCommandContext(state({ selectedMessageIds: new Set(['row-a', 'row-b']) }), { + translate, platform: 'linux', + }); + assert.deepEqual(context.selectedConversationIds, ['acct-1:', 'acct-1:']); + assert.deepEqual(context.visibleConversationIds, ['acct-1:', 'acct-1:']); + assert.equal(context.gtdAvailable, true); + assert.equal(context.cardDavConnected, true); + assert.deepEqual(context.carddavStatus, { connected: true }); + assert.equal(context.carddavStatusLoaded, true); + }); + + it('requires every targeted account to support GTD and never reads CardDAV from accounts', () => { + const context = buildAppCommandContext(state({ + messages: [ + { id: 'row-a', message_id: '', account_id: 'acct-1' }, + { id: 'row-c', message_id: '', account_id: 'acct-2' }, + ], + selectedMessageIds: new Set(['row-a', 'row-c']), + selectedAccountId: null, + accounts: [{ id: 'acct-1', gtd_enabled: true }, { id: 'acct-2', gtd_enabled: false }], + carddavStatus: null, + }), { translate, platform: 'linux' }); + assert.equal(context.gtdAvailable, false); + assert.equal(context.cardDavConnected, false); + assert.equal(context.carddavStatusLoaded, true); + }); + + it('exposes active message and current undo availability', () => { + const context = buildAppCommandContext(state({ + selectedMessageId: 'row-a', notifications: [{ id: 'undo-1', onUndo() {} }], + }), { translate, platform: 'mac' }); + assert.equal(context.activeMessage.id, 'row-a'); + assert.equal(context.undoAvailable, true); + }); + + it('describes application, single, and bulk targets', () => { + assert.deepEqual(commandTargetLabel(buildAppCommandContext(state(), { translate, platform: 'mac' })), { + key: 'commandPalette.target.application', values: {}, + }); + assert.equal(commandTargetLabel(buildAppCommandContext(state({ selectedMessageId: 'row-a' }), { + translate, platform: 'mac', + })).key, 'commandPalette.target.conversation'); + assert.deepEqual(commandTargetLabel(buildAppCommandContext(state({ selectedMessageIds: new Set(['row-a', 'row-b']) }), { + translate, platform: 'mac', + })), { key: 'commandPalette.target.selected', values: { count: 2 } }); + }); + + it('detects all three supported platform values', () => { + assert.equal(detectCommandPlatform({ userAgentData: { platform: 'macOS' } }), 'mac'); + assert.equal(detectCommandPlatform({ userAgentData: { platform: 'Windows' } }), 'windows'); + assert.equal(detectCommandPlatform({ platform: 'Linux x86_64' }), 'linux'); + }); + + it('maps existing persisted shortcut keys without mutating stored preferences', () => { + const shortcuts = { compose: 'q', focusSearch: 'ctrl+f', goInbox: 'g u' }; + const context = buildAppCommandContext(state({ shortcuts }), { translate, platform: 'linux' }); + assert.deepEqual(context.shortcutOverrides, { + compose: 'q', focusSearch: 'ctrl+f', goInbox: 'g u', + 'compose.new': 'q', 'navigation.search': 'ctrl+f', 'navigation.inbox': 'g u', + }); + assert.deepEqual(shortcuts, { compose: 'q', focusSearch: 'ctrl+f', goInbox: 'g u' }); + }); +}); diff --git a/frontend/src/commands/composeSessionCommands.js b/frontend/src/commands/composeSessionCommands.js new file mode 100644 index 00000000..cf83a01e --- /dev/null +++ b/frontend/src/commands/composeSessionCommands.js @@ -0,0 +1,143 @@ +const keys = (primary = null, secondary = []) => Object.freeze({ + primary, + secondary: Object.freeze(secondary), +}); +const modKey = (mac, key) => Object.freeze({ + mac, + windows: `ctrl+${key}`, + linux: `ctrl+${key}`, + default: `ctrl+${key}`, +}); + +const lifecycleAvailable = context => { + const sessionId = context.draft?.id; + const sessions = context.composeSlots || []; + if (!sessionId) return false; + const session = sessions.find(item => item.id === sessionId); + return Boolean(session) && !session.terminalPending; +}; + +const occupiedSlot = (context, slot) => (context.composeSlots || []) + .some(session => session.slot === slot && !session.terminalPending); + +function definition({ + id, + titleKey, + icon = 'compose', + defaultKeys = keys(), + rank = 80, + isAvailable, + executorId, + params = {}, + aliasKeys = [], + targetMode = 'global', +}) { + return Object.freeze({ + id, + titleKey, + aliasKeys: Object.freeze(aliasKeys), + icon, + group: 'compose', + defaultKeys, + rank: Object.freeze({ base: rank }), + isAvailable, + targetMode, + executorId, + params: Object.freeze(params), + }); +} + +const lifecycle = (id, titleKey, executorId, rank, defaultKeys) => definition({ + id, + titleKey, + executorId, + rank, + isAvailable: lifecycleAvailable, + targetMode: 'draft', + ...(defaultKeys ? { defaultKeys } : {}), +}); + +const slotDefinitions = Array.from({ length: 9 }, (_, index) => { + const slot = index + 1; + return definition({ + id: `compose.activateSlot${slot}`, + titleKey: 'commands.compose.activateSlot', + executorId: 'compose.activateSlot', + params: { slot }, + rank: 70 - slot, + isAvailable: context => occupiedSlot(context, slot), + }); +}); + +export const composeSessionCommandDefinitions = Object.freeze([ + definition({ + id: 'compose.new', + titleKey: 'commands.compose.new.title', + executorId: 'compose.create', + defaultKeys: keys('c'), + aliasKeys: ['commands.compose.new.alias.write'], + rank: 100, + isAvailable: context => !['settings', 'picker'].includes(context.surface) + && new Set((context.composeSlots || []).map(session => session.slot)).size < 9, + }), + lifecycle('compose.minimize', 'commands.compose.minimize', 'compose.minimize', 85), + lifecycle('compose.close', 'commands.compose.close', 'compose.close', 84), + lifecycle('compose.discard', 'commands.compose.discard', 'compose.discard', 50), + lifecycle('compose.send', 'compose.send', 'compose.send', 90, keys(modKey('meta+enter', 'enter'))), + ...slotDefinitions, +]); + +const success = value => ({ status: 'success', value }); + +function requestedSession(context, input, fallback = 'recent') { + const sessionId = input?.sessionId || context?.draft?.id; + const sessions = context?.composeSlots || []; + if (sessionId && sessions.some(session => session.id === sessionId)) return sessionId; + const visible = sessions.filter(session => session.visible !== false + && session.presentationState !== 'minimized'); + if (fallback === 'leftmost') { + return [...visible].sort((left, right) => Date.parse(left.createdAt || 0) - Date.parse(right.createdAt || 0) + || left.slot - right.slot)[0]?.id || null; + } + return [...visible].sort((left, right) => Date.parse(right.lastFocusedAt || 0) - Date.parse(left.lastFocusedAt || 0) + || right.slot - left.slot)[0]?.id || null; +} + +export function createComposeSessionCommandExecutors({ getController, openCompose }) { + const controller = () => getController?.() || null; + const runSession = (method, fallback) => async ({ context, input } = {}) => { + const workspace = controller(); + const sessionId = requestedSession(context, input, fallback); + if (!workspace?.[method] || !sessionId) return { status: 'cancelled' }; + const value = await workspace[method](sessionId); + return success(value); + }; + + return Object.freeze({ + 'compose.create': async ({ context, input } = {}) => { + const changes = input || (context?.accountId ? { accountId: context.accountId } : {}); + if (openCompose) return success(await openCompose(changes)); + const workspace = controller(); + if (!workspace?.createSession) return { status: 'cancelled' }; + return success(await workspace.createSession(changes)); + }, + 'compose.minimize': runSession('minimizeSession', 'leftmost'), + 'compose.close': runSession('closeSession'), + 'compose.discard': runSession('discardSession'), + 'compose.send': async ({ context, input } = {}) => { + const workspace = controller(); + const sessionId = requestedSession(context, input); + if (!workspace?.sendSession || !sessionId) return { status: 'cancelled' }; + const options = { ...(input || {}) }; + delete options.sessionId; + return success(await workspace.sendSession(sessionId, options)); + }, + 'compose.activateSlot': async ({ command, context } = {}) => { + const workspace = controller(); + const session = (context?.composeSlots || []) + .find(item => item.slot === command?.params?.slot && !item.terminalPending); + if (!workspace?.focusSession || !session) return { status: 'cancelled' }; + return success(await workspace.focusSession(session.id)); + }, + }); +} diff --git a/frontend/src/commands/composeSessionCommands.test.js b/frontend/src/commands/composeSessionCommands.test.js new file mode 100644 index 00000000..a8d3d1b5 --- /dev/null +++ b/frontend/src/commands/composeSessionCommands.test.js @@ -0,0 +1,193 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import i18next from 'i18next'; +import { createCommandRegistry } from './registry.js'; +import { + composeSessionCommandDefinitions, + createComposeSessionCommandExecutors, +} from './composeSessionCommands.js'; + +function deferred() { + let resolve; + let reject; + const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); + return { promise, resolve, reject }; +} + +const context = ({ slots = [], draft = null, surface = 'list' } = {}) => ({ + surface, + draft, + composeSlots: slots, + accountId: null, + activeConversationId: null, + selectedConversationIds: [], + conversationsById: {}, + platform: 'linux', + shortcutOverrides: {}, + translate: key => key, +}); + +describe('compose session commands', () => { + it('defines one stable command for every compose action and slot', () => { + assert.deepEqual(composeSessionCommandDefinitions.map(command => command.id), [ + 'compose.new', 'compose.minimize', 'compose.close', 'compose.discard', 'compose.send', + ...Array.from({ length: 9 }, (_, index) => `compose.activateSlot${index + 1}`), + ]); + assert.equal(new Set(composeSessionCommandDefinitions).size, 14); + assert.ok(composeSessionCommandDefinitions.every(Object.isFrozen)); + for (const id of ['compose.minimize', 'compose.close', 'compose.discard', 'compose.send']) { + assert.equal( + composeSessionCommandDefinitions.find(command => command.id === id).targetMode, + 'draft', + ); + } + assert.deepEqual( + composeSessionCommandDefinitions.find(command => command.id === 'compose.send').defaultKeys.primary, + { mac: 'meta+enter', windows: 'ctrl+enter', linux: 'ctrl+enter', default: 'ctrl+enter' }, + ); + }); + + it('offers create below capacity and omits it at nine occupied slots', () => { + const registry = createCommandRegistry(composeSessionCommandDefinitions); + const eight = Array.from({ length: 8 }, (_, index) => ({ id: `session-${index + 1}`, slot: index + 1 })); + assert.ok(registry.list(context({ slots: eight })).some(result => result.command.id === 'compose.new')); + const nine = [...eight, { id: 'session-9', slot: 9 }]; + assert.equal(registry.list(context({ slots: nine })).some(result => result.command.id === 'compose.new'), false); + }); + + it('offers activation only for occupied slots', () => { + const registry = createCommandRegistry(composeSessionCommandDefinitions); + const ids = registry.list(context({ + slots: [{ id: 'session-2', slot: 2 }, { id: 'session-7', slot: 7 }], + })).map(result => result.command.id); + assert.ok(ids.includes('compose.activateSlot2')); + assert.ok(ids.includes('compose.activateSlot7')); + assert.equal(ids.includes('compose.activateSlot1'), false); + assert.equal(ids.includes('compose.activateSlot9'), false); + }); + + it('renders an occupied slot title through configured i18next interpolation', async () => { + const translation = JSON.parse(readFileSync( + new URL('../locales/en.json', import.meta.url), 'utf8', + )); + const instance = i18next.createInstance(); + await instance.init({ + resources: { en: { translation } }, + lng: 'en', + fallbackLng: 'en', + interpolation: { escapeValue: false }, + }); + const registry = createCommandRegistry(composeSessionCommandDefinitions); + const commandContext = context({ slots: [{ id: 'session-7', slot: 7 }] }); + commandContext.translate = instance.t.bind(instance); + const title = registry.list(commandContext) + .find(result => result.command.id === 'compose.activateSlot7')?.title; + assert.equal(title, 'Activate draft 7'); + assert.doesNotMatch(title, /[{}]|/); + }); + + it('withholds terminal-pending slot activation and restores it after completion', () => { + const registry = createCommandRegistry(composeSessionCommandDefinitions); + const ids = slots => registry.list(context({ slots })).map(result => result.command.id); + assert.equal(ids([{ id: 'session-7', slot: 7, terminalPending: 'send' }]) + .includes('compose.activateSlot7'), false); + assert.equal(ids([{ id: 'session-7', slot: 7, terminalPending: null }]) + .includes('compose.activateSlot7'), true); + }); + + it('uses the focused draft for lifecycle commands and freezes them while terminal work is pending', () => { + const registry = createCommandRegistry(composeSessionCommandDefinitions); + const cleanSlot = { id: 'session-4', slot: 4, terminalPending: null }; + const available = registry.list(context({ + slots: [cleanSlot], draft: { id: cleanSlot.id, slot: cleanSlot.slot }, surface: 'compose', + })).map(result => result.command.id); + for (const id of ['compose.minimize', 'compose.close', 'compose.discard', 'compose.send']) { + assert.ok(available.includes(id), `missing ${id}`); + } + + const frozen = registry.list(context({ + slots: [{ ...cleanSlot, terminalPending: 'send' }], + draft: { id: cleanSlot.id, slot: cleanSlot.slot }, + surface: 'compose', + })).map(result => result.command.id); + for (const id of ['compose.minimize', 'compose.close', 'compose.discard', 'compose.send']) { + assert.equal(frozen.includes(id), false, `${id} must freeze`); + } + }); + + it('routes every executor exclusively through the workspace controller', async () => { + const calls = []; + const controller = { + createSession: value => { calls.push(['create', value]); return { id: 'created' }; }, + minimizeSession: id => { calls.push(['minimize', id]); return { id }; }, + closeSession: id => { calls.push(['close', id]); return { id }; }, + discardSession: id => { calls.push(['discard', id]); return { id }; }, + sendSession: (id, options) => { calls.push(['send', id, options]); return { id }; }, + focusSession: id => { calls.push(['focus', id]); return { id }; }, + }; + const executors = createComposeSessionCommandExecutors({ getController: () => controller }); + const draftContext = context({ + slots: [{ id: 'session-3', slot: 3 }], + draft: { id: 'session-3', slot: 3 }, + surface: 'compose', + }); + assert.equal((await executors['compose.create']({ input: { accountId: 'synthetic-account' } })).status, 'success'); + assert.equal((await executors['compose.minimize']({ context: draftContext })).status, 'success'); + assert.equal((await executors['compose.close']({ context: draftContext })).status, 'success'); + assert.equal((await executors['compose.discard']({ context: draftContext })).status, 'success'); + assert.equal((await executors['compose.send']({ context: draftContext, input: { undoSeconds: 5 } })).status, 'success'); + assert.equal((await executors['compose.activateSlot']({ + context: draftContext, + command: { params: { slot: 3 } }, + })).status, 'success'); + assert.deepEqual(calls, [ + ['create', { accountId: 'synthetic-account' }], + ['minimize', 'session-3'], + ['close', 'session-3'], + ['discard', 'session-3'], + ['send', 'session-3', { undoSeconds: 5 }], + ['focus', 'session-3'], + ]); + }); + + it('returns cancelled outcomes when a controller or requested session is unavailable', async () => { + const withoutController = createComposeSessionCommandExecutors({ getController: () => null }); + assert.deepEqual(await withoutController['compose.create']({}), { status: 'cancelled' }); + + const executors = createComposeSessionCommandExecutors({ + getController: () => ({ focusSession() { throw new Error('must not execute'); } }), + }); + assert.deepEqual(await executors['compose.activateSlot']({ + context: context({ slots: [] }), command: { params: { slot: 8 } }, + }), { status: 'cancelled' }); + assert.deepEqual(await executors['compose.activateSlot']({ + context: context({ + slots: [{ id: 'session-8', slot: 8, terminalPending: 'discard' }], + }), + command: { params: { slot: 8 } }, + }), { status: 'cancelled' }); + }); + + it('routes compose.new through the readiness-aware store action and awaits its outcome', async () => { + const ready = deferred(); + const calls = []; + const executors = createComposeSessionCommandExecutors({ + getController: () => null, + openCompose: changes => { calls.push(changes); return ready.promise; }, + }); + const pending = executors['compose.create']({ + context: context(), input: { subject: 'Queued command' }, + }); + assert.deepEqual(calls, [{ subject: 'Queued command' }]); + ready.resolve('created-after-ready'); + assert.deepEqual(await pending, { status: 'success', value: 'created-after-ready' }); + + const failure = new Error('synthetic command create failure'); + const rejecting = createComposeSessionCommandExecutors({ + getController: () => null, + openCompose: async () => { throw failure; }, + }); + await assert.rejects(rejecting['compose.create']({ context: context() }), error => error === failure); + }); +}); diff --git a/frontend/src/commands/contextMenuCommands.js b/frontend/src/commands/contextMenuCommands.js new file mode 100644 index 00000000..e3e0f636 --- /dev/null +++ b/frontend/src/commands/contextMenuCommands.js @@ -0,0 +1,31 @@ +const COMMANDS = Object.freeze({ + markRead: 'mail.read', + markUnread: 'mail.unread', + toggleStar: 'mail.toggleStar', + reply: 'mail.reply', + replyAll: 'mail.replyAll', + forward: 'mail.forward', + archive: 'mail.archive', + delete: 'mail.trash', + markSpam: 'mail.spam', + markHam: 'mail.notSpam', +}); + +const SINGLE_CONVERSATION_COMMANDS = new Set(['mail.reply', 'mail.replyAll', 'mail.forward']); + +export function contextMenuTargetMessages(commandId, message, selectedMessages) { + if (SINGLE_CONVERSATION_COMMANDS.has(commandId)) return [message]; + return selectedMessages.length > 1 && selectedMessages.some(candidate => candidate.id === message.id) + ? selectedMessages + : [message]; +} + +export function toContextMenuCommand(action, data) { + if (COMMANDS[action]) return { commandId: COMMANDS[action] }; + if (action === 'moveTo' && data) return { commandId: 'mail.move', input: { folder: data } }; + if (action === 'snooze' && data) return { commandId: 'mail.snooze', input: { until: data } }; + if (action === 'gtdClassify' && ['todo', 'watch', 'delegated', 'someday', 'reference'].includes(data)) { + return { commandId: data === 'delegated' ? 'gtd.delegate' : `gtd.${data}` }; + } + return null; +} diff --git a/frontend/src/commands/contextMenuCommands.test.js b/frontend/src/commands/contextMenuCommands.test.js new file mode 100644 index 00000000..00e8d5dd --- /dev/null +++ b/frontend/src/commands/contextMenuCommands.test.js @@ -0,0 +1,75 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { contextMenuTargetMessages, toContextMenuCommand } from './contextMenuCommands.js'; + +test('maps migrated context actions to command IDs and typed input', () => { + assert.deepEqual(toContextMenuCommand('archive'), { commandId: 'mail.archive' }); + assert.deepEqual(toContextMenuCommand('markRead'), { commandId: 'mail.read' }); + assert.deepEqual(toContextMenuCommand('markUnread'), { commandId: 'mail.unread' }); + assert.deepEqual(toContextMenuCommand('toggleStar'), { commandId: 'mail.toggleStar' }); + assert.deepEqual(toContextMenuCommand('replyAll'), { commandId: 'mail.replyAll' }); + assert.deepEqual(toContextMenuCommand('moveTo', 'Archive'), { + commandId: 'mail.move', input: { folder: 'Archive' }, + }); + assert.deepEqual(toContextMenuCommand('snooze', '2026-08-01T09:00:00.000Z'), { + commandId: 'mail.snooze', input: { until: '2026-08-01T09:00:00.000Z' }, + }); + assert.deepEqual(toContextMenuCommand('gtdClassify', 'todo'), { commandId: 'gtd.todo' }); + assert.deepEqual(toContextMenuCommand('gtdClassify', 'delegated'), { commandId: 'gtd.delegate' }); +}); + +test('returns null for intentionally unmigrated utilities', () => { + assert.equal(toContextMenuCommand('copy'), null); + assert.equal(toContextMenuCommand('createRuleFromMessage'), null); + assert.equal(toContextMenuCommand('setCategory', 'social'), null); +}); + +test('targets the clicked row for responses and the frozen selection for bulk-safe actions', () => { + const clicked = { id: 'clicked' }; + const selected = [{ id: 'first' }, clicked, { id: 'third' }]; + assert.deepEqual(contextMenuTargetMessages('mail.reply', clicked, selected), [clicked]); + assert.deepEqual(contextMenuTargetMessages('mail.forward', clicked, selected), [clicked]); + assert.deepEqual(contextMenuTargetMessages('mail.archive', clicked, selected), selected); + assert.deepEqual(contextMenuTargetMessages('mail.archive', clicked, [{ id: 'first' }]), [clicked]); +}); + +test('routes list, context-menu, bulk, hover, and swipe mail actions through one controller', () => { + const list = fs.readFileSync(new URL('../components/MessageList.jsx', import.meta.url), 'utf8'); + const menu = fs.readFileSync(new URL('../components/ContextMenu.jsx', import.meta.url), 'utf8'); + assert.match(list, /useCommandRuntimeContext\(\)/); + assert.match(list, /actionableMessages\.map\(stableConversationId\)/); + assert.match(list, /source,\s*input,\s*frozenTargetIds/); + assert.match(menu, /toContextMenuCommand\(action, data\)/); + assert.match(menu, /source: 'context-menu'/); + assert.match(menu, /onCommand\(invocation\.commandId, invocation\.input\)/); + assert.match(list, /contextMenuTargetMessages/); + assert.match(list, /executeForMessages\(commandId, 'context-menu', targetMessages, input\)/); + assert.doesNotMatch( + list, + /api\.(bulkArchive|bulkDelete|bulkMove|bulkRead|markStarred|markSpam|markHam|snoozeMessage)/, + ); + assert.doesNotMatch( + list, + /shortcutBus\.on\('(archive|delete|toggleRead|gtdTodo|gtdWatch|gtdDelegated)'/, + ); +}); + +test('routes pane toolbar and menu mail actions through the shared controller', () => { + const pane = fs.readFileSync(new URL('../components/MessagePane.jsx', import.meta.url), 'utf8'); + assert.match(pane, /useCommandRuntimeContext\(\)/); + assert.match(pane, /stableConversationId\(message\)/); + assert.match(pane, /source,\s*input,\s*frozenTargetIds/); + assert.match(pane, /executeForMessage\('gtd\.delegate', 'visible-message-menu'\)/); + assert.match(pane, /account\?\.gtd_enabled && \(\s* { + const list = fs.readFileSync(new URL('../components/MessageList.jsx', import.meta.url), 'utf8'); + assert.match(list, /THREAD_EXPANDING_COMMANDS = new Set\(\[[\s\S]*'gtd\.delegate'/); +}); diff --git a/frontend/src/commands/contracts.js b/frontend/src/commands/contracts.js new file mode 100644 index 00000000..aca5ae51 --- /dev/null +++ b/frontend/src/commands/contracts.js @@ -0,0 +1,136 @@ +export const TARGET_MODES = Object.freeze({ + GLOBAL: 'global', + ACCOUNT: 'account', + DRAFT: 'draft', + SINGLE_CONVERSATION: 'single_conversation', + BULK_SAFE: 'bulk_safe', +}); + +export const SURFACES = Object.freeze(['list', 'conversation', 'compose', 'settings', 'picker']); +export const PLATFORMS = Object.freeze(['mac', 'windows', 'linux']); + +/** @typedef {string | {default?: string, mac?: string, windows?: string, linux?: string}} KeySpec */ + +/** + * @typedef {object} CommandDefinition + * @property {string} id + * @property {string} titleKey + * @property {string[]} aliasKeys + * @property {string} icon + * @property {string} group + * @property {{primary: KeySpec | null, secondary: KeySpec[]}} defaultKeys + * @property {{base: number, boost?: (context: CommandContext) => number}} rank + * @property {(context: CommandContext) => boolean} isAvailable + * @property {'global'|'account'|'draft'|'single_conversation'|'bulk_safe'} targetMode + * @property {string} executorId + * @property {Readonly>} [params] + */ + +/** + * @typedef {object} CommandContext + * @property {'list'|'conversation'|'compose'|'settings'|'picker'} surface + * @property {string | null} activeConversationId + * @property {object | null} activeMessage + * @property {readonly string[]} selectedConversationIds + * @property {readonly string[]} visibleConversationIds + * @property {Readonly>} conversationsById + * @property {string | null} accountId + * @property {string | null} folder + * @property {object | null} draft + * @property {readonly object[]} composeSlots + * @property {boolean} gtdAvailable + * @property {boolean} cardDavConnected + * @property {Readonly} carddavStatus + * @property {boolean} carddavStatusLoaded + * @property {object | null} modal + * @property {boolean} editing + * @property {boolean} undoAvailable + * @property {'mac'|'windows'|'linux'} platform + * @property {Readonly>} shortcutOverrides + * @property {(key: string, values?: object) => string} translate + */ + +function freezeKeySpec(spec) { + return spec && typeof spec === 'object' ? Object.freeze({ ...spec }) : spec; +} + +export function stableConversationId(message) { + const accountId = message?.account_id; + const withinAccountId = message?.message_id || message?.id; + return accountId && withinAccountId ? `${accountId}:${withinAccountId}` : null; +} + +export function validateCommandDefinition(input) { + if (!input?.id?.includes('.')) throw new TypeError('command id must be namespaced'); + for (const key of ['titleKey', 'icon', 'group']) { + if (typeof input[key] !== 'string' || !input[key]) throw new TypeError(`${key} must be a non-empty string`); + } + if (!Array.isArray(input.aliasKeys) || input.aliasKeys.some(key => typeof key !== 'string')) { + throw new TypeError('aliasKeys must be an array of localization keys'); + } + if (!Object.values(TARGET_MODES).includes(input.targetMode)) { + throw new TypeError(`unsupported targetMode "${input.targetMode}"`); + } + if (typeof input.executorId !== 'string' || !input.executorId) { + throw new TypeError('executorId must be a non-empty string'); + } + if (typeof input.isAvailable !== 'function') throw new TypeError('isAvailable must be a function'); + if (!Number.isFinite(input.rank?.base)) throw new TypeError('rank.base must be a finite number'); + if (input.rank.boost != null && typeof input.rank.boost !== 'function') { + throw new TypeError('rank.boost must be a function'); + } + const primary = freezeKeySpec(input.defaultKeys?.primary ?? null); + const secondary = Object.freeze((input.defaultKeys?.secondary || []).map(freezeKeySpec)); + return Object.freeze({ + ...input, + aliasKeys: Object.freeze([...input.aliasKeys]), + defaultKeys: Object.freeze({ primary, secondary }), + rank: Object.freeze({ ...input.rank }), + params: Object.freeze({ ...(input.params || {}) }), + }); +} + +export function createCommandContext(input) { + if (!SURFACES.includes(input.surface)) throw new TypeError(`unsupported surface "${input.surface}"`); + if (!PLATFORMS.includes(input.platform)) throw new TypeError(`unsupported platform "${input.platform}"`); + if (typeof input.translate !== 'function') throw new TypeError('translate must be a function'); + + const conversationsById = {}; + for (const message of input.conversations || []) { + const id = stableConversationId(message); + if (!id || conversationsById[id]) continue; + conversationsById[id] = Object.freeze({ + id, + rowId: message.id, + accountId: message.account_id ?? null, + message, + }); + } + const selectedConversationIds = [...new Set(input.selectedConversationIds || [])]; + const carddavStatus = Object.freeze({ + ...(input.carddavStatus || {}), + connected: input.carddavStatus?.connected === true || input.cardDavConnected === true, + }); + return Object.freeze({ + surface: input.surface, + activeConversationId: input.activeConversationId || null, + activeMessage: input.activeMessage || null, + selectedConversationIds: Object.freeze(selectedConversationIds), + visibleConversationIds: Object.freeze([...new Set(input.visibleConversationIds || [])]), + conversationsById: Object.freeze(conversationsById), + accountId: input.accountId || null, + folder: input.folder || null, + draft: input.draft || null, + composeSlots: Object.freeze((input.composeSlots || []).map(session => Object.freeze({ ...session }))), + gtdAvailable: Boolean(input.gtdAvailable), + cardDavConnected: carddavStatus.connected, + carddavStatus, + carddavStatusLoaded: Boolean(input.carddavStatusLoaded), + modal: input.modal || null, + editing: Boolean(input.editing), + undoAvailable: Boolean(input.undoAvailable), + platform: input.platform, + shortcutOverrides: Object.freeze({ ...(input.shortcutOverrides || {}) }), + translate: input.translate, + }); +} diff --git a/frontend/src/commands/contracts.test.js b/frontend/src/commands/contracts.test.js new file mode 100644 index 00000000..923e132a --- /dev/null +++ b/frontend/src/commands/contracts.test.js @@ -0,0 +1,105 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + TARGET_MODES, + createCommandContext, + stableConversationId, + validateCommandDefinition, +} from './contracts.js'; + +const validDefinition = { + id: 'mail.archive', + titleKey: 'commands.mail.archive.title', + aliasKeys: ['commands.mail.archive.alias.done'], + icon: 'archive', + group: 'mail', + defaultKeys: { primary: 'e', secondary: ['o'] }, + rank: { base: 100, boost: context => context.selectedConversationIds.length ? 25 : 0 }, + isAvailable: context => context.surface !== 'settings', + targetMode: 'bulk_safe', + executorId: 'mail.archive', +}; + +describe('command contracts', () => { + it('uses the five approved target-mode values', () => { + assert.deepEqual(Object.values(TARGET_MODES), [ + 'global', 'account', 'draft', 'single_conversation', 'bulk_safe', + ]); + }); + + it('validates and deeply freezes a complete definition', () => { + const definition = validateCommandDefinition(validDefinition); + assert.equal(definition.id, 'mail.archive'); + assert.ok(Object.isFrozen(definition)); + assert.ok(Object.isFrozen(definition.aliasKeys)); + assert.ok(Object.isFrozen(definition.defaultKeys.secondary)); + assert.throws(() => definition.aliasKeys.push('commands.other')); + }); + + it('rejects missing, unnamespaced, and unsupported fields with exact errors', () => { + assert.throws( + () => validateCommandDefinition({ ...validDefinition, id: 'archive' }), + /command id must be namespaced/, + ); + assert.throws( + () => validateCommandDefinition({ ...validDefinition, targetMode: 'message' }), + /unsupported targetMode "message"/, + ); + assert.throws( + () => validateCommandDefinition({ ...validDefinition, executorId: '' }), + /executorId must be a non-empty string/, + ); + }); + + it('scopes RFC Message-ID and row-id fallback identities by account', () => { + assert.equal(stableConversationId({ account_id: 'acct-1', id: 'row-1', message_id: '' }), 'acct-1:'); + assert.equal(stableConversationId({ account_id: 'acct-2', id: 'row-1', message_id: '' }), 'acct-2:'); + assert.equal(stableConversationId({ account_id: 'acct-1', id: 'row-2' }), 'acct-1:row-2'); + assert.equal(stableConversationId({ id: 'row-2' }), null); + assert.equal(stableConversationId({}), null); + }); + + it('normalizes and freezes a complete command context snapshot', () => { + const context = createCommandContext({ + surface: 'list', + activeConversationId: 'acct-1:', + activeMessage: { id: 'row-a', message_id: '', account_id: 'acct-1' }, + selectedConversationIds: ['acct-1:', 'acct-1:'], + visibleConversationIds: ['acct-1:', 'acct-1:'], + conversations: [ + { id: 'row-a', message_id: '', account_id: 'acct-1' }, + { id: 'row-b', message_id: '', account_id: 'acct-1' }, + ], + accountId: 'acct-1', + folder: 'INBOX', + draft: null, + gtdAvailable: true, + cardDavConnected: false, + modal: null, + editing: false, + undoAvailable: true, + platform: 'mac', + shortcutOverrides: { 'mail.archive': 'x' }, + translate: key => ({ 'commands.mail.archive.title': 'Archive' })[key] || key, + }); + + assert.deepEqual(context.selectedConversationIds, ['acct-1:']); + assert.deepEqual(context.visibleConversationIds, ['acct-1:', 'acct-1:']); + assert.equal(context.conversationsById['acct-1:'].rowId, 'row-a'); + assert.equal(context.activeMessage.id, 'row-a'); + assert.equal(context.undoAvailable, true); + assert.equal(context.translate('commands.mail.archive.title'), 'Archive'); + assert.ok(Object.isFrozen(context)); + assert.ok(Object.isFrozen(context.selectedConversationIds)); + }); + + it('preserves the legacy CardDAV boolean while exposing a frozen status snapshot', () => { + const context = createCommandContext({ + surface: 'list', platform: 'linux', translate: key => key, + cardDavConnected: true, + }); + assert.equal(context.cardDavConnected, true); + assert.deepEqual(context.carddavStatus, { connected: true }); + assert.ok(Object.isFrozen(context.carddavStatus)); + }); +}); diff --git a/frontend/src/commands/controller.js b/frontend/src/commands/controller.js new file mode 100644 index 00000000..e1936bf4 --- /dev/null +++ b/frontend/src/commands/controller.js @@ -0,0 +1,95 @@ +import { hasTargets, resolveTargetIds } from './targets.js'; + +function failed(commandId, error, targetIds = []) { + return { status: 'failed', commandId, targetIds, error: error instanceof Error ? error : new Error(String(error)) }; +} + +function terminalOutcome(commandId, targetIds, result) { + if (!['success', 'cancelled', 'partial', 'failed'].includes(result?.status)) { + return failed(commandId, new Error(`Invalid executor outcome for "${commandId}"`), targetIds); + } + return { commandId, targetIds, ...result }; +} + +export function createCommandController({ + registry, + getContext, + executors, + onContinuation = () => {}, + onOutcome = () => {}, +}) { + const inFlight = new Map(); + + function execute(commandId, { source = 'unknown', input, frozenTargetIds } = {}) { + const initialContext = getContext(); + const command = registry.get(commandId); + if (!command) { + const outcome = failed(commandId, new Error(`Unknown command "${commandId}"`)); + onOutcome(outcome); + return Promise.resolve(outcome); + } + const resumingFrozenTargets = frozenTargetIds != null && input !== undefined; + if ((!resumingFrozenTargets && !command.isAvailable(initialContext)) + || (!frozenTargetIds && !hasTargets(command, initialContext))) { + const outcome = failed(commandId, new Error(`Command "${commandId}" is not available`)); + onOutcome(outcome); + return Promise.resolve(outcome); + } + + const frozen = frozenTargetIds == null + ? resolveTargetIds(command, initialContext).targetIds + : [...new Set(frozenTargetIds)]; + if (command.targetMode === 'single_conversation' && frozen.length !== 1) { + const outcome = failed( + commandId, + new Error(`Command "${commandId}" requires exactly one conversation`), + frozen, + ); + onOutcome(outcome); + return Promise.resolve(outcome); + } + const dedupeKey = `${commandId}:${[...frozen].sort().join('|')}`; + if (inFlight.has(dedupeKey)) return inFlight.get(dedupeKey); + + const promise = (async () => { + const context = getContext(); + const { targetIds, missingTargetIds } = resolveTargetIds(command, context, frozen); + const executor = executors[command.executorId]; + if (!executor) return failed(commandId, new Error(`Missing executor "${command.executorId}"`), targetIds); + if (frozen.length && !targetIds.length) { + return { status: 'partial', commandId, targetIds, missingTargetIds, succeededIds: [], failed: [] }; + } + try { + const result = await executor({ command, context, source, input, targetIds }); + if (result?.status === 'needs_input') { + const continuation = Object.freeze({ + commandId, + kind: result.continuation.kind, + targetIds: Object.freeze([...frozen]), + props: Object.freeze({ ...(result.continuation.props || {}) }), + }); + onContinuation(continuation); + return { status: 'needs_input', continuation }; + } + const outcome = terminalOutcome(commandId, targetIds, result); + if (missingTargetIds.length && outcome.status === 'success') { + return { + status: 'partial', commandId, targetIds, missingTargetIds, + succeededIds: [...targetIds], failed: [], value: outcome.value, + }; + } + return missingTargetIds.length ? { ...outcome, missingTargetIds } : outcome; + } catch (error) { + return failed(commandId, error, targetIds); + } + })().then(outcome => { + if (outcome.status !== 'needs_input') onOutcome(outcome); + return outcome; + }).finally(() => inFlight.delete(dedupeKey)); + + inFlight.set(dedupeKey, promise); + return promise; + } + + return Object.freeze({ execute }); +} diff --git a/frontend/src/commands/controller.test.js b/frontend/src/commands/controller.test.js new file mode 100644 index 00000000..c63d0a21 --- /dev/null +++ b/frontend/src/commands/controller.test.js @@ -0,0 +1,194 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createCommandContext } from './contracts.js'; +import { createCommandRegistry } from './registry.js'; +import { createCommandController } from './controller.js'; + +const definition = (overrides = {}) => ({ + id: 'mail.move', titleKey: 'move', aliasKeys: [], icon: 'move', group: 'mail', + defaultKeys: { primary: 'v', secondary: [] }, rank: { base: 1 }, + isAvailable: () => true, targetMode: 'bulk_safe', executorId: 'mail.move', ...overrides, +}); +const makeContext = (ids = ['acct:', 'acct:']) => createCommandContext({ + surface: 'list', activeConversationId: 'acct:', selectedConversationIds: ids, + conversations: ids.map((id, index) => ({ id: `row-${index}`, message_id: id.slice('acct:'.length), account_id: 'acct' })), + accountId: 'acct', folder: 'INBOX', draft: null, gtdAvailable: false, + cardDavConnected: false, modal: null, editing: false, platform: 'mac', + shortcutOverrides: {}, translate: key => key, +}); + +describe('createCommandController', () => { + it('rejects unknown, unavailable, and missing executors as failed outcomes', async () => { + const registry = createCommandRegistry([definition({ isAvailable: () => false })]); + const controller = createCommandController({ registry, getContext: () => makeContext(), executors: {} }); + assert.match((await controller.execute('missing.command')).error.message, /Unknown command/); + assert.match((await controller.execute('mail.move')).error.message, /not available/); + + const available = createCommandRegistry([definition()]); + const noExecutor = createCommandController({ registry: available, getContext: () => makeContext(), executors: {} }); + assert.match((await noExecutor.execute('mail.move')).error.message, /Missing executor/); + }); + + it('freezes targets into a continuation and reuses them on resume', async () => { + let context = makeContext(); + const calls = []; + const continuations = []; + const registry = createCommandRegistry([definition()]); + const controller = createCommandController({ + registry, + getContext: () => context, + executors: { + 'mail.move': args => { + calls.push(args); + return args.input + ? { status: 'success', value: { folder: args.input.folder } } + : { status: 'needs_input', continuation: { kind: 'move', props: { titleKey: 'move.title' } } }; + }, + }, + onContinuation: value => continuations.push(value), + }); + + const first = await controller.execute('mail.move', { source: 'palette' }); + assert.deepEqual(first.continuation, { + commandId: 'mail.move', kind: 'move', targetIds: ['acct:', 'acct:'], props: { titleKey: 'move.title' }, + }); + assert.deepEqual(continuations, [first.continuation]); + + context = makeContext(['acct:']); + const resumed = await controller.execute('mail.move', { + source: 'palette', input: { folder: 'Archive' }, frozenTargetIds: first.continuation.targetIds, + }); + assert.equal(resumed.status, 'partial'); + assert.deepEqual(resumed.targetIds, ['acct:']); + assert.deepEqual(resumed.missingTargetIds, ['acct:']); + assert.deepEqual(calls[1].targetIds, ['acct:']); + }); + + it('resumes frozen targets after the current selection becomes unavailable', async () => { + let context = makeContext(['acct:']); + const registry = createCommandRegistry([definition({ + isAvailable: current => current.gtdAvailable, + })]); + context = Object.freeze({ ...context, gtdAvailable: true }); + const controller = createCommandController({ + registry, + getContext: () => context, + executors: { 'mail.move': () => ({ status: 'success' }) }, + }); + const frozenTargetIds = ['acct:']; + context = Object.freeze({ ...context, gtdAvailable: false }); + const outcome = await controller.execute('mail.move', { + source: 'continuation', input: { contactId: 'contact-1' }, frozenTargetIds, + }); + assert.equal(outcome.status, 'success'); + assert.deepEqual(outcome.targetIds, frozenTargetIds); + }); + + it('deduplicates only concurrent execution of the same command and target set', async () => { + let release; + let callCount = 0; + const gate = new Promise(resolve => { release = resolve; }); + const registry = createCommandRegistry([definition()]); + const controller = createCommandController({ + registry, getContext: () => makeContext(), + executors: { 'mail.move': async () => { callCount += 1; await gate; return { status: 'success' }; } }, + }); + const one = controller.execute('mail.move'); + const two = controller.execute('mail.move'); + assert.strictEqual(one, two); + release(); + await one; + await controller.execute('mail.move'); + assert.equal(callCount, 2); + }); + + it('deduplicates draft commands per focused compose session', async () => { + const releases = new Map(); + const calls = []; + let draftId = 'draft-a'; + const draftDefinition = definition({ + id: 'compose.close', targetMode: 'draft', executorId: 'compose.close', + }); + const draftContext = () => createCommandContext({ + surface: 'compose', activeConversationId: null, selectedConversationIds: [], + conversations: [], accountId: null, folder: null, + draft: { id: draftId, subject: 'must-not-enter-dedupe-key' }, + composeSlots: [{ id: draftId, slot: draftId === 'draft-a' ? 1 : 2 }], + gtdAvailable: false, cardDavConnected: false, modal: null, editing: true, + platform: 'mac', shortcutOverrides: {}, translate: key => key, + }); + const controller = createCommandController({ + registry: createCommandRegistry([draftDefinition]), + getContext: draftContext, + executors: { + 'compose.close': ({ context }) => new Promise(resolve => { + calls.push(context.draft.id); + releases.set(context.draft.id, resolve); + }).then(() => ({ status: 'success' })), + }, + }); + + const a1 = controller.execute('compose.close'); + const a2 = controller.execute('compose.close'); + assert.strictEqual(a1, a2); + draftId = 'draft-b'; + const b = controller.execute('compose.close'); + assert.notStrictEqual(a1, b); + assert.deepEqual(calls, ['draft-a', 'draft-b']); + releases.get('draft-a')(); + releases.get('draft-b')(); + await Promise.all([a1, b]); + }); + + it('rejects a frozen multi-selection for single-conversation commands', async () => { + let callCount = 0; + const registry = createCommandRegistry([definition({ + id: 'mail.reply', + targetMode: 'single_conversation', + executorId: 'mail.reply', + })]); + const controller = createCommandController({ + registry, + getContext: () => makeContext(), + executors: { + 'mail.reply': () => { + callCount += 1; + return { status: 'success' }; + }, + }, + }); + + const outcome = await controller.execute('mail.reply', { + source: 'context-menu', + frozenTargetIds: ['acct:', 'acct:'], + }); + + assert.equal(outcome.status, 'failed'); + assert.match(outcome.error.message, /exactly one conversation/); + assert.equal(callCount, 0); + }); + + it('normalizes success, cancelled, partial, thrown failures, and callback delivery', async () => { + const terminal = []; + const outcomes = ['success', 'cancelled', 'partial']; + for (const status of outcomes) { + const registry = createCommandRegistry([definition()]); + const controller = createCommandController({ + registry, getContext: () => makeContext(['acct:']), + executors: { 'mail.move': () => status === 'partial' + ? { status, succeededIds: [], failed: [{ targetId: 'acct:', error: new Error('nope') }] } + : { status } }, + onOutcome: outcome => terminal.push(outcome.status), + }); + assert.equal((await controller.execute('mail.move')).status, status); + } + const registry = createCommandRegistry([definition()]); + const failed = createCommandController({ + registry, getContext: () => makeContext(['acct:']), + executors: { 'mail.move': () => { throw new Error('boom'); } }, + onOutcome: outcome => terminal.push(outcome.status), + }); + assert.equal((await failed.execute('mail.move')).status, 'failed'); + assert.deepEqual(terminal, ['success', 'cancelled', 'partial', 'failed']); + }); +}); diff --git a/frontend/src/commands/mailActions.js b/frontend/src/commands/mailActions.js new file mode 100644 index 00000000..32d61428 --- /dev/null +++ b/frontend/src/commands/mailActions.js @@ -0,0 +1,403 @@ +import { openForwardFromMessage, openReplyFromMessage } from '../utils/composeFromMessage.js'; +import { delegateNeedsContact, normalizeDelegateOutcome } from '../utils/delegation.js'; + +const DEFAULT_KEYS = Object.freeze({ + 'mail.archive': { primary: 'e', secondary: [] }, + 'mail.snooze': { primary: 'h', secondary: [] }, + 'mail.move': { primary: 'v', secondary: [] }, + 'mail.toggleRead': { primary: 'u', secondary: ['m'] }, + 'mail.toggleStar': { primary: 's', secondary: [] }, + 'mail.trash': { primary: '#', secondary: [] }, + 'mail.spam': { primary: '!', secondary: [] }, + 'mail.reply': { primary: 'r', secondary: [] }, + 'mail.replyAll': { primary: 'enter', secondary: ['a'] }, + 'mail.forward': { primary: 'f', secondary: [] }, + 'gtd.todo': { primary: 't', secondary: [] }, + 'gtd.watch': { primary: 'w', secondary: [] }, + 'gtd.delegate': { primary: 'd', secondary: [] }, +}); + +const definition = (id, titleKey, aliasKeys, icon, group, targetMode, executorId, rank = 50) => ({ + id, + titleKey, + aliasKeys, + icon, + group, + defaultKeys: DEFAULT_KEYS[id] || { primary: null, secondary: [] }, + rank: { base: rank }, + targetMode, + executorId, + isAvailable: () => true, +}); + +const baseMailCommandDefinitions = [ + definition('mail.archive', 'shortcuts.actions.archive.label', ['commands.mail.archive.aliasDone'], 'archive', 'mail', 'bulk_safe', 'mail.archive', 90), + definition('mail.snooze', 'contextMenu.snooze.label', ['commands.mail.snooze.aliasRemind'], 'clock', 'mail', 'bulk_safe', 'mail.snooze', 85), + definition('mail.move', 'contextMenu.moveToFolder', [], 'folder', 'mail', 'bulk_safe', 'mail.move', 75), + definition('mail.read', 'contextMenu.markRead', [], 'mail-open', 'mail', 'bulk_safe', 'mail.read'), + definition('mail.unread', 'contextMenu.markUnread', [], 'mail', 'mail', 'bulk_safe', 'mail.unread'), + definition('mail.toggleRead', 'shortcuts.actions.toggleRead.label', [], 'mail', 'mail', 'bulk_safe', 'mail.toggleRead', 80), + definition('mail.star', 'message.star', [], 'star', 'mail', 'bulk_safe', 'mail.star'), + definition('mail.unstar', 'message.unstar', [], 'star', 'mail', 'bulk_safe', 'mail.unstar'), + definition('mail.toggleStar', 'shortcuts.actions.toggleStar.label', [], 'star', 'mail', 'bulk_safe', 'mail.toggleStar', 80), + definition('mail.trash', 'shortcuts.actions.delete.label', ['commands.mail.trash.aliasDelete'], 'trash', 'mail', 'bulk_safe', 'mail.trash', 80), + definition('mail.spam', 'contextMenu.markAsSpam', ['commands.mail.spam.aliasJunk'], 'shield', 'mail', 'bulk_safe', 'mail.spam'), + definition('mail.notSpam', 'contextMenu.markAsNotSpam', [], 'shield-check', 'mail', 'bulk_safe', 'mail.notSpam'), + definition('mail.reply', 'shortcuts.actions.reply.label', [], 'reply', 'respond', 'single_conversation', 'mail.reply', 80), + definition('mail.replyAll', 'shortcuts.actions.replyAll.label', [], 'reply-all', 'respond', 'single_conversation', 'mail.replyAll', 80), + definition('mail.forward', 'shortcuts.actions.forward.label', [], 'forward', 'respond', 'single_conversation', 'mail.forward', 70), + definition('gtd.todo', 'shortcuts.actions.gtdTodo.label', [], 'check-square', 'gtd', 'bulk_safe', 'gtd.todo'), + definition('gtd.watch', 'shortcuts.actions.gtdWatch.label', [], 'eye', 'gtd', 'bulk_safe', 'gtd.watch'), + definition('gtd.delegate', 'gtd.delegate.command', [], 'user-check', 'gtd', 'bulk_safe', 'gtd.delegate'), + definition('gtd.someday', 'gtd.states.someday', [], 'calendar', 'gtd', 'bulk_safe', 'gtd.someday'), + definition('gtd.reference', 'gtd.states.reference', [], 'bookmark', 'gtd', 'bulk_safe', 'gtd.reference'), +].map(command => Object.freeze({ + ...command, + isAvailable: context => command.id.startsWith('gtd.') ? context.gtdAvailable === true : true, +})); + +export const mailCommandDefinitions = Object.freeze([...baseMailCommandDefinitions, Object.freeze({ + id: 'mail.unsubscribe', + titleKey: 'message.unsubscribe.button', + aliasKeys: [], + icon: 'mail', + group: 'mail', + defaultKeys: { + primary: { mac: 'meta+u', windows: 'ctrl+u', linux: 'ctrl+u', default: 'ctrl+u' }, + secondary: [], + }, + rank: { base: 60 }, + targetMode: 'single_conversation', + executorId: 'mail.unsubscribe', + isAvailable: context => context.surface === 'conversation' + && !!context.activeMessage?.list_unsubscribe + && !context.activeMessage?.unsubscribed_at, +})]); + +const idOf = target => target.id; +const idsOf = targets => targets.map(idOf); +const errorText = error => error instanceof Error ? error.message : String(error); +const failedOutcome = (targets, error) => ({ + status: 'failed', + succeededIds: [], + failed: targets.map(target => ({ id: idOf(target), error: errorText(error) })), +}); +const settledOutcome = (targets, results) => { + const succeededIds = []; + const failed = []; + results.forEach((result, index) => { + if (result.status === 'fulfilled') succeededIds.push(idOf(targets[index])); + else failed.push({ id: idOf(targets[index]), error: errorText(result.reason) }); + }); + return { + status: failed.length === 0 ? 'success' : succeededIds.length === 0 ? 'failed' : 'partial', + succeededIds, + failed, + }; +}; + +const continuation = (commandId, kind, targets, props) => ({ + status: 'needs_input', + continuation: { commandId, kind, targetIds: idsOf(targets), props }, +}); + +function scheduleOptimisticRemoval(deps, { + targets, + call, + successIds, + onSuccess, + successNotice, + failureNotice, + flushOnUnload, +}) { + const rowIds = targets.map(target => target.message.id); + const unreadTargets = targets.filter(target => !target.message.is_read); + deps.guardPending(rowIds); + deps.removeMessages(targets); + deps.adjustUnread(unreadTargets, true); + let undone = false; + + let unregisterPendingRemoval = () => {}; + const run = async () => { + unregisterPendingRemoval(); + if (undone) return; + try { + const response = await call(); + const succeededRowIds = new Set(successIds(response)); + const succeededTargets = targets.filter(target => succeededRowIds.has(target.message.id)); + const failedTargets = targets.filter(target => !succeededRowIds.has(target.message.id)); + deps.guardCompleted(succeededTargets.map(target => target.message.id)); + deps.clearGuards(failedTargets.map(target => target.message.id)); + if (failedTargets.length > 0) { + deps.restoreMessages(failedTargets); + deps.adjustUnread(failedTargets.filter(target => !target.message.is_read), false); + deps.notify({ + type: 'error', + titleKey: failureNotice, + succeededCount: succeededTargets.length, + failedCount: failedTargets.length, + }); + } + onSuccess?.(response, succeededTargets); + } catch (error) { + deps.clearGuards(rowIds); + deps.restoreMessages(targets); + deps.adjustUnread(unreadTargets, false); + deps.notify({ + type: 'error', + titleKey: failureNotice, + body: errorText(error), + succeededCount: 0, + failedCount: targets.length, + }); + } + }; + const timer = deps.timers.setTimeout(run, 4500); + if (flushOnUnload) { + unregisterPendingRemoval = deps.registerPendingRemoval({ + timer, + run, + unload: () => flushOnUnload(rowIds), + }); + } + + deps.notify({ + titleKey: successNotice, + count: targets.length, + onUndo: () => { + undone = true; + unregisterPendingRemoval(); + deps.timers.clearTimeout(timer); + deps.clearGuards(rowIds); + deps.restoreMessages(targets); + deps.adjustUnread(unreadTargets, false); + }, + }); + + return { status: 'success', succeededIds: idsOf(targets), failed: [] }; +} + +export function createMailActionExecutors(deps) { + const resolveTargets = ({ context, targetIds }) => targetIds + .map(targetId => context.conversationsById[targetId]) + .filter(Boolean); + const setRead = async (targets, read) => { + const changedTargets = targets.filter(target => target.message.is_read !== read); + if (read) deps.guardReadPending(changedTargets); + else deps.clearReadGuards(changedTargets); + deps.patchMessages(targets, { is_read: read }); + deps.adjustUnread(changedTargets, read); + try { + await deps.api.bulkRead(targets.map(target => target.message.id), read); + if (read) deps.guardReadCompleted(changedTargets); + return { status: 'success', succeededIds: idsOf(targets), failed: [] }; + } catch (error) { + deps.clearReadGuards(changedTargets); + changedTargets.forEach(target => deps.patchMessages([target], { is_read: target.message.is_read })); + deps.restoreMessages(targets); + deps.adjustUnread(changedTargets, !read); + return failedOutcome(targets, error); + } + }; + const setStarred = async (targets, starred) => { + deps.patchMessages(targets, { is_starred: starred }); + const results = await Promise.allSettled( + targets.map(target => deps.api.markStarred(target.message.id, starred)), + ); + const outcome = settledOutcome(targets, results); + const failedIds = new Set(outcome.failed.map(item => item.id)); + const failedTargets = targets.filter(target => failedIds.has(idOf(target))); + failedTargets.forEach(target => deps.patchMessages([target], { is_starred: target.message.is_starred })); + deps.restoreMessages(failedTargets); + return outcome; + }; + const classify = state => async ({ targets }) => { + const results = await Promise.allSettled( + targets.map(target => deps.api.gtdClassify(target.message.id, state)), + ); + deps.scheduleGtdRefresh(); + return settledOutcome(targets, results); + }; + const delegate = async ({ context, input, targets }) => { + const carddavStatus = context.carddavStatusLoaded + ? context.carddavStatus + : await deps.refreshCarddavStatus(); + if (input === undefined && delegateNeedsContact(carddavStatus)) { + return continuation('gtd.delegate', 'contact', targets, { targetCount: targets.length }); + } + + const contactId = input?.contactId ?? null; + const result = await deps.api.gtd.delegate( + targets.map(target => target.message.id), + contactId, + ); + const resultsByMessageId = new Map( + (result?.results || []).map(item => [item.messageId, item]), + ); + targets.forEach(target => { + const item = resultsByMessageId.get(target.message.id); + if (item?.ok) deps.patchMessages([target], { delegation: item.delegation ?? null }); + }); + await Promise.all([deps.refreshMessages(), deps.refreshGtdSections()]); + return normalizeDelegateOutcome(result, targets); + }; + + const handlers = { + 'mail.archive': ({ targets }) => scheduleOptimisticRemoval(deps, { + targets, + call: () => deps.api.bulkArchive(targets.map(target => target.message.id)), + successIds: response => response.archived ?? [], + successNotice: 'messageList.bulkArchived.title', + failureNotice: 'messageList.bulkArchived.failTitle', + }), + 'mail.trash': ({ targets }) => scheduleOptimisticRemoval(deps, { + targets, + call: () => deps.api.bulkDelete(targets.map(target => target.message.id)), + successIds: response => response.deleted ?? [], + successNotice: 'messageList.bulkDeleted.title', + failureNotice: 'messageList.bulkDeleted.failTitle', + flushOnUnload: ids => deps.keepaliveDelete(ids), + }), + 'mail.move': ({ targets, input }) => { + const folder = input?.folder ?? input?.value; + if (!folder) { + return continuation('mail.move', 'move', targets, { + accountId: targets[0]?.message.account_id, + targetCount: targets.length, + titleKey: 'contextMenu.moveToFolder', + inputKey: 'folder', + items: deps.moveOptions(targets[0]?.message.account_id, targets), + }); + } + return scheduleOptimisticRemoval(deps, { + targets, + call: () => deps.api.bulkMove(targets.map(target => target.message.id), folder), + successIds: response => response.moved ?? [], + onSuccess: (_response, succeededTargets) => { + if (succeededTargets.length > 0) { + deps.recordRecentFolder(targets[0].message.account_id, folder); + } + }, + successNotice: 'messageList.bulkMoved.title', + failureNotice: 'messageList.bulkMoved.failTitle', + }); + }, + 'mail.snooze': ({ targets, input }) => { + const until = input?.until ?? input?.value; + if (!until) { + return continuation('mail.snooze', 'snooze', targets, { + targetCount: targets.length, + titleKey: 'contextMenu.snooze.label', + inputKey: 'until', + items: deps.snoozeOptions(), + }); + } + return scheduleOptimisticRemoval(deps, { + targets, + call: async () => { + const results = await Promise.allSettled( + targets.map(target => deps.api.snoozeMessage(target.message.id, until)), + ); + return { + snoozed: targets + .filter((_target, index) => results[index].status === 'fulfilled') + .map(target => target.message.id), + }; + }, + successIds: response => response.snoozed, + successNotice: 'message.snoozed.title', + failureNotice: 'message.snoozed.failTitle', + }); + }, + 'mail.spam': ({ targets }) => scheduleOptimisticRemoval(deps, { + targets, + call: async () => { + const results = await Promise.allSettled( + targets.map(target => deps.api.markSpam(target.message.id)), + ); + return { + moved: targets + .filter((_target, index) => results[index].status === 'fulfilled') + .map(target => target.message.id), + }; + }, + successIds: response => response.moved, + successNotice: 'spam.movedToSpamBulk', + failureNotice: 'spam.failTitle', + }), + 'mail.notSpam': ({ targets }) => scheduleOptimisticRemoval(deps, { + targets, + call: async () => { + const results = await Promise.allSettled( + targets.map(target => deps.api.markHam(target.message.id)), + ); + return { + moved: targets + .filter((_target, index) => results[index].status === 'fulfilled') + .map(target => target.message.id), + }; + }, + successIds: response => response.moved, + successNotice: 'spam.movedToInboxBulk', + failureNotice: 'spam.failHamTitle', + }), + 'mail.read': ({ targets }) => setRead(targets, true), + 'mail.unread': ({ targets }) => setRead(targets, false), + 'mail.toggleRead': ({ targets }) => setRead(targets, targets.some(target => !target.message.is_read)), + 'mail.star': ({ targets }) => setStarred(targets, true), + 'mail.unstar': ({ targets }) => setStarred(targets, false), + 'mail.toggleStar': ({ targets }) => setStarred(targets, targets.some(target => !target.message.is_starred)), + 'mail.reply': async ({ targets }) => { + await openReplyFromMessage(targets[0].message, { + accounts: deps.accounts(), + openCompose: deps.openCompose, + getMessageBody: deps.api.getMessageBody, + replyAll: false, + }); + return { status: 'success', succeededIds: idsOf(targets), failed: [] }; + }, + 'mail.replyAll': async ({ targets }) => { + await openReplyFromMessage(targets[0].message, { + accounts: deps.accounts(), + openCompose: deps.openCompose, + getMessageBody: deps.api.getMessageBody, + replyAll: true, + }); + return { status: 'success', succeededIds: idsOf(targets), failed: [] }; + }, + 'mail.forward': async ({ targets }) => { + await openForwardFromMessage(targets[0].message, { + openCompose: deps.openCompose, + getMessageBody: deps.api.getMessageBody, + }); + return { status: 'success', succeededIds: idsOf(targets), failed: [] }; + }, + 'mail.unsubscribe': async ({ targets }) => { + try { + const result = await deps.api.unsubscribeMessage(targets[0].message.id); + const isHandled = result?.type === 'one-click' + || (result?.type === 'url' && result.url) + || (result?.type === 'mailto' && result.mailto); + if (!isHandled) throw new Error('Unsupported unsubscribe response'); + if (result.type === 'url' && result.url) deps.openExternal(result.url); + if (result.type === 'mailto' && result.mailto) deps.openExternal(result.mailto); + deps.patchMessages(targets, { unsubscribed_at: new Date().toISOString() }); + deps.notify({ titleKey: 'message.unsubscribe.done' }); + return { status: 'success', succeededIds: idsOf(targets), failed: [] }; + } catch (error) { + deps.notify({ type: 'error', titleKey: 'message.unsubscribe.error' }); + return failedOutcome(targets, error); + } + }, + 'gtd.todo': classify('todo'), + 'gtd.watch': classify('watch'), + 'gtd.delegate': delegate, + 'gtd.someday': classify('someday'), + 'gtd.reference': classify('reference'), + }; + + return Object.fromEntries(Object.entries(handlers).map(([id, handler]) => [ + id, + args => handler({ ...args, targets: resolveTargets(args) }), + ])); +} diff --git a/frontend/src/commands/mailActions.test.js b/frontend/src/commands/mailActions.test.js new file mode 100644 index 00000000..605c262e --- /dev/null +++ b/frontend/src/commands/mailActions.test.js @@ -0,0 +1,492 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createMailActionExecutors, mailCommandDefinitions } from './mailActions.js'; + +const target = (rowId, patch = {}) => { + const id = `account-1:<${rowId}@example.test>`; + return { + id, + rowId, + accountId: 'account-1', + message: { + id: rowId, + account_id: 'account-1', + message_id: `<${rowId}@example.test>`, + is_read: false, + is_starred: false, + subject: `Subject ${rowId}`, + ...patch, + }, + }; +}; + +function harness(apiPatch = {}, depsPatch = {}) { + const events = []; + const api = { + bulkRead: async (ids, read) => ({ ok: true, ids, read }), + markStarred: async (id, starred) => ({ ok: true, id, starred }), + gtdClassify: async (id, state) => ({ ok: true, id, state }), + gtd: { + delegate: async (ids, contactId) => ({ + status: 'success', successCount: ids.length, failureCount: 0, + results: ids.map(messageId => ({ messageId, ok: true, delegation: contactId })), + }), + }, + getMessageBody: async () => ({ text: 'body', html: '

body

', attachments: [] }), + ...apiPatch, + }; + const deps = { + api, + accounts: () => [{ id: 'account-1', email_address: 'me@example.test', gtd_enabled: true }], + openCompose: payload => events.push(['compose', payload]), + openExternal: value => events.push(['external', value]), + patchMessages: (targets, patch) => events.push(['patch', targets.map(item => item.id), patch]), + restoreMessages: targets => events.push(['restore', targets.map(item => item.id)]), + adjustUnread: (targets, read) => events.push(['unread', targets.map(item => item.id), read]), + guardReadPending: targets => events.push(['read-pending', targets.map(item => item.id)]), + guardReadCompleted: targets => events.push(['read-complete', targets.map(item => item.id)]), + clearReadGuards: targets => events.push(['read-clear', targets.map(item => item.id)]), + scheduleGtdRefresh: () => events.push(['gtd-refresh']), + refreshCarddavStatus: async () => ({ connected: false }), + refreshMessages: async () => events.push(['messages-refresh']), + refreshGtdSections: async () => events.push(['gtd-refresh']), + notify: notification => events.push(['notify', notification]), + moveOptions: () => [{ id: 'Archive', label: 'Archive' }], + snoozeOptions: () => [{ id: '2026-08-01T09:00:00.000Z', label: 'Tomorrow morning' }], + ...depsPatch, + }; + const executors = createMailActionExecutors(deps); + const invoke = (executorId, targets, rest = {}) => executors[executorId]({ + context: { conversationsById: Object.fromEntries(targets.map(item => [item.id, item])) }, + targetIds: targets.map(item => item.id), + source: 'test', + ...rest, + }); + return { events, executors, invoke }; +} + +describe('mailCommandDefinitions', () => { + it('declares bulk-safe mutations and single-only response commands', () => { + const byId = new Map(mailCommandDefinitions.map(definition => [definition.id, definition])); + assert.equal(byId.get('mail.archive').targetMode, 'bulk_safe'); + assert.equal(byId.get('mail.toggleRead').targetMode, 'bulk_safe'); + assert.equal(byId.get('mail.reply').targetMode, 'single_conversation'); + assert.equal(byId.get('mail.replyAll').targetMode, 'single_conversation'); + assert.equal(byId.get('mail.forward').targetMode, 'single_conversation'); + assert.equal(byId.get('mail.move').executorId, 'mail.move'); + assert.deepEqual(byId.get('mail.archive').aliasKeys, ['commands.mail.archive.aliasDone']); + assert.equal(byId.get('mail.snooze').titleKey, 'contextMenu.snooze.label'); + assert.deepEqual(byId.get('mail.toggleRead').defaultKeys, { primary: 'u', secondary: ['m'] }); + assert.deepEqual(byId.get('mail.replyAll').defaultKeys, { primary: 'enter', secondary: ['a'] }); + }); + + it('offers Unsubscribe only for one usable active message', () => { + const command = mailCommandDefinitions.find(item => item.id === 'mail.unsubscribe'); + assert.equal(command.targetMode, 'single_conversation'); + assert.equal(command.isAvailable({ surface: 'conversation', activeMessage: { list_unsubscribe: '' } }), true); + assert.equal(command.isAvailable({ surface: 'list', activeMessage: { list_unsubscribe: '' } }), false); + assert.equal(command.isAvailable({ surface: 'conversation', activeMessage: { list_unsubscribe: null } }), false); + }); +}); + +describe('non-destructive mail executors', () => { + it('accepts the backend one-click unsubscribe result and patches the message', async () => { + const h = harness({ unsubscribeMessage: async () => ({ ok: true, type: 'one-click' }) }); + const message = target('a', { list_unsubscribe: '' }); + const result = await h.invoke('mail.unsubscribe', [message]); + assert.equal(result.status, 'success'); + assert.ok(h.events.find(event => event[0] === 'patch')); + assert.equal(h.events.some(event => event[0] === 'external'), false); + }); + + it('marks the complete resolved target set read with one optimistic patch', async () => { + const h = harness(); + const result = await h.invoke('mail.read', [target('a'), target('b')], { source: 'toolbar' }); + assert.equal(result.status, 'success'); + assert.deepEqual(result.succeededIds, ['account-1:', 'account-1:']); + assert.deepEqual(h.events.find(event => event[0] === 'patch'), [ + 'patch', result.succeededIds, { is_read: true }, + ]); + assert.deepEqual(h.events.find(event => event[0] === 'unread'), [ + 'unread', result.succeededIds, true, + ]); + assert.deepEqual(h.events.filter(event => event[0].startsWith('read-')), [ + ['read-pending', result.succeededIds], + ['read-complete', result.succeededIds], + ]); + }); + + it('restores read state and returns failed when the API rejects', async () => { + const h = harness({ bulkRead: async () => { throw new Error('read failed'); } }); + const result = await h.invoke('mail.read', [target('a')], { source: 'shortcut' }); + assert.equal(result.status, 'failed'); + assert.deepEqual(result.failed, [{ id: 'account-1:', error: 'read failed' }]); + assert.deepEqual(h.events.find(event => event[0] === 'restore'), [ + 'restore', + ['account-1:'], + ]); + assert.deepEqual(h.events.filter(event => event[0].startsWith('read-')), [ + ['read-pending', ['account-1:']], + ['read-clear', ['account-1:']], + ]); + }); + + it('clears read guards before marking a message unread', async () => { + const h = harness(); + await h.invoke('mail.unread', [target('a', { is_read: true })]); + assert.equal(h.events.findIndex(event => event[0] === 'read-clear') + < h.events.findIndex(event => event[0] === 'patch'), true); + assert.deepEqual(h.events.filter(event => event[0].startsWith('read-')), [ + ['read-clear', ['account-1:']], + ]); + }); + + it('reverses the optimistic unread delta when a read request fails', async () => { + const h = harness({ bulkRead: async () => { throw new Error('read failed'); } }); + await h.invoke('mail.read', [target('a')]); + assert.deepEqual(h.events.slice(-2), [ + ['restore', ['account-1:']], + ['unread', ['account-1:'], false], + ]); + }); + + it('reports only failed star targets and rolls those targets back', async () => { + const h = harness({ + markStarred: async id => { + if (id === 'b') throw new Error('flag failed'); + return { ok: true }; + }, + }); + const result = await h.invoke('mail.star', [target('a'), target('b')], { source: 'palette' }); + assert.equal(result.status, 'partial'); + assert.deepEqual(result.succeededIds, ['account-1:']); + assert.deepEqual(result.failed, [{ id: 'account-1:', error: 'flag failed' }]); + assert.deepEqual(h.events.at(-1), ['restore', ['account-1:']]); + }); + + it('opens one reply-all composer through the existing payload builder', async () => { + const h = harness(); + const result = await h.invoke('mail.replyAll', [target('a')], { source: 'context-menu' }); + assert.equal(result.status, 'success'); + assert.equal(h.events[0][0], 'compose'); + assert.equal(h.events[0][1].isReplyAll, true); + }); + + it('does not report a mail compose action successful before creation acknowledges', async () => { + let rejectCompose; + const compose = new Promise((_resolve, reject) => { rejectCompose = reject; }); + const failure = new Error('synthetic create failure'); + const h = harness({}, { openCompose: () => compose }); + const pending = h.invoke('mail.reply', [target('a')], { source: 'shortcut' }); + let settled = false; + pending.finally(() => { settled = true; }).catch(() => {}); + await Promise.resolve(); + assert.equal(settled, false); + + rejectCompose(failure); + await assert.rejects(pending, error => error === failure); + }); + + it('does not depend on a pane-cached body when replying or forwarding', async () => { + const h = harness(); + const current = target('a', { thread_id: 'thread-a' }); + await h.invoke('mail.reply', [current], { source: 'pane-toolbar' }); + await h.invoke('mail.forward', [current], { source: 'list-context-menu' }); + const payloads = h.events.filter(event => event[0] === 'compose').map(event => event[1]); + assert.equal(payloads[0].threadId, 'thread-a'); + assert.equal(payloads[0].quotedBody.includes('body'), true); + assert.equal(payloads[1].quotedBody.includes('body'), true); + }); + + it('returns partial GTD classification and refreshes sections once', async () => { + const h = harness({ + gtdClassify: async id => { + if (id === 'b') throw new Error('copy failed'); + return { ok: true }; + }, + }); + const result = await h.invoke('gtd.todo', [target('a'), target('b')], { source: 'toolbar' }); + assert.equal(result.status, 'partial'); + assert.deepEqual(result.failed, [{ id: 'account-1:', error: 'copy failed' }]); + assert.equal(h.events.filter(event => event[0] === 'gtd-refresh').length, 1); + }); + + it('requests contact input with exact frozen targets when CardDAV is connected', async () => { + const h = harness(); + const targets = [target('a'), target('b')]; + const context = { + conversationsById: Object.fromEntries(targets.map(item => [item.id, item])), + carddavStatus: { connected: true }, + carddavStatusLoaded: true, + }; + const result = await h.invoke('gtd.delegate', targets, { context, source: 'shortcut' }); + assert.deepEqual(result, { + status: 'needs_input', + continuation: { + commandId: 'gtd.delegate', kind: 'contact', targetIds: targets.map(item => item.id), + props: { targetCount: 2 }, + }, + }); + assert.equal(h.events.length, 0); + }); + + it('refreshes an unknown CardDAV status before choosing the delegation workflow', async () => { + const calls = []; + const h = harness({}, { + refreshCarddavStatus: async () => { + calls.push('status'); + return { connected: true }; + }, + }); + const current = target('a'); + const context = { + conversationsById: { [current.id]: current }, + carddavStatus: { connected: false }, + carddavStatusLoaded: false, + }; + const result = await h.invoke('gtd.delegate', [current], { context }); + assert.deepEqual(calls, ['status']); + assert.equal(result.status, 'needs_input'); + assert.equal(result.continuation.kind, 'contact'); + }); + + it('delegates immediately without a person when CardDAV is disconnected', async () => { + const calls = []; + const h = harness({ + gtd: { delegate: async (...args) => { + calls.push(args); + return { + status: 'success', successCount: 1, failureCount: 0, + results: [{ messageId: 'a', ok: true }], + }; + } }, + }); + const current = target('a'); + const context = { + conversationsById: { [current.id]: current }, + carddavStatus: { connected: false }, + carddavStatusLoaded: true, + }; + const result = await h.invoke('gtd.delegate', [current], { context }); + assert.deepEqual(calls, [[['a'], null]]); + assert.equal(result.status, 'success'); + }); + + it('resumes with a stable contact ID and sends only database message UUIDs', async () => { + const calls = []; + const h = harness({ + gtd: { delegate: async (...args) => { + calls.push(args); + return { + status: 'success', successCount: 2, failureCount: 0, + results: ['a', 'b'].map(messageId => ({ messageId, ok: true })), + }; + } }, + }); + const targets = [target('a'), target('b')]; + const context = { + conversationsById: Object.fromEntries(targets.map(item => [item.id, item])), + carddavStatus: { connected: true }, + carddavStatusLoaded: true, + }; + const result = await h.invoke('gtd.delegate', targets, { + context, source: 'continuation', input: { contactId: 'contact-1' }, + }); + assert.deepEqual(calls, [[['a', 'b'], 'contact-1']]); + assert.deepEqual(result.succeededIds, targets.map(item => item.id)); + }); + + it('patches successful delegation metadata into every cached message surface', async () => { + const delegation = { + contact_id: 'contact-1', display_name: 'Casey Rivera', + primary_email: 'casey@example.test', + }; + const h = harness({ + gtd: { delegate: async () => ({ + status: 'partial', successCount: 1, failureCount: 1, + results: [ + { messageId: 'a', ok: true, delegation }, + { messageId: 'b', ok: false, error: { code: 'operation_failed' } }, + ], + }) }, + }); + const targets = [target('a'), target('b')]; + const context = { + conversationsById: Object.fromEntries(targets.map(item => [item.id, item])), + carddavStatus: { connected: true }, carddavStatusLoaded: true, + }; + await h.invoke('gtd.delegate', targets, { + context, input: { contactId: 'contact-1' }, source: 'continuation', + }); + assert.deepEqual(h.events.filter(event => event[0] === 'patch'), [ + ['patch', [targets[0].id], { delegation }], + ]); + }); +}); + +function removalHarness(apiPatch = {}) { + const events = []; + const scheduled = []; + const timers = { + setTimeout(fn, ms) { + const item = { fn, ms, cleared: false }; + scheduled.push(item); + return item; + }, + clearTimeout(item) { item.cleared = true; }, + }; + const api = { + bulkArchive: async ids => ({ archived: ids, noArchiveFolder: [] }), + bulkDelete: async ids => ({ deleted: ids }), + bulkMove: async ids => ({ moved: ids }), + snoozeMessage: async id => ({ ok: true, id }), + markSpam: async id => ({ ok: true, id }), + markHam: async id => ({ ok: true, id }), + ...apiPatch, + }; + const deps = { + api, + accounts: () => [], + openCompose() {}, + patchMessages() {}, + adjustUnread: (targets, read) => events.push(['unread', targets.map(item => item.id), read]), + removeMessages: targets => events.push(['remove', targets.map(item => item.id)]), + restoreMessages: targets => events.push(['restore', targets.map(item => item.id)]), + guardPending: ids => events.push(['guard-pending', ids]), + guardCompleted: ids => events.push(['guard-complete', ids]), + clearGuards: ids => events.push(['guard-clear', ids]), + recordRecentFolder: (accountId, folder) => events.push(['recent', accountId, folder]), + scheduleGtdRefresh() {}, + notify: notification => events.push(['notify', notification]), + moveOptions: () => [{ id: 'Archive', label: 'Archive' }], + snoozeOptions: () => [{ id: '2026-08-01T09:00:00.000Z', label: 'Tomorrow morning' }], + registerPendingRemoval: operation => { + events.push(['register-pending-removal', operation]); + return () => events.push(['unregister-pending-removal']); + }, + keepaliveDelete: ids => events.push(['keepalive-delete', ids]), + timers, + }; + const executors = createMailActionExecutors(deps); + const invoke = (executorId, targets, rest = {}) => executors[executorId]({ + context: { conversationsById: Object.fromEntries(targets.map(item => [item.id, item])) }, + targetIds: targets.map(item => item.id), + source: 'test', + ...rest, + }); + return { events, scheduled, executors, invoke }; +} + +describe('removal and continuation executors', () => { + it('requests Move input with the exact frozen target IDs', async () => { + const h = removalHarness(); + const targets = [target('a'), target('b')]; + const result = await h.invoke('mail.move', targets, { source: 'shortcut' }); + assert.deepEqual(result, { + status: 'needs_input', + continuation: { + commandId: 'mail.move', + kind: 'move', + targetIds: targets.map(item => item.id), + props: { + accountId: 'account-1', + targetCount: 2, + titleKey: 'contextMenu.moveToFolder', + inputKey: 'folder', + items: [{ id: 'Archive', label: 'Archive' }], + }, + }, + }); + assert.equal(h.events.length, 0); + }); + + it('requests Snooze input without mutating targets', async () => { + const h = removalHarness(); + const result = await h.invoke('mail.snooze', [target('a')], { source: 'palette' }); + assert.equal(result.status, 'needs_input'); + assert.equal(result.continuation.kind, 'snooze'); + assert.deepEqual(result.continuation.targetIds, ['account-1:']); + assert.equal(result.continuation.props.inputKey, 'until'); + assert.equal(result.continuation.props.items[0].label, 'Tomorrow morning'); + assert.equal(h.events.length, 0); + }); + + it('undoes Archive before the delayed API call', async () => { + const h = removalHarness(); + const result = await h.invoke('mail.archive', [target('a')], { source: 'hover' }); + assert.equal(result.status, 'success'); + assert.equal(h.scheduled[0].ms, 4500); + const notification = h.events.find(event => event[0] === 'notify')[1]; + notification.onUndo(); + assert.equal(h.scheduled[0].cleared, true); + assert.ok(h.events.some(event => event[0] === 'restore')); + assert.ok(h.events.some(event => event[0] === 'guard-clear')); + }); + + it('restores only failed Move targets and records the destination once', async () => { + const h = removalHarness({ bulkMove: async () => ({ moved: ['a'] }) }); + const result = await h.invoke('mail.move', [target('a'), target('b')], { + input: { folder: 'Archive' }, + source: 'context-menu', + }); + assert.equal(result.status, 'success'); + await h.scheduled[0].fn(); + assert.deepEqual(h.events.find(event => event[0] === 'restore'), ['restore', ['account-1:']]); + assert.equal(h.events.filter(event => event[0] === 'recent').length, 1); + }); + + it('reports per-target Spam failure after the undo window', async () => { + const h = removalHarness({ + markSpam: async id => { + if (id === 'b') throw new Error('spam failed'); + return { ok: true }; + }, + }); + await h.invoke('mail.spam', [target('a'), target('b')], { source: 'toolbar' }); + await h.scheduled[0].fn(); + const errorNotice = h.events.filter(event => event[0] === 'notify').at(-1)[1]; + assert.equal(errorNotice.failedCount, 1); + assert.equal(errorNotice.succeededCount, 1); + assert.deepEqual(h.events.find(event => event[0] === 'restore'), ['restore', ['account-1:']]); + }); + + it('restores every target and reports exact counts when a delayed request throws', async () => { + const h = removalHarness({ bulkArchive: async () => { throw new Error('archive failed'); } }); + await h.invoke('mail.archive', [target('a'), target('b')]); + await h.scheduled[0].fn(); + assert.deepEqual(h.events.find(event => event[0] === 'restore'), [ + 'restore', + ['account-1:', 'account-1:'], + ]); + const errorNotice = h.events.filter(event => event[0] === 'notify').at(-1)[1]; + assert.equal(errorNotice.succeededCount, 0); + assert.equal(errorNotice.failedCount, 2); + }); + + it('registers Trash so lifecycle cleanup can flush it normally or with keepalive', async () => { + const h = removalHarness(); + await h.invoke('mail.trash', [target('a')]); + const operation = h.events.find(event => event[0] === 'register-pending-removal')[1]; + await operation.run(); + operation.unload(); + assert.deepEqual(h.events.find(event => event[0] === 'keepalive-delete'), [ + 'keepalive-delete', + ['a'], + ]); + }); +}); + +it('constructs every executor with the documented dependency adapter', () => { + const required = [ + 'api', 'accounts', 'openCompose', 'patchMessages', 'removeMessages', + 'restoreMessages', 'adjustUnread', 'guardPending', 'guardCompleted', + 'clearGuards', 'recordRecentFolder', 'scheduleGtdRefresh', 'notify', 'timers', + 'moveOptions', 'snoozeOptions', + 'refreshCarddavStatus', 'refreshMessages', 'refreshGtdSections', + 'guardReadPending', 'guardReadCompleted', 'clearReadGuards', + 'registerPendingRemoval', 'keepaliveDelete', + ]; + const deps = Object.fromEntries(required.map(key => [key, key === 'api' ? {} : () => {}])); + deps.timers = { setTimeout, clearTimeout }; + assert.doesNotThrow(() => createMailActionExecutors(deps)); +}); diff --git a/frontend/src/commands/outcomeNotification.js b/frontend/src/commands/outcomeNotification.js new file mode 100644 index 00000000..58788617 --- /dev/null +++ b/frontend/src/commands/outcomeNotification.js @@ -0,0 +1,35 @@ +const errorText = error => error instanceof Error ? error.message : error == null ? '' : String(error); + +export function commandOutcomeNotification(outcome, t) { + if (outcome.status === 'partial' && outcome.value?.messageKey?.startsWith('gtd.delegate.')) { + const succeeded = outcome.succeededIds?.length ?? outcome.value.messageParams?.succeeded ?? 0; + const failed = (outcome.failed?.length ?? outcome.value.messageParams?.failed ?? 0) + + (outcome.missingTargetIds?.length || 0); + return { + title: t('gtd.delegate.partial', { count: succeeded + failed, succeeded, failed }), + }; + } + if (outcome.value?.messageKey) { + return { + ...(outcome.status === 'failed' ? { type: 'error' } : {}), + title: t(outcome.value.messageKey, outcome.value.messageParams), + }; + } + if (outcome.status === 'failed') { + return { + type: 'error', + title: t('commandPalette.outcome.failedTitle'), + body: errorText(outcome.error) || errorText(outcome.failed?.[0]?.error), + }; + } + if (outcome.status === 'partial') { + return { + title: t('commandPalette.outcome.partialTitle'), + body: t('commandPalette.outcome.partialBody', { + succeeded: outcome.succeededIds?.length || 0, + failed: (outcome.failed?.length || 0) + (outcome.missingTargetIds?.length || 0), + }), + }; + } + return null; +} diff --git a/frontend/src/commands/outcomeNotification.test.js b/frontend/src/commands/outcomeNotification.test.js new file mode 100644 index 00000000..8a6c4b52 --- /dev/null +++ b/frontend/src/commands/outcomeNotification.test.js @@ -0,0 +1,55 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { commandOutcomeNotification } from './outcomeNotification.js'; + +const t = (key, values) => values ? `${key}:${JSON.stringify(values)}` : key; + +test('formats executor-returned failures that do not carry a top-level error', () => { + assert.deepEqual(commandOutcomeNotification({ + status: 'failed', + failed: [{ id: 'acct:', error: 'read failed' }], + }, t), { + type: 'error', + title: 'commandPalette.outcome.failedTitle', + body: 'read failed', + }); +}); + +test('formats partial counts including targets that disappeared before execution', () => { + assert.deepEqual(commandOutcomeNotification({ + status: 'partial', + succeededIds: ['acct:'], + failed: [{ id: 'acct:', error: 'move failed' }], + missingTargetIds: ['acct:'], + }, t), { + title: 'commandPalette.outcome.partialTitle', + body: 'commandPalette.outcome.partialBody:{"succeeded":1,"failed":2}', + }); +}); + +test('uses executor-provided localized success and partial messages', () => { + assert.deepEqual(commandOutcomeNotification({ + status: 'success', + value: { messageKey: 'gtd.delegate.success', messageParams: { count: 2 } }, + }, t), { + title: 'gtd.delegate.success:{"count":2}', + }); + assert.deepEqual(commandOutcomeNotification({ + status: 'partial', + value: { messageKey: 'gtd.delegate.partial', messageParams: { succeeded: 1, failed: 1 } }, + }, t), { + title: 'gtd.delegate.partial:{"count":2,"succeeded":1,"failed":1}', + }); +}); + +test('folds vanished frozen targets into the localized delegation partial outcome', () => { + assert.deepEqual(commandOutcomeNotification({ + status: 'partial', + succeededIds: ['acct:'], + failed: [], + missingTargetIds: ['acct:'], + value: { messageKey: 'gtd.delegate.success', messageParams: { count: 1 } }, + }, t), { + title: 'gtd.delegate.partial:{"count":2,"succeeded":1,"failed":1}', + }); +}); diff --git a/frontend/src/commands/paletteFocus.js b/frontend/src/commands/paletteFocus.js new file mode 100644 index 00000000..0dc36603 --- /dev/null +++ b/frontend/src/commands/paletteFocus.js @@ -0,0 +1,14 @@ +export function isRestorableFocus(element) { + return Boolean( + element?.isConnected + && !element.disabled + && element.tabIndex !== -1 + && element.getClientRects?.().length, + ); +} + +export function nextFocusIndex(count, currentIndex, backwards) { + if (count <= 0) return -1; + if (backwards) return currentIndex <= 0 ? count - 1 : currentIndex - 1; + return currentIndex >= count - 1 ? 0 : currentIndex + 1; +} diff --git a/frontend/src/commands/paletteFocus.test.js b/frontend/src/commands/paletteFocus.test.js new file mode 100644 index 00000000..cac24155 --- /dev/null +++ b/frontend/src/commands/paletteFocus.test.js @@ -0,0 +1,21 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { isRestorableFocus, nextFocusIndex } from './paletteFocus.js'; + +describe('palette focus helpers', () => { + it('restores only connected, enabled, visible, focusable elements', () => { + const base = { isConnected: true, disabled: false, tabIndex: 0, getClientRects: () => [{ width: 1 }] }; + assert.equal(isRestorableFocus(base), true); + assert.equal(isRestorableFocus({ ...base, isConnected: false }), false); + assert.equal(isRestorableFocus({ ...base, disabled: true }), false); + assert.equal(isRestorableFocus({ ...base, tabIndex: -1 }), false); + assert.equal(isRestorableFocus({ ...base, getClientRects: () => [] }), false); + }); + + it('wraps Tab and Shift+Tab indices inside the dialog', () => { + assert.equal(nextFocusIndex(3, 0, false), 1); + assert.equal(nextFocusIndex(3, 2, false), 0); + assert.equal(nextFocusIndex(3, 0, true), 2); + assert.equal(nextFocusIndex(0, -1, false), -1); + }); +}); diff --git a/frontend/src/commands/paletteShortcut.js b/frontend/src/commands/paletteShortcut.js new file mode 100644 index 00000000..893dc943 --- /dev/null +++ b/frontend/src/commands/paletteShortcut.js @@ -0,0 +1,12 @@ +export function commandPaletteShortcut(event, previousEditorPress, now = Date.now()) { + if (event.isComposing || event.keyCode === 229) { + return { handled: false, toggle: false, nextEditorPress: previousEditorPress }; + } + const chord = (event.metaKey || event.ctrlKey) && !event.altKey && event.key.toLowerCase() === 'k'; + if (!chord || event.isMobile) return { handled: false, toggle: false, nextEditorPress: previousEditorPress }; + if (!event.target?.isContentEditable) return { handled: true, toggle: true, nextEditorPress: null }; + const second = previousEditorPress?.target === event.target && now - previousEditorPress.at <= 1500; + return second + ? { handled: true, toggle: true, nextEditorPress: null } + : { handled: true, toggle: false, nextEditorPress: { target: event.target, at: now } }; +} diff --git a/frontend/src/commands/paletteShortcut.test.js b/frontend/src/commands/paletteShortcut.test.js new file mode 100644 index 00000000..6b33638e --- /dev/null +++ b/frontend/src/commands/paletteShortcut.test.js @@ -0,0 +1,36 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { commandPaletteShortcut } from './paletteShortcut.js'; + +describe('commandPaletteShortcut', () => { + it('ignores unrelated and mobile events', () => { + assert.equal(commandPaletteShortcut({ key: 'x', metaKey: true }, null, 1).handled, false); + assert.equal(commandPaletteShortcut({ key: 'k', metaKey: true, isMobile: true }, null, 1).handled, false); + }); + + it('ignores Cmd/Ctrl+K while an IME is composing', () => { + assert.equal(commandPaletteShortcut({ key: 'k', metaKey: true, isComposing: true }, null, 1).handled, false); + assert.equal(commandPaletteShortcut({ key: 'k', ctrlKey: true, keyCode: 229 }, null, 1).handled, false); + }); + + it('toggles ordinary Cmd/Ctrl+K immediately', () => { + assert.deepEqual(commandPaletteShortcut({ key: 'k', metaKey: true, target: {} }, null, 1), { + handled: true, toggle: true, nextEditorPress: null, + }); + }); + + it('preserves first rich-editor Insert Link and opens on the next press', () => { + const editor = { isContentEditable: true }; + const first = commandPaletteShortcut({ key: 'k', ctrlKey: true, target: editor }, null, 1000); + assert.equal(first.toggle, false); + const second = commandPaletteShortcut({ key: 'k', ctrlKey: true, target: editor }, first.nextEditorPress, 2000); + assert.equal(second.toggle, true); + assert.equal(second.nextEditorPress, null); + }); + + it('expires the editor second-press window after 1500 ms', () => { + const editor = { isContentEditable: true }; + const old = { target: editor, at: 1000 }; + assert.equal(commandPaletteShortcut({ key: 'k', metaKey: true, target: editor }, old, 2501).toggle, false); + }); +}); diff --git a/frontend/src/commands/paletteState.js b/frontend/src/commands/paletteState.js new file mode 100644 index 00000000..0c605c38 --- /dev/null +++ b/frontend/src/commands/paletteState.js @@ -0,0 +1,29 @@ +export function createPaletteState() { + return { query: '', activeIndex: 0, resultCount: 0, continuation: null }; +} + +export function paletteKeyIntent(event) { + if (event.isComposing || event.nativeEvent?.isComposing || event.keyCode === 229 || event.nativeEvent?.keyCode === 229) { + return null; + } + const key = event.key.toLowerCase(); + if (event.key === 'ArrowDown' || (event.ctrlKey && (key === 'n' || key === 'j'))) return 'next'; + if (event.key === 'ArrowUp' || (event.ctrlKey && (key === 'p' || key === 'k'))) return 'previous'; + if (event.key === 'Enter') return 'execute'; + if (event.key === 'Escape') return 'back'; + return null; +} + +export function reducePaletteState(state, event) { + switch (event.type) { + case 'open': return { ...createPaletteState(), resultCount: state.resultCount }; + case 'query': return { ...state, query: event.query, activeIndex: 0 }; + case 'results': return { ...state, resultCount: event.count, activeIndex: Math.min(state.activeIndex, Math.max(0, event.count - 1)) }; + case 'move': return { ...state, activeIndex: Math.max(0, Math.min(state.resultCount - 1, state.activeIndex + event.delta)) }; + case 'continuation': return { ...state, continuation: event.value, query: '', activeIndex: 0 }; + case 'back': return state.continuation + ? { ...state, continuation: null, query: '', activeIndex: 0 } + : state; + default: return state; + } +} diff --git a/frontend/src/commands/paletteState.test.js b/frontend/src/commands/paletteState.test.js new file mode 100644 index 00000000..4a3702c9 --- /dev/null +++ b/frontend/src/commands/paletteState.test.js @@ -0,0 +1,49 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createPaletteState, paletteKeyIntent, reducePaletteState } from './paletteState.js'; + +describe('palette state', () => { + it('maps every approved local navigation key', () => { + assert.equal(paletteKeyIntent({ key: 'ArrowDown', ctrlKey: false }), 'next'); + assert.equal(paletteKeyIntent({ key: 'n', ctrlKey: true }), 'next'); + assert.equal(paletteKeyIntent({ key: 'j', ctrlKey: true }), 'next'); + assert.equal(paletteKeyIntent({ key: 'ArrowUp', ctrlKey: false }), 'previous'); + assert.equal(paletteKeyIntent({ key: 'p', ctrlKey: true }), 'previous'); + assert.equal(paletteKeyIntent({ key: 'k', ctrlKey: true }), 'previous'); + assert.equal(paletteKeyIntent({ key: 'Enter', ctrlKey: false }), 'execute'); + assert.equal(paletteKeyIntent({ key: 'Escape', ctrlKey: false }), 'back'); + }); + + it('ignores navigation and execution keys while an IME is composing', () => { + assert.equal(paletteKeyIntent({ key: 'Enter', isComposing: true }), null); + assert.equal(paletteKeyIntent({ key: 'ArrowDown', nativeEvent: { isComposing: true } }), null); + assert.equal(paletteKeyIntent({ key: 'Enter', keyCode: 229 }), null); + }); + + it('resets highlight on query/results and clamps movement', () => { + let state = reducePaletteState(createPaletteState(), { type: 'results', count: 2 }); + state = reducePaletteState(state, { type: 'move', delta: 1 }); + state = reducePaletteState(state, { type: 'move', delta: 1 }); + assert.equal(state.activeIndex, 1); + state = reducePaletteState(state, { type: 'query', query: 'arch' }); + assert.equal(state.activeIndex, 0); + }); + + it('opens with the current result count so keyboard movement works immediately', () => { + let state = reducePaletteState(createPaletteState(), { type: 'results', count: 38 }); + state = reducePaletteState(state, { type: 'open' }); + state = reducePaletteState(state, { type: 'move', delta: 1 }); + assert.equal(state.resultCount, 38); + assert.equal(state.activeIndex, 1); + }); + + it('backs out one continuation without latching a close request across reopen', () => { + let state = reducePaletteState(createPaletteState(), { type: 'continuation', value: { kind: 'move' } }); + state = reducePaletteState(state, { type: 'back' }); + assert.equal(state.continuation, null); + state = reducePaletteState(state, { type: 'back' }); + assert.equal('closeRequested' in state, false); + state = reducePaletteState(state, { type: 'open' }); + assert.equal('closeRequested' in state, false); + }); +}); diff --git a/frontend/src/commands/pendingOperationManager.js b/frontend/src/commands/pendingOperationManager.js new file mode 100644 index 00000000..05a6de44 --- /dev/null +++ b/frontend/src/commands/pendingOperationManager.js @@ -0,0 +1,20 @@ +export function createPendingOperationManager(timers) { + const operations = new Set(); + + const register = operation => { + operations.add(operation); + return () => operations.delete(operation); + }; + + const flush = async (mode = 'normal') => { + const pending = [...operations]; + operations.clear(); + await Promise.allSettled(pending.map(operation => { + timers.clearTimeout(operation.timer); + if (mode === 'unload' && operation.unload) return operation.unload(); + return operation.run(); + })); + }; + + return Object.freeze({ register, flush }); +} diff --git a/frontend/src/commands/pendingOperationManager.test.js b/frontend/src/commands/pendingOperationManager.test.js new file mode 100644 index 00000000..3653aef8 --- /dev/null +++ b/frontend/src/commands/pendingOperationManager.test.js @@ -0,0 +1,34 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createPendingOperationManager } from './pendingOperationManager.js'; + +test('flushes registered operations once using normal or unload behavior', async () => { + const events = []; + const timers = { clearTimeout: timer => events.push(['clear', timer]) }; + const manager = createPendingOperationManager(timers); + manager.register({ + timer: 'normal-timer', + run: async () => events.push(['run']), + unload: () => events.push(['unload-unused']), + }); + await manager.flush('normal'); + assert.deepEqual(events, [['clear', 'normal-timer'], ['run']]); + + manager.register({ + timer: 'unload-timer', + run: async () => events.push(['run-unused']), + unload: () => events.push(['unload']), + }); + await manager.flush('unload'); + await manager.flush('normal'); + assert.deepEqual(events.slice(2), [['clear', 'unload-timer'], ['unload']]); +}); + +test('unregister prevents a pending operation from being flushed', async () => { + let calls = 0; + const manager = createPendingOperationManager({ clearTimeout() {} }); + const unregister = manager.register({ timer: 1, run: () => { calls += 1; } }); + unregister(); + await manager.flush('normal'); + assert.equal(calls, 0); +}); diff --git a/frontend/src/commands/registry.js b/frontend/src/commands/registry.js new file mode 100644 index 00000000..0da4eea9 --- /dev/null +++ b/frontend/src/commands/registry.js @@ -0,0 +1,34 @@ +import { validateCommandDefinition } from './contracts.js'; +import { hasTargets } from './targets.js'; +import { rankCommands } from './search.js'; +import { effectiveCommandKeys } from './shortcuts.js'; + +export function createCommandRegistry(definitions) { + const ordered = []; + const byId = new Map(); + for (const input of definitions) { + const command = validateCommandDefinition(input); + if (byId.has(command.id)) throw new TypeError(`duplicate command id "${command.id}"`); + ordered.push(command); + byId.set(command.id, command); + } + Object.freeze(ordered); + + const available = context => ordered.filter(command => command.isAvailable(context) && hasTargets(command, context)); + const decorate = (results, context) => results.map(result => ({ + ...result, + bindings: effectiveCommandKeys(result.command, context).bindings, + })); + + return Object.freeze({ + get(id) { + return byId.get(id) || null; + }, + list(context) { + return decorate(rankCommands(available(context), '', context), context); + }, + search(query, context) { + return decorate(rankCommands(available(context), query, context), context); + }, + }); +} diff --git a/frontend/src/commands/registry.test.js b/frontend/src/commands/registry.test.js new file mode 100644 index 00000000..c93bb3c7 --- /dev/null +++ b/frontend/src/commands/registry.test.js @@ -0,0 +1,48 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createCommandContext } from './contracts.js'; +import { createCommandRegistry } from './registry.js'; + +const raw = (id, overrides = {}) => ({ + id, titleKey: `${id}.title`, aliasKeys: [], icon: 'test', group: 'test', + defaultKeys: { primary: null, secondary: [] }, rank: { base: 0 }, + isAvailable: () => true, targetMode: 'global', executorId: id, ...overrides, +}); +const context = createCommandContext({ + surface: 'list', activeConversationId: null, selectedConversationIds: [], conversations: [], + accountId: null, folder: null, draft: null, gtdAvailable: false, cardDavConnected: false, + modal: null, editing: false, platform: 'mac', shortcutOverrides: {}, translate: key => key, +}); + +describe('createCommandRegistry', () => { + it('rejects duplicate IDs before exposing a partial registry', () => { + assert.throws(() => createCommandRegistry([raw('test.one'), raw('test.one')]), /duplicate command id "test.one"/); + }); + + it('returns definitions by ID without exposing mutation', () => { + const registry = createCommandRegistry([raw('test.one')]); + assert.equal(registry.get('test.one').executorId, 'test.one'); + assert.equal(registry.get('test.missing'), null); + assert.ok(Object.isFrozen(registry.get('test.one'))); + }); + + it('omits explicit and target-mode-unavailable commands', () => { + const registry = createCommandRegistry([ + raw('global.visible'), + raw('global.hidden', { isAvailable: () => false }), + raw('mail.archive', { targetMode: 'bulk_safe' }), + ]); + assert.deepEqual(registry.list(context).map(entry => entry.command.id), ['global.visible']); + }); + + it('returns effective bindings and localized alias search metadata', () => { + const registry = createCommandRegistry([raw('mail.archive', { + titleKey: 'archive', aliasKeys: ['done'], defaultKeys: { primary: 'e', secondary: [] }, + })]); + const localized = { ...context, translate: key => ({ archive: 'Archive', done: 'Done' })[key] || key }; + const [result] = registry.search('done', localized); + assert.equal(result.title, 'Archive'); + assert.equal(result.matchedAlias, 'Done'); + assert.deepEqual(result.bindings, [{ key: 'e', kind: 'primary' }]); + }); +}); diff --git a/frontend/src/commands/search.js b/frontend/src/commands/search.js new file mode 100644 index 00000000..f754123e --- /dev/null +++ b/frontend/src/commands/search.js @@ -0,0 +1,60 @@ +function normalize(value) { + return String(value || '').normalize('NFKD').replace(/\p{Diacritic}/gu, '').toLowerCase().trim(); +} + +function editDistance(a, b) { + const rows = Array.from({ length: a.length + 1 }, (_, i) => [i]); + for (let j = 1; j <= b.length; j += 1) rows[0][j] = j; + for (let i = 1; i <= a.length; i += 1) { + for (let j = 1; j <= b.length; j += 1) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + rows[i][j] = Math.min(rows[i - 1][j] + 1, rows[i][j - 1] + 1, rows[i - 1][j - 1] + cost); + if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) { + rows[i][j] = Math.min(rows[i][j], rows[i - 2][j - 2] + 1); + } + } + } + return rows[a.length][b.length]; +} + +function fuzzyScore(candidate, query) { + const text = normalize(candidate); + const needle = normalize(query); + if (!needle) return 0; + if (text === needle) return 1000; + if (text.startsWith(needle)) return 900 - (text.length - needle.length); + if (text.includes(needle)) return 800 - text.indexOf(needle); + const distance = editDistance(text, needle); + const limit = Math.max(1, Math.floor(Math.max(text.length, needle.length) * 0.34)); + return distance <= limit ? 600 - (distance * 40) - Math.abs(text.length - needle.length) : null; +} + +export function rankCommands(commands, query, context) { + return commands.map((command, index) => { + const title = context.translate(command.titleKey, command.params); + const aliases = command.aliasKeys.map(key => ({ key, value: context.translate(key, command.params) })); + const candidates = [{ key: null, value: title }, ...aliases] + .map(item => ({ ...item, fuzzy: fuzzyScore(item.value, query) })) + .filter(item => item.fuzzy != null) + .sort((a, b) => b.fuzzy - a.fuzzy); + if (query.trim() && !candidates.length) return null; + const match = candidates[0] || { key: null, value: title, fuzzy: 0 }; + const boost = command.rank.boost ? command.rank.boost(context) : 0; + return { + command, + title, + matchedAlias: match.key ? match.value : null, + matchedAliasKey: match.key, + score: match.fuzzy + command.rank.base + boost, + index, + }; + }).filter(Boolean) + .sort((a, b) => b.score - a.score || a.index - b.index) + .map(result => ({ + command: result.command, + title: result.title, + matchedAlias: result.matchedAlias, + matchedAliasKey: result.matchedAliasKey, + score: result.score, + })); +} diff --git a/frontend/src/commands/search.test.js b/frontend/src/commands/search.test.js new file mode 100644 index 00000000..65573612 --- /dev/null +++ b/frontend/src/commands/search.test.js @@ -0,0 +1,55 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createCommandContext, validateCommandDefinition } from './contracts.js'; +import { rankCommands } from './search.js'; + +const make = (id, titleKey, aliasKeys, base, boost = () => 0) => validateCommandDefinition({ + id, titleKey, aliasKeys, icon: 'test', group: 'test', + defaultKeys: { primary: null, secondary: [] }, rank: { base, boost }, + isAvailable: () => true, targetMode: 'global', executorId: id, +}); +const strings = { + archive: 'Archive', done: 'Done', compose: 'Compose', search: 'Search', settings: 'Settings', +}; +const context = createCommandContext({ + surface: 'list', activeConversationId: null, selectedConversationIds: [], conversations: [], + accountId: null, folder: null, draft: null, gtdAvailable: false, cardDavConnected: false, + modal: null, editing: false, platform: 'mac', shortcutOverrides: {}, + translate: key => strings[key] || key, +}); +const archive = make('mail.archive', 'archive', ['done'], 50, ctx => ctx.selectedConversationIds.length ? 30 : 0); +const compose = make('compose.new', 'compose', [], 60); +const settings = make('settings.open', 'settings', [], 10); + +describe('rankCommands', () => { + it('finds case-insensitive title matches', () => { + assert.equal(rankCommands([archive, compose], 'ARCHIVE', context)[0].command.id, 'mail.archive'); + }); + + it('keeps the Mailflow title and discloses the matching alias', () => { + const [result] = rankCommands([archive, compose], 'done', context); + assert.equal(result.title, 'Archive'); + assert.equal(result.matchedAlias, 'Done'); + assert.equal(result.matchedAliasKey, 'done'); + }); + + it('tolerates a close transposition misspelling', () => { + assert.equal(rankCommands([archive, compose], 'arhcive', context)[0].command.id, 'mail.archive'); + }); + + it('uses stable base priority for an empty query', () => { + assert.deepEqual(rankCommands([settings, compose, archive], '', context).map(x => x.command.id), [ + 'compose.new', 'mail.archive', 'settings.open', + ]); + }); + + it('adds contextual boost and preserves definition order for exact ties', () => { + const selected = { ...context, selectedConversationIds: ['acct:'] }; + const equalA = make('navigation.a', 'search', [], 1); + const equalB = make('navigation.b', 'search', [], 1); + assert.equal(rankCommands([compose, archive], '', selected)[0].command.id, 'mail.archive'); + assert.deepEqual(rankCommands([equalA, equalB], 'search', context).map(x => x.command.id), [ + 'navigation.a', 'navigation.b', + ]); + }); +}); diff --git a/frontend/src/commands/selection.js b/frontend/src/commands/selection.js new file mode 100644 index 00000000..ee4ca9d9 --- /dev/null +++ b/frontend/src/commands/selection.js @@ -0,0 +1,5 @@ +export function nextSelection(current, nextOrUpdater) { + const currentCopy = new Set(current || []); + const next = typeof nextOrUpdater === 'function' ? nextOrUpdater(currentCopy) : nextOrUpdater; + return new Set(next || []); +} diff --git a/frontend/src/commands/selection.test.js b/frontend/src/commands/selection.test.js new file mode 100644 index 00000000..0a2275b3 --- /dev/null +++ b/frontend/src/commands/selection.test.js @@ -0,0 +1,20 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { nextSelection } from './selection.js'; + +describe('nextSelection', () => { + it('clones direct Sets and deduplicates iterables', () => { + const input = new Set(['row-a']); + const output = nextSelection(input, input); + assert.deepEqual([...output], ['row-a']); + assert.notStrictEqual(output, input); + assert.deepEqual([...nextSelection(input, ['row-a', 'row-a', 'row-b'])], ['row-a', 'row-b']); + }); + + it('runs updater functions against a defensive Set copy', () => { + const input = new Set(['row-a']); + const output = nextSelection(input, current => current.add('row-b')); + assert.deepEqual([...input], ['row-a']); + assert.deepEqual([...output], ['row-a', 'row-b']); + }); +}); diff --git a/frontend/src/commands/shortcutCommands.js b/frontend/src/commands/shortcutCommands.js new file mode 100644 index 00000000..5a43f768 --- /dev/null +++ b/frontend/src/commands/shortcutCommands.js @@ -0,0 +1,101 @@ +const keys = (primary = null, secondary = []) => ({ primary, secondary }); +const modKey = (mac, key) => ({ mac, windows: `ctrl+${key}`, linux: `ctrl+${key}`, default: `ctrl+${key}` }); +const ICONS = { + 'app.undo': 'clock', + 'navigation.nextConversation': 'mail-open', 'navigation.previousConversation': 'mail-open', + 'navigation.nextThreadMessage': 'mail-open', 'navigation.previousThreadMessage': 'mail-open', + 'navigation.scrollDown': 'eye', 'navigation.scrollUp': 'eye', + 'navigation.openConversation': 'mail-open', 'navigation.inbox': 'inbox', + 'navigation.sent': 'folder', 'navigation.drafts': 'folder', 'navigation.contacts': 'contacts', + 'selection.toggle': 'check-square', 'selection.extendNext': 'check-square', + 'selection.extendPrevious': 'check-square', 'selection.focusedAndOlder': 'check-square', + 'selection.all': 'check-square', 'selection.clear': 'check-square', + 'navigation.back': 'mail-open', + 'help.shortcuts': 'settings', 'layout.toggleRightSidebar': 'appearance', 'mail.print': 'mail', + 'compose.send': 'compose', +}; +const command = (id, titleKey, targetMode, defaultKeys, executorId, rank = 40, isAvailable = () => true) => ({ + id, titleKey, aliasKeys: [], icon: ICONS[id] || 'settings', group: id.split('.')[0], targetMode, defaultKeys, + executorId, rank: { base: rank }, isAvailable, +}); + +export const shortcutCommandDefinitions = [ + command('app.undo', 'commands.shortcuts.undo', 'global', keys('z'), 'app.undo', 90, context => context.undoAvailable), + command('navigation.nextConversation', 'shortcuts.actions.nextMessage.label', 'global', keys('j'), 'navigation.nextConversation', 70, context => ['list', 'conversation'].includes(context.surface)), + command('navigation.previousConversation', 'shortcuts.actions.prevMessage.label', 'global', keys('k'), 'navigation.previousConversation', 70, context => ['list', 'conversation'].includes(context.surface)), + command('navigation.nextThreadMessage', 'commands.shortcuts.nextThreadMessage', 'single_conversation', keys('n'), 'navigation.nextThreadMessage', 70, context => context.surface === 'conversation'), + command('navigation.previousThreadMessage', 'commands.shortcuts.previousThreadMessage', 'single_conversation', keys('p'), 'navigation.previousThreadMessage', 70, context => context.surface === 'conversation'), + command('navigation.scrollDown', 'commands.shortcuts.scrollDown', 'global', keys('space'), 'navigation.scrollDown', 30, context => context.surface === 'conversation'), + command('navigation.scrollUp', 'commands.shortcuts.scrollUp', 'global', keys('shift+space'), 'navigation.scrollUp', 30, context => context.surface === 'conversation'), + command('navigation.openConversation', 'commands.shortcuts.openConversation', 'global', keys('enter', ['o']), 'navigation.openConversation', 100, context => context.surface === 'list' && context.visibleConversationIds.length > 0), + command('selection.toggle', 'shortcuts.actions.selectMessage.label', 'global', keys('x'), 'selection.toggle', 60, context => context.surface === 'list' && context.visibleConversationIds.length > 0), + command('selection.extendNext', 'commands.shortcuts.extendNext', 'global', keys('shift+j'), 'selection.extendNext', 60, context => context.surface === 'list'), + command('selection.extendPrevious', 'commands.shortcuts.extendPrevious', 'global', keys('shift+k'), 'selection.extendPrevious', 60, context => context.surface === 'list'), + command('selection.focusedAndOlder', 'commands.shortcuts.selectOlder', 'global', keys(modKey('meta+a', 'a')), 'selection.focusedAndOlder', 80, context => context.surface === 'list' && context.visibleConversationIds.length > 0), + command('selection.all', 'commands.shortcuts.selectAll', 'global', keys(modKey('meta+shift+a', 'shift+a')), 'selection.all', 85, context => context.surface === 'list'), + command('selection.clear', 'commands.shortcuts.clearSelection', 'global', keys('escape'), 'selection.clear', 110, context => context.selectedConversationIds.length > 0), + command('navigation.back', 'common.back', 'global', keys('escape'), 'navigation.back', 50, context => context.surface === 'conversation'), + command('navigation.inbox', 'commands.navigation.unifiedInbox.title', 'global', keys('g i'), 'navigation.shortcutInbox'), + command('navigation.sent', 'commands.shortcuts.sent', 'account', keys('g t'), 'navigation.sent', 40, context => !!context.accountId), + command('navigation.drafts', 'commands.shortcuts.drafts', 'account', keys('g d'), 'navigation.drafts', 40, context => !!context.accountId), + command('navigation.contacts', 'commands.shortcuts.contacts', 'global', keys('g c'), 'navigation.contacts'), + command('help.shortcuts', 'shortcuts.title', 'global', keys('?'), 'help.shortcuts'), + command('layout.toggleRightSidebar', 'shortcuts.actions.toggleRightSidebar.label', 'global', keys(modKey('meta+/', '/')), 'layout.toggleRightSidebar'), + command('mail.print', 'shortcuts.actions.printMessage.label', 'single_conversation', keys(modKey('meta+p', 'p')), 'mail.print', 60, context => context.surface === 'conversation'), + command('compose.send', 'compose.send', 'draft', keys(modKey('meta+enter', 'enter')), 'editor.send', 0, () => false), +]; + +const success = (ids = []) => ({ status: 'success', succeededIds: ids.filter(Boolean), failed: [] }); + +export function createShortcutCommandExecutors(deps) { + const stepConversation = direction => ({ context }) => { + const ids = context.visibleConversationIds; + if (!ids.length) return { status: 'cancelled' }; + const index = ids.indexOf(context.activeConversationId); + const nextIndex = index < 0 + ? (direction > 0 ? 0 : ids.length - 1) + : Math.max(0, Math.min(ids.length - 1, index + direction)); + const next = ids[nextIndex]; + deps.selectTarget(next, context); + return success([next]); + }; + const stepThread = direction => async ({ context }) => { + const ids = await deps.loadThreadTargets(context); + if (ids.length < 2) return { status: 'cancelled' }; + const index = Math.max(0, ids.indexOf(context.activeConversationId)); + const next = ids[Math.max(0, Math.min(ids.length - 1, index + direction))]; + deps.selectTarget(next, context); + return success([next]); + }; + const specialFolder = specialUse => async ({ context }) => { + const folders = await deps.getFolders(context.accountId); + const folder = folders.find(item => item.special_use?.toLowerCase() === specialUse.toLowerCase()); + if (!folder) return { status: 'failed', failed: [{ id: context.accountId, error: `${specialUse} folder unavailable` }] }; + deps.navigate({ accountId: context.accountId, folder: folder.path }); + return success([folder.path]); + }; + return { + 'app.undo': () => { deps.undoLatest(); return success(); }, + 'navigation.nextConversation': stepConversation(1), + 'navigation.previousConversation': stepConversation(-1), + 'navigation.nextThreadMessage': stepThread(1), + 'navigation.previousThreadMessage': stepThread(-1), + 'navigation.scrollDown': () => { deps.scrollConversation(0.8); return success(); }, + 'navigation.scrollUp': () => { deps.scrollConversation(-0.8); return success(); }, + 'navigation.openConversation': ({ context }) => { const id = context.activeConversationId || context.visibleConversationIds[0]; deps.selectTarget(id, context); return success([id]); }, + 'selection.toggle': ({ context }) => { const id = context.activeConversationId || context.visibleConversationIds[0]; deps.selection.toggle(id, context); return success([id]); }, + 'selection.extendNext': ({ context }) => { deps.selection.extend(1, context); return success(); }, + 'selection.extendPrevious': ({ context }) => { deps.selection.extend(-1, context); return success(); }, + 'selection.focusedAndOlder': ({ context }) => { deps.selection.selectFocusedAndOlder(context); return success(); }, + 'selection.all': ({ context }) => { deps.selection.selectAllVisible(context); return success(); }, + 'selection.clear': () => { deps.selection.clear(); return success(); }, + 'navigation.back': () => { deps.goBack(); return success(); }, + 'navigation.shortcutInbox': () => { deps.navigate({ accountId: null, folder: 'INBOX' }); return success(); }, + 'navigation.sent': specialFolder('\\Sent'), + 'navigation.drafts': specialFolder('\\Drafts'), + 'navigation.contacts': () => { deps.navigate({ contacts: true }); return success(); }, + 'help.shortcuts': () => { deps.emitShortcut('showHelp'); return success(); }, + 'layout.toggleRightSidebar': () => { deps.emitShortcut('toggleRightSidebar'); return success(); }, + 'mail.print': () => { deps.emitShortcut('printMessage'); return success(); }, + }; +} diff --git a/frontend/src/commands/shortcutCommands.test.js b/frontend/src/commands/shortcutCommands.test.js new file mode 100644 index 00000000..8b5ce3ae --- /dev/null +++ b/frontend/src/commands/shortcutCommands.test.js @@ -0,0 +1,46 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createShortcutCommandExecutors, shortcutCommandDefinitions } from './shortcutCommands.js'; + +function harness() { + const calls = []; + const deps = { + selection: { + toggle: id => calls.push(['toggle', id]), extend: direction => calls.push(['extend', direction]), + selectFocusedAndOlder: () => calls.push(['older']), selectAllVisible: () => calls.push(['all']), + clear: () => calls.push(['clear']), + }, + selectTarget: id => calls.push(['select', id]), + loadThreadTargets: async () => ['a', 'b', 'c'], scrollConversation: n => calls.push(['scroll', n]), + navigate: value => calls.push(['navigate', value]), + getFolders: async accountId => [{ accountId, path: 'Sent', special_use: '\\Sent' }, { accountId, path: 'Drafts', special_use: '\\Drafts' }], + undoLatest: () => calls.push(['undo']), + }; + return { calls, executors: createShortcutCommandExecutors(deps) }; +} + +describe('shortcut commands', () => { + it('owns selection directions and thread navigation', async () => { + const h = harness(); + await h.executors['selection.extendNext']({ context: {} }); + await h.executors['selection.extendPrevious']({ context: {} }); + await h.executors['navigation.nextThreadMessage']({ context: { activeConversationId: 'b' } }); + assert.deepEqual(h.calls, [['extend', 1], ['extend', -1], ['select', 'c']]); + }); + + it('resolves special folders and the latest undo', async () => { + const h = harness(); + await h.executors['navigation.sent']({ context: { accountId: 'account-1' } }); + await h.executors['app.undo']({ context: {} }); + assert.deepEqual(h.calls, [['navigate', { accountId: 'account-1', folder: 'Sent' }], ['undo']]); + }); + + it('declares contextual Enter, selection modifiers, and G sequences', () => { + const byId = new Map(shortcutCommandDefinitions.map(command => [command.id, command])); + assert.equal(byId.get('navigation.openConversation').defaultKeys.primary, 'enter'); + assert.deepEqual(byId.get('navigation.openConversation').defaultKeys.secondary, ['o']); + assert.equal(byId.get('selection.focusedAndOlder').defaultKeys.primary.mac, 'meta+a'); + assert.equal(byId.get('selection.all').defaultKeys.primary.windows, 'ctrl+shift+a'); + assert.equal(byId.get('navigation.inbox').defaultKeys.primary, 'g i'); + }); +}); diff --git a/frontend/src/commands/shortcutDispatcher.js b/frontend/src/commands/shortcutDispatcher.js new file mode 100644 index 00000000..fa7d56b9 --- /dev/null +++ b/frontend/src/commands/shortcutDispatcher.js @@ -0,0 +1,74 @@ +const isTypingTarget = target => ['INPUT', 'TEXTAREA', 'SELECT'].includes(target?.tagName) + || target?.isContentEditable || target?.closest?.('[data-shortcut-recorder="true"]'); + +export function shortcutEventChord(event) { + const key = event.key === ' ' ? 'space' : event.key.toLowerCase(); + const parts = []; + if (event.metaKey) parts.push('meta'); + if (event.ctrlKey) parts.push('ctrl'); + if (event.altKey) parts.push('alt'); + if (event.shiftKey && (key === 'space' || /^[a-z0-9]$/.test(key))) parts.push('shift'); + parts.push(key); + return parts.join('+'); +} + +export function createShortcutDispatcher({ registry, getContext, getBindings, execute, timers }) { + let pending = null; + let pendingTimer = null; + let pendingContextKey = null; + + const reset = () => { + pending = null; + pendingContextKey = null; + if (pendingTimer) timers.clearTimeout(pendingTimer); + pendingTimer = null; + }; + + const handleKeyDown = event => { + const context = getContext(); + const contextKey = JSON.stringify([ + context.surface, context.modal?.kind || '', context.editing, + context.accountId, context.folder, context.activeConversationId, + context.selectedConversationIds, context.shortcutOverrides, + ]); + if ((pending && pendingContextKey !== contextKey) || context.modal || context.editing + || isTypingTarget(event.target) || event.isComposing || event.keyCode === 229) { + reset(); + return false; + } + const chord = shortcutEventChord(event); + if (chord === 'escape' && pending) { + event.preventDefault(); + reset(); + return true; + } + const available = new Map(registry.list(context) + .map(result => result.command) + .map(command => [command.id, command])); + const candidates = getBindings().flatMap(item => available.has(item.commandId) + ? item.bindings.map(binding => ({ ...binding, commandId: item.commandId })) + : []); + const resolved = pending ? `${pending} ${chord}` : chord; + const exact = candidates.filter(candidate => candidate.keys.toLowerCase() === resolved) + .sort((a, b) => (available.get(b.commandId).rank?.base ?? 0) + - (available.get(a.commandId).rank?.base ?? 0)); + if (exact.length) { + event.preventDefault(); + reset(); + execute(exact[0].commandId, { source: 'shortcut' }); + return true; + } + if (candidates.some(candidate => candidate.keys.toLowerCase().startsWith(`${resolved} `))) { + event.preventDefault(); + reset(); + pending = resolved; + pendingContextKey = contextKey; + pendingTimer = timers.setTimeout(reset, 1000); + return true; + } + reset(); + return false; + }; + + return Object.freeze({ handleKeyDown, reset }); +} diff --git a/frontend/src/commands/shortcutDispatcher.test.js b/frontend/src/commands/shortcutDispatcher.test.js new file mode 100644 index 00000000..89107041 --- /dev/null +++ b/frontend/src/commands/shortcutDispatcher.test.js @@ -0,0 +1,59 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createShortcutDispatcher } from './shortcutDispatcher.js'; + +const event = (key, patch = {}) => ({ + key, ctrlKey: false, metaKey: false, shiftKey: false, altKey: false, + target: { tagName: 'DIV', isContentEditable: false }, prevented: false, + preventDefault() { this.prevented = true; }, ...patch, +}); + +function harness(context, available, bindings) { + const calls = []; + const scheduled = []; + const dispatcher = createShortcutDispatcher({ + registry: { list: () => available.map(command => ({ command, title: command.id, score: 0, bindings: [] })) }, + getContext: () => context, + getBindings: () => bindings, execute: (id, options) => calls.push([id, options]), + timers: { setTimeout(fn, ms) { const item = { fn, ms }; scheduled.push(item); return item; }, clearTimeout() {} }, + }); + return { calls, scheduled, dispatcher }; +} + +describe('shortcut dispatcher', () => { + it('resolves contextual Enter by availability and rank', () => { + const h = harness({ surface: 'list', modal: null, editing: false }, [ + { id: 'navigation.openConversation', rank: { base: 100 } }, + ], [ + { commandId: 'navigation.openConversation', bindings: [{ keys: 'enter' }] }, + { commandId: 'mail.replyAll', bindings: [{ keys: 'enter' }] }, + ]); + h.dispatcher.handleKeyDown(event('Enter')); + assert.equal(h.calls[0][0], 'navigation.openConversation'); + }); + + it('dispatches a one-second sequence and cancels it on Escape', () => { + const h = harness({ surface: 'list', modal: null, editing: false }, [ + { id: 'navigation.inbox', rank: { base: 40 } }, + ], [{ commandId: 'navigation.inbox', bindings: [{ keys: 'g i' }] }]); + const first = event('g'); + h.dispatcher.handleKeyDown(first); + assert.equal(h.scheduled[0].ms, 1000); + h.dispatcher.handleKeyDown(event('i')); + assert.equal(h.calls[0][0], 'navigation.inbox'); + h.dispatcher.handleKeyDown(event('g')); + h.dispatcher.handleKeyDown(event('Escape')); + h.dispatcher.handleKeyDown(event('i')); + assert.equal(h.calls.length, 1); + }); + + it('normalizes modifiers and yields to typing, modals, and IME composition', () => { + const available = [{ id: 'mail.unsubscribe', rank: { base: 50 } }]; + const bindings = [{ commandId: 'mail.unsubscribe', bindings: [{ keys: 'meta+u' }] }]; + const h = harness({ surface: 'conversation', modal: null, editing: false }, available, bindings); + h.dispatcher.handleKeyDown(event('u', { metaKey: true })); + h.dispatcher.handleKeyDown(event('u', { metaKey: true, target: { tagName: 'INPUT' } })); + h.dispatcher.handleKeyDown(event('u', { metaKey: true, isComposing: true })); + assert.equal(h.calls.length, 1); + }); +}); diff --git a/frontend/src/commands/shortcuts.js b/frontend/src/commands/shortcuts.js new file mode 100644 index 00000000..27b3bdad --- /dev/null +++ b/frontend/src/commands/shortcuts.js @@ -0,0 +1,60 @@ +function selectKey(spec, platform) { + if (!spec) return null; + if (typeof spec === 'string') return spec; + return spec[platform] || spec.default || null; +} + +export function effectiveCommandKeys(command, context) { + const hasOverride = Object.hasOwn(context.shortcutOverrides, command.id); + const override = hasOverride ? context.shortcutOverrides[command.id] : undefined; + const bindings = []; + const primary = hasOverride ? override : selectKey(command.defaultKeys.primary, context.platform); + if (primary) bindings.push({ + key: primary, + kind: hasOverride ? 'user' : typeof command.defaultKeys.primary === 'object' ? 'platform' : 'primary', + }); + for (const spec of command.defaultKeys.secondary) { + const key = selectKey(spec, context.platform); + if (key && !bindings.some(binding => binding.key === key)) bindings.push({ key, kind: 'secondary' }); + } + return { bindings, conflicts: [] }; +} + +function formatChord(key, platform) { + const mac = platform === 'mac'; + const labels = mac + ? { meta: '⌘', ctrl: '⌃', alt: '⌥', shift: '⇧', enter: '↵' } + : { meta: 'Win+', ctrl: 'Ctrl+', alt: 'Alt+', shift: 'Shift+', enter: 'Enter' }; + return key.split('+').map((part, index, all) => { + const normalized = part.toLowerCase(); + const label = labels[normalized] || (part.length === 1 ? part.toUpperCase() : part); + return mac || index === all.length - 1 ? label : label; + }).join(mac ? '' : ''); +} + +export function formatCommandKey(key, platform) { + return key.split(' ').map(chord => formatChord(chord, platform)).join(' then '); +} + +export function getEffectiveCommandBindings(definitions, context) { + return definitions.map(command => ({ + commandId: command.id, + bindings: effectiveCommandKeys(command, context).bindings.map(binding => ({ + keys: binding.key, + source: binding.kind, + })), + })); +} + +export function findBindingConflicts(commands, context) { + const owners = new Map(); + for (const command of commands) { + for (const { key } of effectiveCommandKeys(command, context).bindings) { + if (!owners.has(key)) owners.set(key, []); + owners.get(key).push(command.id); + } + } + return [...owners.entries()] + .filter(([, commandIds]) => commandIds.length > 1) + .map(([key, commandIds]) => ({ key, commandIds })); +} diff --git a/frontend/src/commands/shortcuts.test.js b/frontend/src/commands/shortcuts.test.js new file mode 100644 index 00000000..0b6b1c22 --- /dev/null +++ b/frontend/src/commands/shortcuts.test.js @@ -0,0 +1,60 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { effectiveCommandKeys, findBindingConflicts, formatCommandKey, getEffectiveCommandBindings } from './shortcuts.js'; +import { normalizeLegacyShortcutOverrides } from '../utils/defaultShortcuts.js'; + +const command = { + id: 'palette.toggle', + defaultKeys: { + primary: { mac: 'meta+k', windows: 'ctrl+k', linux: 'ctrl+k' }, + secondary: ['alt+k'], + }, +}; + +describe('command shortcut metadata', () => { + it('selects the platform primary and retains secondary bindings', () => { + assert.deepEqual(effectiveCommandKeys(command, { platform: 'mac', shortcutOverrides: {} }).bindings, [ + { key: 'meta+k', kind: 'platform' }, { key: 'alt+k', kind: 'secondary' }, + ]); + assert.equal(effectiveCommandKeys(command, { platform: 'linux', shortcutOverrides: {} }).bindings[0].key, 'ctrl+k'); + }); + + it('uses the unchanged user override as primary without deleting secondary keys', () => { + assert.deepEqual(effectiveCommandKeys(command, { + platform: 'windows', shortcutOverrides: { 'palette.toggle': 'ctrl+shift+k' }, + }).bindings, [ + { key: 'ctrl+shift+k', kind: 'user' }, { key: 'alt+k', kind: 'secondary' }, + ]); + }); + + it('supports explicit primary unbinding and formats platform labels', () => { + assert.deepEqual(effectiveCommandKeys(command, { + platform: 'mac', shortcutOverrides: { 'palette.toggle': null }, + }).bindings, [{ key: 'alt+k', kind: 'secondary' }]); + assert.equal(formatCommandKey('meta+shift+k', 'mac'), '⌘⇧K'); + assert.equal(formatCommandKey('ctrl+shift+k', 'windows'), 'Ctrl+Shift+K'); + }); + + it('reports conflicts instead of silently dropping either command', () => { + const commands = [command, { ...command, id: 'navigation.search', defaultKeys: { primary: 'alt+k', secondary: [] } }]; + assert.deepEqual(findBindingConflicts(commands, { platform: 'linux', shortcutOverrides: {} }), [ + { key: 'alt+k', commandIds: ['palette.toggle', 'navigation.search'] }, + ]); + }); +}); + +it('aggregates sequences, platform keys, secondary keys, and legacy overrides', () => { + const definitions = [ + { ...command, id: 'mail.toggleRead', defaultKeys: { primary: 'u', secondary: ['m'] } }, + { ...command, id: 'navigation.inbox', defaultKeys: { primary: 'g i', secondary: [] } }, + ]; + const context = { + platform: 'mac', + shortcutOverrides: normalizeLegacyShortcutOverrides({ toggleRead: 'q' }), + }; + assert.deepEqual(getEffectiveCommandBindings(definitions, context)[0].bindings, [ + { keys: 'q', source: 'user' }, + { keys: 'm', source: 'secondary' }, + ]); + assert.equal(formatCommandKey('g i', 'linux'), 'G then I'); +}); diff --git a/frontend/src/commands/targets.js b/frontend/src/commands/targets.js new file mode 100644 index 00000000..36e1d23a --- /dev/null +++ b/frontend/src/commands/targets.js @@ -0,0 +1,38 @@ +import { TARGET_MODES } from './contracts.js'; + +function contextualIds(context) { + return context.selectedConversationIds.length + ? context.selectedConversationIds + : context.activeConversationId ? [context.activeConversationId] : []; +} + +export function hasTargets(command, context) { + switch (command.targetMode) { + case TARGET_MODES.GLOBAL: return true; + case TARGET_MODES.ACCOUNT: return Boolean(context.accountId); + case TARGET_MODES.DRAFT: return Boolean(context.draft?.id); + case TARGET_MODES.SINGLE_CONVERSATION: return contextualIds(context).length === 1; + case TARGET_MODES.BULK_SAFE: return contextualIds(context).length > 0; + default: return false; + } +} + +export function resolveTargetIds(command, context, frozenTargetIds) { + if (command.targetMode === TARGET_MODES.DRAFT) { + const currentId = context.draft?.id || null; + const requested = frozenTargetIds == null + ? (currentId ? [currentId] : []) + : [...new Set(frozenTargetIds)]; + return { + targetIds: requested.filter(id => id === currentId), + missingTargetIds: requested.filter(id => id !== currentId), + }; + } + if (![TARGET_MODES.SINGLE_CONVERSATION, TARGET_MODES.BULK_SAFE].includes(command.targetMode)) { + return { targetIds: [], missingTargetIds: [] }; + } + const requested = frozenTargetIds == null ? contextualIds(context) : [...new Set(frozenTargetIds)]; + const targetIds = requested.filter(id => context.conversationsById[id]); + const missingTargetIds = requested.filter(id => !context.conversationsById[id]); + return { targetIds, missingTargetIds }; +} diff --git a/frontend/src/commands/targets.test.js b/frontend/src/commands/targets.test.js new file mode 100644 index 00000000..f6227503 --- /dev/null +++ b/frontend/src/commands/targets.test.js @@ -0,0 +1,65 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { createCommandContext, validateCommandDefinition } from './contracts.js'; +import { hasTargets, resolveTargetIds } from './targets.js'; + +const command = targetMode => validateCommandDefinition({ + id: `test.${targetMode}`, + titleKey: 'test.title', aliasKeys: [], icon: 'test', group: 'test', + defaultKeys: { primary: null, secondary: [] }, rank: { base: 0 }, + isAvailable: () => true, targetMode, executorId: 'test.execute', +}); +const context = overrides => createCommandContext({ + surface: 'list', activeConversationId: null, selectedConversationIds: [], + conversations: [ + { id: 'row-a', message_id: '', account_id: 'acct' }, + { id: 'row-b', message_id: '', account_id: 'acct' }, + ], + accountId: 'acct', folder: 'INBOX', draft: null, gtdAvailable: false, + cardDavConnected: false, modal: null, editing: false, platform: 'linux', + shortcutOverrides: {}, translate: key => key, ...overrides, +}); + +describe('command targets', () => { + it('omits conversation commands without an active row or checkbox selection', () => { + assert.equal(hasTargets(command('single_conversation'), context({})), false); + assert.equal(hasTargets(command('bulk_safe'), context({})), false); + }); + + it('makes single and bulk-safe commands available for one active conversation', () => { + const ctx = context({ activeConversationId: 'acct:' }); + assert.equal(hasTargets(command('single_conversation'), ctx), true); + assert.deepEqual(resolveTargetIds(command('bulk_safe'), ctx), { + targetIds: ['acct:'], missingTargetIds: [], + }); + }); + + it('uses the complete selection and hides single-conversation commands in bulk', () => { + const ctx = context({ activeConversationId: 'acct:', selectedConversationIds: ['acct:', 'acct:'] }); + assert.equal(hasTargets(command('single_conversation'), ctx), false); + assert.deepEqual(resolveTargetIds(command('bulk_safe'), ctx).targetIds, ['acct:', 'acct:']); + }); + + it('re-resolves a frozen continuation target set and reports missing targets', () => { + assert.deepEqual(resolveTargetIds(command('bulk_safe'), context({}), ['acct:', 'acct:']), { + targetIds: ['acct:'], missingTargetIds: ['acct:'], + }); + }); + + it('requires current account and draft state for their scoped modes', () => { + assert.equal(hasTargets(command('account'), context({ accountId: null })), false); + assert.equal(hasTargets(command('account'), context({ accountId: 'acct' })), true); + assert.equal(hasTargets(command('draft'), context({ draft: null })), false); + assert.equal(hasTargets(command('draft'), context({ draft: { id: 'draft-1' } })), true); + }); + + it('targets draft commands by session id without editor content', () => { + const ctx = context({ draft: { id: 'draft-1', subject: 'private editor content' } }); + assert.deepEqual(resolveTargetIds(command('draft'), ctx), { + targetIds: ['draft-1'], missingTargetIds: [], + }); + assert.deepEqual(resolveTargetIds(command('draft'), ctx, ['draft-2']), { + targetIds: [], missingTargetIds: ['draft-2'], + }); + }); +}); diff --git a/frontend/src/components/AdminPanel.jsx b/frontend/src/components/AdminPanel.jsx index c61b1eba..d6722b93 100644 --- a/frontend/src/components/AdminPanel.jsx +++ b/frontend/src/components/AdminPanel.jsx @@ -25,10 +25,13 @@ import { NOTIFICATION_SOUNDS, playNotificationSound, playCustomSound, warmUpAudi import { usePushNotifications } from '../hooks/usePushNotifications.js'; import SignatureEditor from './SignatureEditor.jsx'; import GtdZeroPet from './GtdZeroPet.jsx'; -import { getEffectiveShortcuts, getGroupedActions, ACTION_DEFS, SPECIAL_KEY_LABELS, parseModKey, modLabel } from '../utils/defaultShortcuts.js'; import { DEFAULT_GTD_FOLDERS, GTD_STATES, resolveAccountGtdFolders, diffGtdFolders, findGtdFolderCollisions } from '../utils/gtd.js'; import { unifiedUnreadTotal } from '../utils/unifiedInbox.js'; import { isValidForwardAddress } from '../utils/ruleActions.js'; +import { useCommandRuntimeContext } from '../commands/CommandRuntimeContext.jsx'; +import { formatCommandKey, getEffectiveCommandBindings } from '../commands/shortcuts.js'; +import { shortcutEventChord } from '../commands/shortcutDispatcher.js'; +import { emptyEmbeddingsForm, embeddingsFormFromConfig, buildEmbeddingsPayload, embeddingsDirty, isSameAsChatProvider, reconcileDimension, embeddingsJob, canSaveAiConfig, EMBEDDING_MODEL_HINTS } from '../utils/embeddingsSettings.js'; // ─── Shared field component ─────────────────────────────────────────────────── function Field({ label, required, children }) { @@ -1474,7 +1477,7 @@ function SwipeActionIcon({ action, size = 17 }) { function LayoutsTab() { const { t } = useTranslation(); const isMobile = useMobile(); - const { layout, setLayout, pageSize, setPageSize, scrollMode, setScrollMode, swipeActions, setSwipeAction, syncInterval, setSyncInterval, folderSyncInterval, setFolderSyncInterval, threadedView, setThreadedView, plaintextEmail, setPlaintextEmail, hoverQuickActions, setHoverQuickActions, showMobileAvatars, setShowMobileAvatars, gravatarAvatars, setGravatarAvatars, replyDefault, setReplyDefault, markReadBehavior, setMarkReadBehavior, markReadDelay, setMarkReadDelay, senderFavicons, senderFaviconsSaving, setSenderFavicons } = useStore(); + const { layout, setLayout, pageSize, setPageSize, scrollMode, setScrollMode, swipeActions, setSwipeAction, syncInterval, setSyncInterval, folderSyncInterval, setFolderSyncInterval, threadedView, setThreadedView, plaintextEmail, setPlaintextEmail, hoverQuickActions, setHoverQuickActions, showMobileAvatars, setShowMobileAvatars, gravatarAvatars, setGravatarAvatars, replyDefault, setReplyDefault, undoSendSeconds, setUndoSendSeconds, markReadBehavior, setMarkReadBehavior, markReadDelay, setMarkReadDelay, senderFavicons, senderFaviconsSaving, setSenderFavicons } = useStore(); const [senderFaviconsError, setSenderFaviconsError] = useState(''); // "Set MailFlow as your default email app": registerProtocolHandler is the @@ -1992,6 +1995,41 @@ function LayoutsTab() { + {/* Undo send */} +
+
+ {t('admin.messageList.undoSend')} +
+
+ {[ + { id: 0, label: t('admin.messageList.undoSendOff') }, + { id: 10, label: t('admin.messageList.undoSend10') }, + { id: 30, label: t('admin.messageList.undoSend30') }, + { id: 60, label: t('admin.messageList.undoSend60') }, + { id: 120, label: t('admin.messageList.undoSend120') }, + ].map(({ id, label }) => { + const active = undoSendSeconds === id; + return ( + + ); + })} +
+
+ {/* Mark as read behaviour */}
@@ -2080,6 +2118,7 @@ function LayoutsTab() { // CardDAV contact sync (e.g. Nextcloud). One-way, read-only pull. function CardDavCard() { const { t } = useTranslation(); + const setCarddavStatus = useStore(state => state.setCarddavStatus); const [status, setStatus] = useState(null); // null while loading const [expanded, setExpanded] = useState(false); const [form, setForm] = useState({ serverUrl: '', username: '', password: '', dupMode: 'separate', intervalMin: 60 }); @@ -2088,7 +2127,14 @@ function CardDavCard() { const [disconnecting, setDisconnecting] = useState(false); const [error, setError] = useState(''); - useEffect(() => { api.carddav.status().then(setStatus).catch(() => setStatus({ connected: false })); }, []); + const applyStatus = useCallback((next) => { + setStatus(next); + setCarddavStatus(next); + }, [setCarddavStatus]); + + useEffect(() => { + api.carddav.status().then(applyStatus).catch(() => applyStatus({ connected: false })); + }, [applyStatus]); const connected = status?.connected; const loading = status === null; @@ -2100,24 +2146,24 @@ function CardDavCard() { serverUrl: form.serverUrl.trim(), username: form.username.trim(), password: form.password, dupMode: form.dupMode, intervalMin: Number(form.intervalMin), }); - setStatus(s); setForm(f => ({ ...f, password: '' })); + applyStatus(s); setForm(f => ({ ...f, password: '' })); } catch (e) { setError(e.message || t('admin.integrations.carddav.connectFailed')); } finally { setConnecting(false); } }; const handleSync = async () => { setSyncing(true); setError(''); - try { const r = await api.carddav.sync(); setStatus(r.status); if (!r.ok && r.error) setError(r.error); } + try { const r = await api.carddav.sync(); applyStatus(r.status); if (!r.ok && r.error) setError(r.error); } catch (e) { setError(e.message); } finally { setSyncing(false); } }; const handleDisconnect = async () => { setDisconnecting(true); setError(''); - try { await api.carddav.disconnect(); setStatus({ connected: false }); } + try { await api.carddav.disconnect(); applyStatus({ connected: false }); } catch (e) { setError(e.message); } finally { setDisconnecting(false); } }; const updateSetting = async (patch) => { - setStatus(s => ({ ...s, ...patch })); + applyStatus({ ...status, ...patch }); try { await api.carddav.update(patch); } catch (e) { setError(e.message); } }; @@ -3567,6 +3613,7 @@ function AISection() { const [config, setConfig] = useState(null); const [loading, setLoading] = useState(true); const [form, setForm] = useState(() => normalizeAiForm()); + const [emb, setEmb] = useState(emptyEmbeddingsForm); const [saving, setSaving] = useState(false); const [testing, setTesting] = useState(false); const [connecting, setConnecting] = useState(false); @@ -3576,21 +3623,40 @@ function AISection() { const [deviceState, setDeviceState] = useState(null); const [copied, setCopied] = useState(false); const [msg, setMsg] = useState(null); + const [job, setJob] = useState(null); + const [vectorAvailable, setVectorAvailable] = useState(true); + const [testingEmb, setTestingEmb] = useState(false); + const [building, setBuilding] = useState(false); const pollerRef = useRef(null); const formRef = useRef(form); + const embRef = useRef(emb); const tRef = useRef(t); + const refreshJob = useCallback(async () => { + try { + const { jobs } = await api.ai.indexingStatus(); + setJob(embeddingsJob(jobs)); + } catch { /* status is best-effort; leave the last known job in place */ } + }, []); + const persistForm = useCallback(async (nextForm) => { - const payload = buildAiSavePayload(nextForm); + const payload = { + ...buildAiSavePayload(nextForm), + embeddings: buildEmbeddingsPayload(embRef.current), + }; const result = await api.ai.saveConfig(payload); const saved = result.config || payload; const normalized = normalizeAiForm(saved); setConfig(saved); formRef.current = normalized; setForm(normalized); + const nextEmb = embeddingsFormFromConfig(saved); + embRef.current = nextEmb; + setEmb(nextEmb); }, []); formRef.current = form; + embRef.current = emb; tRef.current = t; useEffect(() => { @@ -3641,8 +3707,15 @@ function AISection() { setConfig(cfg); formRef.current = normalized; setForm(normalized); + const nextEmb = embeddingsFormFromConfig(cfg || {}); + embRef.current = nextEmb; + setEmb(nextEmb); }), refreshCodexStatus(), + refreshJob(), + api.ai.status().then(s => { + if (active) setVectorAvailable(s?.vectorAvailable !== false); + }), ]) .catch((error) => { if (active) setMsg({ type: 'error', text: error.message }); @@ -3654,7 +3727,14 @@ function AISection() { pollerRef.current?.dispose(); pollerRef.current = null; }; - }, [persistForm]); + }, [persistForm, refreshJob]); + + // Poll the indexing status while a build is running so progress ticks live. + useEffect(() => { + if (!job?.active) return; + const id = setInterval(refreshJob, 2000); + return () => clearInterval(id); + }, [job?.active, refreshJob]); const handleSave = async (e) => { e.preventDefault(); @@ -3677,6 +3757,18 @@ function AISection() { } finally { setTesting(false); } }; + const handleRemove = async () => { + await api.ai.deleteConfig(); + setConfig(null); + const nextForm = normalizeAiForm(); + formRef.current = nextForm; + setForm(nextForm); + const nextEmb = emptyEmbeddingsForm(); + embRef.current = nextEmb; + setEmb(nextEmb); + setMsg(null); + }; + const handleConnect = async () => { setConnecting(true); setMsg(null); setDeviceState(null); setCopied(false); try { @@ -3723,6 +3815,30 @@ function AISection() { } }; + const handleTestEmbeddings = async () => { + setTestingEmb(true); setMsg(null); + try { + const { dimension } = await api.ai.testEmbeddings(); + const { dimension: next, changed } = reconcileDimension(emb.dimension, dimension); + if (changed) setEmb(e => ({ ...e, dimension: String(next) })); + setMsg({ type: 'ok', text: t('admin.ai.emb.testOk', { dimension }) }); + } catch (err) { + setMsg({ type: 'error', text: `${t('admin.ai.testFail')}: ${err.message}` }); + } finally { setTestingEmb(false); } + }; + + const handleBuild = async () => { + setBuilding(true); setMsg(null); + try { + await api.ai.buildEmbeddings(); + setMsg({ type: 'ok', text: t('admin.ai.emb.buildStarted') }); + await refreshJob(); + } catch (err) { + setMsg({ type: 'error', text: err.message }); + await refreshJob(); + } finally { setBuilding(false); } + }; + const field = (label, value, onChange, type = 'text', placeholder = '', help = null) => (
@@ -3740,6 +3856,22 @@ function AISection() {
); + // Embeddings-form counterpart of `field` — reads/writes the `emb` state. + const embField = (label, key, type = 'text', placeholder = '', hint = '') => ( +
+ + setEmb(f => ({ ...f, [key]: e.target.value }))} + placeholder={placeholder} + autoComplete={type === 'password' ? 'new-password' : 'off'} + style={{ width: '100%', background: 'var(--bg-tertiary)', border: '1px solid var(--border)', borderRadius: 6, padding: '7px 10px', color: 'var(--text-primary)', fontSize: 13 }} + /> + {hint &&
{hint}
} +
+ ); + const toggle = (label, checked, onChange) => (
)} + {config && ( + + )} +
{toggle(t('admin.ai.enabled'), form.enabled, () => setForm(f => ({ ...f, enabled: !f.enabled })))} @@ -3920,8 +4065,98 @@ function AISection() {
)} - )} +
+ setEmb(f => ({ ...f, endpoint: e.target.value }))} + placeholder={t('admin.ai.emb.endpointPh')} + autoComplete="off" + style={{ width: '100%', background: 'var(--bg-tertiary)', border: '1px solid var(--border)', borderRadius: 6, padding: '7px 10px', color: 'var(--text-primary)', fontSize: 13 }} + /> +
{t('admin.ai.emb.endpointHint')}
+ + {embField(t('admin.ai.emb.model'), 'model', 'text', t('admin.ai.emb.modelPh'), modelHint)} + {embField(t('admin.ai.emb.dimension'), 'dimension', 'number', t('admin.ai.emb.dimensionPh'), t('admin.ai.emb.dimensionHint'))} + {embField(t('admin.ai.emb.apiKey'), 'apiKey', 'password', t('admin.ai.apiKeyPh'))} + +
+ {t('admin.ai.emb.localTitle')} +

{t('admin.ai.emb.localHelp')}

+
+ +
+ + +
+ + {embDirty && ( +
{t('admin.ai.emb.saveHint')}
+ )} + + {job && ( +
+ {job.state === 'running' && ( + <> +
+ {t('admin.ai.emb.progressLabel', { processed: job.processed, total: job.total })} +
+
+
+
+ + )} + {job.state === 'done' && ( +
{t('admin.ai.emb.progressDone', { total: job.total })}
+ )} + {job.state === 'error' && ( +
{t('admin.ai.emb.progressError', { error: job.lastError || '' })}
+ )} +
+ )} +
+ )} + +
+ + {msgBox} + + @@ -6681,11 +6916,31 @@ const TABS = [ function ShortcutsTab() { const { t } = useTranslation(); const { shortcuts, setShortcuts } = useStore(); + const { commandDefinitions, getContext } = useCommandRuntimeContext(); const [recording, setRecording] = useState(null); // action name currently being recorded - const [pendingConflict, setPendingConflict] = useState(null); // { action: conflictingAction, key } - - const effective = getEffectiveShortcuts(shortcuts); - const groups = getGroupedActions(); + const [pendingConflict, setPendingConflict] = useState(null); // { actions: conflictingCommandIds, key } + + const context = getContext(); + const effectiveRows = getEffectiveCommandBindings(commandDefinitions, context); + const bindingsById = Object.fromEntries(effectiveRows.map(item => [item.commandId, item.bindings])); + const effective = Object.fromEntries(effectiveRows.map(item => [item.commandId, item.bindings[0]?.keys || null])); + const sources = Object.fromEntries(effectiveRows.map(item => [item.commandId, item.bindings[0]?.source])); + const definitionById = new Map(commandDefinitions.map(definition => [definition.id, definition])); + const groupKeys = { + compose: 'shortcuts.groups.composeSearch', help: 'shortcuts.groups.composeSearch', + navigation: 'shortcuts.groups.navigation', selection: 'shortcuts.groups.navigation', + layout: 'shortcuts.groups.navigation', mail: 'shortcuts.groups.messageActions', + respond: 'shortcuts.groups.messageActions', gtd: 'shortcuts.groups.gtd', app: 'shortcuts.groups.navigation', + }; + const groups = commandDefinitions.filter(definition => effective[definition.id] + || definition.id === 'gtd.someday' || definition.id === 'gtd.reference').reduce((result, definition) => { + const groupKey = groupKeys[definition.group] || 'shortcuts.groups.navigation'; + (result[groupKey] ||= []).push({ + action: definition.id, + descriptionKey: definition.titleKey, + }); + return result; + }, {}); // Listen for key presses while recording useEffect(() => { @@ -6701,12 +6956,13 @@ function ShortcutsTab() { return; } - const key = (e.ctrlKey || e.metaKey) ? `ctrl+${e.key.toLowerCase()}` : e.key; + const key = shortcutEventChord(e); // Detect conflicts with other actions (excluding the one being edited) - const conflictEntry = Object.entries(effective).find(([a, k]) => k === key && a !== recording); - if (conflictEntry) { - setPendingConflict({ action: conflictEntry[0], key }); + const conflicts = effectiveRows + .filter(item => item.commandId !== recording && item.bindings.some(binding => binding.keys === key)); + if (conflicts.length) { + setPendingConflict({ actions: conflicts.map(item => item.commandId), key }); } else { setPendingConflict(null); } @@ -6717,7 +6973,7 @@ function ShortcutsTab() { }; window.addEventListener('keydown', handler); return () => window.removeEventListener('keydown', handler); - }, [recording, effective, shortcuts]); // eslint-disable-line react-hooks/exhaustive-deps + }, [recording, effective, shortcuts, context.platform]); // eslint-disable-line react-hooks/exhaustive-deps const clearShortcut = (action) => { const updated = { ...shortcuts, [action]: null }; @@ -6764,35 +7020,7 @@ function ShortcutsTab() { if (!key) { return —; } - // Modifier combos like 'ctrl+p' - const mod = parseModKey(key); - if (mod) { - return ( - - {modLabel(mod.mod)} - + - {mod.bare.toUpperCase()} - - ); - } - // Special key names like 'Delete', 'ArrowUp' — single keypress, render as one badge - if (SPECIAL_KEY_LABELS[key]) { - return {SPECIAL_KEY_LABELS[key]}; - } - // Multi-char keys like 'gi': render each character as separate kbd with "then" - if (key.length > 1) { - return ( - - {[...key].map((c, i) => ( - - {c} - {i < key.length - 1 && {t('shortcuts.then')}} - - ))} - - ); - } - return {key}; + return {formatCommandKey(key, context.platform)}; }; return ( @@ -6824,7 +7052,11 @@ function ShortcutsTab() { background: 'rgba(234, 179, 8, 0.1)', border: '1px solid rgba(234, 179, 8, 0.4)', borderRadius: 7, fontSize: 12, color: 'var(--text-secondary)', }}> - {t('admin.shortcuts.conflict', { key: pendingConflict.key, action: t(ACTION_DEFS[pendingConflict.action]?.labelKey) })} + {t('admin.shortcuts.conflict', { + key: pendingConflict.key, + action: pendingConflict.actions + .map(action => t(definitionById.get(action)?.titleKey)).join(', '), + })}
)} @@ -6839,6 +7071,7 @@ function ShortcutsTab() {
{actions.map(({ action, descriptionKey }, i) => { const key = effective[action]; + const bindings = bindingsById[action] || []; const isDefault = !(action in shortcuts); const isRec = recording === action; return ( @@ -6857,6 +7090,7 @@ function ShortcutsTab() {
+ {!isRec && sources[action] && ( + + {t(`admin.shortcuts.sources.${sources[action]}`)} + + )} + {!isRec && bindings.slice(1).map(binding => ( + + {formatCommandKey(binding.keys, context.platform)} + + {t(`admin.shortcuts.sources.${binding.source}`)} + + + ))} {!isDefault && ( )} +
; +} diff --git a/frontend/src/components/CommandIcon.jsx b/frontend/src/components/CommandIcon.jsx new file mode 100644 index 00000000..0e319159 --- /dev/null +++ b/frontend/src/components/CommandIcon.jsx @@ -0,0 +1,32 @@ +const paths = { + compose: <>, + search: <>, + contacts: <>, + inbox: <>, + folder: , + archive: <>, + clock: <>, + 'mail-open': <>, + mail: <>, + star: , + trash: <>, + shield: <>, + 'shield-check': <>, + reply: <>, + 'reply-all': <>, + forward: <>, + 'check-square': <>, + eye: <>, + 'user-check': <>, + calendar: <>, + bookmark: , + gtd: <>, + appearance: <>, + settings: <>, +}; + +export default function CommandIcon({ name }) { + return ; +} diff --git a/frontend/src/components/CommandPalette.jsx b/frontend/src/components/CommandPalette.jsx index 35e5d86a..ba6f27db 100644 --- a/frontend/src/components/CommandPalette.jsx +++ b/frontend/src/components/CommandPalette.jsx @@ -1,215 +1,126 @@ -import { useState, useEffect, useRef, useCallback } from 'react'; +import { useEffect, useMemo, useReducer, useRef } from 'react'; import { useTranslation } from 'react-i18next'; -import { useStore } from '../store/index.js'; -import { useMobile } from '../hooks/useMobile.js'; -import { THEMES } from '../themes.js'; - -const THEME_NAMES = Object.keys(THEMES); - -function buildActions({ t, openCompose, setSelectedAccount, setShowAdmin, setAdminTab, theme, setTheme, accounts, selectedAccountId }) { - const actions = [ - { - id: 'compose', - label: t('commandPalette.actions.compose'), - icon: , - run: () => openCompose({ accountId: selectedAccountId || undefined }), - }, - { - id: 'inbox', - label: t('commandPalette.actions.inbox'), - icon: , - run: () => setSelectedAccount(null, 'INBOX'), - }, - { - id: 'settings', - label: t('commandPalette.actions.settings'), - icon: , - run: () => { setShowAdmin(true); setAdminTab('accounts'); }, - }, - { - id: 'themes', - label: t('commandPalette.actions.themes'), - icon: , - run: () => { setShowAdmin(true); setAdminTab('appearance'); }, - }, - ]; - - // Theme switch actions - for (const themeKey of THEME_NAMES) { - const label = THEMES[themeKey]?.label || themeKey; - actions.push({ - id: `theme:${themeKey}`, - label: t('commandPalette.actions.switchTheme', { theme: label }), - icon: , - active: theme === themeKey, - run: () => setTheme(themeKey), - }); - } - - // Per-account inbox shortcuts - for (const a of accounts) { - actions.push({ - id: `account:${a.id}`, - label: t('commandPalette.actions.accountInbox', { name: a.name }), - icon: , - run: () => setSelectedAccount(a.id, 'INBOX'), - }); - } - - return actions; -} +import { commandTargetLabel } from '../commands/appContext.js'; +import { formatCommandKey } from '../commands/shortcuts.js'; +import { createPaletteState, paletteKeyIntent, reducePaletteState } from '../commands/paletteState.js'; +import { isRestorableFocus, nextFocusIndex } from '../commands/paletteFocus.js'; +import { useCommandRuntimeContext } from '../commands/CommandRuntimeContext.jsx'; +import CommandContinuation from './CommandContinuation.jsx'; +import CommandIcon from './CommandIcon.jsx'; export default function CommandPalette({ open, onClose }) { const { t } = useTranslation(); - const isMobile = useMobile(); - const { openCompose, setSelectedAccount, setShowAdmin, setAdminTab, theme, setTheme, accounts, selectedAccountId } = useStore(); - const [query, setQuery] = useState(''); - const [activeIdx, setActiveIdx] = useState(0); - const [listScrolled, setListScrolled] = useState(false); + const { registry, controller, getContext, continuation, clearContinuation } = useCommandRuntimeContext(); + const [state, dispatch] = useReducer(reducePaletteState, undefined, createPaletteState); const inputRef = useRef(null); - const listRef = useRef(null); - - const actions = buildActions({ t, openCompose, setSelectedAccount, setShowAdmin, setAdminTab, theme, setTheme, accounts, selectedAccountId }); - - const filtered = query.trim() - ? actions.filter(a => a.label.toLowerCase().includes(query.toLowerCase())) - : actions; + const priorFocusRef = useRef(null); + const dialogRef = useRef(null); + const context = getContext(); + const results = useMemo( + () => registry.search(state.query, context), + [registry, state.query, context], + ); + const target = commandTargetLabel(context); + const resultCount = continuation?.props.items?.length ?? results.length; + useEffect(() => { dispatch({ type: 'results', count: resultCount }); }, [resultCount]); useEffect(() => { - if (open) { - setQuery(''); - setActiveIdx(0); - setTimeout(() => inputRef.current?.focus(), 30); - } + if (!open) return; + priorFocusRef.current = document.activeElement; + dispatch({ type: 'open' }); + queueMicrotask(() => (inputRef.current || dialogRef.current?.querySelector('button'))?.focus()); + return () => { + const prior = priorFocusRef.current; + if (isRestorableFocus(prior)) prior.focus(); + }; }, [open]); - - useEffect(() => { setActiveIdx(0); }, [query]); - - const runAction = useCallback((action) => { - action.run(); - onClose(); - }, [onClose]); - - const handleKeyDown = (e) => { - if (e.key === 'ArrowDown') { - e.preventDefault(); - setActiveIdx(i => Math.min(i + 1, filtered.length - 1)); - } else if (e.key === 'ArrowUp') { - e.preventDefault(); - setActiveIdx(i => Math.max(i - 1, 0)); - } else if (e.key === 'Enter') { - e.preventDefault(); - if (filtered[activeIdx]) runAction(filtered[activeIdx]); - } else if (e.key === 'Escape') { - onClose(); - } - }; - - // Scroll active item into view + useEffect(() => { dispatch({ type: 'continuation', value: continuation }); }, [continuation]); useEffect(() => { - const el = listRef.current?.children[activeIdx]; - el?.scrollIntoView({ block: 'nearest' }); - }, [activeIdx]); + if (!open) return; + dialogRef.current?.querySelectorAll('[role="option"]')[state.activeIndex] + ?.scrollIntoView({ block: 'nearest' }); + }, [continuation, open, resultCount, state.activeIndex, state.query]); if (!open) return null; - return ( -
-
e.stopPropagation()} - > - {/* Search input */} -
- - - - setQuery(e.target.value)} - onKeyDown={handleKeyDown} - placeholder={t('commandPalette.placeholder')} - style={{ - flex: 1, background: 'none', border: 'none', outline: 'none', - color: 'var(--text-primary)', fontSize: 15, - }} - /> - {!isMobile && Esc} -
- - {/* Results */} -
setListScrolled(e.currentTarget.scrollTop > 4)} - style={{ - maxHeight: 360, overflowY: 'auto', padding: '6px 0', - boxShadow: listScrolled ? 'inset 0 8px 8px -8px rgba(0,0,0,0.25)' : 'none', - transition: 'box-shadow 0.2s ease', - }} - > - {filtered.length === 0 ? ( -
- {t('commandPalette.noResults')} -
- ) : filtered.map((action, i) => ( -
runAction(action)} - onMouseEnter={() => setActiveIdx(i)} - onMouseDown={e => { e.currentTarget.style.transform = 'scale(0.98)'; }} - onMouseUp={e => { e.currentTarget.style.transform = ''; }} - onMouseLeave={e => { e.currentTarget.style.transform = ''; }} - style={{ - display: 'flex', alignItems: 'center', gap: 12, - padding: '9px 16px', cursor: 'pointer', - background: i === activeIdx ? 'var(--bg-hover)' : 'transparent', - transition: 'background 0.08s, transform 0.08s', - }} - > - - {action.icon} - - - {action.label} - - {action.active && ( - - - - )} -
- ))} -
+ const execute = async result => { + const outcome = await controller.execute(result.command.id, { source: 'palette' }); + if (['success', 'cancelled', 'partial'].includes(outcome.status)) onClose(); + }; + const onKeyDown = event => { + const intent = paletteKeyIntent(event); + if (!intent) { + if (event.key === 'Tab') { + const nodes = [...dialogRef.current.querySelectorAll('input,button,[tabindex]:not([tabindex="-1"])')]; + const index = nodes.indexOf(document.activeElement); + const next = nodes[nextFocusIndex(nodes.length, index, event.shiftKey)]; + event.preventDefault(); + event.stopPropagation(); + next?.focus(); + } + return; + } + event.preventDefault(); + event.stopPropagation(); + if (intent === 'next') dispatch({ type: 'move', delta: 1 }); + if (intent === 'previous') dispatch({ type: 'move', delta: -1 }); + if (intent === 'execute' && continuation) { + dialogRef.current.querySelectorAll('[role="option"]')[state.activeIndex]?.click(); + } + if (intent === 'execute' && !continuation && results[state.activeIndex]) execute(results[state.activeIndex]); + if (intent === 'back') { + if (continuation) clearContinuation(); + else onClose(); + } + }; - {!isMobile && ( -
- ↑↓ {t('commandPalette.hint.navigate')} - ↵ {t('commandPalette.hint.select')} - Esc {t('commandPalette.hint.close')} -
- )} + return
event.target === event.currentTarget && onClose()}> +
+

{t('commandPalette.title')}

+
+ + {!continuation ? dispatch({ type: 'query', query: event.target.value })} + placeholder={t('commandPalette.placeholder')} + /> : + {t(continuation.props.titleKey)} + } + Esc
-
- ); +
+ {t('commandPalette.announcement', { count: continuation?.props.items?.length ?? results.length, target: t(target.key, target.values) })} +
+ {continuation ? dispatch({ type: 'move', delta: index - state.activeIndex })} + onFinished={() => { clearContinuation(); onClose(); }} /> : +
+ {results.map((result, index) => )} + {!results.length &&

{t('commandPalette.noResults')}

} +
} +
+ {t(target.key, target.values)} + +
+ +
; } diff --git a/frontend/src/components/CommandPalette.test.js b/frontend/src/components/CommandPalette.test.js new file mode 100644 index 00000000..817ff0ec --- /dev/null +++ b/frontend/src/components/CommandPalette.test.js @@ -0,0 +1,75 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +describe('CommandPalette source wiring', () => { + const source = fs.readFileSync(new URL('./CommandPalette.jsx', import.meta.url), 'utf8'); + const continuation = fs.readFileSync(new URL('./CommandContinuation.jsx', import.meta.url), 'utf8'); + const icons = fs.readFileSync(new URL('./CommandIcon.jsx', import.meta.url), 'utf8'); + const mailActions = fs.readFileSync(new URL('../commands/mailActions.js', import.meta.url), 'utf8'); + const css = fs.readFileSync(new URL('../index.css', import.meta.url), 'utf8'); + + it('consumes the shared runtime and required accessible semantics', () => { + assert.match(source, /useCommandRuntimeContext\(\)/); + for (const token of [ + 'role="dialog"', 'aria-modal="true"', 'role="combobox"', 'aria-controls="command-palette-results"', + 'aria-activedescendant', 'role="listbox"', 'role="option"', 'aria-live="polite"', + ]) assert.ok(source.includes(token), `missing ${token}`); + }); + + it('uses registry results, alias hints, shortcut formatting, and focus helpers', () => { + assert.match(source, /registry\.search\(state\.query, context\)/); + assert.match(source, /result\.matchedAlias/); + assert.match(source, /formatCommandKey/); + assert.match(source, /isRestorableFocus/); + assert.match(source, /nextFocusIndex/); + }); + + it('closes directly so a rapid reopen cannot inherit a stale close request', () => { + assert.match(source, /if \(continuation\) clearContinuation\(\);\s*else onClose\(\);/); + assert.doesNotMatch(source, /closeRequested/); + }); + + it('keeps the active keyboard option inside the bounded scroll window', () => { + assert.match(source, /scrollIntoView\(\{ block: 'nearest' \}\)/); + assert.match(source, /\[state\.activeIndex\]/); + }); + + it('resumes continuations with their frozen account-scoped targets', () => { + assert.match(continuation, /frozenTargetIds: continuation\.targetIds/); + assert.match(continuation, /source: 'palette'/); + assert.match(continuation, /\[continuation\.props\.inputKey \|\| 'value'\]: item\.id/); + }); + + it('localizes and bounds continuation options inside the palette', () => { + assert.match(continuation, /useTranslation\(\)/); + assert.match(continuation, /aria-label=\{t\(continuation\.props\.titleKey\)\}/); + assert.match(continuation, /className="command-palette__results"/); + }); + + it('uses the MailFlow search header, escape chip, and keyboard footer', () => { + assert.match(source, /className="command-palette__search"/); + assert.match(source, /Esc<\/kbd>/); + assert.match(source, /className="command-palette__footer"/); + assert.match(source, /className="command-palette__hints"/); + for (const key of ['navigate', 'select', 'close']) { + assert.ok(source.includes(`commandPalette.hint.${key}`)); + } + }); + + it('keeps the approved 5.5-row viewport and theme-aware selected row', () => { + assert.match(css, /\.command-palette__results\s*\{[^}]*max-height:\s*286px/); + assert.match(css, /\.command-palette__row\s*\{[^}]*min-height:\s*52px/); + assert.match(css, /\.command-palette__row\[aria-selected="true"\]\s*\{[^}]*var\(--accent-dim\)/); + }); + + it('renders the native mail-action icons instead of the settings fallback', () => { + const iconNames = [...mailActions.matchAll(/definition\([^\n]+?,\s*'([^']+)',\s*'(?:mail|respond|gtd)'/g)] + .map(match => match[1]); + assert.ok(iconNames.length > 10); + for (const iconName of new Set(iconNames)) { + assert.match(icons, new RegExp(`\\b${iconName.replace('-', "['-]")}['"]?:`), `missing ${iconName}`); + } + }); +}); diff --git a/frontend/src/components/ComposeChip.jsx b/frontend/src/components/ComposeChip.jsx new file mode 100644 index 00000000..09018145 --- /dev/null +++ b/frontend/src/components/ComposeChip.jsx @@ -0,0 +1,77 @@ +import { useTranslation } from 'react-i18next'; +import { + composeChipInteraction, + composeChipPresentation, +} from './composePresentationModel.js'; + +function persistenceLabel(session, t) { + const presentation = composeChipPresentation(session); + return { + ...presentation, + label: t( + presentation.code === 'terminalPending' + ? 'compose.sessions.terminalPending' + : `compose.${presentation.code}`, + { defaultValue: presentation.defaultLabel }, + ), + }; +} + +export default function ComposeChip({ session, onRestore, onFocus, platform }) { + const { t } = useTranslation(); + const title = session.subject?.trim() || t('common.noSubject'); + const platformName = platform + || globalThis.navigator?.userAgentData?.platform + || globalThis.navigator?.platform + || ''; + const isMac = /mac/i.test(platformName); + const keycap = isMac ? `⌘${session.slot}` : `Ctrl+${session.slot}`; + const persistence = persistenceLabel(session, t); + const interaction = composeChipInteraction(session); + const ariaLabel = `${session.slot}. ${title}. ${persistence.label}. ${keycap}`; + + const activate = () => { + if (interaction.action === 'restore') onRestore(session.id); + else if (interaction.action === 'focus') onFocus(session.id); + }; + + return ( + + ); +} diff --git a/frontend/src/components/ComposeChip.test.js b/frontend/src/components/ComposeChip.test.js new file mode 100644 index 00000000..b2469402 --- /dev/null +++ b/frontend/src/components/ComposeChip.test.js @@ -0,0 +1,75 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { + composeChipInteraction, + composeChipPresentation, +} from './composePresentationModel.js'; + +const source = fs.readFileSync(new URL('./ComposeChip.jsx', import.meta.url), 'utf8'); + +test('renders a lightweight native button with the existing compose icon', () => { + assert.match(source, / - -
- - )} - {/* Empty subject warning sheet */} {showEmptySubjectWarn && ( <> @@ -1465,7 +1671,7 @@ export default function ComposeModal() { onClick={() => setShowEmptySubjectWarn(false)} style={{ position: 'fixed', inset: 0, zIndex: 2100, background: 'rgba(0,0,0,0.4)' }} /> -
setShowForgottenAttachWarn(false)} style={{ position: 'fixed', inset: 0, zIndex: 2100, background: 'rgba(0,0,0,0.4)' }} /> -
setShowPrioritySheet(false)} style={{ position: 'fixed', inset: 0, zIndex: 2100, background: 'rgba(0,0,0,0.4)' }} /> -
+
{t('compose.priority')}
@@ -1571,36 +1777,13 @@ export default function ComposeModal() { outline: 'none', }; - if (minimized) { - return ( -
setMinimized(false)} - style={{ - position: 'fixed', bottom: 0, right: 24, - background: 'var(--bg-elevated)', border: '1px solid var(--border)', - borderBottom: 'none', borderRadius: '8px 8px 0 0', - padding: '10px 16px', cursor: 'pointer', - display: 'flex', alignItems: 'center', gap: 10, - color: 'var(--text-primary)', fontSize: 13, fontWeight: 500, - boxShadow: 'var(--shadow-soft)', zIndex: 1000, - }} - > - - - - - {subject || modeLabel} -
- ); - } - return ( <> {maximized && (
setMaximized(false)} style={{ - position: 'fixed', inset: 0, zIndex: 999, + position: 'absolute', inset: 0, zIndex: 999, background: 'rgba(0,0,0,0.35)', backdropFilter: 'blur(4px)', WebkitBackdropFilter: 'blur(4px)', }} @@ -1608,23 +1791,36 @@ export default function ComposeModal() { )}
{ + if (!terminalPending) onFocus(session.id); + }} style={maximized ? { - position: 'fixed', top: 28, left: 28, right: 28, bottom: 28, + position: 'absolute', top: 28, left: 28, right: 28, bottom: 28, background: 'var(--bg-secondary)', border: '1px solid var(--border)', borderRadius: 12, boxShadow: 'var(--shadow-modal)', zIndex: 1000, display: 'flex', flexDirection: 'column', - } : pos ? { - position: 'fixed', top: pos.y, left: pos.x, + } : tiled ? { + position: 'relative', width: '100%', height: 'min(72vh, 720px)', + minWidth: 0, maxWidth: 'none', overflow: 'hidden', + background: 'var(--bg-secondary)', border: '1px solid var(--border)', + borderRadius: 10, boxShadow: 'var(--shadow-modal)', + display: 'flex', flexDirection: 'column', + } : pos && canFreeform ? { + position: 'absolute', top: pos.y, left: pos.x, width: customSize?.width || 540, ...(customSize?.height ? { height: customSize.height } : { maxHeight: '75vh' }), - maxWidth: 'calc(100vw - 16px)', + maxWidth: 'calc(100% - 16px)', background: 'var(--bg-secondary)', border: '1px solid var(--border)', borderRadius: 10, boxShadow: 'var(--shadow-modal)', zIndex: 1000, display: 'flex', flexDirection: 'column', } : { - position: 'fixed', bottom: 0, right: 24, - width: customSize?.width || 540, maxWidth: 'calc(100vw - 48px)', + position: 'absolute', bottom: 0, right: 24, + width: customSize?.width || 540, maxWidth: 'calc(100% - 48px)', ...(customSize?.height ? { height: customSize.height } : { maxHeight: '75vh' }), background: 'var(--bg-secondary)', border: '1px solid var(--border)', borderRadius: 10, @@ -1633,11 +1829,18 @@ export default function ComposeModal() { animation: 'compose-enter var(--motion-normal) var(--ease-emphasized) backwards', }} > - + { const f = e.target.files?.[0]; if (f) insertImageIntoEditor(f); e.target.value = ''; }} style={{ display: 'none' }} /> {/* Title bar */}
{showReplyType && ( -
{ setToChips(parseChips(composeData?.originalFrom || composeData?.to)); setToInput(''); - const allRecipients = parseChips(composeData?.allRecipients || []); + const allRecipients = parseChips(replyAllRecipientsForSession(composeData)); if (allRecipients.length) { setCcChips(allRecipients); setCcInput(''); setShowCc(true); } setReplyAll(true); setShowReplyType(false); @@ -1715,12 +1918,12 @@ export default function ComposeModal() { )}
- setMinimized(true)} title={t('compose.toolbar.minimize')}> + runAfterAttachmentUploads(() => onMinimize(session.id))} title={t('compose.toolbar.minimize')}> - setMaximized(m => !m)} title={maximized ? t('compose.toolbar.restore') : t('compose.toolbar.maximize')}> + setMaximized(m => !m)} title={maximized ? t('compose.toolbar.restore') : t('compose.toolbar.maximize')}> {maximized ? ( @@ -1733,7 +1936,7 @@ export default function ComposeModal() { )} - + @@ -1781,6 +1984,7 @@ export default function ComposeModal() {
{t('compose.to')} {t('compose.cc')} {t('compose.bcc')} {/* Toolbar — sits outside overflow container so dropdowns are never clipped */} - {!plaintextEmail && fileInputRef.current?.click()} onInsertImage={() => imageInputRef.current?.click()} + {!plaintextEmail && fileInputRef.current?.click()} + attachmentDisabled={attachmentControlsDisabled} + onInsertImage={() => imageInputRef.current?.click()} htmlMode={htmlMode} onToggleHtml={() => { if (!htmlMode) { setHtmlSource(editor?.getHTML() ?? ''); setHtmlMode(true); } @@ -1888,10 +2097,10 @@ export default function ComposeModal() {
)} {fwdAttachments.length > 0 && ( - ({ name: a.filename, size: a.size }))} onRemove={i => setFwdAttachments(prev => prev.filter((_, j) => j !== i))} /> + ({ name: a.filename, size: a.size }))} onRemove={i => setFwdAttachments(prev => prev.filter((_, j) => j !== i))} disabled={attachmentControlsDisabled} /> )} {attachments.length > 0 && ( - setAttachments(prev => prev.filter((_, j) => j !== i))} /> + )} {/* Scrollable body area */} @@ -1934,7 +2143,7 @@ export default function ComposeModal() {
)} - {fromSignature ? ( + {(fromSignature || richSignature) ? (
-- signature @@ -1950,6 +2159,7 @@ export default function ComposeModal() { contentEditable suppressContentEditableWarning spellCheck={false} + onInput={event => setQuotedBodyHtml(event.currentTarget.innerHTML)} style={{ width: '100%', minHeight: 120, padding: '10px 14px', @@ -1986,30 +2196,34 @@ export default function ComposeModal() { }}> {plaintextEmail && (
- {!maximized && ( + {!maximized && canFreeform && (
setShowEmptySubjectWarn(false)} style={{ position: 'fixed', inset: 0, zIndex: 2100, background: 'rgba(0,0,0,0.4)' }} /> -
setShowForgottenAttachWarn(false)} style={{ position: 'fixed', inset: 0, zIndex: 2100, background: 'rgba(0,0,0,0.4)' }} /> -
setShowCloseDialog(false)} style={{ position: 'fixed', inset: 0, zIndex: 2100, background: 'rgba(0,0,0,0.4)' }} /> -
{ setShowCloseDialog(false); - if (draftUid != null && draftFolder != null && draftAccountId) { - api.deleteDraft(draftAccountId, draftUid, draftFolder).catch(() => {}); - } - closeCompose(); + runAfterAttachmentUploads(() => onDiscard(session.id)); }} style={{ padding: '8px 16px', background: 'none', border: '1px solid var(--border)', borderRadius: 7, color: 'var(--red)', fontSize: 13, cursor: 'pointer', textAlign: 'center' }} > @@ -2182,32 +2393,6 @@ export default function ComposeModal() { )} - {/* Desktop draft attachment warning */} - {showAttachWarnForDraft && ( - <> -
setShowAttachWarnForDraft(false)} style={{ position: 'fixed', inset: 0, zIndex: 2100, background: 'rgba(0,0,0,0.4)' }} /> -
-
- {t('compose.saveDraft')} -
-
- {t('compose.draftHasAttachments')} -
-
- - -
-
- - )} ); } @@ -2304,7 +2489,7 @@ function Sep() { return ; } -function RichToolbar({ editor, onAttach, onInsertImage, htmlMode, onToggleHtml, isMobile, aiEnabled, onAiAction, aiPanelOpen }) { +function RichToolbar({ composeSessionId, editor, onAttach, attachmentDisabled = false, onInsertImage, htmlMode, onToggleHtml, isMobile, aiEnabled, onAiAction, aiPanelOpen }) { const { t } = useTranslation(); const uiScale = useUiScale(); const savedSelectionRef = useRef(null); @@ -2374,6 +2559,25 @@ function RichToolbar({ editor, onAttach, onInsertImage, htmlMode, onToggleHtml, }; }, [colorPos, highlightPos, emojiPos, linkPos, tablePos, aiMenuPos]); + useEffect(() => { + if (!composeSessionId + || (!colorPos && !highlightPos && !emojiPos && !linkPos && !tablePos && !aiMenuPos)) { + return undefined; + } + return composeEscapeOwners.register({ + sessionId: composeSessionId, + priority: 20, + dismiss: () => { + if (aiMenuPos) setAiMenuPos(null); + else if (tablePos) setTablePos(null); + else if (linkPos) setLinkPos(null); + else if (emojiPos) setEmojiPos(null); + else if (highlightPos) setHighlightPos(null); + else if (colorPos) setColorPos(null); + }, + }); + }, [composeSessionId, colorPos, highlightPos, emojiPos, linkPos, tablePos, aiMenuPos]); + const es = useEditorState({ editor, selector: ({ editor: ed }) => ed ? { @@ -2488,8 +2692,8 @@ function RichToolbar({ editor, onAttach, onInsertImage, htmlMode, onToggleHtml, {mtb(es.underline, 'Underline', e => { e.preventDefault(); editor.chain().focus().toggleUnderline().run(); }, U)} {mtb(es.strike, 'Strikethrough', e => { e.preventDefault(); editor.chain().focus().toggleStrike().run(); }, S)} {onAttach && ( - + {recovery ? ( + + ) : ( + + )} +
+
+ ); +} + +export default function ComposeWorkspace({ uiScale = 1 }) { + const { t } = useTranslation(); + const [bounds, setBounds] = useState(() => ({ + top: 0, left: 0, + width: (globalThis.innerWidth || 0) / uiScale, + height: (globalThis.innerHeight || 0) / uiScale, + })); + const { + sessions, visibleSessions, chipSessions, focusedSessionId, capacity: rawCapacity, + changeSession, saveSession, focusSession, minimizeSession, restoreSession, closeSession, + discardSession, sendSession, undoQueuedSend, addAttachment, removeAttachment, + resolveConflict, + } = useComposeWorkspace(); + const { controller: commandController, registry, getContext } = useCommandRuntimeContext(); + const capacity = Math.min(3, Math.max(1, rawCapacity || 1)); + const announcement = useWorkspaceAnnouncement(sessions, focusedSessionId, t); + const atSessionLimit = new Set(sessions.map(session => session.slot)).size >= 9; + + useComposeShortcuts({ + commandController, + registry, + getContext, + getSessions: () => sessions, + getVisibleSessions: () => visibleSessions, + getFocusedSessionId: () => focusedSessionId, + }); + + useLayoutEffect(() => { + const workspaceElement = globalThis.document?.querySelector('[data-mail-workspace]'); + if (!workspaceElement) return undefined; + const measure = () => { + const rect = workspaceElement.getBoundingClientRect(); + const containingBlockRect = workspaceElement.parentElement?.getBoundingClientRect?.() + || { top: 0, left: 0 }; + setBounds(current => { + const next = localDockBounds(rect, containingBlockRect, uiScale); + return Object.keys(next).every(key => current[key] === next[key]) ? current : next; + }); + }; + measure(); + const observer = typeof ResizeObserver === 'function' ? new ResizeObserver(measure) : null; + observer?.observe(workspaceElement); + globalThis.window?.addEventListener('resize', measure); + return () => { + observer?.disconnect(); + globalThis.window?.removeEventListener('resize', measure); + }; + }, [uiScale]); + + if (!visibleSessions.length && !chipSessions.length) return null; + + return ( +
+
+ {announcement} +
+ + {atSessionLimit && ( +
+ {t('compose.sessions.limit')} +
+ )} + + {visibleSessions.length > 0 && ( +
+ {visibleSessions.map(session => session.snapshot ? (() => { + const titleId = `compose-session-title-${session.id}`; + return ( +
{ + if (composeSessionCanFocus(session)) focusSession(session.id); + }} + > +

+ {composeSessionRegionLabel(session, t)} +

+ {(session.conflict || session.recoveryConflict) && ( + + )} + 1} + allowFreeform={visibleSessions.length === 1} + onChange={changeSession} + onSave={(id, changes) => saveSession(id, changes)} + onFocus={id => { + if (composeSessionCanFocus(session)) focusSession(id); + }} + onMinimize={id => safely(() => minimizeSession(id))} + onClose={id => safely(() => closeSession(id))} + onDiscard={id => safely(() => discardSession(id))} + onSend={(id, options) => safely(() => sendSession(id, options))} + onUndoQueuedSend={outboxId => safely(() => undoQueuedSend(outboxId))} + onAddAttachment={(id, file) => safely(() => addAttachment(id, file))} + onRemoveAttachment={(id, attachmentId) => safely(() => removeAttachment(id, attachmentId))} + /> +
+ ); + })() : null)} +
+ )} + + {chipSessions.length > 0 && ( +
+ {chipSessions.map(session => ( + safely(() => restoreSession(id))} + onFocus={id => { + if (composeSessionCanFocus(session)) focusSession(id); + }} + /> + ))} +
+ )} +
+ ); +} diff --git a/frontend/src/components/ComposeWorkspace.test.js b/frontend/src/components/ComposeWorkspace.test.js new file mode 100644 index 00000000..6bae17f5 --- /dev/null +++ b/frontend/src/components/ComposeWorkspace.test.js @@ -0,0 +1,455 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import i18next from 'i18next'; +import { + attachmentChipView, + composeHydrationStep, + composeFormPatch, + initialComposeForm, + localDockBounds, + composeSessionAnnouncement, + composeSessionCanFocus, + composeSessionRegionLabel, + persistComposeDraft, + reconcileComposeForm, + replyAllRecipientsForSession, + uploadFilesSequentially, +} from './composePresentationModel.js'; + +const source = fs.readFileSync(new URL('./ComposeWorkspace.jsx', import.meta.url), 'utf8'); +const modal = fs.readFileSync(new URL('./ComposeModal.jsx', import.meta.url), 'utf8'); +const mailApp = fs.readFileSync(new URL('./MailApp.jsx', import.meta.url), 'utf8'); + +function deferred() { + let resolve; + let reject; + const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); + return { promise, resolve, reject }; +} + +async function englishTranslator() { + const translation = JSON.parse(fs.readFileSync( + new URL('../locales/en.json', import.meta.url), 'utf8', + )); + const instance = i18next.createInstance(); + await instance.init({ + resources: { en: { translation } }, + lng: 'en', + fallbackLng: 'en', + interpolation: { escapeValue: false }, + }); + return instance.t.bind(instance); +} + +test('mounts one controller-backed workspace at the mail shell boundary', () => { + assert.match(mailApp, /const ComposeWorkspace = lazy/); + assert.equal((mailApp.match(//g) || []).length, 1); + assert.doesNotMatch(mailApp, /\{composing && \}/); + assert.match(mailApp, /data-mail-workspace/); + assert.match(source, /useComposeWorkspace\(\)/); +}); + +test('renders only visible snapshots in stable order and lightweight chips separately', () => { + assert.match(source, /visibleSessions\.map\(session =>/); + assert.match(source, /visibleSessions\.map\(session => session\.snapshot \?/); + assert.match(source, //); + assert.match(source, / { + assert.match(source, /data-compose-capacity=\{capacity\}/); + assert.match(source, /repeat\(\$\{capacity\}, minmax\(0, 1fr\)\)/); + assert.match(source, /gap: 12/); + assert.match(source, /Math\.min\(3, Math\.max\(1/); + assert.match(source, /focusedSessionId === session\.id/); + assert.match(source, /var\(--accent\)/); + assert.match(source, /allowFreeform=\{visibleSessions\.length === 1\}/); + assert.match(source, /tiled=\{visibleSessions\.length > 1\}/); +}); + +test('gives the absolute single-composer surface a visible containing block', () => { + assert.match(source, /position: 'relative'/); + assert.match(source, /height: visibleSessions\.length === 1 \? 'min\(72vh, 720px\)' : undefined/); +}); + +test('renders slot 7 region and status labels through i18next without template residue', async () => { + const t = await englishTranslator(); + const session = { id: 'session-7', slot: 7, subject: 'Synthetic subject', status: 'idle' }; + assert.equal(composeSessionRegionLabel(session, t), 'Draft 7: Synthetic subject'); + assert.equal(composeSessionAnnouncement(session, t), 'Draft 7: Saved'); + assert.doesNotMatch(composeSessionRegionLabel(session, t), /[{}]|/); +}); + +test('announces offline and generic persistence errors truthfully and distinctly', async () => { + const t = await englishTranslator(); + const offline = composeSessionAnnouncement({ id: 'offline', slot: 2, status: 'offline' }, t); + const error = composeSessionAnnouncement({ id: 'error', slot: 3, status: 'error' }, t); + assert.equal(offline, 'Draft 2: Offline — changes not saved'); + assert.equal(error, 'Draft 3: Draft could not be saved'); + assert.notEqual(offline, error); + assert.doesNotMatch(offline, /retry/i); +}); + +test('guards workspace and modal focus while terminal work is pending', () => { + assert.equal(composeSessionCanFocus({ id: 'session-7', terminalPending: 'send' }), false); + assert.equal(composeSessionCanFocus({ id: 'session-7', terminalPending: null }), true); + assert.match(source, /composeSessionCanFocus\(session\)/); + assert.match(modal, /if \(!terminalPending\) onFocus\(session\.id\)/); +}); + +test('constrains the fixed dock to the measured mail workspace rectangle', () => { + assert.match(source, /querySelector\('\[data-mail-workspace\]'\)/); + assert.match(source, /getBoundingClientRect\(\)/); + assert.match(source, /position: 'fixed'/); + for (const edge of ['top', 'left', 'width', 'height']) { + assert.match(source, new RegExp(`${edge}: bounds\\.${edge}`)); + } +}); + +test('delegates all persistence and terminal actions to the controller', () => { + for (const callback of [ + 'changeSession', 'focusSession', 'minimizeSession', 'closeSession', + 'discardSession', 'sendSession', 'addAttachment', 'removeAttachment', + ]) assert.ok(source.includes(callback), `missing ${callback}`); + assert.match(source, /onSave=\{\(id, changes\) => saveSession\(id, changes\)\}/); + assert.match(source, /function safely[\s\S]*console\.error\('Compose action failed'/); +}); + +test('adapts ComposeModal to a server session without local terminal authority', () => { + assert.match(modal, /function ComposeModal\(\{[\s\S]*session,[\s\S]*tiled = false,[\s\S]*allowFreeform = false/); + for (const callback of [ + 'onChange', 'onFocus', 'onMinimize', 'onClose', 'onDiscard', 'onSend', 'onUndoQueuedSend', + 'onAddAttachment', 'onRemoveAttachment', 'onSave', + ]) assert.ok(modal.includes(callback), `missing ${callback}`); + assert.doesNotMatch(modal, /api\.saveDraft/); + assert.doesNotMatch(modal, /api\.post\('\/mail\/send'/); + assert.doesNotMatch(modal, /closeCompose\(/); + assert.doesNotMatch(modal, /\[minimized, setMinimized\]/); + assert.match(modal, /dirtyFieldsRef/); + assert.match(modal, /if \(!hydratedRef\.current\) return/); + assert.match(modal, /onChange\(session\.id, buildEditablePatch\(\)\)/); + assert.match(modal, /initialComposeForm\(composeData/); + assert.match(modal, /reconcileComposeForm\(nextPatch, session, protectedFields\)/); + assert.match(modal, /uploadFilesSequentially/); + assert.doesNotMatch(modal, /inReplyTo: composeData\?/); + assert.doesNotMatch(modal, /references: composeData\?/); + assert.doesNotMatch(modal, /api\.cancelOutbox/); + assert.doesNotMatch(modal, /openCompose\(capturedPayload\)/); + assert.doesNotMatch(modal, /showAttachWarnForDraft|attachWarnDraftCloseAfter|draftHasAttachments/); +}); + +test('manual Save awaits attachment completion and durable persistence acknowledgement', async () => { + const persisted = deferred(); + const events = []; + const changes = { + subject: 'Complete manual snapshot', body: '

Complete body

', + forwardedAttachments: [{ filename: 'forwarded.txt', size: 12 }], + }; + const saving = persistComposeDraft({ + sessionId: 'session-save', changes, + waitForAttachments: async action => { events.push('attachments'); return action(); }, + onChange: () => { throw new Error('manual Save must use the flush boundary'); }, + onSave: (id, snapshotValue) => { + events.push(['save', id, snapshotValue]); + return persisted.promise; + }, + onClose: () => { throw new Error('manual Save must not close'); }, + onSavingChange: value => events.push(['saving', value]), + onError: error => events.push(['error', error]), + }); + await Promise.resolve(); + assert.deepEqual(events, [ + ['saving', true], 'attachments', ['save', 'session-save', changes], + ]); + persisted.resolve('saved'); + assert.equal(await saving, 'saved'); + assert.deepEqual(events.at(-1), ['saving', false]); +}); + +test('manual Save captures edits after the attachment drain instead of replaying an old snapshot', async () => { + const attachment = deferred(); + const initialChanges = { subject: 'Before drain', body: '

Initial body

' }; + let currentChanges = initialChanges; + const saved = []; + const saving = persistComposeDraft({ + sessionId: 'session-latest', + changes: initialChanges, + getChanges: () => structuredClone(currentChanges), + waitForAttachments: async action => { + await attachment.promise; + return action(); + }, + onSave: async (id, changes) => { + saved.push([id, changes]); + return 'saved-latest'; + }, + }); + await Promise.resolve(); + currentChanges = { subject: 'Edited during drain', body: '

Latest body

' }; + attachment.resolve(); + + assert.equal(await saving, 'saved-latest'); + assert.deepEqual(saved, [[ + 'session-latest', + { subject: 'Edited during drain', body: '

Latest body

' }, + ]]); +}); + +test('attachment drain waits for mutations appended while an earlier mutation is pending', async () => { + const presentation = await import('./composePresentationModel.js'); + assert.equal(typeof presentation.runAfterStableAttachmentMutations, 'function'); + const first = deferred(); + const second = deferred(); + let pending = first.promise; + const events = []; + const draining = presentation.runAfterStableAttachmentMutations( + () => pending, + async () => { + events.push('saved'); + return 'stable'; + }, + ); + await Promise.resolve(); + pending = first.promise.then(() => second.promise); + first.resolve(); + await Promise.resolve(); + await Promise.resolve(); + assert.deepEqual(events, []); + second.resolve(); + assert.equal(await draining, 'stable'); + assert.deepEqual(events, ['saved']); +}); + +test('manual Save exposes rejection while reliably clearing the saving state', async () => { + const failure = new Error('synthetic persistence failure'); + const events = []; + const result = await persistComposeDraft({ + sessionId: 'session-save', changes: { subject: 'Rejected' }, + waitForAttachments: action => action(), + onSave: async () => { throw failure; }, + onSavingChange: value => events.push(['saving', value]), + onError: error => events.push(['error', error]), + }); + assert.equal(result, null); + assert.deepEqual(events, [ + ['saving', true], ['error', failure], ['saving', false], + ]); +}); + +test('close-after-save drains attachments, captures the final patch, and calls only atomic close', async () => { + const attachment = deferred(); + const initialChanges = { subject: 'Before close drain', body: '

Initial body

' }; + let changes = initialChanges; + const events = []; + const closing = persistComposeDraft({ + sessionId: 'session-close', changes: initialChanges, getChanges: () => changes, + closeAfter: true, + waitForAttachments: async action => { + events.push('attachments'); + await attachment.promise; + return action(); + }, + onChange: (id, snapshotValue) => events.push(['change', id, snapshotValue]), + onSave: () => { throw new Error('close must not pre-flush'); }, + onClose: async id => { events.push(['close', id]); return 'closed'; }, + onSavingChange: value => events.push(['saving', value]), + onError: error => events.push(['error', error]), + }); + await Promise.resolve(); + changes = { subject: 'Atomic close', body: '

Final body

' }; + attachment.resolve(); + const result = await closing; + assert.equal(result, 'closed'); + assert.deepEqual(events, [ + ['saving', true], + 'attachments', + ['change', 'session-close', changes], + ['close', 'session-close'], + ['saving', false], + ]); +}); + +test('manual saving disables attachment additions and removals until the boundary settles', () => { + assert.match(modal, /savingDraftRef/); + assert.match(modal, /attachmentControlsDisabled/); + assert.match(modal, /if \(terminalPending \|\| savingDraftRef\.current\) return/); + assert.match(modal, /disabled=\{attachmentControlsDisabled\}/); + assert.match(modal, / { + assert.match(modal, /allowFreeform && !tiled/); + assert.match(modal, /onPointerDown=\{canFreeform \? handleTitleDragStart : undefined\}/); + assert.match(modal, /onPointerDown=\{canFreeform \? handleResizeDragStart : undefined\}/); +}); + +test('routes Android back through safe close for the focused composer', () => { + assert.match(mailApp, /focusedComposeSessionId/); + assert.match(mailApp, /composeWorkspaceControllerRef\.current\.closeSession\(sessionId\)/); + assert.match(mailApp, /handleComposeRequest\(\s*\(\) => composeWorkspaceControllerRef\.current\.closeSession\(sessionId\)/); + assert.doesNotMatch(mailApp, /closeSession\(sessionId\)\.catch\(\(\) => \{\}\)/); +}); + +test('uses authoritative body format and preserves quoted HTML in complete patches', () => { + const rich = initialComposeForm({ + body: '

Rich

', bodyIsHtml: true, quotedBodyHtml: '

Quoted

', + editedSignature: '

Custom

', to: [], cc: [], bcc: [], + }, { defaultPlaintext: true }); + assert.equal(rich.bodyIsHtml, true); + assert.equal(rich.editedSignature, '

Custom

'); + + const plain = initialComposeForm({ + body: 'Plain', bodyIsHtml: false, quotedBodyHtml: '

Quoted

', + editedSignature: 'Custom plain', to: [], cc: [], bcc: [], + }, { defaultPlaintext: false }); + assert.equal(plain.bodyIsHtml, false); + assert.equal(plain.editedSignature, 'Custom plain'); + + const switched = { ...rich, bodyIsHtml: false, body: 'Now plain' }; + const patch = composeFormPatch(switched, { accountId: 'account-synthetic' }); + assert.equal(patch.bodyIsHtml, false); + assert.equal(patch.body, 'Now plain'); + assert.equal(patch.quotedBodyHtml, '

Quoted

'); + assert.equal(patch.editedSignature, '

Custom

'); +}); + +test('reconciles only clean authoritative format and signatures', () => { + const current = { + ...initialComposeForm({ + body: 'Local', bodyIsHtml: false, editedSignature: 'Local signature', + quotedBodyHtml: '

Local quote

', to: [], cc: [], bcc: [], + }), + }; + const remote = { + body: '

Remote

', bodyIsHtml: true, editedSignature: '

Remote signature

', + quotedBodyHtml: '

Remote quote

', + }; + const retained = reconcileComposeForm(current, remote, new Set(['body', 'bodyIsHtml', 'editedSignature'])); + assert.equal(retained.body, 'Local'); + assert.equal(retained.bodyIsHtml, false); + assert.equal(retained.editedSignature, 'Local signature'); + assert.equal(retained.quotedBodyHtml, '

Remote quote

'); + + const clean = reconcileComposeForm(current, remote, new Set()); + assert.equal(clean.body, '

Remote

'); + assert.equal(clean.bodyIsHtml, true); + assert.equal(clean.editedSignature, '

Remote signature

'); +}); + +test('hydrates a summary-only form atomically before any complete patch can emit', () => { + let model = composeHydrationStep(undefined, { + type: 'summary', + session: { + id: 'session-synthetic', mode: 'reply', to: [], body: '', + editedSignature: '', inReplyTo: null, references: [], + }, + }); + assert.equal(model.ready, false); + assert.equal(model.emitted, null); + model = composeHydrationStep(model, { + type: 'user-change', changes: { subject: 'Must not autosave yet' }, + }); + assert.equal(model.ready, false); + assert.equal(model.emitted, null); + + const snapshot = { + id: 'session-synthetic', snapshot: {}, mode: 'reply', + accountId: 'account-synthetic', aliasId: null, + to: ['Synthetic Sender '], cc: [], bcc: [], + subject: 'Re: Synthetic', body: '

Authoritative reply

', bodyIsHtml: true, + quotedBody: 'Quoted plain', quotedBodyHtml: '

Quoted rich

', + editedSignature: '

Server signature

', forwardedAttachments: [], + priority: 'normal', inReplyTo: '', + references: ['', ''], fromChanged: false, + }; + model = composeHydrationStep(model, { type: 'snapshot', session: snapshot }); + assert.equal(model.ready, true); + assert.equal(model.emitted, null); + assert.equal(model.form.body, '

Authoritative reply

'); + assert.equal(model.form.bodyIsHtml, true); + assert.equal(model.form.editedSignature, '

Server signature

'); + assert.equal(model.form.inReplyTo, ''); + assert.deepEqual(model.form.references, ['', '']); + + model = composeHydrationStep(model, { + type: 'user-change', changes: { subject: 'Re: Edited synthetic' }, session: snapshot, + }); + assert.equal(model.emitted.subject, 'Re: Edited synthetic'); + assert.equal(model.emitted.body, '

Authoritative reply

'); + assert.equal(model.emitted.editedSignature, '

Server signature

'); + assert.equal(model.emitted.inReplyTo, ''); + assert.deepEqual(model.emitted.references, ['', '']); + assert.deepEqual(Object.keys(model.emitted), [ + 'accountId', 'aliasId', 'mode', 'to', 'cc', 'bcc', 'subject', 'body', + 'bodyIsHtml', 'quotedBody', 'quotedBodyHtml', 'editedSignature', + 'forwardedAttachments', 'priority', 'inReplyTo', 'references', 'fromChanged', + ]); + + model = composeHydrationStep(model, { + type: 'snapshot', + session: { ...snapshot, body: '

Clean remote update

' }, + dirtyFields: new Set(['subject']), + }); + assert.equal(model.emitted, null); + assert.equal(model.form.subject, 'Re: Edited synthetic'); + assert.equal(model.form.body, '

Clean remote update

'); +}); + +test('serializes multi-file uploads in order and stops after a failure', async () => { + const calls = []; + await uploadFilesSequentially(['one', 'two', 'three'], async file => { + calls.push(file); + await Promise.resolve(); + }); + assert.deepEqual(calls, ['one', 'two', 'three']); + + calls.length = 0; + await assert.rejects(uploadFilesSequentially(['one', 'two', 'three'], async file => { + calls.push(file); + if (file === 'two') throw new Error('synthetic upload failure'); + }), /synthetic upload failure/); + assert.deepEqual(calls, ['one', 'two']); +}); + +test('converts scaled viewport geometry to containing-block-local coordinates', () => { + assert.deepEqual(localDockBounds( + { top: 120, left: 300, width: 900, height: 600 }, + { top: 60, left: 120 }, + 1.5, + ), { top: 40, left: 120, width: 600, height: 400 }); + assert.deepEqual(localDockBounds( + { top: 12, left: 24, width: 800, height: 500 }, + { top: 0, left: 0 }, + 1, + ), { top: 12, left: 24, width: 800, height: 500 }); +}); + +test('maps server byteCount to the existing attachment chip size', () => { + assert.deepEqual(attachmentChipView({ + id: 'attachment-synthetic', filename: 'synthetic.bin', byteCount: 321, mediaType: 'application/octet-stream', + }), { + id: 'attachment-synthetic', filename: 'synthetic.bin', byteCount: 321, + mediaType: 'application/octet-stream', name: 'synthetic.bin', size: 321, + type: 'application/octet-stream', + }); +}); + +test('uses durable reply-all metadata and preserves the legacy transition fallback', () => { + assert.deepEqual( + replyAllRecipientsForSession({ + replyAllRecipients: ['Synthetic Primary '], + allRecipients: ['Synthetic Legacy '], + }), + ['Synthetic Primary '], + ); + assert.deepEqual( + replyAllRecipientsForSession({ allRecipients: ['Synthetic Legacy '] }), + ['Synthetic Legacy '], + ); + assert.match(modal, /composeData\?\.mode === 'reply_all'/); + assert.equal((modal.match(/replyAllRecipientsForSession\(composeData\)/g) || []).length, 2); + assert.match(source, /onUndoQueuedSend=\{outboxId => safely\(\(\) => undoQueuedSend\(outboxId\)\)\}/); +}); diff --git a/frontend/src/components/ContextMenu.jsx b/frontend/src/components/ContextMenu.jsx index 1d1c4adf..f1219e8d 100644 --- a/frontend/src/components/ContextMenu.jsx +++ b/frontend/src/components/ContextMenu.jsx @@ -4,6 +4,9 @@ import { useStore } from '../store/index.js'; import { api } from '../utils/api.js'; import { GTD_STATES, GTD_COLORS, resolveAccountGtdFolders, gtdStatesInFolders } from '../utils/gtd.js'; import { getContextMenuPolicy, resolveContextMenuMessage } from '../utils/contextMenuPolicy.js'; +import { stableConversationId } from '../commands/contracts.js'; +import { toContextMenuCommand } from '../commands/contextMenuCommands.js'; +import { useCommandRuntimeContext } from '../commands/CommandRuntimeContext.jsx'; import MessageHeaderModal from './MessageHeaderModal.jsx'; import { useUiScale, descale } from '../hooks/useUiScale.js'; @@ -15,8 +18,19 @@ const SPAM_NAME_RE = /(spam|junk|bulk|indesiderata|spamverdacht|courrier\s*ind|p // ─── Context Menu ───────────────────────────────────────────────────────────── const CATEGORIES = ['primary', 'newsletter', 'promotion', 'automated', 'social']; -export default function ContextMenu({ x, y, message, onClose, onAction, defaultMoveView = false, variant = 'inbox' }) { +export default function ContextMenu({ + x, + y, + message, + targetIds = [stableConversationId(message)].filter(Boolean), + onClose, + onAction, + defaultMoveView = false, + variant = 'inbox', + onCommand, +}) { const { t } = useTranslation(); + const { controller } = useCommandRuntimeContext(); const uiScale = useUiScale(); // Variants share one menu; the policy removes actions that depend on the center // list or conflict with GTD's Done contract while preserving ordinary mail actions. @@ -49,6 +63,18 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM const [folderSearch, setFolderSearch] = useState(''); const unreadCount = Number.parseInt(message.unread_count, 10); const hasUnread = Number.isFinite(unreadCount) ? unreadCount > 0 : !message.is_read; + const runAction = (action, data) => { + const invocation = toContextMenuCommand(action, data); + if (invocation) { + if (onCommand) return onCommand(invocation.commandId, invocation.input); + return controller.execute(invocation.commandId, { + source: 'context-menu', + input: invocation.input, + frozenTargetIds: targetIds, + }); + } + return onAction(action, data); + }; // A folder is "spam-like" when either the user mapped it as spam or the IMAP // server tagged it with \Junk special-use. Falls back to a multilingual name @@ -117,14 +143,14 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM { label: t('contextMenu.open'), icon: , - action: () => onAction('open'), + action: () => runAction('open'), }, { label: hasUnread ? t('contextMenu.markRead') : t('contextMenu.markUnread'), icon: hasUnread ? : , - action: () => onAction(hasUnread ? 'markRead' : 'markUnread'), + action: () => runAction(hasUnread ? 'markRead' : 'markUnread'), }, { label: message.is_starred ? t('contextMenu.unstar') : t('contextMenu.star'), @@ -133,12 +159,12 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM stroke={message.is_starred ? 'var(--amber)' : 'currentColor'} strokeWidth="1.75"> , - action: () => onAction('toggleStar'), + action: () => runAction('toggleStar'), }, ...(!menuPolicy.select ? [] : [{ label: t('contextMenu.select'), icon: , - action: () => onAction('bulkSelect'), + action: () => runAction('bulkSelect'), }]), ] }, @@ -148,17 +174,17 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM ...(!menuPolicy.compose ? [] : [{ label: t('contextMenu.reply'), icon: , - action: () => onAction('reply'), + action: () => runAction('reply'), }, { label: t('contextMenu.replyAll'), icon: , - action: () => onAction('replyAll'), + action: () => runAction('replyAll'), }, { label: t('contextMenu.forward'), icon: , - action: () => onAction('forward'), + action: () => runAction('forward'), }]), { label: t('contextMenu.moveToFolder'), @@ -170,7 +196,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM ...(!menuPolicy.archive ? [] : [{ label: t('contextMenu.archive'), icon: , - action: () => onAction('archive'), + action: () => runAction('archive'), }]), ...(message.folder !== 'Snoozed' && menuPolicy.snooze ? [{ label: t('contextMenu.snooze.label'), @@ -198,17 +224,17 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM ...(menuPolicy.done ? [{ label: t('gtd.done'), icon: , - action: () => onAction('gtdDone'), + action: () => runAction('gtdDone'), }] : []), ...(!menuPolicy.rules ? [] : [{ label: t('contextMenu.createRule'), icon: , - action: () => onAction('createRuleFromMessage'), + action: () => runAction('createRuleFromMessage'), }, { label: t('contextMenu.addToBlockList'), icon: , - action: () => onAction('addToBlockList'), + action: () => runAction('addToBlockList'), }]), // Spam / ham are only shown when there's a real destination for the // action: "Mark as Spam" when the message isn't already in a spam-like @@ -217,12 +243,12 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM ...(spamFolderPaths.size > 0 && !inSpamFolder && menuPolicy.spam ? [{ label: t('contextMenu.markAsSpam'), icon: , - action: () => onAction('markSpam'), + action: () => runAction('markSpam'), }] : []), ...(inSpamFolder && menuPolicy.spam ? [{ label: t('contextMenu.markAsHam'), icon: , - action: () => onAction('markHam'), + action: () => runAction('markHam'), }] : []), ] }, @@ -232,12 +258,12 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM { label: t('contextMenu.copySubject'), icon: , - action: () => { navigator.clipboard.writeText(message.subject || ''); onAction('copy'); }, + action: () => { navigator.clipboard.writeText(message.subject || ''); runAction('copy'); }, }, { label: t('contextMenu.copySender'), icon: , - action: () => { navigator.clipboard.writeText(message.from_email || ''); onAction('copy'); }, + action: () => { navigator.clipboard.writeText(message.from_email || ''); runAction('copy'); }, }, ] }, @@ -264,7 +290,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM { label: t('contextMenu.delete'), icon: , - action: () => onAction('delete'), + action: () => runAction('delete'), danger: true, }, ] @@ -336,7 +362,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM {GTD_STATES.map(state => (
{ onAction('gtdClassify', state); onClose(); }} + onClick={() => { runAction('gtdClassify', state); onClose(); }} style={{ display: 'flex', alignItems: 'center', gap: 10, padding: '7px 14px', cursor: 'pointer', fontSize: 13, color: 'var(--text-primary)' }} onMouseEnter={e => e.currentTarget.style.background = 'var(--bg-hover)'} onMouseLeave={e => e.currentTarget.style.background = 'transparent'} @@ -351,7 +377,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM {gtdRemovableStates.map(state => (
{ onAction('gtdRemove', state); onClose(); }} + onClick={() => { runAction('gtdRemove', state); onClose(); }} style={{ display: 'flex', alignItems: 'center', gap: 10, padding: '7px 14px', cursor: 'pointer', fontSize: 13, color: 'var(--text-secondary)' }} onMouseEnter={e => e.currentTarget.style.background = 'var(--bg-hover)'} onMouseLeave={e => e.currentTarget.style.background = 'transparent'} @@ -386,7 +412,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM return (
{ if (!isCurrent) { onAction('setCategory', cat); onClose(); } }} + onClick={() => { if (!isCurrent) { runAction('setCategory', cat); onClose(); } }} style={{ display: 'flex', alignItems: 'center', gap: 10, padding: '7px 14px', cursor: isCurrent ? 'default' : 'pointer', @@ -454,7 +480,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM onClick={() => { const d = new Date(`${customDate}T${customTime}`); if (isNaN(d.getTime())) return; - onAction('snooze', d.toISOString()); + runAction('snooze', d.toISOString()); onClose(); }} style={{ @@ -503,7 +529,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM ].map(({ label, getDate }) => (
{ onAction('snooze', getDate().toISOString()); onClose(); }} + onClick={() => { runAction('snooze', getDate().toISOString()); onClose(); }} style={{ display: 'flex', alignItems: 'center', gap: 10, padding: '8px 14px', cursor: 'pointer', fontSize: 13, color: 'var(--text-primary)' }} onMouseEnter={e => e.currentTarget.style.background = 'var(--bg-hover)'} onMouseLeave={e => e.currentTarget.style.background = 'transparent'} @@ -600,7 +626,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM { onAction('moveTo', folder.path); onClose(); }} + onClick={() => { runAction('moveTo', folder.path); onClose(); }} /> ))} @@ -626,7 +652,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM { onAction('moveTo', folder.path); onClose(); }} + onClick={() => { runAction('moveTo', folder.path); onClose(); }} /> ))}
@@ -641,7 +667,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM { onAction('moveTo', folder.path); onClose(); }} + onClick={() => { runAction('moveTo', folder.path); onClose(); }} /> ))}
@@ -653,7 +679,7 @@ export default function ContextMenu({ x, y, message, onClose, onAction, defaultM { onAction('moveTo', folder.path); onClose(); }} + onClick={() => { runAction('moveTo', folder.path); onClose(); }} /> )) } diff --git a/frontend/src/components/DelegateContactPicker.jsx b/frontend/src/components/DelegateContactPicker.jsx new file mode 100644 index 00000000..f6d69709 --- /dev/null +++ b/frontend/src/components/DelegateContactPicker.jsx @@ -0,0 +1,204 @@ +import { useCallback, useEffect, useId, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useCommandRuntimeContext } from '../commands/CommandRuntimeContext.jsx'; +import { api } from '../utils/api.js'; +import { + contactOption, + createPickerRequestGate, + nextPickerIndex, +} from '../utils/delegation.js'; + +function focusableElements(container) { + return [...container.querySelectorAll( + 'button:not([disabled]), input:not([disabled]), [tabindex]:not([tabindex="-1"])', + )]; +} + +export default function DelegateContactPicker({ targetCount, onSelect, onCancel }) { + const { t } = useTranslation(); + const titleId = useId(); + const listboxId = useId(); + const dialogRef = useRef(null); + const searchRef = useRef(null); + const optionRefs = useRef([]); + const gateRef = useRef(createPickerRequestGate()); + const restoreFocusRef = useRef(document.activeElement); + const [query, setQuery] = useState(''); + const [contacts, setContacts] = useState([]); + const [activeIndex, setActiveIndex] = useState(-1); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [retryKey, setRetryKey] = useState(0); + + useEffect(() => { + const previousFocus = restoreFocusRef.current; + searchRef.current?.focus(); + return () => previousFocus?.focus?.(); + }, []); + + useEffect(() => { + const requestId = gateRef.current.start(); + setLoading(true); + setError(null); + setContacts([]); + setActiveIndex(-1); + const timer = setTimeout(async () => { + try { + const response = await api.getContacts({ q: query.trim(), limit: 30, offset: 0 }); + if (!gateRef.current.isCurrent(requestId)) return; + const next = (response?.contacts || (Array.isArray(response) ? response : [])) + .map(contactOption) + .filter(contact => contact.id && contact.label); + setContacts(next); + setActiveIndex(next.length ? 0 : -1); + } catch (cause) { + if (gateRef.current.isCurrent(requestId)) setError(cause); + } finally { + if (gateRef.current.isCurrent(requestId)) setLoading(false); + } + }, 200); + return () => clearTimeout(timer); + }, [query, retryKey]); + + useEffect(() => { + optionRefs.current[activeIndex]?.scrollIntoView?.({ block: 'nearest' }); + }, [activeIndex]); + + const chooseActive = useCallback(() => { + const contact = contacts[activeIndex]; + if (contact) onSelect(contact.id); + }, [activeIndex, contacts, onSelect]); + + const handleKeyDown = (event) => { + if (event.isComposing || event.nativeEvent?.isComposing || event.keyCode === 229) return; + if (event.key === 'Escape') { + event.preventDefault(); + onCancel(); + } else if (event.key === 'ArrowDown') { + event.preventDefault(); + setActiveIndex(index => nextPickerIndex(index, 1, contacts.length)); + } else if (event.key === 'ArrowUp') { + event.preventDefault(); + setActiveIndex(index => nextPickerIndex(index, -1, contacts.length)); + } else if (event.key === 'Enter' && activeIndex >= 0 && event.target === searchRef.current) { + event.preventDefault(); + chooseActive(); + } else if (event.key === 'Tab' && dialogRef.current) { + const focusable = focusableElements(dialogRef.current); + if (!focusable.length) return; + const first = focusable[0]; + const last = focusable.at(-1); + if (event.shiftKey && document.activeElement === first) { + event.preventDefault(); + last.focus(); + } else if (!event.shiftKey && document.activeElement === last) { + event.preventDefault(); + first.focus(); + } + } + }; + + return ( +
{ + if (event.target === event.currentTarget) onCancel(); + }}> +
+
+
+

{t('gtd.delegate.pickerTitle', { count: targetCount })}

+

{t('gtd.delegate.pickerHint')}

+
+ +
+ + + +
+ {loading &&
{t('common.loading')}
} + {!loading && error &&
+ {t('gtd.delegate.loadFailed')} + +
} + {!loading && !error && contacts.length === 0 && ( +
{t('gtd.delegate.empty')}
+ )} + {!loading && !error && contacts.map((contact, index) => ( + + ))} +
+ +
+ + Esc {t('common.cancel')} +
+
+
+ ); +} + +export function DelegateContactContinuationHost({ onOpen }) { + const { controller, continuation, clearContinuation } = useCommandRuntimeContext(); + const isContact = continuation?.kind === 'contact'; + useEffect(() => { if (isContact) onOpen?.(); }, [isContact, onOpen]); + if (!isContact) return null; + + return ( + { + const { commandId, targetIds } = continuation; + clearContinuation(); + void controller.execute(commandId, { + source: 'continuation', + input: { contactId }, + frozenTargetIds: targetIds, + }); + }} + /> + ); +} diff --git a/frontend/src/components/DelegateContactPicker.test.js b/frontend/src/components/DelegateContactPicker.test.js new file mode 100644 index 00000000..b9f0a61f --- /dev/null +++ b/frontend/src/components/DelegateContactPicker.test.js @@ -0,0 +1,26 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +const source = fs.readFileSync(new URL('./DelegateContactPicker.jsx', import.meta.url), 'utf8'); + +test('provides dialog, combobox, listbox, and active-descendant semantics', () => { + for (const token of [ + 'role="dialog"', 'aria-modal="true"', 'role="combobox"', + 'aria-activedescendant', 'role="listbox"', 'role="option"', + ]) assert.ok(source.includes(token), `missing ${token}`); +}); + +test('keeps keyboard navigation visible and inert during IME composition', () => { + assert.match(source, /scrollIntoView\?\.\(\{ block: 'nearest' \}\)/); + assert.match(source, /event\.isComposing/); + assert.match(source, /event\.nativeEvent\?\.isComposing/); + assert.match(source, /event\.keyCode === 229/); + assert.match(source, /event\.target === searchRef\.current/); +}); + +test('resumes with frozen target IDs while cancellation only clears the continuation', () => { + assert.match(source, /onCancel=\{clearContinuation\}/); + assert.match(source, /frozenTargetIds: targetIds/); + assert.match(source, /input: \{ contactId \}/); +}); diff --git a/frontend/src/components/DelegatePill.jsx b/frontend/src/components/DelegatePill.jsx new file mode 100644 index 00000000..8545609d --- /dev/null +++ b/frontend/src/components/DelegatePill.jsx @@ -0,0 +1,23 @@ +import { useTranslation } from 'react-i18next'; +import { delegateLabel, delegateTooltip } from '../utils/delegation.js'; + +export default function DelegatePill({ delegation, compact = false }) { + const { t } = useTranslation(); + const label = delegateLabel(delegation); + if (!label) return null; + return ( + + + {label} + + ); +} diff --git a/frontend/src/components/DelegatePill.test.js b/frontend/src/components/DelegatePill.test.js new file mode 100644 index 00000000..6e459de2 --- /dev/null +++ b/frontend/src/components/DelegatePill.test.js @@ -0,0 +1,12 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +test('renders delegate identity on GTD, thread-list, flat-list, and open-message surfaces', () => { + const gtd = fs.readFileSync(new URL('./GtdEntryRow.jsx', import.meta.url), 'utf8'); + const list = fs.readFileSync(new URL('./MessageList.jsx', import.meta.url), 'utf8'); + const pane = fs.readFileSync(new URL('./MessagePane.jsx', import.meta.url), 'utf8'); + assert.match(gtd, / state.addNotification); const openCompose = useStore(state => state.openCompose); const setSelectedAccount = useStore(state => state.setSelectedAccount); @@ -148,7 +151,7 @@ export default function ElectronNotificationBridge() { const originalMessageId = message.message_id || message.messageId; const priorInReplyTo = message.in_reply_to || message.inReplyTo; - openCompose({ + return handleComposeRequest(() => openCompose({ to: sender, cc: [], subject, @@ -159,7 +162,7 @@ export default function ElectronNotificationBridge() { isReply: true, originalFrom: sender, allRecipients: [], - }); + }), { addNotification, t }); }; const runNativeAction = async (payload) => { @@ -177,7 +180,10 @@ export default function ElectronNotificationBridge() { try { if (action === 'new-mail') { - openCompose(payload?.composeData || {}); + await handleComposeRequest( + () => openCompose(payload?.composeData || {}), + { addNotification, t }, + ); return; } @@ -187,7 +193,7 @@ export default function ElectronNotificationBridge() { } if (action === 'reply-message') { - openReplyFromPayload(payload); + await openReplyFromPayload(payload); return; } @@ -273,7 +279,7 @@ export default function ElectronNotificationBridge() { window.removeEventListener('message', handleNativeMessage); if (typeof unsubscribe === 'function') unsubscribe(); }; - }, [addNotification, nativeBridgeReady, openCompose, setSearchQuery, setSelectedAccount, setSelectedMessage]); + }, [addNotification, nativeBridgeReady, openCompose, setSearchQuery, setSelectedAccount, setSelectedMessage, t]); return null; } diff --git a/frontend/src/components/GtdEntryRow.jsx b/frontend/src/components/GtdEntryRow.jsx index 74e4e701..774f1092 100644 --- a/frontend/src/components/GtdEntryRow.jsx +++ b/frontend/src/components/GtdEntryRow.jsx @@ -3,6 +3,7 @@ import { GTD_COLORS, GTD_CHIP_BG, agingLabel, resolveRowDisplay, } from '../utils/gtd.js'; import { formatDate } from '../utils/formatDate.js'; +import DelegatePill from './DelegatePill.jsx'; // One GTD entry row, shared by both display surfaces: the GTD browse list that // replaces the message list (GtdTabList, roomier) and the denser right-sidebar @@ -88,6 +89,7 @@ export default function GtdEntryRow({ }}> {sender} + {/* Aging pill carries the row's kind color (watch yellow / delegated orange), except when stale — staleness outranks kind and keeps the red styling. */} {isWaiting && days != null && ( diff --git a/frontend/src/components/MailApp.jsx b/frontend/src/components/MailApp.jsx index 8e2df0e8..d7be636c 100644 --- a/frontend/src/components/MailApp.jsx +++ b/frontend/src/components/MailApp.jsx @@ -4,22 +4,27 @@ import { useStore } from '../store/index.js'; import { api } from '../utils/api.js'; import { useWebSocket } from '../hooks/useWebSocket.js'; import { useMobile } from '../hooks/useMobile.js'; +import { useCommandRuntime } from '../hooks/useCommandRuntime.js'; import { LAYOUTS } from '../layouts.js'; import { updateFaviconBadge } from '../themes.js'; import { shortcutBus } from '../utils/shortcutBus.js'; import { setPending, pendingMarkReadMap, completedMarkReadMap } from '../utils/pendingReads.js'; -import { buildKeyMap, buildModKeyMap, getEffectiveShortcuts, getGroupedActions, parseModKey, modLabel, SPECIAL_KEYS, SPECIAL_KEY_LABELS } from '../utils/defaultShortcuts.js'; +import { formatCommandKey, getEffectiveCommandBindings } from '../commands/shortcuts.js'; import Sidebar from './Sidebar.jsx'; import MessageList from './MessageList.jsx'; import MessagePane from './MessagePane.jsx'; import GtdSidebarContent from './GtdSidebarContent.jsx'; import NotificationToasts from './NotificationToasts.jsx'; import CommandPalette from './CommandPalette.jsx'; +import { DelegateContactContinuationHost } from './DelegateContactPicker.jsx'; import { gtdActiveForContext } from '../utils/gtd.js'; +import { CommandRuntimeProvider } from '../commands/CommandRuntimeContext.jsx'; +import { commandPaletteShortcut } from '../commands/paletteShortcut.js'; +import { handleComposeRequest } from '../utils/composeRequest.js'; const ContactsPage = lazy(() => import('./ContactsPage.jsx')); -const ComposeModal = lazy(() => import('./ComposeModal.jsx')); +const ComposeWorkspace = lazy(() => import('./ComposeWorkspace.jsx')); const AdminPanel = lazy(() => import('./AdminPanel.jsx')); const ElectronNotificationBridge = lazy(() => import('./ElectronNotificationBridge.jsx')); @@ -59,22 +64,27 @@ const lazyFallback = ( export default function MailApp() { const { t } = useTranslation(); + const editorPalettePressRef = useRef(null); const { setAccounts, setUnreadCounts, showAdmin, - setShowAdmin, setAdminTab, composing, sidebarCollapsed, layout, - unreadCounts, selectedAccountId, openCompose, setSelectedAccount, - shortcuts, selectedMessageId, setSelectedMessage, + setShowAdmin, setAdminTab, sidebarCollapsed, layout, + unreadCounts, selectedAccountId, openCompose, + selectedMessageId, setSelectedMessage, mobileSidebarOpen, setMobileSidebarOpen, addNotification, fontSize, showAppBadge, showFaviconBadge, sidebarWidth, setSidebarWidth, setIsSidebarResizing, showContacts, setTodoistConnected, accounts, rightSidebarWidth, setRightSidebarWidth, isRightSidebarResizing, setIsRightSidebarResizing, fetchGtdSections, rightSidebarHidden, toggleRightSidebarHidden, + refreshCarddavStatus, + focusedComposeSessionId, composeWorkspaceController, } = useStore(); const syncInterval = useStore(s => s.syncInterval); const autoLockMinutes = useStore(s => s.autoLockMinutes); const lockScreen = useStore(s => s.lockScreen); + useEffect(() => { void refreshCarddavStatus(); }, [refreshCarddavStatus]); + // Auto-lock after inactivity (#235). MailApp only mounts while unlocked, so this // timer runs only when unlocked; hitting the timeout locks and unmounts this tree. useEffect(() => { @@ -131,6 +141,11 @@ export default function MailApp() { const [showShortcutHelp, setShowShortcutHelp] = useState(false); const [paletteOpen, setPaletteOpen] = useState(false); + const commandRuntime = useCommandRuntime({ t, shortcutHelpOpen: showShortcutHelp, paletteOpen }); + useEffect(() => { + if (!commandRuntime.continuation) return; + setPaletteOpen(commandRuntime.continuation.kind !== 'contact'); + }, [commandRuntime.continuation]); const isMobile = useMobile(); const sidebarDragRef = useRef(null); const sidebarResizeRef = useRef(null); @@ -197,8 +212,13 @@ export default function MailApp() { // Shortcut hint (e.g. "⌘/") for the collapse/expand tooltips, derived from the // live shortcut map via the existing helpers — no new plumbing. '' when unbound. - const rightSidebarToggleParsed = parseModKey(getEffectiveShortcuts(shortcuts).toggleRightSidebar); - const rightSidebarToggleHint = rightSidebarToggleParsed ? `${modLabel(rightSidebarToggleParsed.mod)}${rightSidebarToggleParsed.bare}` : ''; + const rightSidebarToggleKey = getEffectiveCommandBindings( + commandRuntime.commandDefinitions, + commandRuntime.getContext(), + ).find(item => item.commandId === 'layout.toggleRightSidebar')?.bindings[0]?.keys; + const rightSidebarToggleHint = rightSidebarToggleKey + ? formatCommandKey(rightSidebarToggleKey, commandRuntime.getContext().platform) + : ''; // The right sidebar renders when a feature supplies content. GTD is the // current (only) provider; the layout/shortcut infrastructure below is // feature-agnostic and keys off the seam, not the feature. @@ -413,18 +433,18 @@ export default function MailApp() { // A mailto body is plain text (RFC 6068); escape it so it renders literally in // the HTML editor and can't inject markup. const bodyText = mt.searchParams.get('body') || ''; - openCompose({ + void handleComposeRequest(() => openCompose({ accountId: useStore.getState().selectedAccountId || undefined, to: [...splitAddrs(mt.pathname, true), ...splitAddrs(mt.searchParams.get('to'), false)], cc: splitAddrs(mt.searchParams.get('cc'), false), bcc: splitAddrs(mt.searchParams.get('bcc'), false), subject: mt.searchParams.get('subject') || '', body: bodyText ? esc(bodyText).replace(/\r?\n/g, '
') : '', - }); + }), { addNotification, t }); } catch (err) { console.warn('Invalid mailto link:', err.message); } - }, [openCompose]); + }, [addNotification, openCompose, t]); useEffect(() => { // Load accounts @@ -441,8 +461,8 @@ export default function MailApp() { // Sync Todoist connection state — localStorage alone isn't enough across devices/sessions api.todoist.status().then(({ connected }) => setTodoistConnected(connected)).catch(() => {}); - // Preload ComposeModal chunk so first open is instant - import('./ComposeModal.jsx'); + // Preload the workspace (and its editor chunk) so first open is instant. + import('./ComposeWorkspace.jsx'); // Load unread counts const refreshCounts = () => { @@ -457,6 +477,35 @@ export default function MailApp() { return () => clearInterval(interval); }, [setAccounts, setUnreadCounts, setTodoistConnected]); + useEffect(() => { + api.getOutbox() + .then(({ pending = [] }) => { + for (const row of pending) { + const countdownUntil = new Date(row.sendAt ?? row.send_at).getTime(); + if (!Number.isFinite(countdownUntil)) continue; + addNotification({ + persistent: true, + durationMs: Math.max(0, countdownUntil - Date.now()), + title: t('compose.sending.title'), + body: row.subject || t('common.noSubject'), + countdownUntil, + onUndo: async () => { + try { + await api.cancelOutbox(row.id); + } catch { + addNotification({ + type: 'error', + title: t('compose.sending.tooLate'), + body: row.subject || t('common.noSubject'), + }); + } + }, + }); + } + }) + .catch(err => console.error('Failed to restore outbox:', err)); + }, []); // eslint-disable-line react-hooks/exhaustive-deps -- restore pending sends once per app mount + // WebSocket-independent periodic refresh of the open message list, at the user's chosen sync // interval. Only fires when the tab is visible AND the socket is not OPEN — a true fallback so // read state and new mail still converge if the WebSocket is down, without redundant refetching @@ -489,12 +538,14 @@ export default function MailApp() { }, [unreadCounts, selectedAccountId, showAppBadge, showFaviconBadge]); // ── Global keyboard shortcut listener ────────────────────────────────────── - // Uses refs for composing/showAdmin so the listener doesn't need to + // Uses refs for focused compose/showAdmin so the listener doesn't need to // re-register every time those values change — only re-registers when the // user's custom shortcut map changes. - const composingRef = useRef(composing); + const focusedComposeSessionIdRef = useRef(focusedComposeSessionId); + const composeWorkspaceControllerRef = useRef(composeWorkspaceController); const showAdminRef = useRef(showAdmin); - useEffect(() => { composingRef.current = composing; }, [composing]); + useEffect(() => { focusedComposeSessionIdRef.current = focusedComposeSessionId; }, [focusedComposeSessionId]); + useEffect(() => { composeWorkspaceControllerRef.current = composeWorkspaceController; }, [composeWorkspaceController]); useEffect(() => { showAdminRef.current = showAdmin; }, [showAdmin]); const mobileSidebarOpenRef = useRef(mobileSidebarOpen); @@ -506,8 +557,14 @@ export default function MailApp() { useEffect(() => { window.__mailflowHandleAndroidBack = () => { - if (composingRef.current) { - useStore.getState().closeCompose(); + const sessionId = focusedComposeSessionIdRef.current; + if (sessionId && composeWorkspaceControllerRef.current) { + // Safe close is atomic: the controller sends the local final patch at + // its base revision and retains the editor if the server rejects it. + void handleComposeRequest( + () => composeWorkspaceControllerRef.current.closeSession(sessionId), + { addNotification, t }, + ); return true; } @@ -517,6 +574,7 @@ export default function MailApp() { } if (paletteOpenRef.current) { + commandRuntime.clearContinuation(); setPaletteOpen(false); return true; } @@ -542,103 +600,14 @@ export default function MailApp() { return () => { if (window.__mailflowHandleAndroidBack) delete window.__mailflowHandleAndroidBack; }; - }, [setMobileSidebarOpen, setSelectedMessage, setShowAdmin]); - - useEffect(() => { - if (isMobile) return; - const keyMap = buildKeyMap(shortcuts); - const modKeyMap = buildModKeyMap(shortcuts); - // Keys that are prefixes of two-key sequences (e.g. 'g' for 'gi'). - // Special keys like 'Delete' have length > 1 but are single keypresses — exclude them. - const prefixKeys = new Set( - Object.keys(keyMap).filter(k => k.length > 1 && !SPECIAL_KEYS.has(k)).map(k => k[0]) - ); - - let pendingKey = null; - let pendingTimer = null; - - const clearPending = () => { - pendingKey = null; - if (pendingTimer) { clearTimeout(pendingTimer); pendingTimer = null; } - }; - - const handler = (e) => { - // Never intercept when the compose modal or admin panel is open, or an input is focused - if (composingRef.current || showAdminRef.current) return; - const tag = e.target.tagName; - if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || e.target.isContentEditable) return; - // Modifier combos: emit registered actions, pass everything else through - if (e.ctrlKey || e.metaKey) { - const action = modKeyMap[e.key.toLowerCase()]; - if (action) { e.preventDefault(); shortcutBus.emit(action); } - return; - } - if (e.altKey) return; - - const key = e.key; - - // Pure modifier keys — never intercept - if (['CapsLock', 'Control', 'Meta', 'Alt', 'Shift'].includes(key)) return; - - // Escape cancels any pending prefix sequence - if (key === 'Escape') { clearPending(); return; } - - // Resolve two-key sequences - let resolved = key; - if (pendingKey !== null) { - resolved = pendingKey + key; - clearPending(); - } - - // Check the keymap first — bound actions take priority, including special - // keys like Delete that would otherwise be skipped below. - const action = keyMap[resolved]; - if (action) { - e.preventDefault(); - shortcutBus.emit(action); - return; - } - - // Skip non-character keys that aren't bound (arrow keys, F-keys, etc.) - if (['Tab', 'Enter', 'Backspace', 'Delete', - 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight', - 'Home', 'End', 'PageUp', 'PageDown', 'Insert', - 'F1', 'F2', 'F3', 'F4', 'F5', 'F6', - 'F7', 'F8', 'F9', 'F10', 'F11', 'F12'].includes(key)) { - return; - } - - // Check if this single key could start a two-key sequence - if (prefixKeys.has(resolved) && resolved.length === 1) { - e.preventDefault(); - pendingKey = resolved; - pendingTimer = setTimeout(clearPending, 1000); - return; - } - - // Typed key didn't match anything — clear any stale pending state - if (pendingKey !== null) clearPending(); - }; - - document.addEventListener('keydown', handler); - return () => { - document.removeEventListener('keydown', handler); - clearPending(); - }; - }, [shortcuts, isMobile]); // Re-build key map only when shortcuts or device type changes + }, [addNotification, commandRuntime, setMobileSidebarOpen, setSelectedMessage, setShowAdmin, t]); // Subscribe to global actions that MailApp owns useEffect(() => { - const onCompose = () => openCompose({ accountId: useStore.getState().selectedAccountId || undefined }); - const onGoInbox = () => setSelectedAccount(null, 'INBOX'); const onShowHelp = () => { if (!isMobile) setShowShortcutHelp(v => !v); }; - shortcutBus.on('compose', onCompose); - shortcutBus.on('goInbox', onGoInbox); shortcutBus.on('showHelp', onShowHelp); return () => { - shortcutBus.off('compose', onCompose); - shortcutBus.off('goInbox', onGoInbox); shortcutBus.off('showHelp', onShowHelp); }; }, []); // eslint-disable-line react-hooks/exhaustive-deps @@ -651,25 +620,49 @@ export default function MailApp() { return () => shortcutBus.off('toggleRightSidebar', onToggleRightSidebar); }, [rightSidebarApplicable, toggleRightSidebarHidden]); - // Close help overlay on Escape + // Close help overlay on Escape or the same ? shortcut that opened it. useEffect(() => { if (!showShortcutHelp) return; - const handler = (e) => { if (e.key === 'Escape') { e.preventDefault(); setShowShortcutHelp(false); } }; + const handler = (e) => { + if (e.isComposing || e.keyCode === 229) return; + if (e.key === 'Escape' || e.key === '?') { + e.preventDefault(); + e.stopImmediatePropagation(); + setShowShortcutHelp(false); + } + }; document.addEventListener('keydown', handler); return () => document.removeEventListener('keydown', handler); }, [showShortcutHelp]); - // Cmd+K / Ctrl+K opens command palette + // Cmd+K / Ctrl+K toggles the command palette. In a rich-text editor, the + // first press remains available to the editor and a second press opens it. useEffect(() => { - const handler = (e) => { - if ((e.metaKey || e.ctrlKey) && e.key === 'k') { - e.preventDefault(); - setPaletteOpen(v => !v); - } + if (isMobile) return undefined; + const handler = event => { + const decision = commandPaletteShortcut({ + metaKey: event.metaKey, + ctrlKey: event.ctrlKey, + altKey: event.altKey, + key: event.key, + keyCode: event.keyCode, + isComposing: event.isComposing, + target: event.target, + isMobile, + }, editorPalettePressRef.current); + if (!decision.handled) return; + editorPalettePressRef.current = decision.nextEditorPress; + if (!decision.toggle) return; + event.preventDefault(); + event.stopPropagation(); + setPaletteOpen(open => { + if (open) commandRuntime.clearContinuation(); + return !open; + }); }; document.addEventListener('keydown', handler); return () => document.removeEventListener('keydown', handler); - }, []); + }, [commandRuntime, isMobile]); // Handle same-tab OAuth callback redirects (e.g. /?oauth_success=microsoft). // The popup case (window.opener present) is handled earlier in App.jsx before @@ -702,6 +695,7 @@ export default function MailApp() { }, []); // eslint-disable-line react-hooks/exhaustive-deps return ( +
{ e.currentTarget.style.background = 'var(--border-subtle)'; }} /> )} -
)} - {composing && } + {showAdmin && } {hasNativeBridge && } - setPaletteOpen(false)} /> + { + commandRuntime.clearContinuation(); + setPaletteOpen(false); + }} /> + setPaletteOpen(false)} /> {/* Keyboard shortcut help overlay — toggled by the '?' key */} {showShortcutHelp && ( setShowShortcutHelp(false)} /> )}
+ ); } -function ShortcutHelpOverlay({ shortcuts, onClose }) { +function ShortcutHelpOverlay({ definitions, context, onClose }) { const { t } = useTranslation(); - const effective = getEffectiveShortcuts(shortcuts); - const groups = getGroupedActions(); - - const keyBadge = (key) => { - if (!key) return —; - // Modifier combos like 'ctrl+p' - const mod = parseModKey(key); - if (mod) { - return ( - - {modLabel(mod.mod)} - + - {mod.bare.toUpperCase()} - - ); - } - // Special key names like 'Delete', 'ArrowUp' — single keypress, render as one badge - if (SPECIAL_KEY_LABELS[key]) { - return {SPECIAL_KEY_LABELS[key]}; - } - // For two-key sequences like 'gi', render each key separately - const parts = key.length > 1 - ? [...key].map((c, i) => ( - - {c} - {i < key.length - 1 && {t('shortcuts.then')}} - - )) - : [{key}]; - return {parts}; + const effective = new Map(getEffectiveCommandBindings(definitions, context) + .map(item => [item.commandId, item.bindings])); + const groups = definitions.filter(definition => effective.get(definition.id)?.length).reduce((result, definition) => { + (result[definition.group] ||= []).push(definition); + return result; + }, {}); + + const keyBadge = (bindings = []) => { + if (!bindings.length) return —; + return {bindings.map(binding => ( + {formatCommandKey(binding.keys, context.platform)} + ))}; }; return ( @@ -963,15 +945,15 @@ function ShortcutHelpOverlay({ shortcuts, onClose }) { {Object.entries(groups).map(([groupName, actions]) => (
- {t(groupName)} + {groupName}
- {actions.map(({ action, descriptionKey }) => ( -
( +
- {t(descriptionKey)} - {keyBadge(effective[action])} + {t(definition.titleKey, definition.params)} + {keyBadge(effective.get(definition.id))}
))}
diff --git a/frontend/src/components/MessageList.jsx b/frontend/src/components/MessageList.jsx index 43ebe780..207e57d7 100644 --- a/frontend/src/components/MessageList.jsx +++ b/frontend/src/components/MessageList.jsx @@ -13,15 +13,34 @@ import GtdTabList from './GtdTabList.jsx'; import { useUiScale, descale } from '../hooks/useUiScale.js'; import { gtdActiveForContext, buildGtdDisplaySections, GTD_COLORS, GTD_CHIP_BG, sectionBadge, isSelectedRow, - classifyThread, unclassifyThread, + unclassifyThread, } from '../utils/gtd.js'; import { formatDate } from '../utils/formatDate.js'; -import { openReplyFromMessage, openForwardFromMessage } from '../utils/composeFromMessage.js'; import SenderAvatarImage from './SenderAvatarImage.jsx'; +import { + openDraftFromMessage, +} from '../utils/composeFromMessage.js'; import { shortcutBus } from '../utils/shortcutBus.js'; import { createLatestRequest } from '../utils/latestRequest.js'; -import { pendingMarkReadMap, completedMarkReadMap, setPending } from '../utils/pendingReads.js'; -import { applyDeleteGuard, clearDeleteGuard, clearPendingDelete, setCompletedDelete, setPendingDelete } from '../utils/pendingDeletes.js'; +import { pendingMarkReadMap, completedMarkReadMap } from '../utils/pendingReads.js'; +import { applyDeleteGuard } from '../utils/pendingDeletes.js'; +import { stableConversationId } from '../commands/contracts.js'; +import { contextMenuTargetMessages } from '../commands/contextMenuCommands.js'; +import DelegatePill from './DelegatePill.jsx'; +import { useCommandRuntimeContext } from '../commands/CommandRuntimeContext.jsx'; +import { semanticSearchAvailable, semanticToggleState, searchInputRightPad, isCurrentSearchGeneration, LEXICAL_MODE, SEMANTIC_MODE } from '../utils/searchMode.js'; +import { handleComposeRequest } from '../utils/composeRequest.js'; + +// Sparkle-toggle tone → resting/hover glyph colour + background chip. Kept next +// to the presentation (searchMode.js stays framework-free); the tone strings +// come from semanticToggleState(). The chip is what sets ON (faint purple) and +// the amber fallback apart from the greyed OFF state beyond glyph colour alone, +// and the hover variants give the icon a clickable affordance. +const SEMANTIC_TONE = { + off: { color: 'var(--text-secondary)', hoverColor: 'var(--text-primary)', chip: 'transparent', hoverChip: 'var(--bg-hover)' }, + on: { color: 'var(--accent)', hoverColor: 'var(--accent)', chip: 'var(--accent-glow)', hoverChip: 'rgba(124, 106, 247, 0.28)' }, + fallback: { color: 'var(--amber)', hoverColor: 'var(--amber)', chip: 'rgba(251, 191, 36, 0.15)', hoverChip: 'rgba(251, 191, 36, 0.28)' }, +}; // Folder icon for move picker function FolderIcon({ specialUse, size = 13 }) { @@ -59,6 +78,21 @@ const SWIPE_ACTIONS = { disabled: { color: 'transparent' }, }; +const SWIPE_COMMANDS = Object.freeze({ + archive: 'mail.archive', + delete: 'mail.trash', + star: 'mail.toggleStar', + markRead: 'mail.toggleRead', + reply: 'mail.reply', + replyAll: 'mail.replyAll', +}); + +const THREAD_EXPANDING_COMMANDS = new Set([ + 'mail.archive', 'mail.snooze', 'mail.move', 'mail.read', 'mail.unread', 'mail.toggleRead', + 'mail.star', 'mail.unstar', 'mail.toggleStar', 'mail.trash', 'mail.spam', 'mail.notSpam', + 'gtd.delegate', +]); + function getSwipeActionView(action, message, t, unreadCount = null) { const unread = unreadCount != null ? unreadCount > 0 : !message.is_read; if (action === 'archive') return { label: t('message.archive'), color: SWIPE_ACTIONS.archive.color, icon: 'archive' }; @@ -100,13 +134,14 @@ function SwipeBackground({ side, actionView, innerRef }) { export default function MessageList() { const { t } = useTranslation(); + const { controller: commandController } = useCommandRuntimeContext(); const uiScale = useUiScale(); const { selectedAccountId, selectedFolder, messages, setMessages, appendMessages, messagesTotal, setMessagesTotal, setMessagesOffset, hasMoreMessages, setHasMoreMessages, loadingMessages, setLoadingMessages, selectedMessageId, lastViewedMessageId, - setSelectedMessage, updateMessage, removeMessage, removeMessages, + setSelectedMessage, updateMessage, removeMessage, decrementUnread, incrementUnread, addNotification, notifications, removeNotification, searchQuery, setSearchQuery, setIsSearching, searchResults, setSearchResults, openCompose, accountsReady, accounts, @@ -117,14 +152,21 @@ export default function MessageList() { hoverQuickActions, showMobileAvatars, swipeActions, folders, favoriteFolders, addFavoriteFolder, removeFavoriteFolder, setSelectedAccount, - categorizationEnabled, categoryCounts, setCategoryCounts, adjustCategoryCount, + categorizationEnabled, categoryCounts, setCategoryCounts, markReadBehavior, markReadDelay, searchAllFolders, + searchMode, setSearchMode, activeGtdTab, setActiveGtdTab, gtdSections, } = useStore(); // RFC message_id of the open message, so a row highlights when it is a different DB copy // of the selected message (multi-folder model) — e.g. the inbox copy of a GTD sidebar click. const selectedMid = useStore(selectSelectedMessageMid); + const selectedIds = useStore(state => state.selectedMessageIds); + const setSelectedIds = useStore(state => state.setSelectedMessageIds); + const requestCompose = useCallback(request => handleComposeRequest(request, { + addNotification, + t, + }), [addNotification, t]); const isMobile = useMobile(); const isUnified = selectedAccountId === null; @@ -138,7 +180,7 @@ export default function MessageList() { // server-side. undefined = search all folders. const searchFolder = (!isUnified && !searchAllFolders) ? selectedFolder : undefined; const searchPageSize = Math.max(1, Math.min(Number(pageSize) || 50, 200)); - const undoableNotifications = notifications.filter(n => n.onUndo); + const undoableNotifications = notifications.filter(n => n.onUndo && !n.countdownUntil); const currentLayout = LAYOUTS[layout] || LAYOUTS.comfortable; const isColumn = currentLayout.direction === 'column'; @@ -181,14 +223,79 @@ export default function MessageList() { const [searchHasMore, setSearchHasMore] = useState(false); const [searchLoadingMore, setSearchLoadingMore] = useState(false); const searchFetchedOffsetRef = useRef(0); + const [semanticAvailable, setSemanticAvailable] = useState(false); + const [searchFellBack, setSearchFellBack] = useState(false); + // The active mode is searchMode only when the vector-availability probe says + // semantic search is usable; otherwise force lexical regardless of the + // persisted preference (e.g. this Postgres has no pgvector schema). + const effectiveMode = semanticAvailable ? searchMode : LEXICAL_MODE; + + // Right-side control cluster that lives INSIDE the search input, shared by the + // desktop and mobile search boxes. Holds the semantic-search toggle (sparkle + // icon, gated on vector availability) and the clear-query button, in that + // order. The toggle is greyed when off, purple when on, and amber when the + // backend silently fell back to lexical — the fallback hint rides the icon's + // colour + tooltip so no extra row appears below the input. + const renderSearchControls = () => { + const on = searchMode !== LEXICAL_MODE; + const toggle = semanticToggleState({ on, fellBack: searchFellBack, hasQuery: Boolean(searchQuery.trim()) }); + const toggleLabel = t(toggle.titleKey); + const tone = SEMANTIC_TONE[toggle.tone]; + // Shared icon-button shape so the sparkle and clear × read as one cluster + // and their hover chips are the same size. + const iconBtn = { + display: 'flex', alignItems: 'center', justifyContent: 'center', + padding: 4, borderRadius: 6, border: 'none', cursor: 'pointer', + WebkitTapHighlightColor: 'transparent', + transition: 'background 0.15s, color 0.15s', + }; + return ( +
+ {semanticAvailable && ( + + )} + {searchQuery && ( + + )} +
+ ); + }; const listRef = useRef(null); const searchInputRef = useRef(null); // for focusSearch shortcut - const pendingDeleteTimers = useRef(new Map()); // id/thread key -> pending delete metadata const recentMessageOpenUntilRef = useRef(0); const deferredRefreshTimerRef = useRef(null); // Bulk selection state - const [selectedIds, setSelectedIds] = useState(new Set()); const [selectionModeActive, setSelectionModeActive] = useState(false); const [showFolderPicker, setShowFolderPicker] = useState(false); const [pickerFolders, setPickerFolders] = useState([]); @@ -204,7 +311,7 @@ export default function MessageList() { const layoutPickerRef = useRef(null); useEffect(() => { currentPageRef.current = currentPage; }, [currentPage]); - useEffect(() => { setActiveCategory('primary'); setActiveGtdTab(null); }, [selectedAccountId, selectedFolder, setActiveGtdTab]); + useEffect(() => { setActiveCategory('primary'); }, [selectedAccountId, selectedFolder]); useEffect(() => { const markOpening = () => { recentMessageOpenUntilRef.current = Date.now() + 1500; @@ -212,6 +319,16 @@ export default function MessageList() { window.addEventListener('mailflow:message-opening', markOpening); return () => window.removeEventListener('mailflow:message-opening', markOpening); }, []); + + // Probe vector availability once on mount — gates whether the semantic + // toggle renders at all (Task 10). + useEffect(() => { + let alive = true; + api.ai.status() + .then(s => { if (alive) setSemanticAvailable(semanticSearchAvailable(s)); }) + .catch(() => { if (alive) setSemanticAvailable(false); }); + return () => { alive = false; }; + }, []); const searchTimer = useRef(null); // Category tab scroll arrows @@ -267,9 +384,7 @@ export default function MessageList() { // Ref that always holds the latest values needed by shortcut handlers. // Updated synchronously on every render so handlers are never stale. const scRef = useRef({}); - scRef.current = { messages, selectedIds, setSelectedIds, updateMessage, decrementUnread, addNotification }; - const tRef = useRef(t); - useEffect(() => { tRef.current = t; }, [t]); + scRef.current = { selectedIds, setSelectedIds }; // Clear selection whenever the message list resets (nav, folder change, etc.) useEffect(() => { @@ -277,7 +392,7 @@ export default function MessageList() { setSelectionModeActive(false); setShowFolderPicker(false); lastSelectIdxRef.current = -1; - }, [messagesRefreshToken]); + }, [messagesRefreshToken, setSelectedIds]); // Escape clears selection; click-outside closes folder picker useEffect(() => { @@ -303,7 +418,7 @@ export default function MessageList() { document.removeEventListener('keydown', onKey); document.removeEventListener('pointerdown', onPointer); }; - }, []); + }, [setSelectedIds]); useEffect(() => { if (!showFolderPicker) setPickerSearch(''); @@ -460,23 +575,30 @@ export default function MessageList() { // Search useEffect(() => { clearTimeout(searchTimer.current); + // Bump the request generation on EVERY context change (query, mode, folder, + // account, page size — the dep set below — and clearing the query). Async + // appends (load-more, post-delete prefetch) capture this before their await + // and discard themselves if it moves on, so a stale page can't append onto a + // fresh, different-context list. + const seq = ++searchSeq.current; if (!searchQuery.trim()) { setIsSearching(false); setSearchResults([]); setSearchHasMore(false); + setSearchFellBack(false); searchFetchedOffsetRef.current = 0; return; } setIsSearching(true); setSearchHasMore(false); - const seq = ++searchSeq.current; searchTimer.current = setTimeout(async () => { try { - const data = await api.search(searchQuery, selectedAccountId || undefined, { offset: 0, limit: searchPageSize, folder: searchFolder }); + const data = await api.search(searchQuery, selectedAccountId || undefined, { offset: 0, limit: searchPageSize, folder: searchFolder, mode: effectiveMode }); if (searchSeq.current !== seq) return; searchFetchedOffsetRef.current = data.messages.length; setSearchResults(applyReadGuard(data.messages)); setSearchHasMore(data.messages.length === searchPageSize); + setSearchFellBack(Boolean(data.fellBack)); } catch (err) { if (searchSeq.current === seq) console.error('Search failed:', err); } finally { @@ -484,7 +606,7 @@ export default function MessageList() { } }, 300); return () => clearTimeout(searchTimer.current); - }, [searchQuery, selectedAccountId, searchFolder, searchPageSize, searchReloadToken, unifiedInboxAccountKey, applyReadGuard, setIsSearching, setSearchResults]); + }, [searchQuery, selectedAccountId, searchFolder, searchPageSize, searchReloadToken, unifiedInboxAccountKey, effectiveMode, applyReadGuard, setIsSearching, setSearchResults]); // Re-run an active search (and refresh the folder view) after inbox rules run, since // rules can move messages out of the searched folder and a search snapshot would @@ -505,12 +627,14 @@ export default function MessageList() { const loadMoreSearch = useCallback(async () => { if (searchLoadingMore) return; const qSnapshot = searchQuery; // capture before async gap + const seq = searchSeq.current; // request generation at dispatch time setSearchLoadingMore(true); try { const offset = searchFetchedOffsetRef.current; - const data = await api.search(qSnapshot, selectedAccountId || undefined, { offset, limit: searchPageSize, folder: searchFolder }); - // Discard results if the query changed while we were fetching - if (useStore.getState().searchQuery !== qSnapshot) return; + const data = await api.search(qSnapshot, selectedAccountId || undefined, { offset, limit: searchPageSize, folder: searchFolder, mode: effectiveMode }); + // Discard if ANY search context (query, mode, folder, account) changed + // while we were fetching — otherwise a stale page appends onto a fresh list. + if (!isCurrentSearchGeneration(seq, searchSeq.current)) return; searchFetchedOffsetRef.current = offset + data.messages.length; const current = useStore.getState().searchResults; useStore.setState({ searchResults: [...current, ...applyReadGuard(data.messages)] }); @@ -520,30 +644,7 @@ export default function MessageList() { } finally { setSearchLoadingMore(false); } - }, [searchQuery, selectedAccountId, searchFolder, searchPageSize, searchLoadingMore, applyReadGuard]); - - const prefetchSearchAfterRemoval = useCallback(async (offset) => { - const qSnapshot = useStore.getState().searchQuery; - if (!qSnapshot.trim()) return; - try { - const data = await api.search(qSnapshot, selectedAccountId || undefined, { offset, limit: searchPageSize, folder: searchFolder }); - if (useStore.getState().searchQuery !== qSnapshot) return; - searchFetchedOffsetRef.current = Math.max(searchFetchedOffsetRef.current, offset + data.messages.length); - const additions = applyReadGuard(data.messages); - if (!additions.length) { - setSearchHasMore(data.messages.length === searchPageSize); - return; - } - useStore.setState(state => { - const existing = new Set(state.searchResults.map(m => m.id)); - const missing = additions.filter(m => m && !existing.has(m.id)); - return missing.length ? { searchResults: [...state.searchResults, ...missing] } : {}; - }); - setSearchHasMore(data.messages.length === searchPageSize); - } catch (err) { - console.error('Search prefetch after delete failed:', err); - } - }, [selectedAccountId, searchFolder, searchPageSize, applyReadGuard]); + }, [searchQuery, selectedAccountId, searchFolder, searchPageSize, searchLoadingMore, effectiveMode, applyReadGuard]); // Infinite scroll + scroll-to-top visibility const handleScroll = useCallback(() => { @@ -696,168 +797,43 @@ export default function MessageList() { } const effectiveFolder = selectedAccountId ? selectedFolder : 'INBOX'; const data = await api.getThread(tid, effectiveFolder, isUnified); - return data.messages?.length ? data.messages : [message]; - }, [isThreadListRow, threadMessages, selectedAccountId, selectedFolder, isUnified]); - - const setCachedThreadRead = useCallback((message, read) => { - const tid = message.thread_id || message.id; - if (threadMessages[tid]) { - setThreadMessages(tid, threadMessages[tid].map(msg => ({ ...msg, is_read: read }))); - } - }, [threadMessages, setThreadMessages]); - - const setCachedThreadStarred = useCallback((message, starred) => { - const tid = message.thread_id || message.id; - if (threadMessages[tid]) { - setThreadMessages(tid, threadMessages[tid].map(msg => ({ ...msg, is_starred: starred }))); - } - }, [threadMessages, setThreadMessages]); - - const setMessagesReadState = useCallback(async (message, read) => { - const isThreadRow = isThreadListRow(message); - const unreadCount = Number.parseInt(message.unread_count, 10); - // Use the row's own unread_count as the immediate estimate. - // For thread rows this is the aggregate already present on the row; - // for single messages it is always 1 (or 0 if already in the target state). - const estimatedDelta = isThreadRow && Number.isFinite(unreadCount) ? unreadCount : 1; - - // Immediate optimistic update — do not wait for thread resolution. - // For unexpanded thread rows this avoids a visible delay caused by the - // api.getThread call inside resolveMessagesForThreadAction. - if (isThreadRow) { - updateMessage(message.id, { is_read: read, unread_count: read ? 0 : estimatedDelta }); - // setCachedThreadRead intentionally deferred until after resolution so - // that actionMessages still reflects the pre-update sub-message states, - // letting us compute the exact delta for any needed correction. - } else { - updateMessage(message.id, { is_read: read, unread_count: read ? 0 : 1 }); - } - if (read) { - if (estimatedDelta > 0) { - decrementUnread(message.account_id, estimatedDelta); - adjustCategoryCount(message.category, -estimatedDelta); - } - } else { - if (estimatedDelta > 0) { - incrementUnread(message.account_id, estimatedDelta); - adjustCategoryCount(message.category, estimatedDelta); - } - } - - // Resolve the individual sub-messages needed for the bulk API call. - // For unexpanded thread rows this fires api.getThread, but the UI has - // already updated above so the user sees no delay. - let actionMessages; - try { - actionMessages = await resolveMessagesForThreadAction(message); - } catch (err) { - console.error('Failed to load thread for read state change:', err.message); - // Revert the optimistic update - if (isThreadRow) { - updateMessage(message.id, { is_read: !read, unread_count: !read ? 0 : estimatedDelta }); - } else { - updateMessage(message.id, { is_read: !read, unread_count: !read ? 0 : 1 }); - } - if (read && estimatedDelta > 0) { incrementUnread(message.account_id, estimatedDelta); adjustCategoryCount(message.category, estimatedDelta); } - else if (!read && estimatedDelta > 0) { decrementUnread(message.account_id, estimatedDelta); adjustCategoryCount(message.category, -estimatedDelta); } - return; - } - - // Compute exact delta from sub-message states (before mutating the cache). - const actualDelta = read - ? actionMessages.filter(msg => !msg.is_read).length - : actionMessages.filter(msg => msg.is_read).length; - - // Now update the thread cache and correct the parent row if our estimate was off. - if (isThreadRow) { - setCachedThreadRead(message, read); - if (actualDelta !== estimatedDelta) { - updateMessage(message.id, { is_read: read, unread_count: read ? 0 : actionMessages.length }); - } - } - - // Correct the sidebar badge if the estimate differed from the actual count. - if (actualDelta !== estimatedDelta) { - const diff = actualDelta - estimatedDelta; - if (read) { - if (diff > 0) decrementUnread(message.account_id, diff); - else incrementUnread(message.account_id, -diff); - } else { - if (diff > 0) incrementUnread(message.account_id, diff); - else decrementUnread(message.account_id, -diff); - } - adjustCategoryCount(message.category, read ? -diff : diff); - } - - if (read) { - actionMessages.forEach(msg => setPending(msg.id, msg.account_id)); - } else { - actionMessages.forEach(msg => { - pendingMarkReadMap.delete(msg.id); - completedMarkReadMap.delete(msg.id); - }); - } - - try { - await api.bulkRead(actionMessages.map(msg => msg.id), read); - if (read) { - actionMessages.forEach(msg => { - pendingMarkReadMap.delete(msg.id); - completedMarkReadMap.set(msg.id, msg.account_id); - setTimeout(() => completedMarkReadMap.delete(msg.id), 10000); + const resolved = data.messages?.length ? data.messages : [message]; + if (resolved.length > 1) setThreadMessages(tid, resolved); + return resolved; + }, [isThreadListRow, threadMessages, selectedAccountId, selectedFolder, isUnified, setThreadMessages]); + + const executeForMessages = useCallback(async (commandId, source, targetMessages, input) => { + let actionableMessages = targetMessages; + if (THREAD_EXPANDING_COMMANDS.has(commandId)) { + try { + actionableMessages = (await Promise.all( + targetMessages.map(message => resolveMessagesForThreadAction(message)), + )).flat(); + } catch (error) { + addNotification({ + type: 'error', + title: t('commandPalette.outcome.failedTitle'), + body: error instanceof Error ? error.message : String(error), }); - } - } catch (err) { - console.error('markRead failed:', err); - if (isThreadRow) { - updateMessage(message.id, { is_read: !read, unread_count: read ? actualDelta : 0 }); - setCachedThreadRead(message, !read); - } else { - updateMessage(message.id, { is_read: !read, unread_count: read ? 1 : 0 }); - } - if (read) { - if (actualDelta > 0) { incrementUnread(message.account_id, actualDelta); adjustCategoryCount(message.category, actualDelta); } - actionMessages.forEach(msg => pendingMarkReadMap.delete(msg.id)); - } else if (actualDelta > 0) { - decrementUnread(message.account_id, actualDelta); - adjustCategoryCount(message.category, -actualDelta); + return { status: 'failed', error }; } } - }, [ - resolveMessagesForThreadAction, isThreadListRow, updateMessage, setCachedThreadRead, - decrementUnread, incrementUnread, adjustCategoryCount, - ]); + const frozenTargetIds = [...new Set(actionableMessages.map(stableConversationId).filter(Boolean))]; + return commandController.execute(commandId, { + source, + input, + frozenTargetIds, + }); + }, [addNotification, commandController, resolveMessagesForThreadAction, t]); const handleMarkRead = (e, message) => { e.stopPropagation(); - setMessagesReadState(message, !message.is_read); + executeForMessages('mail.toggleRead', 'hover', [message]); }; - const setMessagesStarredState = useCallback(async (message, starred) => { - let actionMessages; - try { - actionMessages = await resolveMessagesForThreadAction(message); - } catch (err) { - console.error('Failed to load thread for star state change:', err.message); - return; - } - - const isThreadRow = isThreadListRow(message); - updateMessage(message.id, { is_starred: starred }); - if (isThreadRow) setCachedThreadStarred(message, starred); - - try { - await Promise.all(actionMessages.map(msg => api.markStarred(msg.id, starred))); - } catch (err) { - console.error('markStarred failed:', err.message); - updateMessage(message.id, { is_starred: !starred }); - if (isThreadRow) setCachedThreadStarred(message, !starred); - } - }, [resolveMessagesForThreadAction, isThreadListRow, updateMessage, setCachedThreadStarred]); - const handleStar = (e, message) => { e.stopPropagation(); - setMessagesStarredState(message, !message.is_starred); + executeForMessages('mail.toggleStar', 'hover', [message]); }; // GTD "done" from the inbox hover cluster (all-states mode): the backend marks the thread @@ -889,420 +865,15 @@ export default function MessageList() { } }, [removeMessage, decrementUnread, incrementUnread, addNotification, t]); - // Undo-able delete: optimistically remove, delay the API call by 4.5s so user can undo - const scheduleDelete = useCallback(async (message) => { - const tid = message.thread_id || message.id; - const isThreadRow = isThreadListRow(message); - const key = isThreadRow ? `thread:${tid}` : message.id; - if (pendingDeleteTimers.current.has(key)) return; - - let deleteMessages = [message]; - try { - deleteMessages = await resolveMessagesForThreadAction(message); - } catch (err) { - console.error('Failed to load thread for delete:', err.message); - addNotification({ type: 'error', title: t('messageList.deleted.failTitle'), body: t('messageList.deleted.failBody') }); - return; - } - - const ids = [...new Set(deleteMessages.map(msg => msg.id).filter(Boolean))]; - const visibleMessage = message; - ids.forEach((id) => setPendingDelete(id)); - - // Advance selection to the next visible message before removing this one - const { selectedMessageId, setSelectedMessage } = useStore.getState(); - if (selectedMessageId === visibleMessage.id) { - const displayMsgs = scRef.current.displayMessages || []; - const idx = displayMsgs.findIndex(m => m.id === visibleMessage.id); - const next = displayMsgs[idx + 1] || displayMsgs[idx - 1] || null; - setSelectedMessage(next?.id ?? null); - } - - removeMessage(visibleMessage.id); - if (expandedThreadId === tid) setExpandedThreadId(null); - - const unreadCount = Number.parseInt(message.unread_count, 10); - const unreadDelta = Number.isFinite(unreadCount) - ? unreadCount - : deleteMessages.filter(msg => !msg.is_read).length; - if (unreadDelta > 0) decrementUnread(message.account_id, unreadDelta); - - const timer = setTimeout(async () => { - pendingDeleteTimers.current.delete(key); - try { - if (ids.length > 1) { - const result = await api.bulkDelete(ids); - const deletedSet = new Set(result.deleted ?? []); - ids.forEach(id => (deletedSet.has(id) ? setCompletedDelete(id) : clearDeleteGuard(id))); - const failedIds = ids.filter(id => !deletedSet.has(id)); - if (failedIds.length > 0) { - const idToMsg = new Map(deleteMessages.map(m => [m.id, m])); - const failedUnreadDelta = failedIds.filter(id => idToMsg.has(id) && !idToMsg.get(id).is_read).length; - useStore.getState().restoreMessages([visibleMessage]); - if (failedUnreadDelta > 0) incrementUnread(message.account_id, failedUnreadDelta); - addNotification({ - type: 'error', - title: t('messageList.bulkDeleted.failTitle'), - body: t('messageList.bulkDeleted.failBody', { count: failedIds.length }), - }); - } - } else { - await api.deleteMessage(ids[0] || visibleMessage.id); - ids.forEach((id) => setCompletedDelete(id)); - } - } catch { - ids.forEach((id) => clearDeleteGuard(id)); - useStore.getState().restoreMessages([visibleMessage]); - if (unreadDelta > 0) incrementUnread(message.account_id, unreadDelta); - addNotification({ - type: 'error', - title: ids.length > 1 ? t('messageList.bulkDeleted.failTitle') : t('messageList.deleted.failTitle'), - body: ids.length > 1 ? t('messageList.bulkDeleted.failBody', { count: ids.length }) : t('messageList.deleted.failBody'), - }); - } - }, 4500); - pendingDeleteTimers.current.set(key, { timer, message: visibleMessage, ids }); - addNotification({ - title: ids.length > 1 ? t('messageList.bulkDeleted.title', { count: ids.length }) : t('messageList.deleted.title'), - body: ids.length > 1 ? t('messageList.bulkDeleted.body') : t('messageList.deleted.body'), - onUndo: () => { - const pending = pendingDeleteTimers.current.get(key); - if (!pending) return; - clearTimeout(pending.timer); - pendingDeleteTimers.current.delete(key); - ids.forEach((id) => clearPendingDelete(id)); - useStore.getState().restoreMessages([visibleMessage]); - if (unreadDelta > 0) incrementUnread(message.account_id, unreadDelta); - }, - }); - }, [ - isThreadListRow, expandedThreadId, resolveMessagesForThreadAction, - removeMessage, setExpandedThreadId, decrementUnread, incrementUnread, - addNotification, t, - ]); - - // Antispam helpers (v0.1). - // - // Strategy for both mark-as-spam and mark-as-ham: - // 1. Optimistically remove the message(s) from the visible list and decrement - // unread counts — same pattern as scheduleDelete above. - // 2. Show a toast with Undo (4.5s window). Undo restores the message locally - // and cancels the API call via a per-message timer. - // 3. After the timer fires, call api.markSpam / api.markHam per id in parallel - // (Promise.allSettled). On any failure, restore the messages that failed - // and show an error toast. - // - // Bulk is handled by collecting `messages` from selectedIds if multiple are - // selected; the caller (handleContextAction) decides which set to pass. - - const performSpamLabel = useCallback(async (messages, label) => { - if (!messages.length) return; - const ids = messages.map(m => m.id); - const isBulk = ids.length > 1; - - // Optimistic local update: remove from view + drop unread badge. - const unreadCount = messages.reduce((sum, m) => sum + (m.is_read ? 0 : 1), 0); - const accountId = messages[0].account_id; - messages.forEach(m => removeMessage(m.id)); - if (unreadCount > 0) decrementUnread(accountId, unreadCount); - - // Folder-aware unread badge updates for the sidebar. - // - // For spam (move INTO the junk folder) we decrement whichever folder the - // messages came from (typically INBOX, but possibly some other folder the - // user is in). For ham (move OUT of junk into inbox) we decrement the - // source folder (junk) and increment the destination folder (inbox). - // - // We aggregate by folder path because a bulk action may touch messages - // from different folders in theory (the UI currently only selects from - // one folder at a time, but the data model allows otherwise). - const { adjustFolderUnread } = useStore.getState(); - const account = accounts.find(a => a.id === accountId); - const spamDest = account?.folder_mappings?.spam; - const inboxDest = account?.folder_mappings?.inbox || 'INBOX'; - const unreadBySource = new Map(); - const unreadByHamSource = new Map(); // for ham: track source folder - messages.forEach(m => { - if (m.is_read) return; - const src = m.folder; - if (!src) return; - if (label === 'spam') { - unreadBySource.set(src, (unreadBySource.get(src) || 0) + 1); - } else if (label === 'ham') { - unreadByHamSource.set(src, (unreadByHamSource.get(src) || 0) + 1); - } - }); - if (label === 'spam') { - // origin folder loses its unread messages; junk gains them - for (const [src, n] of unreadBySource) adjustFolderUnread(accountId, src, -n); - if (spamDest && unreadCount > 0) adjustFolderUnread(accountId, spamDest, +unreadCount); - } else if (label === 'ham') { - // junk loses them; inbox gains them - for (const [src, n] of unreadByHamSource) adjustFolderUnread(accountId, src, -n); - if (inboxDest && unreadCount > 0) adjustFolderUnread(accountId, inboxDest, +unreadCount); - } - - // Per-id timer map so Undo can cancel any pending API call. - const timers = new Map(); - let settled = false; - const undo = () => { - settled = true; - timers.forEach(timer => clearTimeout(timer)); - timers.clear(); - // Restore the messages in their original position (re-sort by date). - useStore.getState().restoreMessages(messages); - if (unreadCount > 0) incrementUnread(accountId, unreadCount); - // Reverse the folder badge adjustments so undo behaves like the move - // never happened. - if (label === 'spam') { - for (const [src, n] of unreadBySource) adjustFolderUnread(accountId, src, +n); - if (spamDest && unreadCount > 0) adjustFolderUnread(accountId, spamDest, -unreadCount); - } else if (label === 'ham') { - for (const [src, n] of unreadByHamSource) adjustFolderUnread(accountId, src, +n); - if (inboxDest && unreadCount > 0) adjustFolderUnread(accountId, inboxDest, -unreadCount); - } - }; - - const performCall = (id) => { - const fn = label === 'spam' ? api.markSpam : api.markHam; - return fn(id).catch(err => ({ __failed: true, id, message: err.message })); - }; - - timers.set('__call__', setTimeout(async () => { - if (settled) return; - timers.delete('__call__'); - const results = await Promise.allSettled(ids.map(performCall)); - const failed = []; - results.forEach((r, i) => { - if (r.status === 'rejected' || r.value?.__failed) failed.push(ids[i]); - }); - if (failed.length) { - const failedMsgs = messages.filter(m => failed.includes(m.id)); - useStore.getState().restoreMessages(failedMsgs); - const failedUnread = failedMsgs.reduce((sum, m) => sum + (m.is_read ? 0 : 1), 0); - if (failedUnread > 0) incrementUnread(accountId, failedUnread); - const titleKey = label === 'spam' ? 'spam.failTitle' : 'spam.failHamTitle'; - const bodyKey = label === 'spam' ? 'spam.failBody' : 'spam.failHamBody'; - addNotification({ - type: 'error', - title: t(titleKey), - body: isBulk ? t('spam.failBodyBulk', { count: failed.length }) : t(bodyKey), - }); - } - // Safety net: after the IMAP move actually completes (or partially - // fails), reconcile sidebar counts and folder badges with the server. - // Even if our optimistic math was right, edge cases like the user - // moving messages between two folders that share a parent, or a - // concurrent IMAP IDLE update, can desync the local counters. - api.getUnreadCounts().then(c => useStore.getState().setUnreadCounts(c)).catch(() => {}); - api.getFolders(accountId).then(f => useStore.getState().setFolders(accountId, f)).catch(() => {}); - }, 4500)); - - addNotification({ - title: label === 'spam' - ? (isBulk ? t('spam.movedToSpamBulk', { count: ids.length }) : t('spam.movedToSpam')) - : (isBulk ? t('spam.movedToInboxBulk', { count: ids.length }) : t('spam.movedToInbox')), - body: messages[0].subject || t('common.noSubject'), - onUndo: undo, - }); - }, [removeMessage, decrementUnread, incrementUnread, addNotification, t, accounts]); - - // On page unload (refresh/close), fire pending deletes with keepalive:true so the - // browser completes the request even after the page tears down. Clears the map so - // the unmount cleanup below does not double-fire on normal navigation. - useEffect(() => { - const handleBeforeUnload = () => { - pendingDeleteTimers.current.forEach(({ timer, message, ids }) => { - clearTimeout(timer); - const deleteIds = ids?.length ? ids : [message.id]; - try { - if (deleteIds.length > 1) { - fetch('/api/mail/messages/bulk-delete', { - method: 'POST', - credentials: 'include', - headers: { 'Content-Type': 'application/json', 'X-Requested-With': 'MailFlow' }, - body: JSON.stringify({ ids: deleteIds }), - keepalive: true, - }); - } else { - fetch(`/api/mail/messages/${deleteIds[0]}`, { - method: 'DELETE', - credentials: 'include', - headers: { 'X-Requested-With': 'MailFlow' }, - keepalive: true, - }); - } - } catch { /* keepalive not supported — best effort */ } - }); - pendingDeleteTimers.current.clear(); - }; - window.addEventListener('beforeunload', handleBeforeUnload); - return () => window.removeEventListener('beforeunload', handleBeforeUnload); - }, []); - - // On normal unmount (navigating away), immediately fire any pending deletes. - // Navigating away during the 4.5s undo window should still delete the message — - // cancelling the timer would silently leave it on the server. - // (Page refresh is handled by the beforeunload listener above which clears the map first.) - useEffect(() => () => { - pendingDeleteTimers.current.forEach(({ timer, message, ids }) => { - clearTimeout(timer); - const deleteIds = ids?.length ? ids : [message.id]; - const deletePromise = - deleteIds.length > 1 - ? api.bulkDelete(deleteIds) - : api.deleteMessage(deleteIds[0]); - deletePromise - .then(result => { - const actuallyDeleted = new Set(result?.deleted ?? deleteIds); - deleteIds.forEach(id => (actuallyDeleted.has(id) ? setCompletedDelete(id) : clearDeleteGuard(id))); - }) - .catch(() => { deleteIds.forEach((id) => clearDeleteGuard(id)); }); - }); - }, []); - const handleDelete = (e, message) => { e.stopPropagation(); - scheduleDelete(message); + executeForMessages('mail.trash', 'hover', [message]); }; - // Mobile swipe action handlers (no event object needed) - const handleSwipeDelete = useCallback((message) => { - scheduleDelete(message); - }, [scheduleDelete]); - - const handleSwipeToggleRead = useCallback(async (message) => { - const unreadCount = Number.parseInt(message.unread_count, 10); - const hasThreadUnreadCount = Number.isFinite(unreadCount); - const isUnread = hasThreadUnreadCount ? unreadCount > 0 : !message.is_read; - await setMessagesReadState(message, isUnread); - }, [setMessagesReadState]); - - const handleSwipeArchive = useCallback(async (message) => { - refreshRequestRef.current.invalidate(); - advanceSelectionAfterRemoval(message.id); - removeMessage(message.id); - if (!message.is_read) decrementUnread(message.account_id); - let undone = false; - const timer = setTimeout(async () => { - if (undone) return; - try { - await api.bulkArchive([message.id]); - } catch (err) { - console.error('swipe archive failed:', err.message); - } - }, 4500); - addNotification({ - title: t('messageList.bulkArchived.title', { count: 1 }), - body: message.subject || '', - onUndo: () => { - undone = true; - clearTimeout(timer); - useStore.getState().restoreMessages([message]); - if (!message.is_read) incrementUnread(message.account_id); - }, - }); - }, [removeMessage, decrementUnread, incrementUnread, addNotification, t]); - - const handleSwipeStar = useCallback((message) => { - setMessagesStarredState(message, !message.is_starred); - }, [setMessagesStarredState]); - - const handleSwipeReply = useCallback((message, replyAll = false) => { - const replyToArr = Array.isArray(message.reply_to) - ? message.reply_to - : (() => { try { return JSON.parse(message.reply_to || '[]'); } catch { return []; } })(); - const replyTarget = (replyToArr.length && replyToArr[0].email) - ? replyToArr[0] - : { name: message.from_name || '', email: message.from_email || '' }; - const sender = replyTarget.email ? [replyTarget] : []; - - const myAccount = accounts.find(a => a.id === message.account_id); - const myEmail = myAccount?.email_address || ''; - const myAddresses = new Set([ - myEmail.toLowerCase(), - ...(myAccount?.aliases || []).map(al => al.email.toLowerCase()), - ]); - - const replyAliasId = (() => { - const aliases = myAccount?.aliases || []; - if (!aliases.length) return null; - try { - const toArr = Array.isArray(message.to_addresses) - ? message.to_addresses - : JSON.parse(message.to_addresses || '[]'); - const ccArr = Array.isArray(message.cc_addresses) - ? message.cc_addresses - : JSON.parse(message.cc_addresses || '[]'); - const allEmails = [...toArr, ...ccArr].map(t => t.email?.toLowerCase()).filter(Boolean); - const fromEmail = (message.from_email || '').toLowerCase(); - const match = aliases.find(al => { - const aliasEmail = al.email.toLowerCase(); - return allEmails.includes(aliasEmail) || fromEmail === aliasEmail; - }); - return match ? match.id : null; - } catch { return null; } - })(); - - const allRecipients = (() => { - try { - const toArr = Array.isArray(message.to_addresses) - ? message.to_addresses - : JSON.parse(message.to_addresses || '[]'); - const ccArr = Array.isArray(message.cc_addresses) - ? message.cc_addresses - : JSON.parse(message.cc_addresses || '[]'); - return [...toArr, ...ccArr].filter( - t => t.email && !myAddresses.has(t.email.toLowerCase()) && t.email !== replyTarget.email - ); - } catch { return []; } - })(); - - const referencesChain = [message.in_reply_to, message.message_id] - .filter(Boolean).join(' ').trim() || null; - const rawSubject = (message.subject || '').trim(); - - openCompose({ - to: sender, - cc: replyAll ? allRecipients : [], - subject: rawSubject.startsWith('Re:') ? rawSubject : rawSubject ? `Re: ${rawSubject}` : 'Re:', - body: '', - quotedBody: '', - inReplyTo: message.message_id, - references: referencesChain, - accountId: message.account_id, - aliasId: replyAliasId, - isReply: true, - isReplyAll: replyAll, - originalFrom: sender, - allRecipients, - }); - }, [accounts, openCompose]); - const runSwipeAction = useCallback((action, message) => { - switch (action) { - case 'archive': - handleSwipeArchive(message); - break; - case 'delete': - handleSwipeDelete(message); - break; - case 'star': - handleSwipeStar(message); - break; - case 'markRead': - handleSwipeToggleRead(message); - break; - case 'reply': - handleSwipeReply(message, false); - break; - case 'replyAll': - handleSwipeReply(message, true); - break; - default: - break; - } - }, [handleSwipeArchive, handleSwipeDelete, handleSwipeReply, handleSwipeStar, handleSwipeToggleRead]); + const commandId = SWIPE_COMMANDS[action]; + if (commandId) executeForMessages(commandId, 'swipe', [message]); + }, [executeForMessages]); // ── Bulk selection helpers ─────────────────────────────────── const toggleSelect = useCallback((id) => { @@ -1311,18 +882,18 @@ export default function MessageList() { next.has(id) ? next.delete(id) : next.add(id); return next; }); - }, []); + }, [setSelectedIds]); const selectAll = useCallback((msgs) => { setSelectedIds(new Set(msgs.map(m => m.id))); - }, []); + }, [setSelectedIds]); const clearSelection = useCallback(() => { setSelectedIds(new Set()); setSelectionModeActive(false); setShowFolderPicker(false); lastSelectIdxRef.current = -1; - }, []); + }, [setSelectedIds]); // Derived from store — must be declared before callbacks that use it in dependency arrays const displayMessages = searchQuery.trim() ? searchResults : messages; @@ -1347,14 +918,16 @@ export default function MessageList() { } return results; })(); - // Keep scRef in sync so scheduleDelete can read displayMessages without a stale closure - scRef.current.displayMessages = displayMessages; - // Arrow-key navigation: intercepts ArrowDown/ArrowUp when the list container has focus. const handleListKeyDown = useCallback((e) => { - if (e.key === 'ArrowDown') { e.preventDefault(); shortcutBus.emit('nextMessage'); } - else if (e.key === 'ArrowUp') { e.preventDefault(); shortcutBus.emit('prevMessage'); } - }, []); + if (e.key === 'ArrowDown') { + e.preventDefault(); + commandController.execute('navigation.nextConversation', { source: 'list-keydown' }); + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + commandController.execute('navigation.previousConversation', { source: 'list-keydown' }); + } + }, [commandController]); // Called when the avatar is clicked: enters selection mode and selects that message const handleAvatarClick = useCallback((id) => { @@ -1366,7 +939,7 @@ export default function MessageList() { next.has(id) ? next.delete(id) : next.add(id); return next; }); - }, [displayMessages]); + }, [displayMessages, setSelectedIds]); // Called for normal (non-shift) row checkbox toggles — tracks anchor for range select const handleRowToggleSelect = useCallback((id) => { @@ -1377,7 +950,7 @@ export default function MessageList() { next.has(id) ? next.delete(id) : next.add(id); return next; }); - }, [displayMessages]); + }, [displayMessages, setSelectedIds]); // Called on shift-click: selects all rows between anchor and current index const handleRangeSelect = useCallback((id) => { @@ -1393,133 +966,19 @@ export default function MessageList() { return next; }); lastSelectIdxRef.current = clickedIdx; - }, [displayMessages]); - - const handleBulkDelete = useCallback(async (ids, msgs) => { - const key = `bulk:${ids[0]}`; - // Selected thread rows delete the whole conversation, matching the - // single-row delete path — without this only each thread's visible - // (newest) message was deleted and the rest of the thread survived. - let deleteIds = ids; - try { - const resolved = await Promise.all(msgs.map(m => resolveMessagesForThreadAction(m))); - deleteIds = [...new Set([...ids, ...resolved.flat().map(m => m?.id).filter(Boolean)])]; - } catch (err) { - console.error('Failed to load thread for bulk delete:', err.message); - } - const searchOffsetBeforeRemoval = searchFetchedOffsetRef.current; - const shouldPrefetchSearch = Boolean(useStore.getState().searchQuery.trim() && searchHasMore); - deleteIds.forEach(id => setPendingDelete(id)); - ids.forEach(id => removeMessage(id)); - if (shouldPrefetchSearch) { - prefetchSearchAfterRemoval(searchOffsetBeforeRemoval); - } - msgs.forEach(msg => { - const delta = parseInt(msg.unread_count) || (msg.is_read ? 0 : 1); - if (delta > 0) decrementUnread(msg.account_id, delta); - }); - setSelectedIds(new Set()); + }, [displayMessages, setSelectedIds]); + + const handleBulkDelete = useCallback((_ids, msgs) => { setSelectionModeActive(false); setShowFolderPicker(false); - let undone = false; - const timer = setTimeout(async () => { - pendingDeleteTimers.current.delete(key); - if (undone) return; - const chunks = []; - for (let i = 0; i < deleteIds.length; i += 500) chunks.push(deleteIds.slice(i, i + 500)); - const results = await Promise.allSettled(chunks.map(chunk => api.bulkDelete(chunk))); - results - .filter(r => r.status === 'rejected') - .forEach(r => console.error('Bulk delete failed:', r.reason?.message)); - const deleted = results - .filter(r => r.status === 'fulfilled') - .flatMap(r => r.value.deleted ?? []); - const deletedSet = new Set(deleted); - deleteIds.forEach(id => (deletedSet.has(id) ? setCompletedDelete(id) : clearDeleteGuard(id))); - const failedIds = deleteIds.filter(id => !deletedSet.has(id)); - if (failedIds.length > 0) { - const failedSet = new Set(failedIds); - const failedMsgs = msgs.filter(msg => failedSet.has(msg.id)); - useStore.getState().restoreMessages(failedMsgs); - failedMsgs.forEach(msg => { - const delta = parseInt(msg.unread_count) || (msg.is_read ? 0 : 1); - if (delta > 0) incrementUnread(msg.account_id, delta); - }); - addNotification({ type: 'error', title: t('messageList.bulkDeleted.failTitle'), body: t('messageList.bulkDeleted.failBody', { count: failedIds.length }) }); - } - if (useStore.getState().searchQuery.trim()) { - setSearchReloadToken(token => token + 1); - } - }, 4500); - pendingDeleteTimers.current.set(key, { timer, message: msgs[0], ids: deleteIds }); - addNotification({ - title: t('messageList.bulkDeleted.title', { count: deleteIds.length }), - body: t('messageList.bulkDeleted.body'), - onUndo: () => { - undone = true; - clearTimeout(timer); - pendingDeleteTimers.current.delete(key); - deleteIds.forEach(id => clearPendingDelete(id)); - useStore.getState().restoreMessages(msgs); - msgs.forEach(msg => { - const delta = parseInt(msg.unread_count) || (msg.is_read ? 0 : 1); - if (delta > 0) incrementUnread(msg.account_id, delta); - }); - }, - }); - }, [searchHasMore, removeMessage, prefetchSearchAfterRemoval, resolveMessagesForThreadAction, decrementUnread, incrementUnread, addNotification, t]); - - const handleBulkMove = useCallback(async (ids, msgs, folder) => { - // Selected thread rows move the whole conversation. A folder path is - // account-specific, so scope each thread's expansion to its row's own - // account — the server would just skip (and previously silently drop) - // another account's copies from a folder that doesn't exist there. - let moveIds = ids; - try { - const resolved = await Promise.all(msgs.map(async (m) => { - const thread = await resolveMessagesForThreadAction(m); - return thread.filter(tm => tm?.account_id === m.account_id); - })); - moveIds = [...new Set([...ids, ...resolved.flat().map(m => m?.id).filter(Boolean)])]; - } catch (err) { - console.error('Failed to load thread for bulk move:', err.message); - } - ids.forEach(id => removeMessage(id)); - msgs.forEach(msg => { if (!msg.is_read) decrementUnread(msg.account_id); }); - setSelectedIds(new Set()); + return executeForMessages('mail.trash', 'bulk-toolbar', msgs); + }, [executeForMessages]); + + const handleBulkMove = useCallback((_ids, msgs, folder) => { setSelectionModeActive(false); setShowFolderPicker(false); - let undone = false; - const timer = setTimeout(async () => { - if (undone) return; - try { - const result = await api.bulkMove(moveIds, folder); - const movedSet = new Set(result.moved ?? []); - const failedCount = moveIds.filter(id => !movedSet.has(id)).length; - if (failedCount > 0) { - const failedMsgs = msgs.filter(msg => !movedSet.has(msg.id)); - if (failedMsgs.length > 0) useStore.getState().restoreMessages(failedMsgs); - addNotification({ title: t('messageList.bulkMoved.failTitle'), body: t('messageList.bulkMoved.failBody', { count: failedCount }) }); - } else if (msgs[0]?.account_id) { - useStore.getState().recordRecentFolder({ accountId: msgs[0].account_id, path: folder }); - } - } catch (err) { - console.error('Bulk move failed:', err); - useStore.getState().restoreMessages(msgs); - addNotification({ title: t('messageList.bulkMoved.failTitle'), body: t('messageList.bulkMoved.failBody', { count: moveIds.length }) }); - } - }, 4500); - addNotification({ - title: t('messageList.bulkMoved.title', { count: moveIds.length }), - body: folder, - onUndo: () => { - undone = true; - clearTimeout(timer); - useStore.getState().restoreMessages(msgs); - msgs.forEach(msg => { if (!msg.is_read) incrementUnread(msg.account_id); }); - }, - }); - }, [removeMessage, decrementUnread, incrementUnread, resolveMessagesForThreadAction, addNotification, t]); + return executeForMessages('mail.move', 'bulk-toolbar', msgs, { folder }); + }, [executeForMessages]); const handleRowMove = useCallback((e, msg) => { e.stopPropagation(); @@ -1537,293 +996,31 @@ export default function MessageList() { e.dataTransfer.effectAllowed = 'move'; }, []); - const handleBulkArchive = useCallback((ids, msgs) => { - refreshRequestRef.current.invalidate(); - // Tombstone the ids so a background refresh/websocket refetch during the undo window - // can't resurrect them — applyDeleteGuard filters pending/completed ids out of refetch - // results — and drop every row in one batched state update rather than one per id. - ids.forEach(id => setPendingDelete(id)); - removeMessages(ids); - msgs.forEach(msg => { if (!msg.is_read) decrementUnread(msg.account_id); }); - setSelectedIds(new Set()); + const handleBulkArchive = useCallback((_ids, msgs) => { setSelectionModeActive(false); setShowFolderPicker(false); - let undone = false; - const timer = setTimeout(async () => { - if (undone) return; - try { - const result = await api.bulkArchive(ids); - const archivedSet = new Set(result.archived ?? []); - // Archived: keep guarding briefly (completed grace) so an in-flight refetch can't - // bring them back; not archived: release the guard so those rows can return. - ids.forEach(id => (archivedSet.has(id) ? setCompletedDelete(id) : clearDeleteGuard(id))); - const failedMsgs = msgs.filter(msg => !archivedSet.has(msg.id)); - if (failedMsgs.length > 0) { - useStore.getState().restoreMessages(failedMsgs); - failedMsgs.forEach(msg => { if (!msg.is_read) incrementUnread(msg.account_id); }); - if (result.noArchiveFolder?.length) { - addNotification({ title: t('messageList.bulkArchived.noFolderTitle'), body: t('messageList.bulkArchived.noFolderBody') }); - } else { - addNotification({ title: t('messageList.bulkArchived.failTitle'), body: t('messageList.bulkArchived.failBody', { count: failedMsgs.length }) }); - } - } - } catch (err) { - console.error('Bulk archive failed:', err); - ids.forEach(id => clearDeleteGuard(id)); - useStore.getState().restoreMessages(msgs); - msgs.forEach(msg => { if (!msg.is_read) incrementUnread(msg.account_id); }); - addNotification({ title: t('messageList.bulkArchived.failTitle'), body: t('messageList.bulkArchived.failBody', { count: ids.length }) }); - } - }, 4500); - addNotification({ - title: t('messageList.bulkArchived.title', { count: ids.length }), - body: t('messageList.bulkArchived.body'), - onUndo: () => { - undone = true; - clearTimeout(timer); - ids.forEach(id => clearPendingDelete(id)); - useStore.getState().restoreMessages(msgs); - msgs.forEach(msg => { if (!msg.is_read) incrementUnread(msg.account_id); }); - }, - }); - }, [removeMessages, decrementUnread, incrementUnread, addNotification, t]); - - const handleBulkMarkRead = useCallback(async (ids, msgs) => { - const markAsRead = msgs.some(m => !m.is_read); - // Compute per-account and per-category unread deltas before mutating state - const deltaByAccount = {}; - const deltaByCategory = {}; - msgs.forEach(msg => { - if (!deltaByAccount[msg.account_id]) deltaByAccount[msg.account_id] = 0; - const catKey = msg.category || 'primary'; - if (!deltaByCategory[catKey]) deltaByCategory[catKey] = 0; - if (markAsRead && !msg.is_read) { deltaByAccount[msg.account_id]++; deltaByCategory[catKey]++; } - if (!markAsRead && msg.is_read) { deltaByAccount[msg.account_id]++; deltaByCategory[catKey]++; } - }); - // Optimistic update - msgs.forEach(msg => updateMessage(msg.id, { is_read: markAsRead, unread_count: markAsRead ? 0 : 1 })); - Object.entries(deltaByAccount).forEach(([accountId, delta]) => { - if (delta > 0) markAsRead ? decrementUnread(accountId, delta) : incrementUnread(accountId, delta); - }); - Object.entries(deltaByCategory).forEach(([cat, delta]) => { - if (delta > 0) adjustCategoryCount(cat, markAsRead ? -delta : delta); - }); - setSelectedIds(new Set()); + return executeForMessages('mail.archive', 'bulk-toolbar', msgs); + }, [executeForMessages]); + + const handleBulkMarkRead = useCallback((_ids, msgs) => { setSelectionModeActive(false); - try { - await api.bulkRead(ids, markAsRead); - } catch (err) { - console.error('Bulk mark read failed:', err); - msgs.forEach(msg => updateMessage(msg.id, { is_read: msg.is_read, unread_count: msg.unread_count })); - Object.entries(deltaByAccount).forEach(([accountId, delta]) => { - if (delta > 0) markAsRead ? incrementUnread(accountId, delta) : decrementUnread(accountId, delta); - }); - Object.entries(deltaByCategory).forEach(([cat, delta]) => { - if (delta > 0) adjustCategoryCount(cat, markAsRead ? delta : -delta); - }); - } - }, [updateMessage, decrementUnread, incrementUnread, adjustCategoryCount]); + return executeForMessages('mail.toggleRead', 'bulk-toolbar', msgs); + }, [executeForMessages]); const autoMarkReadTimerRef = useRef(null); useEffect(() => () => clearTimeout(autoMarkReadTimerRef.current), []); - // Keep refs to bulk handlers so the shortcut effect (registered once) is never stale - const bulkDeleteRef = useRef(handleBulkDelete); - const bulkArchiveRef = useRef(handleBulkArchive); - const scheduleDeleteRef = useRef(scheduleDelete); - const handleContextActionRef = useRef(null); // assigned below, once handleContextAction is defined - useEffect(() => { bulkDeleteRef.current = handleBulkDelete; }, [handleBulkDelete]); - useEffect(() => { bulkArchiveRef.current = handleBulkArchive; }, [handleBulkArchive]); - useEffect(() => { scheduleDeleteRef.current = scheduleDelete; }, [scheduleDelete]); - // Subscribe to keyboard shortcut actions that belong to the message list. - // Registered once ([] deps); all live state is read through scRef/bulkDeleteRef/bulkArchiveRef. useEffect(() => { - const getState = () => useStore.getState(); - - const markRead = (msg) => { - if (msg.is_read) return; - const { updateMessage, decrementUnread, incrementUnread, adjustCategoryCount, markReadBehavior, markReadDelay } = getState(); - if (markReadBehavior === 'manual') return; - clearTimeout(autoMarkReadTimerRef.current); - autoMarkReadTimerRef.current = null; - const doMarkRead = () => { - updateMessage(msg.id, { is_read: true }); - decrementUnread(msg.account_id); - adjustCategoryCount(msg.category, -1); - setPending(msg.id, msg.account_id); - api.bulkRead([msg.id], true) - .then(() => { - pendingMarkReadMap.delete(msg.id); - completedMarkReadMap.set(msg.id, msg.account_id); - setTimeout(() => completedMarkReadMap.delete(msg.id), 10000); - }) - .catch(e => { - console.error('markRead failed:', e.message); - updateMessage(msg.id, { is_read: false }); - incrementUnread(msg.account_id); - adjustCategoryCount(msg.category, 1); - pendingMarkReadMap.delete(msg.id); - }); - }; - if (markReadBehavior === 'delay') { - autoMarkReadTimerRef.current = setTimeout(doMarkRead, (markReadDelay || 1) * 1000); - } else { - doMarkRead(); - } - }; - - const onNext = () => { - const { messages, searchResults, searchQuery, selectedMessageId, setSelectedMessage } = getState(); - const pool = searchQuery.trim() ? searchResults : messages; - if (!pool.length) return; - const idx = pool.findIndex(m => m.id === selectedMessageId); - const next = pool[idx + 1] ?? pool[0]; - setSelectedMessage(next.id); - markRead(next); - }; - - const onPrev = () => { - const { messages, searchResults, searchQuery, selectedMessageId, setSelectedMessage } = getState(); - const pool = searchQuery.trim() ? searchResults : messages; - if (!pool.length) return; - const idx = pool.findIndex(m => m.id === selectedMessageId); - const prev = idx <= 0 ? pool[pool.length - 1] : pool[idx - 1]; - setSelectedMessage(prev.id); - markRead(prev); - }; - - const onOpen = () => { - const { messages, selectedMessageId, setSelectedMessage } = getState(); - if (selectedMessageId || !messages.length) return; - setSelectedMessage(messages[0].id); - }; - - const onSelect = () => { - const { selectedMessageId } = getState(); - if (!selectedMessageId) return; - scRef.current.setSelectedIds(prev => { - const next = new Set(prev); - if (next.has(selectedMessageId)) next.delete(selectedMessageId); - else next.add(selectedMessageId); - return next; - }); - }; - - const onArchive = () => { - const { messages, searchResults, searchQuery, selectedMessageId, removeMessage, decrementUnread, addNotification } = getState(); - const pool = searchQuery.trim() ? searchResults : messages; - const ids = [...scRef.current.selectedIds]; - if (ids.length > 0) { - const msgs = pool.filter(m => ids.includes(m.id)); - bulkArchiveRef.current(ids, msgs); - } else if (selectedMessageId) { - const msg = pool.find(m => m.id === selectedMessageId); - if (!msg) return; - refreshRequestRef.current.invalidate(); - setPendingDelete(selectedMessageId); - advanceSelectionAfterRemoval(selectedMessageId); - removeMessage(selectedMessageId); - if (!msg.is_read) decrementUnread(msg.account_id); - api.bulkArchive([selectedMessageId]).then(result => { - setCompletedDelete(selectedMessageId); - if (result.noArchiveFolder?.length) { - addNotification({ title: tRef.current('messageList.noArchiveFolder.title'), body: tRef.current('messageList.noArchiveFolder.body') }); - } - }).catch(err => { clearDeleteGuard(selectedMessageId); console.error(err); }); - } - }; - - const onDelete = () => { - const { messages, searchResults, searchQuery, selectedMessageId } = getState(); - const pool = searchQuery.trim() ? searchResults : messages; - const ids = [...scRef.current.selectedIds]; - if (ids.length > 0) { - const msgs = pool.filter(m => ids.includes(m.id)); - bulkDeleteRef.current(ids, msgs); - } else if (selectedMessageId) { - const msg = pool.find(m => m.id === selectedMessageId); - if (!msg) return; - scheduleDeleteRef.current(msg); - } - }; - - const onToggleRead = () => { - const { messages, selectedMessageId, updateMessage, decrementUnread, incrementUnread, adjustCategoryCount } = getState(); - if (!selectedMessageId) return; - const msg = messages.find(m => m.id === selectedMessageId); - if (!msg) return; - const newRead = !msg.is_read; - updateMessage(selectedMessageId, { is_read: newRead }); - if (newRead) { - decrementUnread(msg.account_id); - adjustCategoryCount(msg.category, -1); - setPending(selectedMessageId, msg.account_id); - api.bulkRead([selectedMessageId], true) - .then(() => { - pendingMarkReadMap.delete(selectedMessageId); - completedMarkReadMap.set(selectedMessageId, msg.account_id); - setTimeout(() => completedMarkReadMap.delete(selectedMessageId), 10000); - }) - .catch(err => { - console.error('markRead failed:', err); - pendingMarkReadMap.delete(selectedMessageId); - }); - } else { - incrementUnread(msg.account_id); - adjustCategoryCount(msg.category, 1); - pendingMarkReadMap.delete(selectedMessageId); - completedMarkReadMap.delete(selectedMessageId); - api.bulkRead([selectedMessageId], false).catch(console.error); - } - }; - const onFocusSearch = () => { searchInputRef.current?.focus(); searchInputRef.current?.select(); }; - // GTD classify keys (t/w/d): COPY the selected message into a state's label - // folder, reusing the same classify dispatch as the context menu (no parallel - // action path). Silent no-op unless the message's account has GTD enabled, so - // the keys stay inert for non-GTD accounts. - const gtdClassifySelected = (state) => () => { - const { messages, searchResults, searchQuery, selectedMessageId, accounts } = getState(); - if (!selectedMessageId) return; - const pool = searchQuery.trim() ? searchResults : messages; - const msg = pool.find(m => m.id === selectedMessageId); - if (!msg) return; - if (!accounts.find(a => a.id === msg.account_id)?.gtd_enabled) return; - handleContextActionRef.current?.('gtdClassify', msg, state); - }; - const onGtdTodo = gtdClassifySelected('todo'); - const onGtdWatch = gtdClassifySelected('watch'); - const onGtdDelegated = gtdClassifySelected('delegated'); - - shortcutBus.on('nextMessage', onNext); - shortcutBus.on('prevMessage', onPrev); - shortcutBus.on('openMessage', onOpen); - shortcutBus.on('selectMessage', onSelect); - shortcutBus.on('archive', onArchive); - shortcutBus.on('delete', onDelete); - shortcutBus.on('toggleRead', onToggleRead); shortcutBus.on('focusSearch', onFocusSearch); - shortcutBus.on('gtdTodo', onGtdTodo); - shortcutBus.on('gtdWatch', onGtdWatch); - shortcutBus.on('gtdDelegated', onGtdDelegated); return () => { - shortcutBus.off('nextMessage', onNext); - shortcutBus.off('prevMessage', onPrev); - shortcutBus.off('openMessage', onOpen); - shortcutBus.off('selectMessage', onSelect); - shortcutBus.off('archive', onArchive); - shortcutBus.off('delete', onDelete); - shortcutBus.off('toggleRead', onToggleRead); shortcutBus.off('focusSearch', onFocusSearch); - shortcutBus.off('gtdTodo', onGtdTodo); - shortcutBus.off('gtdWatch', onGtdWatch); - shortcutBus.off('gtdDelegated', onGtdDelegated); }; }, []); @@ -1852,189 +1049,14 @@ export default function MessageList() { }, [showFolderPicker]); // ───────────────────────────────────────────────────────────── - const handleContextAction = async (action, message, data) => { + const handleContextUtility = async (action, message, data) => { switch (action) { case 'open': handleSelect(message); break; - case 'markRead': { - const uc = parseInt(message.unread_count); - const threadUnread = Number.isFinite(uc) && uc > 0; - if (!message.is_read || threadUnread) { - await setMessagesReadState(message, true); - } - break; - } - case 'markUnread': { - const uc = parseInt(message.unread_count); - const needsMarkUnread = message.is_read || (Number.isFinite(uc) && uc === 0); - if (needsMarkUnread) { - await setMessagesReadState(message, false); - } - break; - } - case 'toggleStar': { - const newVal = !message.is_starred; - await setMessagesStarredState(message, newVal); - break; - } - case 'reply': - case 'replyAll': - await openReplyFromMessage(message, { - accounts, - openCompose, - getMessageBody: api.getMessageBody, - replyAll: action === 'replyAll', - }); - break; - case 'forward': - await openForwardFromMessage(message, { - openCompose, - getMessageBody: api.getMessageBody, - }); - break; case 'bulkSelect': setSelectedIds(new Set([message.id])); break; - case 'archive': { - const archived = message; - refreshRequestRef.current.invalidate(); - advanceSelectionAfterRemoval(archived.id); - removeMessage(archived.id); - if (!archived.is_read) decrementUnread(archived.account_id); - let archiveUndone = false; - const archiveTimer = setTimeout(async () => { - if (archiveUndone) return; - try { - const result = await api.bulkArchive([archived.id]); - if (result.noArchiveFolder?.length) { - addNotification({ title: t('message.archived.noFolderTitle'), body: t('message.archived.noFolderBody') }); - } - } catch (err) { - console.error('Archive failed:', err.message); - addNotification({ title: t('message.archived.failTitle'), body: t('message.archived.failBody') }); - } - }, 4500); - addNotification({ - title: t('message.archived.title'), - body: archived.subject || t('common.noSubject'), - onUndo: () => { - archiveUndone = true; - clearTimeout(archiveTimer); - useStore.getState().restoreMessages([archived]); - if (!archived.is_read) incrementUnread(archived.account_id); - }, - }); - break; - } - case 'moveTo': { - const folder = data; - if (!folder) break; - // If multiple messages are checked and the right-clicked message is among them, - // delegate to handleBulkMove so all selected messages are moved together. - if (selectedIds.size > 1 && selectedIds.has(message.id)) { - const bulkMsgs = displayMessages.filter(m => selectedIds.has(m.id)); - handleBulkMove([...selectedIds], bulkMsgs, folder); - // handleBulkMove already clears the selection internally. - break; - } - const moved = message; - let moveMessages; - try { - moveMessages = await resolveMessagesForThreadAction(message); - } catch (err) { - console.error('Failed to load thread for move:', err.message); - addNotification({ title: t('message.moved.failTitle'), body: t('message.moved.failBody') }); - break; - } - // A folder path is account-specific: a thread can span accounts (and - // always includes Sent copies), and the server skips messages whose - // account lacks the destination folder. Scope the move to the - // right-clicked message's account so nothing is silently dropped. - moveMessages = moveMessages.filter(msg => msg?.account_id === moved.account_id); - const moveIds = [...new Set(moveMessages.map(msg => msg.id).filter(Boolean))]; - if (!moveIds.length) moveIds.push(moved.id); - removeMessage(moved.id); - if (!moved.is_read) decrementUnread(moved.account_id); - // Remove the moved message from the selection so the action bar doesn't - // stay around claiming "X selected" for messages that are no longer here. - if (selectedIds.has(moved.id)) { - const next = new Set(selectedIds); - next.delete(moved.id); - setSelectedIds(next); - if (next.size === 0) setSelectionModeActive(false); - } - let moveUndone = false; - const moveTimer = setTimeout(async () => { - if (moveUndone) return; - try { - const result = await api.bulkMove(moveIds, folder); - // The server reports per-message success (200 even when some IMAP - // moves fail or are skipped) — surface partial failures instead of - // letting the thread silently reappear on the next sync. - const movedSet = new Set(result.moved ?? []); - const failedCount = moveIds.filter(id => !movedSet.has(id)).length; - if (failedCount > 0) { - if (!movedSet.has(moved.id)) { - useStore.getState().restoreMessages([moved]); - if (!moved.is_read) incrementUnread(moved.account_id); - } - addNotification({ type: 'error', title: t('message.moved.failTitle'), body: t('messageList.bulkMoved.failBody', { count: failedCount }) }); - } else { - useStore.getState().recordRecentFolder({ accountId: moved.account_id, path: folder }); - } - } catch (err) { - console.error('Move failed:', err.message); - useStore.getState().restoreMessages([moved]); - if (!moved.is_read) incrementUnread(moved.account_id); - addNotification({ title: t('message.moved.failTitle'), body: t('message.moved.failBody') }); - } - }, 4500); - addNotification({ - title: t('message.moved.title'), - body: folder, - onUndo: () => { - moveUndone = true; - clearTimeout(moveTimer); - useStore.getState().restoreMessages([moved]); - if (!moved.is_read) incrementUnread(moved.account_id); - }, - }); - break; - } - case 'snooze': { - const snoozedMsg = message; - const untilIso = data; - if (!untilIso) break; - removeMessage(snoozedMsg.id); - if (!snoozedMsg.is_read) decrementUnread(snoozedMsg.account_id); - if (selectedIds.has(snoozedMsg.id)) { - const next = new Set(selectedIds); - next.delete(snoozedMsg.id); - setSelectedIds(next); - if (next.size === 0) setSelectionModeActive(false); - } - addNotification({ title: t('message.snoozed.title'), body: snoozedMsg.subject || t('common.noSubject') }); - api.snoozeMessage(snoozedMsg.id, untilIso).catch(err => { - console.error('Snooze failed:', err.message); - useStore.getState().restoreMessages([snoozedMsg]); - if (!snoozedMsg.is_read) incrementUnread(snoozedMsg.account_id); - addNotification({ title: t('message.snoozed.failTitle'), body: t('message.snoozed.failBody') }); - }); - break; - } - case 'gtdClassify': { - // Classify = COPY into the state's label folder. The message stays put - // (no optimistic removal / undo — it does not leave INBOX), so we just - // fire the copy and poke the GTD sections store instead of waiting on the WS event. - await classifyThread(message.id, data, { - gtdClassify: api.gtdClassify, - addNotification, - scheduleGtdSectionsFetch: useStore.getState().scheduleGtdSectionsFetch, - t, - }); - break; - } case 'gtdRemove': { await unclassifyThread(message.id, data, { gtdUnclassify: api.gtdUnclassify, @@ -2061,36 +1083,6 @@ export default function MessageList() { }); break; } - case 'delete': - if (selectedIds.size > 1 && selectedIds.has(message.id)) { - const bulkMsgs = displayMessages.filter(m => selectedIds.has(m.id)); - handleBulkDelete([...selectedIds], bulkMsgs); - } else { - scheduleDelete(message); - } - break; - case 'markSpam': { - // Bulk when more than one message is selected and the right-clicked - // message is among them; otherwise just the single message. - const targets = (selectedIds.size > 1 && selectedIds.has(message.id)) - ? displayMessages.filter(m => selectedIds.has(m.id)) - : [message]; - performSpamLabel(targets, 'spam'); - // The action bar should not claim "X selected" for messages that have - // just been queued for move-to-Junk. Clear the selection (handleBulk* - // already does this; performSpamLabel doesn't, because it's shared - // with the single-message toolbar path which never had a selection). - if (selectedIds.has(message.id)) clearSelection(); - break; - } - case 'markHam': { - const targets = (selectedIds.size > 1 && selectedIds.has(message.id)) - ? displayMessages.filter(m => selectedIds.has(m.id)) - : [message]; - performSpamLabel(targets, 'ham'); - if (selectedIds.has(message.id)) clearSelection(); - break; - } case 'setCategory': { const newCategory = data || 'primary'; const dbCategory = newCategory === 'primary' ? null : newCategory; @@ -2114,17 +1106,11 @@ export default function MessageList() { break; } }; - // Expose the latest handleContextAction to the once-registered shortcut effect via - // a post-commit effect (the sibling handler refs' pattern), rather than mutating the - // ref during render. No dep array: handleContextAction isn't memoized, so it syncs - // on every commit. - useEffect(() => { handleContextActionRef.current = handleContextAction; }); - const handleThreadMarkRead = (e, message) => { e.stopPropagation(); const uc = parseInt(message.unread_count); const hasUnreadInThread = Number.isFinite(uc) && uc > 0; - handleContextAction(hasUnreadInThread ? 'markRead' : 'markUnread', message); + executeForMessages(hasUnreadInThread ? 'mail.read' : 'mail.unread', 'hover', [message]); }; const isDraftsFolder = (() => { @@ -2137,31 +1123,17 @@ export default function MessageList() { return folderInfo?.special_use === '\\Drafts'; })(); - const formatAddressArray = (arr) => { - if (!Array.isArray(arr)) return []; - return arr.map(a => { - if (typeof a === 'string') return a; - const addr = a.address || a.email || ''; - return (a.name && addr) ? `${a.name} <${addr}>` : (addr || a.name || ''); - }).filter(Boolean); - }; - const handleSelect = async (message) => { if (isDraftsFolder) { try { - const bodyData = await api.getMessageBody(message.id); - openCompose({ - accountId: message.account_id, - draftUid: message.uid, - draftFolder: message.folder, - to: formatAddressArray(message.to_addresses), - cc: formatAddressArray(message.cc_addresses), - subject: message.subject || '', - body: bodyData.html || bodyData.text || '', - bodyIsHtml: !!bodyData.html, - }); + const opened = await requestCompose(() => openDraftFromMessage(message, { + openCompose, + getMessageBody: api.getMessageBody, + })); + if (opened === null) setSelectedMessage(message.id); } catch (err) { - console.error('Failed to open draft:', err.message); + console.error('Failed to load draft:', err.message); + addNotification({ type: 'error', title: t('common.error', { message: err.message }) }); setSelectedMessage(message.id); } return; @@ -2173,27 +1145,7 @@ export default function MessageList() { clearTimeout(autoMarkReadTimerRef.current); autoMarkReadTimerRef.current = null; if (!message.is_read && markReadBehavior !== 'manual') { - const prevUnread = message.unread_count; - const doMarkRead = () => { - updateMessage(message.id, { is_read: true, unread_count: 0 }); - decrementUnread(message.account_id); - adjustCategoryCount(message.category, -1); - setPending(message.id, message.account_id); - api.bulkRead([message.id], true) - .catch(() => api.bulkRead([message.id], true)) - .then(() => { - pendingMarkReadMap.delete(message.id); - completedMarkReadMap.set(message.id, message.account_id); - setTimeout(() => completedMarkReadMap.delete(message.id), 10000); - }) - .catch(e => { - console.error('markRead failed:', e.message); - updateMessage(message.id, { is_read: false, unread_count: prevUnread }); - incrementUnread(message.account_id); - adjustCategoryCount(message.category, 1); - pendingMarkReadMap.delete(message.id); - }); - }; + const doMarkRead = () => executeForMessages('mail.read', 'auto-read', [message]); if (markReadBehavior === 'delay') { autoMarkReadTimerRef.current = setTimeout(doMarkRead, markReadDelay * 1000); } else { @@ -2236,7 +1188,9 @@ export default function MessageList() { const showInboxIcon = !isUnified && selectedFolder === 'INBOX' && !searchQuery.trim(); const label = searchQuery.trim() - ? `Search: "${searchQuery}"` + // No quotes around the query: when the header truncates with a CSS ellipsis + // the closing quote would be lost, stranding an unbalanced opening one. + ? `Search: ${searchQuery}` : isUnified ? t('sidebar.allInboxes') : selectedFolder; // Non-INBOX folders omitted: byAccount is account-total, not folder-specific, so it would mislead. @@ -2311,7 +1265,7 @@ export default function MessageList() { - ) : label} + ) : {label}} {headerUnread > 0 && !searchQuery.trim() && ( - ) : label} + ) : {label}}
{messagesTotal > 0 && !searchQuery && ( @@ -2735,7 +1689,7 @@ export default function MessageList() { value={searchQuery} onChange={e => setSearchQuery(e.target.value)} style={{ - width: '100%', padding: '8px 10px 8px 32px', + width: '100%', padding: `8px ${searchInputRightPad(semanticAvailable)}px 8px 32px`, background: 'var(--bg-tertiary)', border: '1px solid var(--border)', borderRadius: 8, color: 'var(--text-primary)', fontSize: 13, outline: 'none', boxSizing: 'border-box', @@ -2743,20 +1697,7 @@ export default function MessageList() { onFocus={e => { e.target.style.borderColor = 'var(--accent)'; setSearchFocused(true); }} onBlur={e => { e.target.style.borderColor = 'var(--border)'; setSearchFocused(false); }} /> - {searchQuery && ( - - )} + {renderSearchControls()} {/* Operator hints — shown when focused with an empty query */} {searchFocused && !searchQuery && ( @@ -2823,7 +1764,7 @@ export default function MessageList() { value={searchQuery} onChange={e => setSearchQuery(e.target.value)} style={{ - width: '100%', padding: '8px 10px 8px 32px', + width: '100%', padding: `8px ${searchInputRightPad(semanticAvailable)}px 8px 32px`, background: 'var(--bg-tertiary)', border: '1px solid var(--border)', borderRadius: 8, color: 'var(--text-primary)', fontSize: 13, outline: 'none', boxSizing: 'border-box', @@ -2831,20 +1772,7 @@ export default function MessageList() { onFocus={e => e.target.style.borderColor = 'var(--accent)'} onBlur={e => e.target.style.borderColor = 'var(--border)'} /> - {searchQuery && ( - - )} + {renderSearchControls()}
)} @@ -3144,7 +2072,7 @@ export default function MessageList() { accounts={accounts} onClearSearch={() => { setSearchQuery(''); }} onShowAll={() => setUnreadOnly(false)} - onCompose={() => openCompose({ accountId: selectedAccountId || undefined })} + onCompose={() => { void requestCompose(() => openCompose({ accountId: selectedAccountId || undefined })); }} /> )} @@ -3506,9 +2434,22 @@ export default function MessageList() { x={contextMenu.x} y={contextMenu.y} message={contextMenu.message} + targetIds={(selectedIds.size > 1 && selectedIds.has(contextMenu.message.id) + ? displayMessages.filter(candidate => selectedIds.has(candidate.id)) + : [contextMenu.message] + ).map(stableConversationId).filter(Boolean)} defaultMoveView={contextMenu.defaultMoveView} + onCommand={(commandId, input) => { + const selectedMessages = displayMessages.filter(candidate => selectedIds.has(candidate.id)); + const targetMessages = contextMenuTargetMessages( + commandId, + contextMenu.message, + selectedMessages, + ); + return executeForMessages(commandId, 'context-menu', targetMessages, input); + }} onClose={() => setContextMenu(null)} - onAction={(action, data) => handleContextAction(action, contextMenu.message, data)} + onAction={(action, data) => handleContextUtility(action, contextMenu.message, data)} /> )} @@ -3692,7 +2633,7 @@ export default function MessageList() { )}
{/* Row 3: snippet */}
- {message.snippet || ''} + + + {message.snippet || ''} +
{hovered && hoverQuickActions && ( @@ -4417,7 +3363,8 @@ function MessageRow({ message, selected, lastViewed, isChecked, selectionMode, s
{/* Row 3: Snippet */} -
+
+ a.name ? `${a.name} <${a.email}>` : a.email).filter(Boolean).join(', '); - } catch { return ''; } -} +import DelegatePill from './DelegatePill.jsx'; function linkifyText(text) { const escaped = text.replace(/&/g, '&').replace(//g, '>'); @@ -91,24 +84,40 @@ function fileIcon(type) { export default function MessagePane() { const { t } = useTranslation(); + const { controller: commandController, commandDefinitions, getContext } = useCommandRuntimeContext(); const { messages, searchResults, searchQuery, selectedMessageId, setSelectedMessage, - updateMessage, removeMessage, decrementUnread, incrementUnread, openCompose, accounts, addNotification, + updateMessage, accounts, addNotification, imageWhitelist, addToImageWhitelist, blockRemoteImages, threadMessages, - replyDefault, shortcuts, recentFolders, favoriteFolders, todoistConnected, - categorizationEnabled, setCategoryCounts, adjustCategoryCount, + replyDefault, recentFolders, favoriteFolders, todoistConnected, + categorizationEnabled, setCategoryCounts, aiActions, setShowAdmin, setAdminTab, } = useStore(); const isMobile = useMobile(); const defaultReplyAll = replyDefault === 'replyAll'; - const effectiveShortcuts = getEffectiveShortcuts(shortcuts); + const executeForTarget = useCallback((commandId, source, target, input) => { + const targetId = stableConversationId(target); + if (!targetId) return Promise.resolve({ status: 'cancelled' }); + return commandController.execute(commandId, { + source, + input, + frozenTargetIds: [targetId], + }); + }, [commandController]); + + const commandBindings = new Map(getEffectiveCommandBindings(commandDefinitions, getContext()) + .map(item => [item.commandId, item.bindings[0]?.keys])); + const shortcutIds = { + reply: 'mail.reply', replyAll: 'mail.replyAll', forward: 'mail.forward', + archive: 'mail.archive', delete: 'mail.trash', toggleStar: 'mail.toggleStar', + toggleRead: 'mail.toggleRead', printMessage: 'mail.print', + }; const shortcutLabel = (action) => { - const k = effectiveShortcuts[action]; + const k = commandBindings.get(shortcutIds[action]); if (!k) return null; - const mod = parseModKey(k); - return mod ? `${modCompactLabel(mod.mod)}${mod.bare.toUpperCase()}` : k.toUpperCase(); + return formatCommandKey(k, getContext().platform); }; // Navigate to a message and mark it as read in one shot. // Arrow buttons and swipe gestures bypass handleSelect in MessageList, so they @@ -122,32 +131,14 @@ export default function MessagePane() { if (!msg.is_read) { const { markReadBehavior, markReadDelay } = useStore.getState(); if (markReadBehavior === 'manual') return; - const doMarkRead = () => { - updateMessage(msg.id, { is_read: true }); - decrementUnread(msg.account_id); - adjustCategoryCount(msg.category, -1); - setPending(msg.id, msg.account_id); - api.bulkRead([msg.id], true) - .then(() => { - pendingMarkReadMap.delete(msg.id); - completedMarkReadMap.set(msg.id, msg.account_id); - setTimeout(() => completedMarkReadMap.delete(msg.id), 10000); - }) - .catch(e => { - console.error('markRead failed:', e.message); - updateMessage(msg.id, { is_read: false }); - incrementUnread(msg.account_id); - adjustCategoryCount(msg.category, 1); - pendingMarkReadMap.delete(msg.id); - }); - }; + const doMarkRead = () => executeForTarget('mail.read', 'pane-navigation', msg); if (markReadBehavior === 'delay') { autoMarkReadTimerRef.current = setTimeout(doMarkRead, (markReadDelay || 1) * 1000); } else { doMarkRead(); } } - }, [setSelectedMessage, updateMessage, decrementUnread, incrementUnread, adjustCategoryCount]); + }, [executeForTarget, setSelectedMessage]); const paneRef = useRef(null); const mountedRef = useRef(true); @@ -206,6 +197,16 @@ export default function MessagePane() { const message = allMessages.find(m => m.id === selectedMessageId) ?? Object.values(threadMessages).flat().find(m => m.id === selectedMessageId); + const executeForMessage = useCallback((commandId, source, input) => { + if (!message) return Promise.resolve({ status: 'cancelled' }); + const frozenTargetIds = [stableConversationId(message)].filter(Boolean); + return commandController.execute(commandId, { + source, + input, + frozenTargetIds, + }); + }, [commandController, message]); + useEffect(() => { setResolvedSubject(null); }, [message?.id]); @@ -224,42 +225,6 @@ export default function MessagePane() { const inSpamFolder = message ? spamFolderPaths.has(message.folder) : false; const hasSpamFolder = spamFolderPaths.size > 0; - // Mark current message as spam / ham from the MessagePane toolbar. - // Mirrors MessageList.performSpamLabel (single-message variant). Kept inline - // here so the MessagePane doesn't need to reach into MessageList internals. - const performSingleSpamLabel = useCallback(async (label) => { - if (!message) return; - const wasUnread = !message.is_read; - removeMessage(message.id); - if (wasUnread) decrementUnread(message.account_id); - let settled = false; - const undo = () => { - settled = true; - useStore.getState().restoreMessages([message]); - if (wasUnread) incrementUnread(message.account_id); - }; - setTimeout(async () => { - if (settled) return; - try { - const fn = label === 'spam' ? api.markSpam : api.markHam; - await fn(message.id); - } catch (err) { - useStore.getState().restoreMessages([message]); - if (wasUnread) incrementUnread(message.account_id); - addNotification({ - type: 'error', - title: t(label === 'spam' ? 'spam.failTitle' : 'spam.failHamTitle'), - body: err.message || t(label === 'spam' ? 'spam.failBody' : 'spam.failHamBody'), - }); - } - }, 4500); - addNotification({ - title: label === 'spam' ? t('spam.movedToSpam') : t('spam.movedToInbox'), - body: message.subject || t('common.noSubject'), - onUndo: undo, - }); - }, [message, removeMessage, decrementUnread, incrementUnread, addNotification, t]); - const currentIdx = allMessages.findIndex(m => m.id === selectedMessageId); const hasPrev = currentIdx > 0; const hasNext = currentIdx >= 0 && currentIdx < allMessages.length - 1; @@ -295,6 +260,18 @@ export default function MessagePane() { const scrollContainerRef = useRef(null); const iframeRef = useRef(null); const roRef = useRef(null); + useEffect(() => { + const onScrollCommand = event => { + const element = scrollContainerRef.current; + if (!element) return; + element.scrollBy({ + top: (event.detail?.direction || 0) * element.clientHeight, + behavior: 'smooth', + }); + }; + window.addEventListener('mailflow:scroll-conversation', onScrollCommand); + return () => window.removeEventListener('mailflow:scroll-conversation', onScrollCommand); + }, []); // useMemo so prepared is available in the same render as body.html — no extra frame, // no flash of empty content between skeleton-gone and email-shown. const prepared = useMemo(() => { @@ -308,8 +285,7 @@ export default function MessagePane() { const bodyCacheOrder = useRef([]); // insertion-order keys for LRU eviction // Session-scoped set of message IDs where the user has clicked "Load images once" const imagesRequestedRef = useRef(new Set()); - // Ref holding the latest pane action handlers so shortcut subscriptions ([] deps) never go stale - const paneActionsRef = useRef({}); + const printActionRef = useRef(null); const emailScaleRef = useRef(1); // scale applied to wide emails that resist CSS reflow // Track previous blocking policy so we can detect tightening vs loosening. @@ -807,7 +783,7 @@ export default function MessagePane() { el.style.transform = 'translateX(0)'; } } else { - const { messages: msgs, searchResults: sr, searchQuery: sq, selectedMessageId: selId, setSelectedMessage: setSel, updateMessage: updMsg, decrementUnread: decUnread, incrementUnread: incUnread, adjustCategoryCount: adjCat } = useStore.getState(); + const { messages: msgs, searchResults: sr, searchQuery: sq, selectedMessageId: selId } = useStore.getState(); const list = sq.trim() ? sr : msgs; const idx = list.findIndex(m => m.id === selId); let target = null; @@ -817,40 +793,7 @@ export default function MessagePane() { target = list[idx - 1]; } if (target) { - window.dispatchEvent(new CustomEvent(MESSAGE_OPENING_EVENT)); - api.getMessageBody(target.id).catch(() => {}); - setSel(target.id); - clearTimeout(autoMarkReadTimerRef.current); - autoMarkReadTimerRef.current = null; - if (!target.is_read) { - const { markReadBehavior, markReadDelay } = useStore.getState(); - if (markReadBehavior !== 'manual') { - const doMarkRead = () => { - updMsg(target.id, { is_read: true }); - decUnread(target.account_id); - adjCat(target.category, -1); - setPending(target.id, target.account_id); - api.bulkRead([target.id], true) - .then(() => { - pendingMarkReadMap.delete(target.id); - completedMarkReadMap.set(target.id, target.account_id); - setTimeout(() => completedMarkReadMap.delete(target.id), 10000); - }) - .catch(e => { - console.error('markRead failed:', e.message); - updMsg(target.id, { is_read: false }); - incUnread(target.account_id); - adjCat(target.category, 1); - pendingMarkReadMap.delete(target.id); - }); - }; - if (markReadBehavior === 'delay') { - autoMarkReadTimerRef.current = setTimeout(doMarkRead, (markReadDelay || 1) * 1000); - } else { - doMarkRead(); - } - } - } + selectAndMarkRead(target); } } }; @@ -864,124 +807,7 @@ export default function MessagePane() { el.removeEventListener('touchmove', onMove); el.removeEventListener('touchend', onEnd); }; - }, [isMobile, setSelectedMessage, resetPaneSwipeStyles]); - - const handleReply = (replyAll = false) => { - if (!message) return; - const date = message.date ? new Date(message.date).toLocaleString() : ''; - const safeName = (message.from_name || '').replace(/[\r\n]+/g, ' '); - const fromStr = safeName - ? `${safeName} <${message.from_email}>` - : message.from_email || ''; - const quotedText = body?.text - ? `\n\n---\nOn ${date}, ${fromStr} wrote:\n${body.text.split('\n').map(l => '> ' + l).join('\n')}` - : ''; - const quotedBodyHtml = body?.html - ? `

On ${date}, ${fromStr} wrote:

${body.html}
` - : null; - - const replyToArr = Array.isArray(message.reply_to) - ? message.reply_to - : (() => { try { return JSON.parse(message.reply_to || '[]'); } catch { return []; } })(); - const replyTarget = (replyToArr.length && replyToArr[0].email) - ? replyToArr[0] - : { name: message.from_name || '', email: message.from_email || '' }; - const sender = replyTarget.email ? [replyTarget] : []; - - const myAccount = accounts.find(a => a.id === message.account_id); - const myEmail = myAccount?.email_address || ''; - - const replyAliasId = pickReplyAlias({ - aliases: myAccount?.aliases || [], - deliveryAddresses: message.delivery_addresses, - toAddresses: message.to_addresses, - ccAddresses: message.cc_addresses, - fromEmail: message.from_email, - }); - - const myAddresses = new Set([ - myEmail.toLowerCase(), - ...(myAccount?.aliases || []).map(al => al.email.toLowerCase()), - ]); - const allRecipients = (() => { - try { - const toArr = Array.isArray(message.to_addresses) - ? message.to_addresses - : JSON.parse(message.to_addresses || '[]'); - const ccArr = Array.isArray(message.cc_addresses) - ? message.cc_addresses - : JSON.parse(message.cc_addresses || '[]'); - return [...toArr, ...ccArr].filter( - t => t.email && !myAddresses.has(t.email.toLowerCase()) && t.email !== replyTarget.email - ); - } catch { return []; } - })(); - - const referencesChain = [message.in_reply_to, message.message_id] - .filter(Boolean).join(' ').trim() || null; - - const rawSubject = (message.subject || '').trim(); - const reSubject = rawSubject.startsWith('Re:') ? rawSubject : rawSubject ? `Re: ${rawSubject}` : 'Re:'; - - setShowReplyMenu(false); - openCompose({ - to: sender, - cc: replyAll ? allRecipients : [], - subject: reSubject, - body: '', - quotedBody: quotedText, - quotedBodyHtml, - inReplyTo: message.message_id, - references: referencesChain, - accountId: message.account_id, - aliasId: replyAliasId, - isReply: true, - isReplyAll: replyAll, - originalFrom: sender, - allRecipients, - threadId: message.thread_id, - }); - }; - - const handleForward = () => { - if (!message) return; - const date = message.date ? new Date(message.date).toLocaleString() : ''; - const safeName = (message.from_name || '').replace(/[\r\n]+/g, ' '); - const fromStr = safeName - ? `${safeName} <${message.from_email}>` - : message.from_email || ''; - const safeSubject = (message.subject || '').replace(/[\r\n]+/g, ' '); - - const toStr = parseAddressField(message.to_addresses); - const ccStr = parseAddressField(message.cc_addresses); - - const fwdText = `\n\n---------- Forwarded message ----------\nFrom: ${fromStr}\nDate: ${date}\nSubject: ${safeSubject}${toStr ? `\nTo: ${toStr}` : ''}${ccStr ? `\nCc: ${ccStr}` : ''}\n\n${body?.text || ''}`; - const fwdHtml = body?.html - ? `

---------- Forwarded message ----------
From: ${fromStr}
Date: ${date}
Subject: ${safeSubject}${toStr ? `
To: ${toStr}` : ''}${ccStr ? `
Cc: ${ccStr}` : ''}

${body.html}
` - : null; - openCompose({ - subject: message.subject?.startsWith('Fwd:') ? message.subject : `Fwd: ${message.subject}`, - body: '', - quotedBody: fwdText, - quotedBodyHtml: fwdHtml, - accountId: message.account_id, - isForward: true, - forwardedAttachments: (body?.attachments || []).map(att => ({ - messageId: message.id, - part: att.part, - filename: att.filename || 'attachment', - type: att.type || 'application/octet-stream', - size: att.size || 0, - })), - }); - }; - - const handleStarToggle = async () => { - if (!message) return; - const newVal = !message.is_starred; - await api.markStarred(message.id, newVal); - updateMessage(message.id, { is_starred: newVal }); - }; + }, [isMobile, resetPaneSwipeStyles, selectAndMarkRead]); const handlePrint = () => { if (!message) return; @@ -1115,35 +941,14 @@ ${bodyContent}
); - // Keep pane action refs current every render - paneActionsRef.current = { - reply: () => handleReply(defaultReplyAll), - replyAll: () => handleReply(true), - forward: handleForward, - toggleStar: handleStarToggle, - print: handlePrint, - }; + printActionRef.current = handlePrint; - // Subscribe to keyboard shortcut actions that belong to the message pane. - // Registered once ([] deps); live state is accessed through paneActionsRef. + // Print stays on the compatibility bus until shortcut parity migrates it. useEffect(() => { - const onReply = () => paneActionsRef.current.reply(); - const onReplyAll = () => paneActionsRef.current.replyAll(); - const onForward = () => paneActionsRef.current.forward(); - const onToggleStar = () => paneActionsRef.current.toggleStar(); - const onPrintMessage = () => paneActionsRef.current.print?.(); - - shortcutBus.on('reply', onReply); - shortcutBus.on('replyAll', onReplyAll); - shortcutBus.on('forward', onForward); - shortcutBus.on('toggleStar', onToggleStar); + const onPrintMessage = () => printActionRef.current?.(); shortcutBus.on('printMessage', onPrintMessage); return () => { - shortcutBus.off('reply', onReply); - shortcutBus.off('replyAll', onReplyAll); - shortcutBus.off('forward', onForward); - shortcutBus.off('toggleStar', onToggleStar); shortcutBus.off('printMessage', onPrintMessage); }; }, []); @@ -1192,22 +997,6 @@ ${bodyContent} } }, [showMovePicker, message?.account_id]); // eslint-disable-line react-hooks/exhaustive-deps - const handleMarkUnread = useCallback(() => { - if (!message || !message.is_read) return; - updateMessage(message.id, { is_read: false }); - incrementUnread(message.account_id); - adjustCategoryCount(message.category, 1); - completedMarkReadMap.delete(message.id); - pendingMarkReadMap.delete(message.id); - api.bulkRead([message.id], false).catch(e => { - console.error('markUnread failed:', e.message); - updateMessage(message.id, { is_read: true }); - decrementUnread(message.account_id); - adjustCategoryCount(message.category, -1); - }); - if (isMobile) setSelectedMessage(null); - }, [message, updateMessage, incrementUnread, decrementUnread, adjustCategoryCount, isMobile, setSelectedMessage]); - const handleEmailClick = useCallback((ev) => { const anchor = ev.target.closest('a[href]'); if (!anchor) return; @@ -1221,36 +1010,10 @@ ${bodyContent} } }, []); - const handleMoveToFolder = useCallback((folder) => { - if (!message) return; + const moveToFolder = useCallback((folder) => { setShowMovePicker(false); - const moved = message; - removeMessage(moved.id); - if (!moved.is_read) decrementUnread(moved.account_id); - let undone = false; - const timer = setTimeout(async () => { - if (undone) return; - try { - await api.bulkMove([moved.id], folder); - useStore.getState().recordRecentFolder({ accountId: moved.account_id, path: folder }); - } catch (err) { - console.error('Move failed:', err); - useStore.getState().restoreMessages([moved]); - if (!moved.is_read) incrementUnread(moved.account_id); - addNotification({ title: t('message.moved.failTitle'), body: t('message.moved.failBody') }); - } - }, 4500); - addNotification({ - title: t('message.moved.title'), - body: folder, - onUndo: () => { - undone = true; - clearTimeout(timer); - useStore.getState().restoreMessages([moved]); - if (!moved.is_read) incrementUnread(moved.account_id); - }, - }); - }, [message, removeMessage, decrementUnread, incrementUnread, addNotification, t]); + return executeForMessage('mail.move', 'pane-move-picker', { folder }); + }, [executeForMessage]); // Close move picker when the selected message changes and handle click-outside useEffect(() => { @@ -1335,67 +1098,6 @@ ${bodyContent} ); } - const handleDelete = () => { - const deleted = message; - setPendingDelete(deleted.id); - removeMessage(deleted.id); - if (!deleted.is_read) decrementUnread(deleted.account_id); - let undone = false; - const timer = setTimeout(async () => { - if (undone) return; - try { - await api.deleteMessage(deleted.id); - setCompletedDelete(deleted.id); - } catch { - clearDeleteGuard(deleted.id); - useStore.getState().restoreMessages([deleted]); - if (!deleted.is_read) incrementUnread(deleted.account_id); - addNotification({ type: 'error', title: t('messageList.deleted.failTitle'), body: t('messageList.deleted.failBody') }); - } - }, 4500); - addNotification({ - title: t('messageList.deleted.title'), - body: t('messageList.deleted.body'), - onUndo: () => { - undone = true; - clearTimeout(timer); - clearPendingDelete(deleted.id); - useStore.getState().restoreMessages([deleted]); - if (!deleted.is_read) incrementUnread(deleted.account_id); - }, - }); - }; - - const handleArchive = () => { - const archived = message; - removeMessage(archived.id); - if (!archived.is_read) decrementUnread(archived.account_id); - let undone = false; - const timer = setTimeout(async () => { - if (undone) return; - try { - const result = await api.bulkArchive([archived.id]); - if (result.noArchiveFolder?.length) { - addNotification({ title: t('message.archived.noFolderTitle'), body: t('message.archived.noFolderBody') }); - } - } catch (err) { - console.error('Archive failed:', err); - addNotification({ title: t('message.archived.failTitle'), body: t('message.archived.failBody') }); - } - }, 4500); - addNotification({ - title: t('message.archived.title'), - body: archived.subject || t('common.noSubject'), - onUndo: () => { - undone = true; - clearTimeout(timer); - const state = useStore.getState(); - state.setMessages([...state.messages, archived].sort((a, b) => new Date(b.date) - new Date(a.date))); - if (!archived.is_read) incrementUnread(archived.account_id); - }, - }); - }; - const handleLoadImages = () => { imagesRequestedRef.current.add(selectedMessageId); delete bodyCache.current[selectedMessageId]; @@ -1424,31 +1126,8 @@ ${bodyContent} const handleUnsubscribe = async () => { if (!message) return; setUnsubscribeStatus('loading'); - const msg = message; - try { - const result = await api.unsubscribeMessage(msg.id); - const succeeded = result.type === 'one-click' || result.type === 'url' || result.type === 'mailto'; - if (!succeeded) { setUnsubscribeStatus('error'); return; } - if (result.type === 'url' && result.url) window.open(result.url, '_blank', 'noopener,noreferrer'); - else if (result.type === 'mailto' && result.mailto) window.open(result.mailto, '_blank', 'noopener,noreferrer'); - setUnsubscribeStatus('done'); - addNotification({ - title: t('message.unsubscribe.done'), - actionLabel: t('message.unsubscribe.moveToTrash'), - onAction: () => { - const { removeMessage, decrementUnread, restoreMessages, incrementUnread } = useStore.getState(); - removeMessage(msg.id); - if (!msg.is_read) decrementUnread(msg.account_id); - api.deleteMessage(msg.id).catch(() => { - restoreMessages([msg]); - if (!msg.is_read) incrementUnread(msg.account_id); - }); - }, - }); - } catch { - setUnsubscribeStatus('error'); - addNotification({ type: 'error', title: t('message.unsubscribe.error') }); - } + const outcome = await executeForTarget('mail.unsubscribe', 'pane', message); + setUnsubscribeStatus(outcome.status === 'success' ? 'done' : 'error'); }; const handleAiClassify = async () => { @@ -1595,7 +1274,7 @@ ${bodyContent} }}> {/* Split Reply button */}
- handleReply(defaultReplyAll)} style={{ borderRadius: '6px 0 0 6px' }} title={isMobile ? (defaultReplyAll ? t('message.replyAll') : t('message.reply')) : `${defaultReplyAll ? t('message.replyAll') : t('message.reply')}${shortcutLabel(defaultReplyAll ? 'replyAll' : 'reply') ? ` (${shortcutLabel(defaultReplyAll ? 'replyAll' : 'reply')})` : ''}`}> + executeForMessage(defaultReplyAll ? 'mail.replyAll' : 'mail.reply', 'pane-toolbar')} style={{ borderRadius: '6px 0 0 6px' }} title={isMobile ? (defaultReplyAll ? t('message.replyAll') : t('message.reply')) : `${defaultReplyAll ? t('message.replyAll') : t('message.reply')}${shortcutLabel(defaultReplyAll ? 'replyAll' : 'reply') ? ` (${shortcutLabel(defaultReplyAll ? 'replyAll' : 'reply')})` : ''}`}> {defaultReplyAll ? ( @@ -1641,7 +1320,7 @@ ${bodyContent} ].map(opt => (
handleReply(opt.replyAll)} + onClick={() => { setShowReplyMenu(false); executeForMessage(opt.replyAll ? 'mail.replyAll' : 'mail.reply', 'pane-reply-menu'); }} style={{ padding: '9px 14px', cursor: 'pointer', fontSize: 13, color: 'var(--text-primary)', @@ -1656,13 +1335,13 @@ ${bodyContent} )}
- + executeForMessage('mail.forward', 'pane-toolbar')} title={isMobile ? t('message.forward') : `${t('message.forward')}${shortcutLabel('forward') ? ` (${shortcutLabel('forward')})` : ''}`}> - + executeForMessage('mail.archive', 'pane-toolbar')} title={isMobile ? t('message.archive') : `${t('message.archive')}${shortcutLabel('archive') ? ` (${shortcutLabel('archive')})` : ''}`}> @@ -1726,7 +1405,7 @@ ${bodyContent} ) : filtered.map(f => (
{ setShowMoreMenu(false); handleMarkUnread(); }} + onClick={() => { setShowMoreMenu(false); executeForMessage('mail.unread', 'pane-more-menu'); }} style={{ display: 'flex', alignItems: 'center', gap: 10, padding: '11px 14px', cursor: 'pointer', fontSize: 13, color: 'var(--text-primary)', borderBottom: '1px solid var(--border-subtle)' }} onMouseEnter={e => e.currentTarget.style.background = 'var(--bg-hover)'} onMouseLeave={e => e.currentTarget.style.background = 'transparent'} @@ -1835,9 +1514,24 @@ ${bodyContent} {t('contextMenu.markUnread')}
)} + {account?.gtd_enabled && ( + + )} {hasSpamFolder && !inSpamFolder && message && (
{ performSingleSpamLabel('spam'); setShowMoreMenu(false); }} + onClick={() => { setShowMoreMenu(false); executeForMessage('mail.spam', 'pane-more-menu'); }} style={{ display: 'flex', alignItems: 'center', gap: 10, padding: '11px 14px', cursor: 'pointer', fontSize: 13, color: 'var(--text-primary)', borderBottom: '1px solid var(--border-subtle)' }} onMouseEnter={e => e.currentTarget.style.background = 'var(--bg-hover)'} onMouseLeave={e => e.currentTarget.style.background = 'transparent'} @@ -1851,7 +1545,7 @@ ${bodyContent} )} {inSpamFolder && message && (
{ performSingleSpamLabel('ham'); setShowMoreMenu(false); }} + onClick={() => { setShowMoreMenu(false); executeForMessage('mail.notSpam', 'pane-more-menu'); }} style={{ display: 'flex', alignItems: 'center', gap: 10, padding: '11px 14px', cursor: 'pointer', fontSize: 13, color: 'var(--text-primary)', borderBottom: '1px solid var(--border-subtle)' }} onMouseEnter={e => e.currentTarget.style.background = 'var(--bg-hover)'} onMouseLeave={e => e.currentTarget.style.background = 'transparent'} @@ -1935,8 +1629,16 @@ ${bodyContent}
) : ( <> + {account?.gtd_enabled && ( + executeForMessage('gtd.delegate', 'visible-message-menu')} title={t('gtd.delegate.command')}> + + + + + + )} {hasSpamFolder && !inSpamFolder && message && ( - performSingleSpamLabel('spam')} title={t('contextMenu.markAsSpam')}> + executeForMessage('mail.spam', 'pane-toolbar')} title={t('contextMenu.markAsSpam')}> @@ -1944,7 +1646,7 @@ ${bodyContent} )} {inSpamFolder && message && ( - performSingleSpamLabel('ham')} title={t('contextMenu.markAsHam')}> + executeForMessage('mail.notSpam', 'pane-toolbar')} title={t('contextMenu.markAsHam')}> @@ -1959,7 +1661,7 @@ ${bodyContent} )} {message.is_read && ( - + executeForMessage('mail.unread', 'pane-toolbar')} title={t('contextMenu.markUnread')}> @@ -2008,7 +1710,7 @@ ${bodyContent} )} - + executeForMessage('mail.toggleStar', 'pane-toolbar')} title={t('message.star')}> @@ -2016,7 +1718,7 @@ ${bodyContent} - + executeForMessage('mail.trash', 'pane-toolbar')} title={t('message.delete')} danger> @@ -2052,13 +1754,17 @@ ${bodyContent} fontSize: 17, fontWeight: 600, color: 'var(--text-primary)', lineHeight: 1.3, fontFamily: 'var(--font-display)', + display: 'flex', alignItems: 'center', flexWrap: 'wrap', gap: 8, }}> - {(() => { - const paneSubject = resolvedSubject || message.subject; - return (paneSubject && paneSubject !== '(no subject)') - ? paneSubject - : t('message.noSubject'); - })()} + + {(() => { + const paneSubject = resolvedSubject || message.subject; + return (paneSubject && paneSubject !== '(no subject)') + ? paneSubject + : t('message.noSubject'); + })()} + +
@@ -2639,7 +2345,7 @@ ${bodyContent} ) : filtered.map(f => (
{notification.onAction && ( diff --git a/frontend/src/components/ProfileModal.jsx b/frontend/src/components/ProfileModal.jsx index 1470fc2a..5e3cb4bc 100644 --- a/frontend/src/components/ProfileModal.jsx +++ b/frontend/src/components/ProfileModal.jsx @@ -1,8 +1,15 @@ -import { useState, useRef } from 'react'; +import { useState, useRef, useEffect } from 'react'; import { useTranslation } from 'react-i18next'; import { useStore } from '../store/index.js'; import { api } from '../utils/api.js'; +const TOKEN_SCOPE_OPTIONS = [ + { name: 'read', labelKey: 'profile.tokens.scopes.read' }, + { name: 'write', labelKey: 'profile.tokens.scopes.write' }, + { name: 'send', labelKey: 'profile.tokens.scopes.send' }, + { name: 'settings', labelKey: 'profile.tokens.scopes.settings' }, +]; + function resizeImage(file, maxPx = 256) { return new Promise((resolve, reject) => { const reader = new FileReader(); @@ -37,6 +44,55 @@ export default function ProfileModal({ onClose }) { const [saving, setSaving] = useState(false); const [error, setError] = useState(''); + // MCP API tokens (per-user bearer tokens for the /mcp endpoint). + const [tokens, setTokens] = useState([]); + const [tokenName, setTokenName] = useState(''); + const [tokenScopes, setTokenScopes] = useState(['read']); + const [mintedToken, setMintedToken] = useState(''); + const [tokenBusy, setTokenBusy] = useState(false); + + useEffect(() => { + let alive = true; + api.tokens.list().then((r) => { if (alive) setTokens(r.tokens || []); }).catch(() => {}); + return () => { alive = false; }; + }, []); + + async function handleCreateToken() { + const name = tokenName.trim(); + if (!name || tokenBusy) return; + setTokenBusy(true); + setError(''); + try { + const { token } = await api.tokens.create(name, tokenScopes); + setMintedToken(token); // shown once — never retrievable again + setTokenName(''); + setTokenScopes(['read']); + const r = await api.tokens.list(); + setTokens(r.tokens || []); + } catch (e) { + setError(e.message || t('profile.tokens.createFailed')); + } finally { + setTokenBusy(false); + } + } + + async function handleRevokeToken(id) { + try { + await api.tokens.revoke(id); + setTokens((prev) => prev.filter((tok) => tok.id !== id)); + } catch (e) { + setError(e.message || t('profile.tokens.revokeFailed')); + } + } + + function toggleTokenScope(scope) { + setTokenScopes((current) => ( + current.includes(scope) + ? current.filter((candidate) => candidate !== scope) + : [...current, scope] + )); + } + async function handleFileChange(e) { const file = e.target.files[0]; if (!file) return; @@ -193,6 +249,91 @@ export default function ProfileModal({ onClose }) {
+ {/* API tokens (MCP) — bearer tokens for the Streamable-HTTP /mcp endpoint */} +
+ + {tokens.length > 0 && ( +
+ {tokens.map((tok) => ( +
+
+ + {tok.name} + + {tok.last_used_at + ? t('profile.tokens.usedOn', { date: new Date(tok.last_used_at).toLocaleDateString() }) + : t('profile.tokens.neverUsed')} + + +
+ {(tok.scopes || ['read']).map((scope) => ( + + {t(`profile.tokens.scopes.${scope}`)} + + ))} +
+
+ +
+ ))} +
+ )} +
+ {TOKEN_SCOPE_OPTIONS.map(({ name, labelKey }) => ( + + ))} +
+ {tokenScopes.includes('send') && ( + + {t('profile.tokens.scopes.sendRisk')} + + )} +
+ setTokenName(e.target.value)} + onKeyDown={e => e.key === 'Enter' && handleCreateToken()} + placeholder={t('profile.tokens.namePlaceholder')} + maxLength={80} + style={{ flex: 1, padding: '7px 10px', borderRadius: 8, border: '1px solid var(--border)', background: 'var(--bg-primary)', color: 'var(--text-primary)', fontSize: 13, outline: 'none', boxSizing: 'border-box' }} + /> + +
+ {mintedToken && ( +
+ {mintedToken} + + {t('profile.tokens.copyOnce')} + +
+ )} +
+ {error && (
{error} diff --git a/frontend/src/components/Sidebar.jsx b/frontend/src/components/Sidebar.jsx index aac0fe62..93e6057b 100644 --- a/frontend/src/components/Sidebar.jsx +++ b/frontend/src/components/Sidebar.jsx @@ -14,6 +14,7 @@ import { useMobile } from '../hooks/useMobile.js'; import LogoMark from './LogoMark.jsx'; import ProfileModal from './ProfileModal.jsx'; import { useUiScale, descale } from '../hooks/useUiScale.js'; +import { handleComposeRequest } from '../utils/composeRequest.js'; const ICONS = { inbox: ( @@ -266,6 +267,10 @@ export default function Sidebar() { } = useStore(); const isMobile = useMobile(); + const requestCompose = useCallback(changes => handleComposeRequest( + () => openCompose(changes), + { addNotification, t }, + ), [addNotification, openCompose, t]); // On mobile the sidebar is always expanded (shown as an overlay drawer) const sidebarCollapsed = isMobile ? false : sidebarCollapsedPref; @@ -856,7 +861,7 @@ export default function Sidebar() { {/* Compose button */}