diff --git a/apps/api/src/__tests__/account-health-role-access.test.ts b/apps/api/src/__tests__/account-health-role-access.test.ts index 4e58e9f55..c6baf502a 100644 --- a/apps/api/src/__tests__/account-health-role-access.test.ts +++ b/apps/api/src/__tests__/account-health-role-access.test.ts @@ -434,6 +434,22 @@ describe("account-health with an unusable customer role", () => { expect(mockCaptureException).not.toHaveBeenCalled(); }); + it("alerts once per unreachable episode, not once per day", async () => { + // A customer who removed the role on purpose (or an abandoned account) is + // not going to repair it, and a daily "can no longer reach your AWS + // account" email to them is noise. The episode starts at the last good + // check, so a role that recovers and breaks again alerts again. + ACCOUNT_ROW.roleLastReachableAt = new Date("2026-07-01T00:00:00Z"); + mockStsSend.mockRejectedValue(accessDeniedError()); + + await invoke(); + + const dedupe = mockHasRecentNotification.mock.calls.find( + (call) => call[0].type === "aws.role_unreachable" + )?.[0]; + expect(dedupe?.since).toEqual(new Date("2026-07-01T00:00:00Z")); + }); + it("still reports unexpected failures to Sentry", async () => { mockStsSend.mockRejectedValue( Object.assign(new TypeError("cannot read property of undefined"), { diff --git a/apps/api/src/workers/account-health.ts b/apps/api/src/workers/account-health.ts index ad19404f7..48c374b4c 100644 --- a/apps/api/src/workers/account-health.ts +++ b/apps/api/src/workers/account-health.ts @@ -25,6 +25,11 @@ * * Each alert is deduped per account per 24h via hasRecentNotification, so * an ongoing episode notifies once per day rather than once per hour. + * `aws.role_unreachable` is the exception: it notifies once per episode. A + * customer who removed the role on purpose will not repair it, and a daily + * email for a state they chose is noise. The episode starts at the last good + * check (roleLastReachableAt only moves forward on success), so a role that + * recovers and breaks again alerts again. * Per-account errors are logged and skipped — one broken role must not * abort the sweep. * @@ -143,8 +148,10 @@ async function notifyOnce(params: { body: string; href: string; data?: Record; + /** Dedupe window start; defaults to the last 24h. */ + since?: Date; }): Promise { - const since = new Date(Date.now() - DEDUPE_WINDOW_MS); + const since = params.since ?? new Date(Date.now() - DEDUPE_WINDOW_MS); const already = await hasRecentNotification({ organizationId: params.account.organizationId, type: params.type, @@ -383,6 +390,7 @@ async function checkAccount(account: AccountRow): Promise { body: `The wraps-console-access-role in AWS account ${account.accountId} (${account.region}) cannot be assumed or is missing SES permissions. Sending uses this same role, so email is blocked until it is repaired — and health checks (sending paused, reputation, and quota alerts) are not running either. Open this account in Wraps and open Connection → Role access for the steps: update your CloudFormation stack if you deployed one, or run \`wraps platform update-role\` if you connected with the CLI (\`wraps platform connect\` if the role was deleted).`, href: connectionHref, data: { reason, lastReachableAt }, + since: account.roleLastReachableAt, }); log.warn("[account-health] Customer role unusable, skipping account", { accountId: account.id, diff --git a/packages/cli/src/commands/__tests__/platform-disconnect.test.ts b/packages/cli/src/commands/__tests__/platform-disconnect.test.ts index ce6c70a72..a53c91058 100644 --- a/packages/cli/src/commands/__tests__/platform-disconnect.test.ts +++ b/packages/cli/src/commands/__tests__/platform-disconnect.test.ts @@ -30,6 +30,7 @@ vi.mock("@pulumi/pulumi", () => ({ })); vi.mock("@clack/prompts"); vi.mock("../../utils/shared/aws.js"); +vi.mock("../../utils/shared/config.js"); vi.mock("../../utils/shared/fs.js"); vi.mock("../../utils/shared/metadata.js"); vi.mock("../../utils/shared/pulumi.js"); @@ -41,6 +42,7 @@ vi.mock("../../infrastructure/email-stack.js"); import * as prompts from "@clack/prompts"; import * as pulumi from "@pulumi/pulumi"; import * as aws from "../../utils/shared/aws.js"; +import * as config from "../../utils/shared/config.js"; import * as fsUtils from "../../utils/shared/fs.js"; import * as jsonOutput from "../../utils/shared/json-output.js"; import * as metadata from "../../utils/shared/metadata.js"; @@ -80,8 +82,20 @@ function connectedMetadata(overrides: Record = {}) { }; } +const SAAS_API = "https://api.wraps.dev"; + +/** A platform-connected account: the connection id is aws_account.id on the server. */ +function platformConnectedMetadata() { + return { + ...connectedMetadata(), + platform: { externalId: "ext-abc", connectionId: "conn-abc" }, + } as any; +} + let capturedOverrides: Record | undefined; let savedMetadata: any; +let fetchMock: ReturnType; +let events: string[]; beforeEach(() => { vi.clearAllMocks(); @@ -90,6 +104,21 @@ beforeEach(() => { iamMock.on(DeleteRoleCommand).resolves({}); capturedOverrides = undefined; savedMetadata = undefined; + events = []; + + fetchMock = vi.fn(async () => { + events.push("api-delete"); + return new Response(JSON.stringify({ success: true }), { status: 200 }); + }); + vi.stubGlobal("fetch", fetchMock); + vi.mocked(config.getApiBaseUrl).mockReturnValue(SAAS_API); + vi.mocked(config.resolveTokenAsync).mockResolvedValue("saas-token"); + vi.mocked(config.readAuthConfig).mockResolvedValue({ + auth: { + token: "saas-token", + organizations: [{ id: "org-1", name: "One", slug: "one" }], + }, + } as any); const mockSpinner = { start: vi.fn(), stop: vi.fn(), message: vi.fn() }; vi.mocked(prompts.spinner).mockReturnValue(mockSpinner as never); @@ -258,3 +287,174 @@ describe("platform disconnect", () => { ).not.toHaveBeenCalled(); }); }); + +describe("platform disconnect — server-side record", () => { + // Without this the dashboard keeps `webhook_secret` set, and account-health + // (which sweeps every account holding one) emails the owner a daily + // "Wraps can no longer reach your AWS account" for a role they deleted on purpose. + + it("tells the platform to drop the connection after the target is gone and before the role is deleted", async () => { + vi.mocked(metadata.loadConnectionMetadata).mockResolvedValue( + platformConnectedMetadata() + ); + vi.mocked( + pulumi.automation.LocalWorkspace.createOrSelectStack + ).mockResolvedValue({ + setConfig: vi.fn().mockResolvedValue(undefined), + refresh: vi.fn().mockResolvedValue(undefined), + exportStack: vi.fn().mockResolvedValue({ deployment: { resources: [] } }), + up: vi.fn(async () => { + events.push("stack-up"); + return {}; + }), + } as any); + iamMock.on(DeleteRoleCommand).callsFake(async () => { + events.push("role-deleted"); + return {}; + }); + + await disconnect({ force: true }); + + // Role deletion is what starts role_unreachable, so the record must be + // cleared first; and only once the target is gone, so no in-flight event + // meets a nulled secret. + expect(events).toEqual(["stack-up", "api-delete", "role-deleted"]); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe(`${SAAS_API}/v1/connections/conn-abc`); + expect(init.method).toBe("DELETE"); + expect(init.headers.Authorization).toBe("Bearer saas-token"); + expect(init.headers["X-Organization-Id"]).toBe("org-1"); + }); + + it("targets the SaaS plane even when signed in to a self-hosted instance", async () => { + // resolveApiTarget() would follow activeInstance here and null the secret + // on the customer's own control plane — the one they are keeping. + vi.mocked(config.readAuthConfig).mockResolvedValue({ + auth: { + token: "saas-token", + organizations: [{ id: "org-1", name: "One", slug: "one" }], + }, + activeInstance: "https://selfhost.example.com", + selfhost: { + "https://selfhost.example.com": { + token: "selfhost-token", + apiUrl: "https://selfhost-api.example.com", + }, + }, + } as any); + vi.mocked(metadata.loadConnectionMetadata).mockResolvedValue( + platformConnectedMetadata() + ); + + await disconnect({ force: true }); + + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe(`${SAAS_API}/v1/connections/conn-abc`); + expect(init.headers.Authorization).toBe("Bearer saas-token"); + }); + + it("tries the next organization when the first does not own the connection", async () => { + vi.mocked(config.readAuthConfig).mockResolvedValue({ + auth: { + token: "saas-token", + organizations: [ + { id: "org-1", name: "One", slug: "one" }, + { id: "org-2", name: "Two", slug: "two" }, + ], + }, + } as any); + fetchMock + .mockResolvedValueOnce(new Response("{}", { status: 404 })) + .mockResolvedValueOnce(new Response("{}", { status: 200 })); + vi.mocked(metadata.loadConnectionMetadata).mockResolvedValue( + platformConnectedMetadata() + ); + + await disconnect({ force: true }); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fetchMock.mock.calls[1][1].headers["X-Organization-Id"]).toBe( + "org-2" + ); + expect(savedMetadata.platform).toBeUndefined(); + }); + + it("still finishes the AWS teardown, keeps the connection id and says so when the platform call fails", async () => { + fetchMock.mockResolvedValue(new Response("{}", { status: 401 })); + vi.mocked(metadata.loadConnectionMetadata).mockResolvedValue( + platformConnectedMetadata() + ); + + await expect(disconnect({ force: true })).resolves.toBeUndefined(); + + expect(iamMock.commandCalls(DeleteRoleCommand)).toHaveLength(1); + // Kept so a re-run can finish the server-side half. + expect(savedMetadata.platform?.connectionId).toBe("conn-abc"); + expect(prompts.log.warn).toHaveBeenCalledWith( + expect.stringContaining("still lists this account as connected") + ); + expect(prompts.log.success).not.toHaveBeenCalledWith( + expect.stringContaining("Disconnected from the Wraps Platform") + ); + }); + + it("reports platformRecordCleared in JSON mode", async () => { + vi.mocked(jsonOutput.isJsonMode).mockReturnValue(true); + fetchMock.mockResolvedValue(new Response("{}", { status: 500 })); + vi.mocked(metadata.loadConnectionMetadata).mockResolvedValue( + platformConnectedMetadata() + ); + + await disconnect({ force: true }); + + expect(jsonOutput.jsonSuccess).toHaveBeenCalledWith( + "platform.disconnect", + expect.objectContaining({ + disconnected: true, + platformRecordCleared: false, + }) + ); + }); + + it("warns instead of calling the API when there is no connection id (manual-paste connect)", async () => { + vi.mocked(metadata.loadConnectionMetadata).mockResolvedValue( + connectedMetadata() as any + ); + + await disconnect({ force: true }); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(prompts.log.warn).toHaveBeenCalledWith( + expect.stringContaining("still lists this account as connected") + ); + }); + + it("warns instead of calling the API when not signed in", async () => { + vi.mocked(config.resolveTokenAsync).mockResolvedValue(null); + vi.mocked(metadata.loadConnectionMetadata).mockResolvedValue( + platformConnectedMetadata() + ); + + await disconnect({ force: true }); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(savedMetadata.platform?.connectionId).toBe("conn-abc"); + }); + + it("finishes the server-side half on a re-run after an earlier partial disconnect", async () => { + // Local secret already cleared, but the connection id survived because the + // platform call failed last time. + vi.mocked(metadata.loadConnectionMetadata).mockResolvedValue({ + ...connectedMetadata({ webhookSecret: undefined }), + platform: { externalId: "ext-abc", connectionId: "conn-abc" }, + } as any); + + await disconnect({ force: true }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect( + pulumi.automation.LocalWorkspace.createOrSelectStack + ).not.toHaveBeenCalled(); + expect(savedMetadata.platform).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/commands/platform/disconnect.ts b/packages/cli/src/commands/platform/disconnect.ts index eaef8a227..a02ec0490 100644 --- a/packages/cli/src/commands/platform/disconnect.ts +++ b/packages/cli/src/commands/platform/disconnect.ts @@ -20,6 +20,13 @@ * the email stack. This command is not a teardown; `wraps email destroy` is, * and it removes shared infrastructure the self-hosted plane still needs. * + * It also tells the platform. The AWS-side teardown alone leaves the dashboard + * holding a `webhook_secret` for the account, and account-health sweeps every + * account that holds one — so with the console role gone it emails the owner a + * daily "Wraps can no longer reach your AWS account" for a role they removed on + * purpose. `DELETE /v1/connections/:id` clears the secret and takes the account + * out of that sweep. + * * Sending is entirely unaffected either way — mail flows from the customer's * own SES with their own credentials, and Wraps is never in that path. */ @@ -36,6 +43,11 @@ import pc from "picocolors"; import { deployEmailStack } from "../../infrastructure/email-stack.js"; import type { PlatformDisconnectOptions } from "../../types/index.js"; import { validateAWSCredentials } from "../../utils/shared/aws.js"; +import { + getApiBaseUrl, + readAuthConfig, + resolveTokenAsync, +} from "../../utils/shared/config.js"; import { isAWSNotFoundError } from "../../utils/shared/errors.js"; import { ensurePulumiWorkDir, @@ -101,6 +113,127 @@ async function removeConsoleAccessRole(region: string): Promise { } } +type PlatformRecordResult = + | { cleared: true } + | { cleared: false; reason: string }; + +/** One DELETE attempt; resolves to the HTTP status, rejects on a network failure. */ +async function deleteConnection( + connectionId: string, + token: string, + orgId: string | undefined +): Promise { + const response = await fetch( + `${getApiBaseUrl()}/v1/connections/${encodeURIComponent(connectionId)}`, + { + method: "DELETE", + headers: { + Authorization: `Bearer ${token}`, + ...(orgId ? { "X-Organization-Id": orgId } : {}), + }, + } + ); + return response.status; +} + +/** + * The organizations to try the DELETE under. A session token resolves to its + * active organization unless told which one, and a mismatch is a 404, not an + * error. The connection's org isn't stored in metadata, so try each one the + * token belongs to; with none known, send no header and let the server decide. + */ +async function organizationAttempts(): Promise> { + const config = await readAuthConfig(); + const orgIds = config?.auth?.organizations?.map((org) => org.id) ?? []; + return orgIds.length ? orgIds : [undefined]; +} + +/** + * Ask the Wraps platform to forget this connection (clears its webhook secret). + * + * Deliberately NOT `resolveApiTarget()`: that follows `activeInstance`, and this + * command's main caller is a customer who also runs a self-hosted control plane + * — so it could aim the DELETE at their own plane and null the secret of the + * one they are keeping. This command only ever concerns the SaaS plane, so it + * pairs `getApiBaseUrl()` with the SaaS-slot token, as `platform connect` does. + * + * Never throws: by the time this runs the AWS side is already torn down, so a + * failure here must be reported, not allowed to abort the rest of the command. + */ +async function clearPlatformRecord( + connectionId: string | undefined +): Promise { + if (!connectionId) { + return { + cleared: false, + reason: "no connection id is recorded for it (it was connected manually)", + }; + } + + const token = await resolveTokenAsync(); + if (!token) { + return { cleared: false, reason: "you are not signed in to Wraps" }; + } + + const attempts = await organizationAttempts(); + + let status = 0; + for (const orgId of attempts) { + try { + status = await deleteConnection(connectionId, token, orgId); + } catch (error) { + return { + cleared: false, + reason: error instanceof Error ? error.message : String(error), + }; + } + if (status >= 200 && status < 300) { + return { cleared: true }; + } + if (status !== 404) { + break; + } + } + + return { + cleared: false, + reason: + status === 404 + ? "none of your organizations owns that connection" + : `the platform answered HTTP ${status}`, + }; +} + +/** + * `clearPlatformRecord` with a spinner that ends on the real outcome. + * `progress.execute` would tick green on a `{ cleared: false }` result, since + * that is a return value, not a throw. + */ +async function removePlatformRecord( + progress: DeploymentProgress, + connectionId: string | undefined +): Promise { + progress.start("Removing platform record"); + const record = await clearPlatformRecord(connectionId); + if (record.cleared) { + progress.succeed("Platform record removed"); + } else { + progress.fail("Platform record not removed"); + } + return record; +} + +/** Loud, because a quiet failure here is exactly what produces the daily emails. */ +function warnPlatformRecordNotCleared(reason: string): void { + log.warn( + [ + `Wraps still lists this account as connected (${reason}).`, + 'Until that is cleared you will keep getting "Wraps can no longer reach your AWS account" emails.', + `Clear it in the dashboard (Settings → AWS accounts → this account → ${pc.cyan("Disconnect streaming")}), or fix the above and re-run ${pc.cyan("wraps platform disconnect")}.`, + ].join("\n ") + ); +} + export async function disconnect( options: PlatformDisconnectOptions ): Promise { @@ -166,6 +299,40 @@ export async function disconnect( // Already disconnected is success, not failure — this command is the kind // of thing people run twice, and re-running a Pulumi up to remove nothing // is a slow way to say "nothing to do". + // + // Unless an earlier run couldn't reach the platform: it keeps + // `platform.connectionId` in that case, and finishing that half is all a + // re-run has left to do. + if (!emailService.webhookSecret && metadata.platform?.connectionId) { + const record = await removePlatformRecord( + progress, + metadata.platform.connectionId + ); + if (record.cleared) { + metadata.platform = undefined; + // This path never deploys; it only drops an id the platform has forgotten. + await saveConnectionMetadata(metadata); // baseline:allow-early-save — no deployment on this path + } + if (json) { + jsonSuccess("platform.disconnect", { + accountId: identity.accountId, + region, + disconnected: false, + reason: "not connected", + platformRecordCleared: record.cleared, + }); + return; + } + if (record.cleared) { + log.success("Wraps no longer lists this account as connected."); + outro("Done"); + } else { + warnPlatformRecordNotCleared(record.reason); + outro("Platform record not cleared"); + } + return; + } + if (!emailService.webhookSecret) { progress.stop(); if (json) { @@ -183,6 +350,8 @@ export async function disconnect( } const stillSelfhosted = !!emailService.selfhostWebhook; + // Read now: `metadata.platform` is cleared before the end of this command. + const connectionId = metadata.platform?.connectionId; if (!(options.force || options.yes || json)) { progress.stop(); @@ -293,6 +462,11 @@ export async function disconnect( progress.succeed("Platform event target removed"); + // Before the role goes: deleting it is what makes account-health start + // reporting the account unreachable, so the record has to be cleared first. + // After `stack.up`, so no event still in flight meets a cleared secret. + const record = await removePlatformRecord(progress, connectionId); + // Revoke cross-account access last: if this fails the event target is // already gone, so the customer is disconnected in the sense that matters // and can re-run to finish the job. Doing it first would leave the reverse @@ -305,8 +479,12 @@ export async function disconnect( // The platform issued this externalId for a connection that no longer // exists. Leaving it behind would let a later connect reuse a stale id. - metadata.platform = undefined; - await saveConnectionMetadata(metadata); + // Unless the platform still lists the connection: then the id is the only + // handle a re-run has to finish the job. + if (record.cleared) { + metadata.platform = undefined; + await saveConnectionMetadata(metadata); + } if (json) { jsonSuccess("platform.disconnect", { @@ -315,12 +493,18 @@ export async function disconnect( disconnected: true, selfhostWebhookRetained: stillSelfhosted, consoleAccessRoleRemoved: roleRemoved, + platformRecordCleared: record.cleared, }); return; } console.log(); - log.success("Disconnected from the Wraps Platform."); + if (record.cleared) { + log.success("Disconnected from the Wraps Platform."); + } else { + log.warn("Disconnected on AWS, but not yet on the Wraps Platform."); + warnPlatformRecordNotCleared(record.reason); + } console.log(); console.log( ` SES events from ${pc.cyan(identity.accountId)} no longer reach app.wraps.dev.`