fix(manager): retry failed static reconciliation

.changeset/calm-ravens-reconcile.md

@@ -0,0 +1,7 @@
+---
+"@cotal-ai/manager": patch
+"@cotal-ai/cli": patch
+"@cotal-ai/connector-core": patch
+---
+
+Report failed static lifecycle reconciliation, retry the same durable terminal with a bounded schedule, and drain accepted reconciliation work during shutdown.

bin/smoke/ci-suites.d/4fac15840cc0d39928185484921a58e0778f546be0344c88b4c05f91beba993a.txt

@@ -0,0 +1 @@
+smoke:manager-reconcile-redrive

bin/smoke/required-arg-seam.smoke.ts

@@ -293,7 +293,10 @@ const SEAMS: Seam[] = [
   // 118/87 -> 119/88: static-lifecycle.smoke.ts's #1274 crash-resume cell (driveTerminalDirect) opens
   // one more lifecycle-executor connection to plant a terminalizing slot and drive runStaticTerminal
   // as a resume would. It is under smoke/, harness residue not product connect, and states tls: false.
-  { fn: "standaloneConnectOpts", key: "tls", sites: 119, untypecheckedSites: 88 },
+  // 119/88 -> 124/92: #774 adds one typechecked provisioner re-read before each static reconcile
+  // retry, plus four smoke-side connections for the isolated broker acceptance fixture (orphan write,
+  // observer, caller, and exact terminal gate inspection). Every site states tls: false.
+  { fn: "standaloneConnectOpts", key: "tls", sites: 124, untypecheckedSites: 92 },
 ];
 
 /**

docs/run-a-mesh.md

@@ -65,6 +65,24 @@ continues, but prints a named `connector <name> unavailable` line and records th
 manager's `status` response. Available connector rows record the absolute paths boot resolved.
 Spawn keeps the same pre-mint check as a backstop for connectors registered after boot.
 
+On an authenticated manager start, unfinished static lifecycle rows reconcile while the control
+endpoint is already serving. The manager `status` response and `cotal status --components` report
+the `staticReconciliation` state, the last sweep counts, and each failed alias with its durable
+phase and literal disposition. A failed exact terminal is retried in the same process after 1, 5,
+and 30 seconds. Each attempt re-reads the durable slot and re-enters the same deterministic terminal
+operation; the delays only schedule work and never release the lifecycle fence.
+
+On shutdown, the manager fences new reconciliation work and waits for an exact terminal that already
+started. The current serial sweep stops before its next alias, and startup cannot publish the manager
+service after `stop()` completes.
+
+The four-attempt budget is per manager process. An exhausted row stays held and reports
+`retry-exhausted` with the remedy to restart the manager. The next process derives a fresh budget
+from the still-authoritative durable row. A `recovered` row remains visible until the next static
+reconciliation sweep, then clears. This component reports reconciliation outcomes. It does not say
+whether footprint cleanup completed independently of the terminal result; that separate durable
+projection remains tracked by #1274.
+
 There is no supported `cotal service install` command yet. Running the manager as a launchd agent or
 systemd user service remains operator-managed; service installation is separate from this boot-time
 detection behavior.

implementations/cli/smoke/component-health.smoke.ts

@@ -166,8 +166,8 @@ try {
   const present = cli("status", "--components", "--space", SPACE, "--server", server);
   const presentText = `${present.stdout}${present.stderr}`;
   check("live lease-holder that never serves exits present-not-serving (2)", present.status === 2, presentText);
-  check("manager row names not-serving, its PID, and its unreported phase",
-    /manager\s+not-serving/.test(presentText) && presentText.includes(`pid ${held.pid}`) && presentText.includes("phase not reported by this manager build"), presentText);
+  check("manager row names not-serving, its PID, and its unreported static reconciliation",
+    /manager\s+not-serving/.test(presentText) && presentText.includes(`pid ${held.pid}`) && presentText.includes("static reconciliation not reported by this manager build"), presentText);
   check("manager row names the lease holder rather than substituting service success",
     /lease holder local\./.test(presentText) && presentText.includes("serve no answer"), presentText);
 

implementations/cli/src/commands/status.ts

@@ -607,7 +607,14 @@ async function componentEp(target: MeshTarget): Promise<{ ep: CotalEndpoint; clo
 async function managerServiceHealth(
   target: MeshTarget,
   auth: { creds?: string; caller: { owner: string; actor: string; uid: string } },
-): Promise<{ instanceId?: unknown; runtime?: unknown }> {
+): Promise<{
+  instanceId?: unknown;
+  runtime?: unknown;
+  staticReconciliation?: {
+    state?: unknown;
+    failures?: Array<{ alias?: unknown; phase?: unknown; disposition?: unknown; nextRetryAt?: unknown; remedy?: unknown }>;
+  };
+}> {
   const nc = await connect({
     servers: target.server,
     ...standaloneConnectOpts(auth.creds ? { creds: auth.creds, tls: target.tlsRequired } : { tls: target.tlsRequired }),
@@ -618,7 +625,14 @@ async function managerServiceHealth(
     const response = await invokeCommand(nc, target.space, service, "status", undefined, { deadlineMs: 3_000 });
     if (response.reply.ok !== true)
       throw new EpEnvelopeError(response.reply.error?.code === "unavailable" ? "unavailable" : "failed-precondition", response.reply.error?.message ?? "manager status refused");
-    return response.reply.data as { instanceId?: unknown; runtime?: unknown };
+    return response.reply.data as {
+      instanceId?: unknown;
+      runtime?: unknown;
+      staticReconciliation?: {
+        state?: unknown;
+        failures?: Array<{ alias?: unknown; phase?: unknown; disposition?: unknown; nextRetryAt?: unknown; remedy?: unknown }>;
+      };
+    };
   } finally {
     await nc.drain().catch(() => nc.close());
   }
@@ -631,7 +645,7 @@ async function managerHealth(target: MeshTarget, context: LocalProcessContext):
   // permission to replace it with a network answer.  Name that failed local control surface first.
   if (record.kind === "unattributable" || record.kind === "unknown") {
     facts.push(record.kind === "unattributable" ? "unattributable pidfile" : "pid liveness unestablishable");
-    facts.push("phase not reported by this manager build");
+    facts.push("static reconciliation not reported by this manager build");
     return { name: "manager", verdict: "refused", facts };
   }
   if (record.kind === "dead") facts.push("stale pidfile");
@@ -657,13 +671,24 @@ async function managerHealth(target: MeshTarget, context: LocalProcessContext):
     });
     facts.push(`service instance ${served.instanceId ?? "unreported"}`);
     facts.push(`runtime ${served.runtime ?? "unreported"}`);
-    facts.push("phase not reported by this manager build");
+    const reconcile = served.staticReconciliation;
+    if (!reconcile) {
+      facts.push("static reconciliation not reported by this manager build");
+    } else {
+      facts.push(`static reconciliation ${String(reconcile.state ?? "unreported")}`);
+      for (const failure of reconcile.failures ?? []) {
+        const next = typeof failure.nextRetryAt === "string" ? ` nextRetryAt=${failure.nextRetryAt}` : "";
+        const remedy = typeof failure.remedy === "string" ? ` remedy=${failure.remedy}` : "";
+        facts.push(`static alias=${String(failure.alias ?? "unreported")} phase=${String(failure.phase ?? "unreported")} disposition=${String(failure.disposition ?? "unreported")}${next}${remedy}`);
+      }
+    }
     facts.push("serve reachable");
     // A manager service without its own liveness lease is a contradicted component surface, not a
     // healthy one. It still reports reachability, but cannot claim the required lease holder.
-    return { name: "manager", verdict: lease ? "serving" : "not-serving", facts };
+    const reconcileFailed = reconcile?.state === "failed";
+    return { name: "manager", verdict: lease && !reconcileFailed ? "serving" : "not-serving", facts };
   } catch (e) {
-    facts.push("phase not reported by this manager build");
+    facts.push("static reconciliation not reported by this manager build");
     // A no-responder service rail or an absent manager registry is definitive no-service evidence.
     // The lease and PID answer whether that missing service belongs to an extant component (not
     // serving) or an absent one; any other failed probe remains a refusal.

implementations/manager/smoke/fixtures/manager-reconcile-redrive.mutations.json

@@ -0,0 +1,54 @@
+{
+  "suite": "implementations/manager/smoke/manager-reconcile-redrive.smoke.ts",
+  "guard": "startup static reconciliation reports a failed sweep and re-drives the same durable exact terminal in process",
+  "command": "pnpm smoke:manager-reconcile-redrive",
+  "proveWith": "node scripts/mutation-proof.mjs --config implementations/manager/smoke/fixtures/manager-reconcile-redrive.mutations.json",
+  "why": [
+    "The suite starts a real authenticated nats-server through serverConfig and an in-process Manager, then plants three durable orphan lifecycle rows.",
+    "The first mutation changes only the failed aggregate branch. The earliest named cell observes the completed first sweep after all three shipped terminal paths ran, so it reddens because failure became success-shaped reporting rather than because an earlier row short-circuited.",
+    "The second mutation removes only the retry scheduling call after the real middle-row terminal failed. Earlier aggregate, durable hold, and status cells still run; the named re-drive cell is the first one that requires a second exact terminal entry.",
+    "The shutdown mutations remove the accepted-sweep drain, the serial stop fence, and the start-task join. The real broker control gates the first terminal and holds service registration after startup has passed its shutdown fence, so the start-task join has its own deterministic observation before the registration gate is released."
+  ],
+  "mutations": [
+    {
+      "name": "report a failed reconciliation sweep with the clean-success aggregate",
+      "file": "implementations/manager/src/manager.ts",
+      "find": "        console.error(`! static reconcile completed: ${sweep.attempted} attempted, ${sweep.succeeded} succeeded, ${sweep.failed} failed; failed=${failed.join(\",\")}`);",
+      "replace": "        console.error(`✓ static reconcile completed: ${sweep.attempted} attempted, ${sweep.attempted} succeeded, 0 failed`);",
+      "expectRed": "the first sweep is non-success and names the failed alias, phase, and disposition",
+      "cell": "the first sweep is non-success and names the failed alias, phase, and disposition"
+    },
+    {
+      "name": "do not schedule an in-process re-drive after the exact terminal fails",
+      "file": "implementations/manager/src/manager.ts",
+      "find": "          if (this.staticReconcileSweepsInFlight > 0) item.disposition = \"retry-scheduled\";\n          else this.scheduleStaticReconcileRetry(key, item);",
+      "replace": "          item.disposition = \"retry-exhausted\";\n          item.remedy = \"restart this manager for a fresh per-process retry budget\";",
+      "expectRed": "the same manager process re-drives the failed alias to retired",
+      "cell": "the same manager process re-drives the failed alias to retired"
+    },
+    {
+      "name": "stop returns while accepted startup reconciliation work is still running",
+      "file": "implementations/manager/src/manager.ts",
+      "find": "    await this.awaitStaticReconcileDrain();\n    await starting?.catch(() => {});",
+      "replace": "",
+      "expectRed": "stop waits for an accepted startup reconciliation terminal",
+      "cell": "stop waits for an accepted startup reconciliation terminal"
+    },
+    {
+      "name": "a shutdown sweep starts the next planned alias after the accepted terminal drains",
+      "file": "implementations/manager/src/manager.ts",
+      "find": "        for (const row of terminalRows) {\n          if (this.staticReconcileStopping) break;\n          sweep.attempted++;",
+      "replace": "        for (const row of terminalRows) {\n          sweep.attempted++;",
+      "expectRed": "the shutdown fence prevents the serial sweep from starting a later terminal",
+      "cell": "the shutdown fence prevents the serial sweep from starting a later terminal"
+    },
+    {
+      "name": "stop does not join startup after draining reconciliation",
+      "file": "implementations/manager/src/manager.ts",
+      "find": "    await this.awaitStaticReconcileDrain();\n    await starting?.catch(() => {});",
+      "replace": "    await this.awaitStaticReconcileDrain();",
+      "expectRed": "stop joins startup service registration admitted before shutdown",
+      "cell": "stop joins startup service registration admitted before shutdown"
+    }
+  ]
+}