diff --git a/.changeset/down-leaves-agents.md b/.changeset/down-leaves-agents.md new file mode 100644 index 000000000..452645c07 --- /dev/null +++ b/.changeset/down-leaves-agents.md @@ -0,0 +1,9 @@ +--- +"@cotal-ai/cli": minor +"@cotal-ai/manager": minor +"@cotal-ai/connector-core": minor +--- + +Bare `cotal down` and `Manager.stop()` leave managed agents running. The previous reap is `cotal down --with-agents` / `stop({ withAgents: true })`. Spare down always signals; listing seats is honesty, not a refuse-to-signal. Listing an unreachable manager must not prune the mesh registry. `--with-agents --dry-run` prints the seats that would be reaped. `--with-agents` that cannot list seats still stops the stack, reaps none, and exits non-zero. A plain `stop()` of a pty seat calls `release()` on the concrete handle: Linux custodial pty closes the unix socket and leaves the child; in-process node-pty (`LegacyPtyRuntime`, used off Linux and as the M1 residual on Linux) throws, because dropping the master would kill the child. That refusal is measured by injecting `LegacyPtyRuntime` on Linux; it does not exercise the macOS or Windows node-pty backend. tmux/cmux/orca/herdr retain no fd, so they spare without a release. `AgentHandle` still has no `close`/`unref`/`release` in core, and the detached set is still not readable, so `stop({ withAgents: true })` still cannot reap what a plain `stop()` spared. `cotal down` returns after signalling; the only `process.exit(0)` in the CLI composition root is the EPIPE guard. The manager daemon's SIGTERM handler does `process.exit(0)` after a successful spare `stop()`. A failed `stop()` sets `exitCode` and falls through, so that path can still stall. Tracked in #1377. + +Refs #964. A PTY child may still die when the manager process exits and the PTY master closes. A successor manager on the same root may still adopt leftover seats; that path is not claimed here. `cotal down` is the listing surface; a failed `cotal up` teardown SIGTERMs the manager without a seat snapshot. An older manager whose `stop()` still reaps will still reap on SIGTERM. diff --git a/README.md b/README.md index ece548b09..019b59f85 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Setup gets your machine ready and **starts nothing**. Then: cotal up --detach # start the mesh cotal spawn # put your agent on it and talk to it (Ctrl-C to leave) cotal web # watch it in the browser -cotal down # stop everything +cotal down # stop the stack; managed agents stay running unless --with-agents ``` One agent, on a real mesh, that you can talk to. Add a second and they can see each other, which diff --git a/bin/smoke/attach-auth-root.smoke.ts b/bin/smoke/attach-auth-root.smoke.ts index d8d43f482..da4b9bc72 100644 --- a/bin/smoke/attach-auth-root.smoke.ts +++ b/bin/smoke/attach-auth-root.smoke.ts @@ -437,7 +437,9 @@ try { // manager stop that hangs must not be able to keep that from happening. Measured once: a leaked // `nats-server` from this rig outlived its run by half an hour, and the reaper attributes a leak // like that to whichever suite was running. - await Promise.race([mgr?.stop().catch(() => {}) ?? Promise.resolve(), sleep(10_000)]); + // Reap the supervised seat. A plain stop leaves the PTY child, and node-pty's waitpid worker + // then holds this process open after the banner (CI shards 0/2 hit the 60m timeout that way). + await Promise.race([mgr?.stop({ withAgents: true }).catch(() => {}) ?? Promise.resolve(), sleep(10_000)]); console.log("attach-auth-root: manager-stop-returned"); await Promise.all(kids.map((k) => { k.kill("SIGKILL"); return awaitExit(k); })); releaseBroker?.(); @@ -446,3 +448,4 @@ try { // process past a green banner until the shard hour cap. process.exit(exitCode); } +process.exit(fail ? 1 : 0); diff --git a/bin/smoke/ci-suites.d/bf212a862b795c37219f6afe1f04968b12781aeb8fd85a30ad993dbab9dbbe20.txt b/bin/smoke/ci-suites.d/bf212a862b795c37219f6afe1f04968b12781aeb8fd85a30ad993dbab9dbbe20.txt new file mode 100644 index 000000000..edbe63404 --- /dev/null +++ b/bin/smoke/ci-suites.d/bf212a862b795c37219f6afe1f04968b12781aeb8fd85a30ad993dbab9dbbe20.txt @@ -0,0 +1 @@ +smoke:manager-stop-spare-guard diff --git a/bin/smoke/ci-suites.d/ccfff544787b57b47dc11dc2d6821baf713d0e1750da3417d53fbe344f7c1b02.txt b/bin/smoke/ci-suites.d/ccfff544787b57b47dc11dc2d6821baf713d0e1750da3417d53fbe344f7c1b02.txt new file mode 100644 index 000000000..fcbbfeb3e --- /dev/null +++ b/bin/smoke/ci-suites.d/ccfff544787b57b47dc11dc2d6821baf713d0e1750da3417d53fbe344f7c1b02.txt @@ -0,0 +1 @@ +smoke:down-partial-reap diff --git a/bin/smoke/fixtures/manager-stop-spare.planted.ts b/bin/smoke/fixtures/manager-stop-spare.planted.ts new file mode 100644 index 000000000..3851bca21 --- /dev/null +++ b/bin/smoke/fixtures/manager-stop-spare.planted.ts @@ -0,0 +1,13 @@ +// Planted positive control for smoke:manager-stop-spare-guard. Not imported. +// A live-PTY Manager smoke that spare-stops: the regex must see this, or a +// zero hit count in the real tree would mean nothing. +// Still compiled: `bin/tsconfig.smoke.json` typechecks every `**/*.ts` under bin/, +// including fixtures nothing imports. tsx never sees this file, so a green +// guard run is not a compile proof. Use the workspace package, not a relative +// path that would resolve to the nonexistent `bin/src/manager.js`. +import { Manager } from "@cotal-ai/manager"; + +const manager = new Manager({ space: "planted", runtime: "pty", workspaceRoot: "/tmp/planted" }); +await manager.start(); +await manager.startAgent({ name: "leak", agent: "seatcon" }); +await manager?.stop().catch(() => {}); diff --git a/bin/smoke/flag-inventory.smoke.ts b/bin/smoke/flag-inventory.smoke.ts index 60950d2d7..ee62c3681 100644 --- a/bin/smoke/flag-inventory.smoke.ts +++ b/bin/smoke/flag-inventory.smoke.ts @@ -53,7 +53,9 @@ const GOLDEN: Record`). - down: { flags: ["dry-run:boolean", "file:string:f", "preserve-state:boolean", "run:string", "space:string", "store-dir:string"], positionals: true }, + // `--with-agents` (2026-09, #964): bare whole-stack reap of managed agents. Default `cotal down` + // leaves those seats running; this flag is the previous always-reap. + down: { flags: ["dry-run:boolean", "file:string:f", "preserve-state:boolean", "run:string", "space:string", "store-dir:string", "with-agents:boolean"], positionals: true }, backup: { flags: ["only:string", "store-dir:string"], positionals: true }, // `meshes` gained the registry-maintenance verbs (2026-08): `add --server … [--root] // [--mode]` registers a mesh this machine did NOT start, `rm …` drops records. `--force` diff --git a/bin/smoke/lang-spawn-live.smoke.ts b/bin/smoke/lang-spawn-live.smoke.ts index ae186f051..b7f0e556f 100644 --- a/bin/smoke/lang-spawn-live.smoke.ts +++ b/bin/smoke/lang-spawn-live.smoke.ts @@ -540,7 +540,7 @@ log("got", v.estimate);`, } rc = fail === 0 ? 0 : 1; } finally { - try { await mgr?.stop(); } catch { /* teardown */ } + try { await mgr?.stop({ withAgents: true }); } catch { /* teardown */ } for (const k of kids) { try { k.kill("SIGKILL"); } catch { /* gone */ } } rmSync(home, { recursive: true, force: true }); rmSync(workspaceRoot, { recursive: true, force: true }); diff --git a/bin/smoke/lang-supervise-live.smoke.ts b/bin/smoke/lang-supervise-live.smoke.ts index 5ec258ba7..64131553f 100644 --- a/bin/smoke/lang-supervise-live.smoke.ts +++ b/bin/smoke/lang-supervise-live.smoke.ts @@ -267,7 +267,7 @@ try { } rc = fail === 0 ? 0 : 1; } finally { - try { await mgr?.stop(); } catch { /* teardown */ } + try { await mgr?.stop({ withAgents: true }); } catch { /* teardown */ } for (const k of kids) { try { k.kill("SIGKILL"); } catch { /* gone */ } } rmSync(home, { recursive: true, force: true }); rmSync(workspaceRoot, { recursive: true, force: true }); diff --git a/bin/smoke/manager-stop-reaps-agents.smoke.ts b/bin/smoke/manager-stop-reaps-agents.smoke.ts index 63b12f721..f886271c5 100644 --- a/bin/smoke/manager-stop-reaps-agents.smoke.ts +++ b/bin/smoke/manager-stop-reaps-agents.smoke.ts @@ -1,55 +1,37 @@ /** - * HONEST REPRODUCTION of #964: a stack stop reaps every managed agent, and there is no way to - * opt out - pnpm smoke:manager-stop-reap - * - * The incident: one stop signal to the stack took six live seats with it, several holding - * uncommitted work, and the logs read as a deliberate teardown. The mechanism is - * `Manager.stop()`: its normal active path unconditionally calls `teardownManagedAgents()` - * (implementations/manager/src/manager.ts, the `maintenanceState === "active"` arm), which - * hard-stops every managed seat and deprovisions its footprint. Bare `cotal down` stops the - * manager and therefore drives exactly this path. There is no flag, mode, or argument that - * stops the stack and leaves the seats running. + * Regression for #964: a stack stop leaves managed agents running unless `--with-agents`. + * Run: pnpm smoke:manager-stop-reap * - * This suite drives the SHIPPED owner of that behavior - a real `Manager` over a real authed - * broker with a real co-located delivery daemon (a direct `deliver` run, never `up`; on an auth - * mesh the deprovision path verify-evicts through it), a real managed seat spawned through the - * real CLI spawn command - and pins today's defective semantics as explicitly-labeled - * "#964 unfixed:" expectations that are GREEN today. + * The incident: one stop signal to the stack took six live seats with it. `Manager.stop()` on + * the normal active path now detaches those seats (no handle.stop, no deprovision). The old + * reap is `stop({ withAgents: true })`, which `cotal down --with-agents` drives by stopping + * each seat and then signalling the manager. * - * THE CONTRACT WITH THE FUTURE FIX (read this before touching the labeled cells): the accepted - * direction for #964 is a three-mode `down` (bare = spare agents, `--with-agents` = today, - * `--preserve-state` = capture-and-restore). When that lands, the "#964 unfixed:" cells below - * MUST NOT stay silently green: - * - if `Manager.stop()` itself learns a sparing default, they go RED and the fix PR flips - * them into assertions that the seat SURVIVES a bare stack stop; - * - if instead `stop()` grows an explicit mode parameter and bare `down` passes the sparing - * choice, the fix PR must RELABEL these cells as the destructive mode's explicit spelling - * (`stop({withAgents: true})` or equivalent) and add the sparing path as new green cells. - * Either way this file changes in the fix PR; a fix that leaves it untouched is incomplete. + * This suite drives a real `Manager` over a real authed broker with a real co-located delivery + * daemon (a direct `deliver` run, never `up`), and a real managed seat spawned through the + * real CLI spawn command. * * What runs here: - * DEFECT phase: manager up, one live managed seat (its child process writes a pidfile, so - * liveness is measured, not inferred), then a plain `mgr.stop()`. The seat's process is dead - * and its minted creds file is gone, with no refusal and no warning. - * CONTROL phase: a fresh manager over the same root, a second seat, and a DELIBERATE per-seat - * stop through the real CLI (`cotal stop --name`). The terminal observables are the SAME - * (process dead, creds gone) - which is the issue's "indistinguishable from deliberate - * teardown" claim made concrete: on disk, a mass reap looks exactly like an operator despawn. + * SPARE phase: manager up, one live managed seat (pidfile), then a plain `mgr.stop()`. + * The seat's process stays alive and its minted creds file stays. The managed table is empty. + * REAP phase: a FRESH workspace root (a successor on the SAME root would run + * `reconcileStaticLifecycles` and terminalize seat A's durable slot; that successor + * hazard is named and out of this PR). A second seat, `mgr.stop({ withAgents: true })`. + * The process is dead and the creds file is gone. * * NAMED GAPS (deliberate, not oversights): * - The CLI `down` surface itself is not driven here: this host must never run `cotal down` - * (or `up`), including against a throwaway root - the #964 architecture review restricts - * local validation to shipped handlers and in-process manager seams, which is what this is. - * - The preservation arm (`stopRetainedAgentsOnExit`) is not driven: it is only reachable - * through preservation state no shipped public path sets in this rig, and hand-poking - * private state would prove nothing about the shipped owner. + * (or `up`). Flag refusals for `--with-agents` live in the hermetic down-target smoke. + * - The preservation arm (`stopRetainedAgentsOnExit`) is not driven. * - The broker-side footprint (dm_/dlv_ durables, ACL row) is not asserted; the on-disk creds * file is the asserted deprovision observable. + * - A PTY child may still die when the manager *process* exits and the PTY master closes. + * This suite keeps the Manager object after a sparing stop so that in-process GC is not + * what is being measured. * - * Throwaway everything: own authed nats-server on an OS-assigned free port (ONE space - each - * space reserves a 4 GiB artifact store on the broker's tmpfs store dir), sandboxed COTAL_HOME, - * scratch workspace root, kills only PIDs it spawned or that its own children wrote to pidfiles. - * No live stack is touched, no `cotal up`/`down` anywhere. Needs nats-server on PATH. + * Throwaway everything: own authed nats-server on an OS-assigned free port (ONE space), + * sandboxed COTAL_HOME, scratch workspace root, kills only PIDs it spawned or that its own + * children wrote to pidfiles. No live stack is touched. Needs nats-server on PATH. * Run: pnpm smoke:manager-stop-reap */ import { spawn as spawnProc, type ChildProcess } from "node:child_process"; @@ -90,9 +72,9 @@ const must = (name: string, cond: boolean, extra?: unknown) => { pass++; console.log(` ✓ ${name}`); }; -/** Cells that ran, before the count cell itself: 6 must + 8 ok. A throw lands in the catch as a +/** Cells that ran, before the count cell itself: 6 must + 11 ok. A throw lands in the catch as a * counted failure, so a partial run can never print the OK banner. */ -const EXPECTED_CELLS = 15; +const EXPECTED_CELLS = 18; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const until = async (cond: () => boolean, ms: number): Promise => { const end = Date.now() + ms; @@ -178,9 +160,9 @@ const cmd = (name: string): Command => { }; /** Spawn a seat through the REAL CLI spawn command, in-process, standing in the workspace root * (the command resolves auth from the cwd root, exactly as an operator's shell would). */ -const spawnSeat = async (name: string): Promise => { +const spawnSeat = async (name: string, cwd: string = root): Promise => { const prev = process.cwd(); - process.chdir(root); + process.chdir(cwd); try { await cmd("spawn").run(parseCommandArgs(cmd("spawn"), ["probe", "--detach", "--agent", "seatcon", "--space", SPACE, "--name", name])); } finally { @@ -188,22 +170,6 @@ const spawnSeat = async (name: string): Promise => { } }; const kids: ChildProcess[] = []; -/** The deliberate despawn control runs the REAL binary as a subprocess: the in-process `stop` - * command exits the whole process on failure, which would kill the suite. */ -const cliStop = (name: string): Promise<{ code: number | null; out: string }> => - new Promise((resolve, reject) => { - const p = spawnProc(TSX, [BIN, "stop", "--name", name, "--space", SPACE], { - cwd: root, - env: { ...cleanEnv, COTAL_HOME: home, XDG_CONFIG_HOME: join(home, "xdg"), COTAL_SKIP_CONNECTOR_SEED: "1", NO_COLOR: "1" }, - stdio: ["ignore", "pipe", "pipe"], - }); - kids.push(p); - let out = ""; - p.stdout?.on("data", (b: Buffer) => void (out += b.toString())); - p.stderr?.on("data", (b: Buffer) => void (out += b.toString())); - p.on("error", reject); - p.on("exit", (code) => resolve({ code, out })); - }); let releaseBroker: (() => void) | undefined; let brokerProc: ChildProcess | undefined; @@ -213,7 +179,7 @@ const daemonSink = { out: "", exited: false }; let mgr1: InstanceType | undefined; let mgr2: InstanceType | undefined; -console.log("\n── #964: a stack stop reaps every managed agent ─────────────\n"); +console.log("\n── #964: a stack stop leaves managed agents running ─────────────\n"); try { console.log("manager-stop-reap: first-line"); // ── the rig: one authed broker, one provisioned space ───────────────────────────────────────── @@ -268,7 +234,7 @@ try { daemonSink.out.slice(-500), ); - // ── DEFECT phase: one live seat, then a plain stack stop ────────────────────────────────────── + // ── SPARE phase: one live seat, then a plain stack stop ────────────────────────────────────── mgr1 = new Manager({ space: SPACE, servers: SERVER, runtime: "pty", workspaceRoot: root }); await mgr1.start(); await spawnSeat("seatA"); @@ -278,7 +244,7 @@ try { const credsA = optsByName.get("seatA")?.creds; ok("seat A's minted creds file exists on disk (the footprint a deprovision removes)", credsA !== undefined && existsSync(credsA), { credsA }); await sleep(1500); - ok("instrument: seat A is still live after a settle window (its death below is stop-caused, not self-inflicted)", pidA !== undefined && alive(pidA)); + ok("instrument: seat A is still live after a settle window (its death below would be stop-caused, not self-inflicted)", pidA !== undefined && alive(pidA)); let stopError: string | undefined; console.log("manager-stop-reap: before-mgr1-stop"); @@ -288,36 +254,39 @@ try { stopError = (e as Error).message; } console.log("manager-stop-reap: manager-stop-returned"); - mgr1 = undefined; - ok( - "#964 unfixed: a plain Manager.stop() - the stack-stop path bare `cotal down` drives - proceeds against a live managed seat with no refusal and no sparing mode", - stopError === undefined, - { stopError }, - ); - const deadA = pidA !== undefined && (await until(() => !alive(pidA), 10_000)); - ok("#964 unfixed: the stack stop hard-stopped the live managed seat (its process is dead)", deadA, { pidA }); - const credsAGone = credsA !== undefined && (await until(() => !existsSync(credsA), 10_000)); - ok("#964 unfixed: the reaped seat was deprovisioned (its minted creds file is gone)", credsAGone, { credsA }); + ok("a plain Manager.stop() proceeds against a live managed seat", stopError === undefined, { stopError }); + ok("default stop empties the managed table", (mgr1 as unknown as { agents: Map }).agents.size === 0); + ok("#964: a plain Manager.stop() leaves the live managed seat running", pidA !== undefined && alive(pidA), { pidA }); + ok("#964: a spared seat is not deprovisioned (its minted creds file remains)", credsA !== undefined && existsSync(credsA), { credsA }); - // ── CONTROL phase: a fresh manager, a second seat, a DELIBERATE despawn ─────────────────────── - mgr2 = new Manager({ space: SPACE, servers: SERVER, runtime: "pty", workspaceRoot: root }); + // ── REAP phase: a different root, so this manager is not seat A's successor ────────────────── + const rootB = join(base, "rootB"); + mkdirSync(join(rootB, ".cotal", "agents"), { recursive: true }); + writeFileSync(join(rootB, ".cotal", "agents", "probe.md"), "---\nname: probe\nrole: worker\nsubscribe: []\n---\nA supervised seat that exists to be reaped.\n"); + saveSpaceAuth(authDir(rootB), auth); + recordMesh({ space: SPACE, server: SERVER, root: rootB, mode: "auth", ts: new Date().toISOString() }); + mgr2 = new Manager({ space: SPACE, servers: SERVER, runtime: "pty", workspaceRoot: rootB }); await mgr2.start(); - must("a fresh manager starts over the same root (control phase)", true); - await spawnSeat("seatB"); + must("a fresh manager starts over a different root (reap phase is not a same-root successor)", true); + ok("#964: seat A is still live after a manager starts on a different root (same-root successor reconcile is out of this PR)", pidA !== undefined && alive(pidA), { pidA }); + await spawnSeat("seatB", rootB); const pidB = await until(() => pidOf(join(pidDir, "seatB.pid")) !== undefined, 15_000) ? pidOf(join(pidDir, "seatB.pid"))! : undefined; must("seat B is live under the fresh manager", pidB !== undefined && alive(pidB) && optsByName.has("seatB"), { pidB }); const credsB = optsByName.get("seatB")?.creds; - ok("seat B's creds file exists before the deliberate despawn", credsB !== undefined && existsSync(credsB), { credsB }); - const stopped = await cliStop("seatB"); - ok("control: a deliberate per-seat stop through the real CLI succeeds", stopped.code === 0, { code: stopped.code, out: stopped.out.slice(-400) }); + ok("seat B's creds file exists before the explicit reap", credsB !== undefined && existsSync(credsB), { credsB }); + let reapError: string | undefined; + try { + await mgr2.stop({ withAgents: true }); + } catch (e) { + reapError = (e as Error).message; + } + mgr2 = undefined; + ok("stop({ withAgents: true }) proceeds", reapError === undefined, { reapError }); const bReaped = pidB !== undefined && (await until(() => !alive(pidB), 10_000)) && credsB !== undefined && (await until(() => !existsSync(credsB), 10_000)); - ok( - "control: the deliberately-despawned seat shows the SAME terminal observables (process dead, creds gone) - on disk a mass reap is indistinguishable from an operator despawn", - bReaped, - { pidB, credsB }, - ); + ok("#964: stop({ withAgents: true }) reaps the seat (process dead, creds gone)", bReaped, { pidB, credsB }); + ok("#964: the spared seat A is still live after the other manager's reap", pidA !== undefined && alive(pidA), { pidA }); ok("every cell ran (silently skipped cells must not read as green)", pass + fail === EXPECTED_CELLS - 1, { pass, fail, expected: EXPECTED_CELLS - 1 }); } catch (e) { diff --git a/bin/smoke/manager-stop-spare-guard.smoke.ts b/bin/smoke/manager-stop-spare-guard.smoke.ts new file mode 100644 index 000000000..69730e64a --- /dev/null +++ b/bin/smoke/manager-stop-spare-guard.smoke.ts @@ -0,0 +1,245 @@ +/** + * After #964, a plain `Manager.stop()` detaches managed seats. A smoke that spawns a real + * PTY child and then spare-stops leaks that child unless something else reaps it. + * + * This guard reddens a live-PTY smoke teardown that still calls `.stop()` without + * `{ withAgents: true }`. Suites that ASSERT the spare path are named below; they are + * the coverage, not the leak. A planted fixture proves the search can still see the + * banned form: a grep that finds nothing and a grep that cannot find anything print + * the same zero. + * + * livePty() inclusion (disclosed, measured at 7b88fb3bfe82b3642037d885df03fc8b50573f37): + * a file is EXAMINED only when it has a literal `new Manager(` plus one of seven spawn + * shapes (`.startAgent(`, `spawnSeat(`, `cmd("spawn")`, `spawnTool.run(`, `MeshHandler`, + * `.startByName(`, `invokeService("manager", "spawn"`), is not `kind: "fake"`, and has + * `runtime: "pty"` or `pty.spawn(`. That admitted 26 of 74 `new Manager(` smokes and + * skipped 32 live-PTY files, 29 of which carry the banned spare-stop form. The spawn-shape + * gate is kept; silence is not. Every skipped live-PTY file that still carries a candidate + * stop is named in the suite output AND must appear on the frozen #1343 inventory below. + * A new skipped candidate-stop path reds. A stale inventory path reds so the list can only + * shrink. Truncating the examined set below the measured floor reds. The receiver-name list + * below is a separate, disclosed boundary (#1310). + * + * Run: pnpm smoke:manager-stop-spare-guard + */ +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; + +const ROOT = join(import.meta.dirname, "..", ".."); +const SELF = "bin/smoke/manager-stop-spare-guard.smoke.ts"; +const SKIP = new Set(["node_modules", "dist", ".git", ".changeset", "coverage", "build", ".internal"]); +const DEREGISTER = "implementations/manager/smoke/manager-deregister.smoke.ts"; +/** Measured examined live-PTY population at 7b88fb3b. A later shrink of this set reds. */ +const EXAMINED_FLOOR = 26; +/** Measured `new Manager(` population at 7b88fb3b. A later walk truncation reds. */ +const NEW_MANAGER_FLOOR = 74; + +/** Suites whose job is to prove the spare path. Each is named, not found by silence. */ +const SPARE_COVERAGE = new Set([ + "bin/smoke/manager-stop-reaps-agents.smoke.ts", + "implementations/manager/smoke/start-model-preflight.smoke.ts", + "implementations/manager/smoke/preserve-state.smoke.ts", + "implementations/manager/smoke/lease-loss-keeps-serving.smoke.ts", +]); + +/** + * Tracked deferral of skipped live-PTY files that still carry a candidate spare-stop. + * Owned by issue #1343. These are not SPARE_COVERAGE and are not safe: livePty() never + * examines them, so a spare-stop there stays green unless this inventory notices growth. + * Do not add new paths here. Audit the file, admit it through the spawn-shape gate, or + * change its teardown to `{ withAgents: true }` and delete the stale entry. + */ +const FROZEN_DROPPED = [ + "bin/smoke/manager-two-root-renewal.smoke.ts", + "bin/smoke/persona-announce.smoke.ts", + "bin/smoke/readiness-window-live.smoke.ts", + "bin/smoke/run-host-live.smoke.ts", + "bin/smoke/spawn-detach-live.smoke.ts", + "extensions/connector-hermes/smoke/boot-requirement.smoke.ts", + "implementations/cli/smoke/scatter-pinned-probe.smoke.ts", + "implementations/manager/smoke/boot-self-heal-gate.smoke.ts", + "implementations/manager/smoke/cli-on-instance-live.smoke.ts", + "implementations/manager/smoke/describe-split-duplicate-effect.smoke.ts", + "implementations/manager/smoke/goal-sibling-race.smoke.ts", + "implementations/manager/smoke/instrument-instance-pin.smoke.ts", + "implementations/manager/smoke/manager-coexist.smoke.ts", + "implementations/manager/smoke/manager-deregister.smoke.ts", + "implementations/manager/smoke/manager-on-route.smoke.ts", + "implementations/manager/smoke/manager-restart-fence.smoke.ts", + "implementations/manager/smoke/manager-restart-live.smoke.ts", + "implementations/manager/smoke/manager-scatter.smoke.ts", + "implementations/manager/smoke/manager-service-invoke.smoke.ts", + "implementations/manager/smoke/manager-service-ops.smoke.ts", + "implementations/manager/smoke/manager-service.smoke.ts", + "implementations/manager/smoke/queue-win-distribution.smoke.ts", + "implementations/manager/smoke/resolve-rtt-probe.smoke.ts", + "implementations/manager/smoke/seat-input-live.smoke.ts", + "implementations/manager/smoke/session-ledger-family.smoke.ts", + "implementations/manager/smoke/sibling-mint-fence.smoke.ts", + "implementations/manager/smoke/spawn-action-auth.smoke.ts", + "implementations/manager/smoke/spawn-action.smoke.ts", + "implementations/manager/smoke/turn-relay-auth.smoke.ts", +] as const; + +let pass = 0, fail = 0; +const check = (name: string, condition: boolean, detail?: unknown) => { + if (condition) { pass++; console.log(` ✓ ${name}`); } + else { fail++; console.log(` ✗ FAIL: ${name}`, detail ?? ""); } +}; + +function smokeSources(dir: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir)) { + if (SKIP.has(entry)) continue; + const p = join(dir, entry); + const st = statSync(p); + if (st.isDirectory()) smokeSources(p, out); + else if (p.endsWith(".ts") && /(^|\/)smoke\//.test(relative(ROOT, p)) && !relative(ROOT, p).startsWith("bin/smoke/fixtures/") && relative(ROOT, p) !== SELF) out.push(p); + } + return out; +} + +/** A Manager.stop / mgr.stop / m1?.stop call whose argument list does not pass withAgents: true. + * Identifier class is the manager-shaped names smokes actually use. delivery/broker/ep stops + * are not this hazard. Conventional names only: `supervisor`, `boss`, `managerB`, `mBoot`, and + * a destructured or helper-returned handle are outside this regex. */ +const SPARE_STOP = /(?:await\s+)?(?:manager|mgr|mgr[0-9A-Z]|m[0-9]|adopting|openMgr|hung|first|next|live|corpse|booting|replacement)\??\.stop\(\s*(?:\{\s*(?!.*withAgents\s*:\s*true)[^}]*\}\s*)?\)/g; + +const SPAWN_SHAPES = [ + /\.startAgent\s*\(/, + /\bspawnSeat\s*\(/, + /cmd\(\s*"spawn"\s*\)/, + /\bspawnTool\.run\s*\(/, + /\bMeshHandler\b/, + /\.startByName\s*\(/, + /invokeService\(\s*"manager"\s*,\s*"spawn"/, +] as const; + +function hasNewManager(text: string): boolean { + return /\bnew\s+Manager\s*\(/.test(text); +} +function hasEnumeratedSpawn(text: string): boolean { + return SPAWN_SHAPES.some((re) => re.test(text)); +} +function isFakeKind(text: string): boolean { + return /\bkind:\s*"fake"/.test(text); +} +function hasLivePtyRuntime(text: string): boolean { + return /\bruntime:\s*"pty"/.test(text) || /\bpty\.spawn\s*\(/.test(text); +} +function livePty(text: string): boolean { + return hasNewManager(text) && hasEnumeratedSpawn(text) && !isFakeKind(text) && hasLivePtyRuntime(text); +} +function livePtySkippedBySpawnShape(text: string): boolean { + return hasNewManager(text) && hasLivePtyRuntime(text) && !isFakeKind(text) && !hasEnumeratedSpawn(text); +} + +function spareStops(text: string): string[] { + return [...text.matchAll(SPARE_STOP)].map((m) => m[0].replace(/\s+/g, " ").trim()); +} + +const files = smokeSources(ROOT); +check("the walk finds a non-trivial population of smoke sources", files.length >= 50, `found ${files.length}`); + +const planted = join(ROOT, "bin", "smoke", "fixtures", "manager-stop-spare.planted.ts"); +const plantedText = readFileSync(planted, "utf8"); +check("the planted control looks like a live-PTY Manager smoke", livePty(plantedText), plantedText.slice(0, 120)); +check("the planted control carries a spare stop and the regex sees it", spareStops(plantedText).length > 0, spareStops(plantedText)); + +const hits: string[] = []; +const examined: string[] = []; +const newManagerFiles: string[] = []; +const droppedWithStop: string[] = []; +for (const f of files) { + const rel = relative(ROOT, f); + const text = readFileSync(f, "utf8"); + if (hasNewManager(text)) newManagerFiles.push(rel); + if (livePty(text)) examined.push(rel); + if (livePtySkippedBySpawnShape(text) && spareStops(text).length && !SPARE_COVERAGE.has(rel)) { + droppedWithStop.push(`${rel}: ${spareStops(text).join(" | ")}`); + } + if (SPARE_COVERAGE.has(rel)) continue; + if (!livePty(text)) continue; + const found = spareStops(text); + if (found.length) hits.push(`${rel}: ${found.join(" | ")}`); +} + +const droppedPaths = droppedWithStop.map((row) => { + const cut = row.indexOf(": "); + return cut < 0 ? row : row.slice(0, cut); +}); +const droppedPathSet = new Set(droppedPaths); +const frozenSet = new Set(FROZEN_DROPPED); +const newDropped = droppedPaths.filter((p) => !frozenSet.has(p)); +const staleFrozen = FROZEN_DROPPED.filter((p) => !droppedPathSet.has(p)); +const frozenOnSpare = FROZEN_DROPPED.filter((p) => SPARE_COVERAGE.has(p)); + +console.log(`examined live-PTY Manager smokes: ${examined.length} of ${newManagerFiles.length} new Manager( files (floor ${EXAMINED_FLOOR})`); +console.log(`dropped-with-candidate-stop (live PTY, not fake, spawn-shape miss): ${droppedWithStop.length}`); +for (const row of droppedWithStop) console.log(` skip ${row}`); + +check( + `the walk still sees at least ${NEW_MANAGER_FLOOR} smokes that construct new Manager(`, + newManagerFiles.length >= NEW_MANAGER_FLOOR, + `found ${newManagerFiles.length}`, +); +check( + `examined live-PTY Manager smokes stay at or above the measured floor (${EXAMINED_FLOOR})`, + examined.length >= EXAMINED_FLOOR, + `examined ${examined.length}`, +); +check(`no live-PTY smoke spare-stops a Manager (${examined.length} files examined)`, hits.length === 0, hits); +check( + "frozen #1343 dropped inventory is not SPARE_COVERAGE and is not a safe list", + frozenOnSpare.length === 0, + frozenOnSpare, +); +check( + "a new dropped-with-candidate-stop path is not on the frozen #1343 inventory", + newDropped.length === 0, + newDropped, +); +check( + "frozen #1343 dropped inventory has no stale paths", + staleFrozen.length === 0, + staleFrozen, +); + +const deregPath = join(ROOT, DEREGISTER); +const deregText = readFileSync(deregPath, "utf8"); +const deregDropped = droppedWithStop.some((row) => row.startsWith(`${DEREGISTER}:`)); +check("manager-deregister is present in the walk", files.some((f) => relative(ROOT, f) === DEREGISTER), DEREGISTER); +check("manager-deregister constructs a live-PTY Manager (not kind fake)", hasNewManager(deregText) && hasLivePtyRuntime(deregText) && !isFakeKind(deregText), DEREGISTER); +check("manager-deregister carries a banned spare-stop form", spareStops(deregText).length > 0, spareStops(deregText)); +check("manager-deregister is not hidden on SPARE_COVERAGE", !SPARE_COVERAGE.has(DEREGISTER), DEREGISTER); +check( + "manager-deregister is examined or named as a dropped-with-candidate-stop exclusion", + livePty(deregText) || deregDropped, + { livePty: livePty(deregText), named: deregDropped, stops: spareStops(deregText) }, +); + +for (const named of SPARE_COVERAGE) { + check(`spare-coverage suite is present: ${named}`, files.some((f) => relative(ROOT, f) === named), named); +} + +const managerSrc = readFileSync(join(ROOT, "implementations/manager/src/manager.ts"), "utf8"); +const detach = managerSrc.match(/private detachManagedAgents\(\): void \{([\s\S]*?)\n \}/)?.[1] ?? ""; +check("detachManagedAgents is present in manager.ts", detach.includes("this.detached.push(a)")); +check("detachManagedAgents does not optional-chain close", !/close\?\./.test(detach), detach); +check( + "detachManagedAgents calls release() on pty without optional chaining", + /handle\.kind === "pty"/.test(detach) && /release\.call\(a\.handle\)/.test(detach) && !/release\?\./.test(detach), + detach, +); +const custodialSrc = readFileSync(join(ROOT, "implementations/manager/src/runtime/custodial-pty.ts"), "utf8"); +check( + "CustodialPtyRuntime exposes release() that closes the seat socket", + /release:\s*\(\)\s*=>\s*\{\s*seat\.close\(\);/.test(custodialSrc), +); +const legacySrc = readFileSync(join(ROOT, "implementations/manager/src/runtime/pty.ts"), "utf8"); +check( + "LegacyPtyRuntime release() throws rather than no-op or kill", + /release:\s*\(\)\s*=>\s*\{[\s\S]*?cannot spare agent[\s\S]*?in-process node-pty cannot release/.test(legacySrc), +); + +console.log(`\nMANAGER-STOP-SPARE-GUARD ${fail === 0 ? "OK" : "FAILED"} (${pass} passed, ${fail} failed)`); +if (fail) process.exitCode = 1; diff --git a/bin/smoke/mutations/attach-auth-root.json b/bin/smoke/mutations/attach-auth-root.json index ea373574e..e8ae50ed7 100644 --- a/bin/smoke/mutations/attach-auth-root.json +++ b/bin/smoke/mutations/attach-auth-root.json @@ -135,7 +135,7 @@ "replace": "cwd,\n env: { ...cleanEnv, COTAL_HOME: home, XDG_CONFIG_HOME: join(home, \"xdg\"), NO_COLOR: \"1\" },\n stdio:", "expectRed": "the attached-session subprocess does not reconcile connector payloads before exercising attach", "cell": "the attached-session subprocess does not reconcile connector payloads before exercising attach", - "note": "The attach subprocess must reach the attach path directly. Without the skip, the command auto-reconciles connector payloads first; its seed output can consume the bounded window and hide both the attached banner and the divergent-root report." + "note": "The attach subprocess must reach the attach path directly. Without the skip, a source-checkout CLI refuses to reconcile the operator-global seed store (or, on a released install, writes seed payloads first). Either output can consume the bounded window and hide both the attached banner and the divergent-root report." } ] } diff --git a/bin/smoke/mutations/lease-loss-keeps-serving.json b/bin/smoke/mutations/lease-loss-keeps-serving.json index 6f159fd26..9440707f0 100644 --- a/bin/smoke/mutations/lease-loss-keeps-serving.json +++ b/bin/smoke/mutations/lease-loss-keeps-serving.json @@ -22,10 +22,10 @@ "thousand identical lines in `manager.log`, which is how the motivating incident was read: by", "searching a log for the one line that mattered.", "", - "MUTATION 4 IS THE CONTROL AND ALSO A REAL INVARIANT. It makes the ORDINARY shutdown path leave", + "MUTATION 4 IS THE CONTROL AND ALSO A REAL INVARIANT. It makes the EXPLICIT reap path leave", "the children running. Every lease cell stays green and only the control cell reddens, which proves", - "the `stops` counter fires (so the zeros in the graded cells are earned) and that `cotal down` /", - "Ctrl-C stay destructive." + "the `stops` counter fires (so the zeros in the graded cells are earned) and that `cotal down", + "--with-agents` still reaps. Ordinary `stop()` leaving children is the #964 default, not a defect." ], "mutations": [ { @@ -56,13 +56,13 @@ "note": "Correct decision, unreadable log. A careless edit rather than a malicious one." }, { - "name": "CONTROL and invariant: the ordinary shutdown path leaves every agent running", + "name": "CONTROL and invariant: the explicit reap path leaves every agent running", "file": "implementations/manager/src/manager.ts", - "find": " await this.teardownManagedAgents();", - "replace": " this.agents.clear();", - "expectRed": "CONTROL: the ordinary stop path stops the child", - "cell": "CONTROL: the ordinary stop path stops the child (instrument fires)", - "note": "Doubles as the positive control for the `stops` counter and as the guard on the opposite failure: an operator who asked for a shutdown must get one. Anchored on the bare call with no trailing comment, which is unique in the file." + "find": " if (opts?.withAgents === true) await this.teardownManagedAgents();", + "replace": " if (opts?.withAgents === true) this.agents.clear();", + "expectRed": "CONTROL: the explicit reap path stops the child", + "cell": "CONTROL: the explicit reap path stops the child (instrument fires)", + "note": "Doubles as the positive control for the `stops` counter and as the guard on the opposite failure: an operator who asked for `--with-agents` must get a reap. Anchored on the withAgents true call, which is unique in the file." } ] } diff --git a/bin/smoke/mutations/manager-stop-spare-guard.mutations.json b/bin/smoke/mutations/manager-stop-spare-guard.mutations.json new file mode 100644 index 000000000..ccf79a833 --- /dev/null +++ b/bin/smoke/mutations/manager-stop-spare-guard.mutations.json @@ -0,0 +1,43 @@ +{ + "suite": "bin/smoke/manager-stop-spare-guard.smoke.ts", + "guard": "skipped live-PTY spare-stops are a frozen #1343 inventory: a new dropped path reds, a stale frozen path reds, and SPARE_COVERAGE is not a hiding place", + "command": "pnpm smoke:manager-stop-spare-guard", + "completionMarker": "MANAGER-STOP-SPARE-GUARD", + "grades": "tool", + "proveWith": "node scripts/mutation-proof.mjs --config bin/smoke/mutations/manager-stop-spare-guard.mutations.json", + "why": [ + "Issue #1343: printing 29 skipped rows still exits green when a NEW skipped live-PTY file", + "carries a candidate stop. new Manager( count can rise, the examined floor still holds, and", + "admitted hits stay zero. The inventory is the gate. N1 introduces a new dropped candidate", + "in a currently skipped live-PTY file that is not on FROZEN_DROPPED. N2 removes the last", + "candidate stop from a frozen path and leaves the inventory entry, so the baseline can only shrink.", + "", + "NO BUILD STEP: the suite reads smoke sources off disk." + ], + "mutations": [ + { + "name": "N1 a new skipped live-PTY file gains a candidate spare-stop", + "file": "implementations/manager/smoke/persona-show-auth.smoke.ts", + "find": "const manager = new Manager({ space: \"persona-show-auth\", runtime: \"pty\", workspaceRoot: root });\n", + "replace": "const manager = new Manager({ space: \"persona-show-auth\", runtime: \"pty\", workspaceRoot: root });\nawait manager.stop();\n", + "expectRed": "a new dropped-with-candidate-stop path is not on the frozen #1343 inventory", + "note": "persona-show-auth already constructs a live-PTY Manager with no enumerated spawn shape. Adding a spare stop makes it a new dropped candidate. It is not on FROZEN_DROPPED and is not SPARE_COVERAGE." + }, + { + "name": "N2 a frozen dropped path loses its candidate stop and stays on the list", + "file": "bin/smoke/persona-announce.smoke.ts", + "find": " await mgr.stop().catch(() => {});", + "replace": " await mgr.stop({ withAgents: true }).catch(() => {});", + "expectRed": "frozen #1343 dropped inventory has no stale paths", + "note": "persona-announce is on the frozen #1343 inventory. Repairing its only matching spare-stop without deleting the inventory row must red so the baseline can only shrink." + }, + { + "name": "N3 pty spare optional-chains close instead of calling release", + "file": "implementations/manager/src/manager.ts", + "find": " if (a.handle.kind === \"pty\") {\n const release = (a.handle as { release?: () => void }).release;\n if (typeof release !== \"function\") {\n throw new Error(\n `runtime \"${a.handle.kind}\" cannot spare agent \"${a.name}\": handle has no release()`,\n );\n }\n release.call(a.handle);\n }", + "replace": " (a.handle as Partial<{ close(): void }>).close?.();", + "expectRed": "detachManagedAgents does not optional-chain close", + "note": "Restores the silent no-op. The source-scan cells on detachManagedAgents redden; the inventory cells stay green." + } + ] +} diff --git a/bin/smoke/persona-agent.smoke.ts b/bin/smoke/persona-agent.smoke.ts index 08e34b57b..aa99a6307 100644 --- a/bin/smoke/persona-agent.smoke.ts +++ b/bin/smoke/persona-agent.smoke.ts @@ -333,7 +333,7 @@ try { console.log(`\npersona-agent 869 reachability smoke: ${pass} passed, ${fail} failed`); process.exit(fail === 0 ? 0 : 1); } finally { - try { await mgr?.stop(); } catch { /* teardown best-effort */ } + try { await mgr?.stop({ withAgents: true }); } catch { /* teardown best-effort */ } for (const k of kids) k.kill("SIGKILL"); releaseBroker?.(); } diff --git a/docs/cli.md b/docs/cli.md index 8c2593021..9bba8b803 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -251,6 +251,7 @@ and the manager's log both name the credential and this repair. ```bash cotal down +cotal down --with-agents cotal down --preserve-state [--store-dir ] cotal down manager [delivery auth web nats ...] cotal down web [--space ] @@ -264,10 +265,25 @@ cotal down -f | --run [--dry-run] | `--space ` | current mesh | With components: the mesh whose target-addressed components (e.g. `web`) to stop | | `--dry-run` | off | Print the manifest teardown or selected components, mutate nothing | | `--preserve-state` | off | Bare whole stack only: fence the manager, retain principals and durable state, stop and prove the stack down, then publish `ready` | +| `--with-agents` | off | Bare whole stack only: also stop every managed agent (the previous default) | | `--store-dir ` | `.cotal/nats` | With `--preserve-state`: the actual store path (required for a custom store) | -Bare `cotal down` stops the whole local stack in dependency order. Positional component names stop -only those self-registered local processes; for example, `cotal down manager` leaves delivery and +Bare `cotal down` stops the whole local stack in dependency order and leaves managed agents +running as unmanaged OS processes. If the manager answers `ps`, it prints their names, pids, and +working directories, and points at `cotal down --with-agents` to take them with the stack. If the +manager cannot be asked, it still signals the stack and says leftovers may remain. +`--with-agents` in that case still stops the stack, reaps no seats, and exits non-zero +naming that the agents are still running unmanaged. A PTY child +may still die when the manager process exits. A later manager on the same root may still take +leftover seats. This listing is `cotal down`. A failed `cotal up` teardown SIGTERMs the manager +without listing leftover seats. An older manager whose `stop()` still reaps will still reap on +SIGTERM. `--with-agents` +cannot be combined with `--preserve-state`, component names, `--space`, `--file`, or `--run`. +`--with-agents --dry-run` prints the seats that would be reaped and mutates nothing. `--with-agents` +waits for each seat's runtime to prove exit before signalling the manager. If only some seats stop, +the command prints the stopped seats and the still-running seats separately, with each failure, and +exits non-zero. Positional +component names stop only those self-registered local processes; for example, `cotal down manager` leaves delivery and the broker running, and `cotal down web` is available when the web extension is installed. A component that starts target-resolved (the web dashboard) is stopped the same way: `cotal down web` resolves the mesh the same way as `cotal web` (registry current mesh first, `--space` to name one), so diff --git a/docs/define-a-team.md b/docs/define-a-team.md index cd4777bca..3b45d1d7e 100644 --- a/docs/define-a-team.md +++ b/docs/define-a-team.md @@ -50,9 +50,12 @@ cotal topology view -f cotal.yaml # validate + render the access graph (no cotal up -f cotal.yaml # broker + channels + agents, all fresh cotal ps --space main # see the agents the manager booted cotal web --space main # ...or watch it in the browser -cotal down # stop the whole mesh +cotal down # stop the stack; managed agents stay running unless --with-agents ``` +A PTY child may still die when the manager process exits. A later manager on the same +root may still take leftover seats. + The manifest introduces no access model of its own; the three verbs are the same ones Cotal uses everywhere: `subscribe` (auto-listen at boot, and implicitly may read), `allowSubscribe` (**read**; defaults to `subscribe`, must be a superset of it), and diff --git a/docs/embedding.md b/docs/embedding.md index bb44d16fd..69308c5a9 100644 --- a/docs/embedding.md +++ b/docs/embedding.md @@ -194,7 +194,7 @@ pre-spawn, and the forever wait. A host composes that lifecycle itself around `M ```ts import { Manager } from "@cotal-ai/manager"; const mgr = new Manager({ space, servers: brokerUrl, workspaceRoot }); -await mgr.start(); // then wire your own SIGINT/SIGTERM -> mgr.stop() +await mgr.start(); // then wire SIGINT/SIGTERM -> mgr.stop() (Linux pty leaves seats; pass { withAgents: true } to reap; in-process node-pty cannot spare) ``` Unlike delivery, the manager is **not** a pre-minted-scoped-cred daemon (auth-service is also a diff --git a/docs/getting-started.md b/docs/getting-started.md index 9b06f8a80..0ac1450dd 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -92,7 +92,7 @@ whole loop is three commands: ```bash cotal up --detach # start the mesh + delivery daemon + manager (JWT-authed by default) cotal spawn # launch your agent here and talk to it (Ctrl-C to leave) -cotal down # stop everything +cotal down # stop the stack; managed agents stay running unless --with-agents ``` Open the browser dashboard with `cotal web` (it ships with `cotal-ai`, seeded automatically). Add the @@ -169,7 +169,7 @@ cotal spawn # your agent (edit .cotal/agents/default.md cotal spawn david # a guided expert, needs `cotal setup --demo` first (also sven, me) cotal console --space main # live mesh view in the terminal (TUI) cotal web --space main # open the browser dashboard -cotal down # stop the background mesh, delivery daemon, and manager +cotal down # stop the stack; managed agents stay running unless --with-agents ``` Feedback flows through your agent too: tell it "send feedback: ..." and it reports it for @@ -201,7 +201,10 @@ npx cotal-ai up --detach # start the mesh + delivery daemon + manager step fails, so an agent or a CI job can check the result (add `--demo` for the guided team). `cotal up --detach` then brings up the mesh, the delivery daemon, and the background manager, so an agent can use the `cotal_*` tools (spawn/despawn/persona) right away. `cotal down` -stops the background processes. +stops those background processes and leaves managed agents running. Pass +`cotal down --with-agents` to take the agents with the stack. A PTY child may +still die when the manager process exits. A later manager on the same root may +still take leftover seats. ## Troubleshooting diff --git a/docs/run-a-mesh.md b/docs/run-a-mesh.md index 70b5006e6..ad0215248 100644 --- a/docs/run-a-mesh.md +++ b/docs/run-a-mesh.md @@ -9,7 +9,10 @@ operator-only maintenance verbs. Every command's full flag set is in the ## The stack -`cotal up` brings up the whole local stack and bare `cotal down` stops it: +`cotal up` brings up the whole local stack and bare `cotal down` stops it. Managed agents +stay running as unmanaged OS processes; pass `--with-agents` to take them with the stack. +A PTY child may still die when the manager process exits. A later manager on the same root +may still take leftover seats. The stack is: - **Broker**: a local `nats-server` (logs to `.cotal/nats.log`). - **Delivery daemon**: the durable backstop, auth mode only @@ -90,7 +93,10 @@ detection behavior. Stop one part without tearing down the mesh by naming its registered component: `cotal down manager`, `cotal down delivery`, or `cotal down web`. Component names from installed extensions -join the same surface; `cotal down` with no names retains whole-stack behavior. +join the same surface; `cotal down` with no names retains whole-stack behavior and leaves +managed agents running as unmanaged OS processes. `cotal down --with-agents` is the previous +reap. A PTY child may still die when the manager process exits. A later manager on the same +root may still take leftover seats. ## Remote supervised agents diff --git a/docs/setup-internals.md b/docs/setup-internals.md index bcd89e860..662a78eb4 100644 --- a/docs/setup-internals.md +++ b/docs/setup-internals.md @@ -101,7 +101,9 @@ with the log path and a non-zero exit. It still launches nothing. The control pl old-manager preflight → **delivery daemon** (auth mode only) → **manager**, via `ensureControlPlane` ([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)). The detached -processes, all stopped by `cotal down`: +processes, all stopped by `cotal down` (managed agents stay running unless +`cotal down --with-agents`). A PTY child may still die when the manager process +exits. A later manager on the same root may still take leftover seats. With no explicit `--server`, `cotal up` auto-selects a free local port when the default broker address is already held by another root or an unrecorded broker; an explicit `--server` remains @@ -135,7 +137,8 @@ separate install. Start it with `cotal web`; it records Safari may need plain `127.0.0.1`). `webUp()` probes the port for setup's status card. All recorded local processes self-register `local-process` descriptors. Bare `cotal down` resolves -the full set and stops it in dependency order; `cotal down manager` (or another component name) +the full set and stops it in dependency order, leaving managed agents running; `cotal down +--with-agents` also stops those seats. `cotal down manager` (or another component name) selects only that descriptor. Installed extensions cache their contributed registry keys, so the base CLI does not hardcode optional package pidfiles. diff --git a/extensions/connector-core/src/docs-bundle.generated.ts b/extensions/connector-core/src/docs-bundle.generated.ts index 288fdc48f..3f8e3003e 100644 --- a/extensions/connector-core/src/docs-bundle.generated.ts +++ b/extensions/connector-core/src/docs-bundle.generated.ts @@ -19,7 +19,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "Quickstart", "kind": "Start here (informative)", "summary": "Paste this into any coding agent (Claude Code, OpenCode, Cursor, Codex) and it will do the whole page for you:", - "body": "# Quickstart\n\n> **Start here** (informative) · **For:** everyone · **Next:** [Connect Claude](connect-claude.md) · [Define a team](define-a-team.md) · [Watch a mesh](watch-a-mesh.md)\n\n## Set up with your agent\n\nPaste this into any coding agent (Claude Code, OpenCode, Cursor, Codex) and it will do\nthe whole page for you:\n\n```text wrap\nRead https://docs.cotal.ai/prompt.md, then set up Cotal on this machine: install it, start a local mesh, and put an agent on it.\n```\n\nTo do it by hand instead, keep reading: this page takes you from install to a running\nlocal mesh with an agent on it, in a few minutes.\n\n## Start a local mesh\n\n```bash\ncurl -fsSL https://get.cotal.ai | sh\n```\n\nThat is the whole install on a machine with nothing on it. The script finds a Node 22+ or\ninstalls a verified one of its own, puts `cotal` in `~/.local/bin`, adds that to your PATH,\nand runs guided setup. It never uses sudo and writes nothing outside your home directory.\nRead it first at [get.cotal.ai](https://get.cotal.ai); it is served as plain text for that\nreason. Useful flags: `--dry-run` to see the plan, `--no-modify-path` to leave your shell rc\nalone, `--no-setup` to install only. Pass them through the pipe as\n`| sh -s -- --dry-run`.\n\nOn Windows, or if you already run Node 22+ and would rather use npm directly:\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH\ncotal setup # one-time, configure-only; launches nothing\n```\n\nCotal runs natively on Windows, but the installer above is a POSIX shell script, so npm is the\nroute there (or run the installer under WSL).\n\nBare `cotal` prints help; `cotal setup` runs the guided setup. `npx cotal-ai setup` works\ntoo and offers to install the global `cotal` at the end. Declining is fine: the hints stay\n`npx cotal-ai …`, and the background processes `cotal up` starts invoke their own resolved\npath rather than a global `cotal`.\n\nRequirements:\n\n- Node 22 or newer. The installer handles this for you; it downloads an official Node build\n and checks it against the SHA-256 sums published beside it on nodejs.org.\n- A glibc system. Cotal's terminal layer ships prebuilt native binaries for glibc only, so\n musl distributions (Alpine) are not supported yet and the installer refuses them rather\n than leaving you with an install that cannot start.\n- A `nats-server` binary, version 2.12 or newer (the control surface uses its message\n schedules and per-message TTLs, and fails loud at connect against an older broker). The\n one that ships with the package is new enough; if you already have `nats-server` on your\n PATH, Cotal uses that instead, so make sure it is 2.12+.\n\nTo uninstall: `rm -rf ~/.local/share/cotal ~/.local/bin/cotal` removes what the installer wrote,\n`rm -rf ~/.cotal` removes your meshes, agents and credentials, and the `# cotal` block it added\nto your shell rc can be deleted.\n\n## First run\n\n`cotal setup` is configure-only: it prepares your machine and starts nothing. The first\ntime, it walks you through:\n\n1. **Checks.** Verifies Node 22+ and locates a `nats-server` (the bundled one, or your\n own on PATH). Located only; nothing starts.\n2. **Picks connectors.** Choose from every installed connector; detected ones are pre-selected.\n The list and its hints come from the connectors themselves, never from a name the CLI knows: a\n connector that declares its own setup runs it (Claude installs its plugin that way), a connector\n missing a required executable is named, and the rest are ready at spawn.\n3. **Seeds one agent.** The generic `default` persona that a bare `cotal spawn` launches;\n edit it to taste. It joins no channels at boot, but may join, create, read, and post to\n channels on demand. `cotal setup --demo` additionally seeds a guided team to talk to:\n **david** (the engineer, how Cotal works), **sven** (the guide, what to build), and\n **me** (the session you drive). Every file setup writes is announced with a\n `→ wrote …` line.\n\n Re-running setup after an upgrade repairs the earlier untouched `default` template that had an\n empty post ACL. The repair requires a byte-for-byte match, so any persona you edited is left\n unchanged.\n4. **Nothing to install for the dashboard.** `@cotal-ai/web` ships inside `cotal-ai` and is\n seeded automatically on first run (like the built-in connectors), so `cotal web` works out\n of the box and tracks your CLI version on upgrade.\n5. **Offers a global install.** Run via `npx` with no global `cotal`, it offers to\n `npm i -g cotal-ai` so you can just type `cotal`.\n\nWhen it finishes, nothing is running yet; it prints the commands to start things. The\nwhole loop is three commands:\n\n```bash\ncotal up --detach # start the mesh + delivery daemon + manager (JWT-authed by default)\ncotal spawn # launch your agent here and talk to it (Ctrl-C to leave)\ncotal down # stop everything\n```\n\nOpen the browser dashboard with `cotal web` (it ships with `cotal-ai`, seeded automatically). Add the\nguided expert team with `cotal setup --demo`, then `cotal spawn\ndavid` (or `sven`, or `me`). Watch the mesh in this terminal anytime with `cotal console`:\n\n![The cotal console: a live roster of agents and their all-activity feed in a terminal TUI](../assets/quickstart.gif)\n\n`cotal up` is JWT-authed by default (sender authenticity plus per-agent ACLs), starts the\nserver-side [delivery daemon](delivery-daemon.md) as the durable backstop, and starts a\ndetached manager so `cotal spawn --detach` / `cotal_spawn` work right after.\n`cotal up --open` gives you an open, loopback-only, live-only mesh instead (no auth, no\ndaemon) for quick local experiments.\n\nFor a mesh where **people sign in** instead of handing out creds files, start it with\n`cotal up --user-auth --idp `: each human runs `cotal login --idp ` once,\nthe operator grants their agents with `cotal actor grant --sub ` (a full\ngrant by default: all channels, may spawn; narrow it with `--allow-subscribe` /\n`--allow-publish` / `--scope`), and every connect is authorized live against that grant\n(revoke and it's gone). See [identity & auth](identity-and-auth.md).\n\nIf a step fails, setup offers to hand you to an interactive Claude session that has the\nfailure context. Type `/exit` to return, and it retries.\n\n## The primitives\n\nThe vocabulary behind those three commands, which every other page builds on:\n\n| Primitive | What it is |\n|---|---|\n| **Space** | One collaboration, isolated from other spaces. Your mesh is a space. |\n| **Endpoint** | Any software on the mesh: a long-lived connection with presence. |\n| **Agent node** | An endpoint with identity, role, and tags (what `cotal spawn` launches). |\n| **Channel** | A named topic participants broadcast on and subscribe to. |\n| **Direct message** | A message addressed to one peer. |\n| **Presence** | The live roster: who is here, `idle` / `waiting` / `working` / `offline`. |\n| **History** | Recent messages a late joiner replays. |\n\nDelivery comes in three modes: **multicast** (to a channel), **unicast** (to one peer),\nand **anycast** (to any one holder of a role). More in\n[Presence & delivery](presence-and-delivery.md); the full term list is in the\n[glossary](glossary.md).\n\n## After the first run\n\nEvery later `cotal setup` prints a **read-only status card**:\n\n```\ncotal · status\n✓ NATS nats://127.0.0.1:4222\n✓ plugin installed\n○ mesh down · start: cotal up --detach\n○ web down · start: cotal web\n○ manager not running · start: cotal up, or: cotal supervise\n```\n\nIt probes the current folder (the mesh, the browser dashboard, and the manager behind\n`cotal_spawn` / `despawn` / `persona`) and shows the exact start command for anything\nthat is down. It starts nothing itself.\n\nThe dashboard ships with `cotal-ai` and is seeded automatically on first run. It runs at\n`http://cotal.localhost:7799` once you start it with `cotal web` (works in Chrome,\nFirefox, and Edge; on Safari use `http://127.0.0.1:7799`). If a seeded copy is damaged,\n`cotal ext seed --repair` restores it.\n\nYou drive Cotal through an agent: spawn one and talk to it. It has the tools to message\npeers, spawn teammates, and send feedback (the full surface is the\n[MCP tool catalog](mcp-tools.md)). The same things are available as commands:\n\n```bash\ncotal up --detach # start the mesh + delivery daemon + manager\ncotal status # detailed setup, process, registry, and live mesh status\ncotal spawn # your agent (edit .cotal/agents/default.md)\ncotal spawn david # a guided expert, needs `cotal setup --demo` first (also sven, me)\ncotal console --space main # live mesh view in the terminal (TUI)\ncotal web --space main # open the browser dashboard\ncotal down # stop the background mesh, delivery daemon, and manager\n```\n\nFeedback flows through your agent too: tell it \"send feedback: ...\" and it reports it for\nyou (built-in `cotal_feedback`), or run `cotal feedback \"\"`.\n\n`cotal setup --demo` adds the guided team (david, sven, me) to an already-configured machine.\n`cotal setup --full` redoes the whole guided flow (team included), for example to repair\nsomething. Defaults (persona, harness, model selection) and day-to-day operation are in\n[Run a mesh](run-a-mesh.md); every command and flag is in the [CLI reference](cli.md).\n\n## Launch a team from a manifest\n\nThe guided flow gives you one agent (or the expert team with `--demo`). To run a **specific\nteam** (your own channels, agents, and who may read and post where), describe it once in a\n`cotal.yaml` and launch it with `cotal up -f cotal.yaml`. The walkthrough is\n**[Define a team](define-a-team.md)**; the file format is the\n[manifest reference](manifest.md).\n\n## Non-interactive setup\n\nA coding agent can set Cotal up for you with two non-interactive commands:\n\n```bash\nnpx cotal-ai setup --yes # configure: install the plugin + seed one agent (launches nothing)\nnpx cotal-ai up --detach # start the mesh + delivery daemon + manager\n```\n\n`setup --yes` accepts every default with no prompts and exits non-zero with the log path if a\nstep fails, so an agent or a CI job can check the result (add `--demo` for the guided team).\n`cotal up --detach` then brings up the mesh, the delivery daemon, and the background manager,\nso an agent can use the `cotal_*` tools (spawn/despawn/persona) right away. `cotal down`\nstops the background processes.\n\n## Troubleshooting\n\n- The full log is at `.cotal/setup.log` (and `.cotal/nats.log` for the server).\n- Re-running setup is safe. It reuses a running web and keeps your files.\n- Set `COTAL_SKIP_ASSIST=1` to disable the Claude handoff offer on failures.\n\nNext: put your own agent on the mesh ([Connectors](connectors.md) compares them:\n[Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n[Hermes](connect-hermes.md) · [pi](connect-pi.md)), declare a team\n([Define a team](define-a-team.md)), or watch it live ([Watch a mesh](watch-a-mesh.md)).\n" + "body": "# Quickstart\n\n> **Start here** (informative) · **For:** everyone · **Next:** [Connect Claude](connect-claude.md) · [Define a team](define-a-team.md) · [Watch a mesh](watch-a-mesh.md)\n\n## Set up with your agent\n\nPaste this into any coding agent (Claude Code, OpenCode, Cursor, Codex) and it will do\nthe whole page for you:\n\n```text wrap\nRead https://docs.cotal.ai/prompt.md, then set up Cotal on this machine: install it, start a local mesh, and put an agent on it.\n```\n\nTo do it by hand instead, keep reading: this page takes you from install to a running\nlocal mesh with an agent on it, in a few minutes.\n\n## Start a local mesh\n\n```bash\ncurl -fsSL https://get.cotal.ai | sh\n```\n\nThat is the whole install on a machine with nothing on it. The script finds a Node 22+ or\ninstalls a verified one of its own, puts `cotal` in `~/.local/bin`, adds that to your PATH,\nand runs guided setup. It never uses sudo and writes nothing outside your home directory.\nRead it first at [get.cotal.ai](https://get.cotal.ai); it is served as plain text for that\nreason. Useful flags: `--dry-run` to see the plan, `--no-modify-path` to leave your shell rc\nalone, `--no-setup` to install only. Pass them through the pipe as\n`| sh -s -- --dry-run`.\n\nOn Windows, or if you already run Node 22+ and would rather use npm directly:\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH\ncotal setup # one-time, configure-only; launches nothing\n```\n\nCotal runs natively on Windows, but the installer above is a POSIX shell script, so npm is the\nroute there (or run the installer under WSL).\n\nBare `cotal` prints help; `cotal setup` runs the guided setup. `npx cotal-ai setup` works\ntoo and offers to install the global `cotal` at the end. Declining is fine: the hints stay\n`npx cotal-ai …`, and the background processes `cotal up` starts invoke their own resolved\npath rather than a global `cotal`.\n\nRequirements:\n\n- Node 22 or newer. The installer handles this for you; it downloads an official Node build\n and checks it against the SHA-256 sums published beside it on nodejs.org.\n- A glibc system. Cotal's terminal layer ships prebuilt native binaries for glibc only, so\n musl distributions (Alpine) are not supported yet and the installer refuses them rather\n than leaving you with an install that cannot start.\n- A `nats-server` binary, version 2.12 or newer (the control surface uses its message\n schedules and per-message TTLs, and fails loud at connect against an older broker). The\n one that ships with the package is new enough; if you already have `nats-server` on your\n PATH, Cotal uses that instead, so make sure it is 2.12+.\n\nTo uninstall: `rm -rf ~/.local/share/cotal ~/.local/bin/cotal` removes what the installer wrote,\n`rm -rf ~/.cotal` removes your meshes, agents and credentials, and the `# cotal` block it added\nto your shell rc can be deleted.\n\n## First run\n\n`cotal setup` is configure-only: it prepares your machine and starts nothing. The first\ntime, it walks you through:\n\n1. **Checks.** Verifies Node 22+ and locates a `nats-server` (the bundled one, or your\n own on PATH). Located only; nothing starts.\n2. **Picks connectors.** Choose from every installed connector; detected ones are pre-selected.\n The list and its hints come from the connectors themselves, never from a name the CLI knows: a\n connector that declares its own setup runs it (Claude installs its plugin that way), a connector\n missing a required executable is named, and the rest are ready at spawn.\n3. **Seeds one agent.** The generic `default` persona that a bare `cotal spawn` launches;\n edit it to taste. It joins no channels at boot, but may join, create, read, and post to\n channels on demand. `cotal setup --demo` additionally seeds a guided team to talk to:\n **david** (the engineer, how Cotal works), **sven** (the guide, what to build), and\n **me** (the session you drive). Every file setup writes is announced with a\n `→ wrote …` line.\n\n Re-running setup after an upgrade repairs the earlier untouched `default` template that had an\n empty post ACL. The repair requires a byte-for-byte match, so any persona you edited is left\n unchanged.\n4. **Nothing to install for the dashboard.** `@cotal-ai/web` ships inside `cotal-ai` and is\n seeded automatically on first run (like the built-in connectors), so `cotal web` works out\n of the box and tracks your CLI version on upgrade.\n5. **Offers a global install.** Run via `npx` with no global `cotal`, it offers to\n `npm i -g cotal-ai` so you can just type `cotal`.\n\nWhen it finishes, nothing is running yet; it prints the commands to start things. The\nwhole loop is three commands:\n\n```bash\ncotal up --detach # start the mesh + delivery daemon + manager (JWT-authed by default)\ncotal spawn # launch your agent here and talk to it (Ctrl-C to leave)\ncotal down # stop the stack; managed agents stay running unless --with-agents\n```\n\nOpen the browser dashboard with `cotal web` (it ships with `cotal-ai`, seeded automatically). Add the\nguided expert team with `cotal setup --demo`, then `cotal spawn\ndavid` (or `sven`, or `me`). Watch the mesh in this terminal anytime with `cotal console`:\n\n![The cotal console: a live roster of agents and their all-activity feed in a terminal TUI](../assets/quickstart.gif)\n\n`cotal up` is JWT-authed by default (sender authenticity plus per-agent ACLs), starts the\nserver-side [delivery daemon](delivery-daemon.md) as the durable backstop, and starts a\ndetached manager so `cotal spawn --detach` / `cotal_spawn` work right after.\n`cotal up --open` gives you an open, loopback-only, live-only mesh instead (no auth, no\ndaemon) for quick local experiments.\n\nFor a mesh where **people sign in** instead of handing out creds files, start it with\n`cotal up --user-auth --idp `: each human runs `cotal login --idp ` once,\nthe operator grants their agents with `cotal actor grant --sub ` (a full\ngrant by default: all channels, may spawn; narrow it with `--allow-subscribe` /\n`--allow-publish` / `--scope`), and every connect is authorized live against that grant\n(revoke and it's gone). See [identity & auth](identity-and-auth.md).\n\nIf a step fails, setup offers to hand you to an interactive Claude session that has the\nfailure context. Type `/exit` to return, and it retries.\n\n## The primitives\n\nThe vocabulary behind those three commands, which every other page builds on:\n\n| Primitive | What it is |\n|---|---|\n| **Space** | One collaboration, isolated from other spaces. Your mesh is a space. |\n| **Endpoint** | Any software on the mesh: a long-lived connection with presence. |\n| **Agent node** | An endpoint with identity, role, and tags (what `cotal spawn` launches). |\n| **Channel** | A named topic participants broadcast on and subscribe to. |\n| **Direct message** | A message addressed to one peer. |\n| **Presence** | The live roster: who is here, `idle` / `waiting` / `working` / `offline`. |\n| **History** | Recent messages a late joiner replays. |\n\nDelivery comes in three modes: **multicast** (to a channel), **unicast** (to one peer),\nand **anycast** (to any one holder of a role). More in\n[Presence & delivery](presence-and-delivery.md); the full term list is in the\n[glossary](glossary.md).\n\n## After the first run\n\nEvery later `cotal setup` prints a **read-only status card**:\n\n```\ncotal · status\n✓ NATS nats://127.0.0.1:4222\n✓ plugin installed\n○ mesh down · start: cotal up --detach\n○ web down · start: cotal web\n○ manager not running · start: cotal up, or: cotal supervise\n```\n\nIt probes the current folder (the mesh, the browser dashboard, and the manager behind\n`cotal_spawn` / `despawn` / `persona`) and shows the exact start command for anything\nthat is down. It starts nothing itself.\n\nThe dashboard ships with `cotal-ai` and is seeded automatically on first run. It runs at\n`http://cotal.localhost:7799` once you start it with `cotal web` (works in Chrome,\nFirefox, and Edge; on Safari use `http://127.0.0.1:7799`). If a seeded copy is damaged,\n`cotal ext seed --repair` restores it.\n\nYou drive Cotal through an agent: spawn one and talk to it. It has the tools to message\npeers, spawn teammates, and send feedback (the full surface is the\n[MCP tool catalog](mcp-tools.md)). The same things are available as commands:\n\n```bash\ncotal up --detach # start the mesh + delivery daemon + manager\ncotal status # detailed setup, process, registry, and live mesh status\ncotal spawn # your agent (edit .cotal/agents/default.md)\ncotal spawn david # a guided expert, needs `cotal setup --demo` first (also sven, me)\ncotal console --space main # live mesh view in the terminal (TUI)\ncotal web --space main # open the browser dashboard\ncotal down # stop the stack; managed agents stay running unless --with-agents\n```\n\nFeedback flows through your agent too: tell it \"send feedback: ...\" and it reports it for\nyou (built-in `cotal_feedback`), or run `cotal feedback \"\"`.\n\n`cotal setup --demo` adds the guided team (david, sven, me) to an already-configured machine.\n`cotal setup --full` redoes the whole guided flow (team included), for example to repair\nsomething. Defaults (persona, harness, model selection) and day-to-day operation are in\n[Run a mesh](run-a-mesh.md); every command and flag is in the [CLI reference](cli.md).\n\n## Launch a team from a manifest\n\nThe guided flow gives you one agent (or the expert team with `--demo`). To run a **specific\nteam** (your own channels, agents, and who may read and post where), describe it once in a\n`cotal.yaml` and launch it with `cotal up -f cotal.yaml`. The walkthrough is\n**[Define a team](define-a-team.md)**; the file format is the\n[manifest reference](manifest.md).\n\n## Non-interactive setup\n\nA coding agent can set Cotal up for you with two non-interactive commands:\n\n```bash\nnpx cotal-ai setup --yes # configure: install the plugin + seed one agent (launches nothing)\nnpx cotal-ai up --detach # start the mesh + delivery daemon + manager\n```\n\n`setup --yes` accepts every default with no prompts and exits non-zero with the log path if a\nstep fails, so an agent or a CI job can check the result (add `--demo` for the guided team).\n`cotal up --detach` then brings up the mesh, the delivery daemon, and the background manager,\nso an agent can use the `cotal_*` tools (spawn/despawn/persona) right away. `cotal down`\nstops those background processes and leaves managed agents running. Pass\n`cotal down --with-agents` to take the agents with the stack. A PTY child may\nstill die when the manager process exits. A later manager on the same root may\nstill take leftover seats.\n\n## Troubleshooting\n\n- The full log is at `.cotal/setup.log` (and `.cotal/nats.log` for the server).\n- Re-running setup is safe. It reuses a running web and keeps your files.\n- Set `COTAL_SKIP_ASSIST=1` to disable the Claude handoff offer on failures.\n\nNext: put your own agent on the mesh ([Connectors](connectors.md) compares them:\n[Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n[Hermes](connect-hermes.md) · [pi](connect-pi.md)), declare a team\n([Define a team](define-a-team.md)), or watch it live ([Watch a mesh](watch-a-mesh.md)).\n" }, { "slug": "architecture", @@ -75,7 +75,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "`cotal` CLI reference", "kind": "Reference: describes the TypeScript reference implementation (the `cotal` CLI), not the wire contract.", "summary": "cotal is the operator command line for the reference implementation: bring a mesh up, mint identities, launch agents, watch what they do, and tear it all down.", - "body": "# `cotal` CLI reference\n\n> **Reference**: describes the TypeScript reference implementation (the `cotal` CLI), not the wire contract. · **For:** operators · **Wire contract:** [SPEC](../SPEC.md)\n\n`cotal` is the operator command line for the reference implementation: bring a mesh up, mint\nidentities, launch agents, watch what they do, and tear it all down. It is a thin client over the\nwire contract: the normative subjects and schemas live in the [SPEC](../SPEC.md); this page is\nlookup material for the commands, not a walkthrough; if you are new, start with\n[Getting started](getting-started.md).\n\n## Running it\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH (needs Node 22+)\ncotal --help # every command, grouped\ncotal --version # cotal-ai version + each installed extension's (also `cotal -v`)\ncotal --help # one command's flags and usage\n```\n\n`npx cotal-ai ` runs it without a global install; in a dev clone, `pnpm cotal `\nruns it through `tsx` with no build step. Bare `cotal` prints help. Every command generates its own\n`--help`, usage, and shell completion from its declared flags.\n\nCommands come from the surfaces the binary composes: the base mesh CLI, the manager\n(`supervise`), and the delivery daemon (`deliver`), plus any operator-installed extensions.\n`cotal ext add ` installs any registry providers a package contributes: commands,\nruntimes, and local process lifecycle descriptors. The `web` dashboard and optional manager\nruntimes ship this way.\n\n## Commands\n\n| Area | Command | Purpose |\n|---|---|---|\n| Set up & lifecycle | [`setup`](#setup) | Guided, configure-only setup (installs, seeds personas; launches nothing) |\n| Set up & lifecycle | [`update`](#update) | Reconcile first-party extensions and check or opt into a coherent CLI upgrade |\n| Set up & lifecycle | [`up`](#up) | Start a local mesh (nats-server + JetStream), or boot a whole manifest with `-f` |\n| Set up & lifecycle | [`down`](#down) | Stop the whole stack, selected registered components, or a manifest deploy |\n| Set up & lifecycle | [`backup`](#backups) | Create an offline full-space or registry-only artifact from a preserved cut |\n| Set up & lifecycle | [`clean`](#clean) | Configurable cleanup: purge history (live), or wipe the local store / identity (stopped) |\n| Set up & lifecycle | [`meshes`](#mesh-registry) | List the running meshes on this machine |\n| Set up & lifecycle | [`use`](#mesh-registry) | Set the default mesh a bare `cotal spawn` joins |\n| Set up & lifecycle | [`status`](#mesh-registry) | Read-only diagnostics for setup, processes, and the selected mesh |\n| Agents & personas | [`spawn`](#spawn) | Launch an agent from a persona (foreground, or `--detach` via the manager) |\n| Agents & personas | [`models`](#models) | List connector model catalogs and variants from the manager |\n| Agents & personas | [`ps`](#managed-seats) | List managed agents and their mesh status |\n| Agents & personas | [`stop`](#managed-seats) | Ask the manager to stop a managed agent |\n| Agents & personas | [`attach`](#managed-seats) | Stream and drive a managed agent's terminal (pty runtime) |\n| Agents & personas | [`input`](#input) | Type one line into a managed agent's terminal without attaching |\n| Agents & personas | [`personas`](#personas) | List, show, edit, create, or remove local personas |\n| Agents & personas | [`supervise`](#supervise) | Run a manager daemon (the agent supervisor / control plane) |\n| Agents & personas | [`runtimes`](#runtimes) | List the agent runtimes the manager can spawn through and whether each is reachable |\n| Agents & personas | [`reconcile-gate`](#reconcile-gate) | Unfreeze an issuance gate left frozen by a crashed restart when the successor cannot boot-heal it (holder gone, complete CONNZ sweep) |\n| Messaging & watching | [`endpoints`](#endpoints) | List every endpoint in the live presence roster, including infrastructure |\n| Messaging & watching | [`describe` / `invoke`](#endpoint-control) | Resolve a v0.4 service's command surface off the wire; invoke one command by name |\n| Messaging & watching | [`send`](#send) | Send one message, then exit: DM a peer, post a channel, or ask a role |\n| Messaging & watching | [`channels`](#channels) | Inspect or set the channel registry |\n| Messaging & watching | [`history`](#history) | Clear retained message history |\n| Messaging & watching | [`console`](#console) | Live protocol view for a space (TUI, or `--plain` line stream) |\n| Messaging & watching | [`web`](#web) | Browser dashboard (installed as the `@cotal-ai/web` extension) |\n| Auth & meshes | [`mint`](#mint) | Mint a creds file for a space (static auth mode) |\n| Auth & meshes | [`login`](#login) | Sign in to a per-user-auth mesh's IdP (once per machine) |\n| Auth & meshes | [`logout`](#login) | Revoke the IdP session and clear the cached login |\n| Auth & meshes | [`actor`](#actor) | Manage a user-auth space's actor ledger (grant / revoke / list) |\n| Auth & meshes | [`doctor`](#doctor) | Credential-health diagnosis and repair (`doctor auth`) |\n| Auth & meshes | [`join`](#join) | Join a space as your own presence (interactive) |\n| Manifest | [`topology`](#manifest-deploys) | Validate and view a mesh manifest's access graph (read-only) |\n| Extensions & misc | [`ext`](#ext) | Install / remove operator CLI extensions |\n| Extensions & misc | [`completion`](#completion) | Print or install shell completion |\n| Extensions & misc | [`feedback`](#feedback) | Send feedback to the Cotal developers |\n| Extensions & misc | [`deliver`](#server-daemons) | Run the server-side Plane-3 delivery daemon |\n| Workflow runs | [`run`](#run) | Operate durable workflow runs: start, resume, list, inspect, answer a checkpoint |\n| Extensions & misc | [`feedback-intake`](#server-daemons) | Run a self-hosted feedback intake server |\n\nThe manifest modes of `up`, `spawn`, and `down` (`-f `) plus `topology` are covered\ntogether under [Manifest deploys](#manifest-deploys).\n\n## setup\n\n```bash\ncotal setup [--full] [--demo] [--yes] [--skills]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--full` | off | Redo the full guided flow (implies `--demo`) |\n| `--demo` | off | Also seed the guided expert team (`david`, `sven`, `me`) |\n| `--yes`, `-y` | off | Non-interactive accept-all (for agents / CI) |\n| `--skills` | off | Reconcile Cotal skills only through installed connector providers, plus `~/.agents/skills`. Refused with `--full` or `--demo`. |\n\nGuided setup is **configure-only**: it checks prerequisites, invokes installed connectors' declared setup providers, and\nseeds persona files, and it launches nothing (no mesh, no web, no manager). First run gets the\nnarrated flow; later runs print a status card. By default it seeds one `default` persona; the\n`david`/`sven`/`me` team is opt-in via `--demo`. `cotal status` points stale Claude skills and\nout-of-date `.agents` skills at `cotal setup --skills`, not unscoped `setup`. See [Getting started](getting-started.md) and, for\nmaintainers, [setup internals](setup-internals.md).\n\nWhen a mesh resolves, setup seeds that mesh's recorded `.cotal/agents` catalog, the same catalog a\nfollowing `cotal spawn` reads. It prints the absolute destination. On a fresh machine with no mesh it\nuses this folder and says why; when several meshes are available and none is selected, it refuses\nrather than choosing a catalog.\n\n## update\n\n```bash\ncotal update [--self] [--space ] [--server ] [--creds ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--self` | off | If a newer release exists, install that exact validated `cotal-ai` version globally and reconcile through the newly installed binary |\n| `--space`, `--server`, `--creds` | resolved mesh | Select the running manager whose continuity state is reported |\n\nWithout `--self`, `update` keeps the installed first-party surfaces coherent with the running\nbinary: it force-reconciles the four built-in connectors, then reinstalls other `@cotal-ai/*`\noperator extensions at the binary's exact version. Each extension runs in an isolated child, so one\nfailure cannot poison later replays. It then checks npm; a newer binary is an informational notice\nwith `cotal update --self` as the next command, not an automatic install.\n\nAfter disk reconciliation, `update` reads the selected running manager. A manager without a\ncustody generation is reported as `legacy`: it cannot preserve its manager-owned PTYs, so the\ncommand says that this is not a hot update and prints `exact`, `fork`, `fresh`, or `drain-only`\nfor every seat. This report sends no stop, preservation-commit, or replacement command.\nIt does not preserve a running PTY on a legacy manager. On Linux a detached custodian\nowns each PTY, so a manager-worker death no longer closes the seat and `status` reports\n`custodied`. Other platforms still spawn in-process and report `legacy`. An incompatible native\n`@lydell/node-pty` or ConPTY ABI break remains an explicit per-seat maintenance cut.\n\nWith `--self`, the selected running manager is reported before any global install. When a newer\nrelease exists, Cotal then installs the exact version it validated, resolves and verifies that\npackage in npm's global root, then launches that binary with the same `--space` / `--server` /\n`--creds` selection to reconcile connectors and first-party extensions to the new generation. An npx\nor dev-clone invocation therefore installs and continues through a separate global copy; it never\nclaims the already-running process changed. If the binary is current, `--self` performs the normal\nlocal reconcile without reinstalling it.\n\nThird-party extensions are listed with their installed version and recorded spec but are not\nauto-updated in v1. Floating third-party updates require `@cotal-ai/*` peer-range validation and are\na future follow-up. A failed connector/extension install, npm metadata check, or requested global\ninstall is reported and makes the command exit nonzero. Independent extension attempts continue so\nthe output includes every failure; an unavailable npm registry does not undo a completed local\nreconcile, but the command still exits nonzero because it could not establish that the install is\ncurrent.\n\n## up\n\n```bash\ncotal up [--detach] [--open] [--space ] [--server ] [--channels ] [--runtime ]\ncotal up --user-auth --idp [--exchange-public-port --exchange-public-url [--exchange-trusted-proxy]]\ncotal up --tls-cert --tls-key # serve broker TLS (both, or neither)\ncotal up --restore [--restore-only registry] [--accept-missing-source]\ncotal up -f [--dry-run] [--runtime ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--server ` | auto (free local port) | Listen URL override |\n| `--host ` | none | Bind host override. With no `--server`, the broker URL is derived from it, so `--host ` alone is enough to make a mesh reachable at that address; a `--host`/`--server` pair naming different addresses is refused. A wildcard bind (`0.0.0.0`, `::`) keeps a dialable loopback URL. Recorded on the mesh and reused by every later manager launch, so a repair or resume keeps remote [`attach`](#managed-seats) working |\n| `--space ` | the folder's name | Space name |\n| `--store-dir ` | none | JetStream store directory |\n| `--channels ` | `.cotal/channels.json` if present | Channel-registry seed file (JSON). An explicit path that is missing is an error |\n| `--restore ` | none | Restore a completed offline backup before exposing the normal listener |\n| `--restore-only registry` | artifact selection | Restore only the registry component |\n| `--accept-missing-source` | off | Explicit disaster consent when the inode-bound preserved source is absent |\n| `--open` | off (auth) | Unauthenticated dev mesh: no JWT, no ACLs |\n| `--user-auth` | off | Per-user auth: people `cotal login`; connects are authorized against the actor ledger |\n| `--idp ` | none | With `--user-auth`: the IdP auth base URL to pin on first enable |\n| `--exchange-public-port ` | none | With `--user-auth`: add the public exchange face on this loopback port, for an HTTPS reverse proxy to forward to |\n| `--exchange-public-url ` | none | With `--exchange-public-port`: advertise the reverse proxy's HTTPS URL in discovery |\n| `--exchange-trusted-proxy` | off | With `--exchange-public-port`: attribute public failure buckets to the last `X-Forwarded-For` hop. Enable only when the listener is reachable solely through a trusted proxy; otherwise the socket address is used |\n| `--detach` | off | Run in the background (stop with `cotal down`) |\n| `--tls-cert ` | none | PEM certificate to serve TLS with. Must be given together with `--tls-key`. Before starting the broker, Cotal checks readability, private-key mode, key/certificate match, the validity window, and host coverage. `nats-server` accepts an expired certificate and leaves the failure to clients, so Cotal performs these checks first. The decision is recorded; a later bare `cotal up` keeps serving TLS |\n| `--tls-key ` | none | PEM private key for `--tls-cert`. Refused if group- or other-readable (tighten to `600`) |\n| `--file `, `-f` | none | Launch a whole mesh from a manifest |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--runtime ` | `pty` (or the manifest's, with `-f`) | Agent runtime for the mesh manager (`pty` built in; others are installed extensions, explicit-only). Resolved + probed before the broker starts; an uninstalled/unreachable runtime fails loud. With `-f`, overrides the manifest's runtime |\n| `--rotate-sys` | off | Rotate the space's system account and re-mint its two `$SYS` creds. Needs a stopped mesh; refused with `--open` |\n\n`cotal up` boots a local nats-server with JetStream and, in auth mode (the default), JWT auth and\nper-agent ACLs; `--detach` records the mesh so `cotal spawn` from any directory can find it. With no\n`--server`, it auto-selects a free port if the default address is taken; an explicit `--server`\nstays fail-loud on collision. `--detach` also brings up the control plane (delivery daemon in auth\nmode, then the manager). The `-f` form is a [manifest deploy](#manifest-deploys); see\n[Run a mesh](run-a-mesh.md).\n\n`--user-auth --idp ` starts the space's auth service alongside the broker: the NATS\nauth callout plus its capability-gated local exchange, and optionally the closed public exchange\nface configured by the three `--exchange-*` flags above. The service is torn down with `cotal down`,\nand a re-run of `cotal up` heals a dead service on a running broker. `--user-auth` and `--open`\ncontradict each other and are refused loudly; a running broker cannot change auth mode\nwithout a `cotal down` first. See [identity & auth](identity-and-auth.md).\n\n`--rotate-sys` renews the two `$SYS` credentials (`membership-observer`, `connection-evictor`).\nThey carry a 30-day expiry and nothing re-signs them in place, because the system-account seed is\nnever persisted, so they are renewed by issuing a **new system account** under the same broker\noperator and minting fresh creds against it. A plain re-`up` does **not** do this: it reuses the\nexisting trust record, and its `$SYS` creds along with it.\n\nThe rotation is safe to run on a real space, with one operational cost. The data account, the account\nsigning key, every agent credential minted from it, and the JetStream store are all untouched; what\ndies is the retired system account, and with it any out-of-band copy of the old `$SYS` creds, on every\nbroker that loads the rotated config. The cost is that **earlier full backups stop being restorable**\n(see below), so this is not a no-consequence operation. It needs the broker to restart on the rewritten\nconfig, so it runs as part of a boot:\n\n```bash\ncotal down\ncotal up --rotate-sys --detach # agents reconnect; nothing is re-provisioned\ncotal doctor auth # both $SYS creds healthy again, 30 days out\n```\n\nA rotation is a stopped, fresh boot, and anything that is not one refuses it, all for the same reason\n(the on-disk material and the broker it runs on must never end up on different generations):\n\n- a live mesh, because the running broker would keep serving the retired account;\n- an open mesh, whether that comes from `--open` or from `broker.auth: false` in a manifest, which\n has no system account at all;\n- `--restore`, because reinstating a trust root and superseding it in one command leaves no way to\n say which authority the mesh came up on;\n- an unfinished restore or resume attempt on this root, including one `cotal up` would recover on\n its own, because those paths can adopt a live listener and return without booting a broker;\n- a root that hosts more than one space, because the system account lives in the shared broker\n record and a rotation would retire every tenant's, while the root holds one `$SYS` cred pair\n pinned to one data account.\n\nTwo things to know before you run it:\n\n- **The retirement is config-load-bound.** Old `$SYS` creds are refused by any broker that loads the\n rotated config. A stale `nats-server` still running the *previous* config in memory would keep\n honouring them, so stop every broker for this root first. `--rotate-sys` refuses if this root's\n mesh is recorded as running, if anything unidentified is answering at the address it was given, or\n if the root's pid file names a live (or unreadable) process. Those are Cotal's own ownership\n records, not a scan of the process table: a `nats-server` you started by hand against this root's\n `server.conf` on some other port writes none of them and will not be seen. Do not run one.\n- **It invalidates earlier full backups.** A full artifact binds to the trust chain it was taken\n against, and that commitment covers the operator JWT and the system account. Every full backup\n taken before a rotation refuses to restore afterwards, so take a fresh `cotal backup` once the\n rotated mesh is up. `cotal up --restore` names this case when the data account still matches.\n\nThe commit is not atomic (a trust-record write plus two credential writes), so an interrupted\nrotation leaves the record ahead of the creds. That split is detected rather than silent: every\n`cotal up` on an auth mesh, and every `cotal doctor auth`, compares each `$SYS` cred's issuer against\nthe persisted record and names the retired account. `up` warns rather than refusing, because these\ncreds power the membership graph and live eviction, both of which degrade fail-soft; the mesh is not\nworth taking down over them. Re-running the rotation heals it, at the cost of one generation.\n\nWhile those creds are expired the mesh keeps delivering messages, but the\n[membership feed](delivery-daemon.md) and live connection eviction stay down; `cotal doctor auth`\nand the manager's log both name the credential and this repair.\n\n## down\n\n```bash\ncotal down\ncotal down --preserve-state [--store-dir ]\ncotal down manager [delivery auth web nats ...]\ncotal down web [--space ]\ncotal down -f | --run [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--file `, `-f` | none | Tear down this manifest's deploy |\n| `--run ` | none | Tear down one `spawn -f` run by id |\n| `--space ` | current mesh | With components: the mesh whose target-addressed components (e.g. `web`) to stop |\n| `--dry-run` | off | Print the manifest teardown or selected components, mutate nothing |\n| `--preserve-state` | off | Bare whole stack only: fence the manager, retain principals and durable state, stop and prove the stack down, then publish `ready` |\n| `--store-dir ` | `.cotal/nats` | With `--preserve-state`: the actual store path (required for a custom store) |\n\nBare `cotal down` stops the whole local stack in dependency order. Positional component names stop\nonly those self-registered local processes; for example, `cotal down manager` leaves delivery and\nthe broker running, and `cotal down web` is available when the web extension is installed. A\ncomponent that starts target-resolved (the web dashboard) is stopped the same way: `cotal down web`\nresolves the mesh the same way as `cotal web` (registry current mesh first, `--space` to name one), so\nit works from any directory; the other components always stop under the folder you run it in. The\n`-f` / `--run` forms tear down a [manifest deploy](#manifest-deploys) without stopping the whole mesh\nand cannot be combined with component names. Stopping `nats` alone is refused while an unselected\nregistered daemon is still live; include those components or use bare `cotal down`.\n\n**Teardown verifies pinned process identity before signalling.** PIDs are recycled by every OS,\nso a recorded pid alone is not a durable target identity. `up` records each stack process's\ncreation identity in a sibling `.identity` pin, which holds the pid and the process start\nreported by the OS. Every stop path, including `down` for the broker, web and extension components,\nand the manager, delivery and auth-service stops, applies the same rule. A pin that names a different\nstart means the pid was reused, so teardown refuses and preserves it. A torn or unreadable pin also\nrefuses. Once the recorded process is stopped, rerunning teardown clears the stale record\nautomatically.\n\nThe first teardown after upgrading a running pre-pin stack has a narrower guarantee. A live record\nwith no identity pin is signalled after a loud warning that it predates identity pinning. Restarting\nthe component writes the pin, so later teardowns receive full match and mismatch protection. The\nsame warning applies on platforms where no stable start token is available.\n\nNormal `down` remains destructive at the logical identity/durable layer. `--preserve-state` is a\ndifferent maintenance transition: it suppresses leave/deprovision cleanup, persists the manager's\nsame-principal resume inventory, stops the entire stack without removing run/auth artifacts, and\npublishes a stable inode-bound cut only after every recorded process is proven stopped and the exact\nrecorded NATS endpoint is unreachable. A missing or stale broker pidfile never counts as stopped. The\nattempt is bound durably before the manager is fenced, the resume document and attempt-bound\n`cut-intent` are fsynced before manager commit, and the manager's commitment itself is journaled\n(`cut-committed`) before any process stops. A retry after a crash at any of those boundaries reuses\nthe exact recorded attempt and finishes the remaining stop and endpoint proofs idempotently, without\nneeding the (by then intentionally dead) manager. A partial cut never publishes `ready`. It cannot\nbe combined with component names, manifest teardown, or `--dry-run`.\n\n## clean\n\n```bash\ncotal clean --force\ncotal clean restore-attempt --attempt --force\ncotal clean restore-fallback --attempt --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | `history`: target mesh |\n| `--dms` | off | `history`: also clear DM history |\n| `--store-dir ` | `.cotal/nats` | `store`/`all`: JetStream store directory |\n| `--force` | none | Required: destructive, no prompting |\n| `--attempt ` | none | `restore-attempt`: exact stale pre-commit attempt; `restore-fallback`: matching healthy committed restore |\n\nOne configurable cleanup verb; every target requires `--force`.\n\n- `history` purges the retained message backlog on the **running** broker (channels, plus DMs\n with `--dms`). The same operation as [`history clear`](#history), which stays as an alias.\n- `store` deletes the **stopped** mesh's JetStream store (`.cotal/nats`): streams, durable\n consumers, and messages. This is the reset for stale on-disk broker state, e.g. durables\n minted by an older, incompatible Cotal generation surviving a `down`/`up` cycle.\n- `all` is `store` plus the space identity (`.cotal/auth`), the local creds and markers tied to\n it, any crash residue a normal `down` would have swept (stale pidfiles, `run/`), and the mesh's\n registry entry; the next `cotal up` mints a fresh identity.\n\n`history` needs the mesh up; `store` and `all` refuse while any recorded mesh process is still\nalive or any same-root recorded broker endpoint remains reachable (run `cotal down` first). They\nalso refuse outright on a root that holds accounts for several spaces: the store and the broker\ntrust record are shared by every space on the broker, so both targets would take out all of them\nand no `--space` can narrow that. `down`, `backup` and `up --restore` refuse there for the same\nreason. `cotal status` lists the tenants on such a root. Personas\n(`.cotal/agents`) and logs are never touched. A custom\nstore location is not recorded anywhere, so `--store-dir` must repeat whatever the mesh was\nlaunched with. Custom cleanup targets must contain either the Cotal store-generation marker or a\nreal `jetstream/` store directory; filesystem roots, project roots, and Cotal auth/maintenance trees\nare always refused.\n\n`store` and `all` also refuse every maintenance journal state. After a healthy committed restore,\n`restore-fallback` is the only supported way to remove the recorded unchanged old-store inode; it\nnever deletes the active target, requires both the exact attempt id and `--force`, and retires the\ncompleted restore journal so a later `down --preserve-state` can start a new backup cycle.\n\n## Backups\n\n```bash\ncotal down --preserve-state [--store-dir ]\ncotal backup create [--only full|registry] [--store-dir ]\ncotal up --restore [--restore-only registry] [--accept-missing-source]\n```\n\nBackup is offline-only. It requires the stable `ready` record from `down --preserve-state`, an exact\nstore match, no live recorded process, and an unreachable exact endpoint from the recorded cut.\nThat endpoint is probed immediately before cloning, so a live broker with a missing or stale pidfile\nis still refused. It claims the cut, reflink/copies the stopped source to a\nprivate attempt clone, and opens only that clone on a random loopback bootstrap broker with an\nindependent parent/deadline watchdog. It validates the canonical stream and pull-consumer inventory,\nwrites native snapshots with consumers excluded, and stores conservative contiguous ACK-floor\ncheckpoints separately. The original store is never opened by the backup broker, and the stack is\nnot restarted implicitly. Artifact destinations must not overlap the preserved source or maintenance\nattempt tree. Restore artifacts and targets likewise cannot nest inside or contain each other, the\npreserved source, or the maintenance attempt tree.\n\nStopped client-managed KV ordered consumers are ephemeral read residue, not backup state. Backup\nignores only the pinned client's exact stopped shapes: ordinary last-value watchers and the\nwhole-bucket scanner that uses all-history delivery to collapse concurrent tombstones. A bound\nconsumer or any lookalike with a different filter, inbox, lifetime, or other config is still refused.\n\n`full` is the default and indivisible: channel registry, CHAT/DM/TASK/INBOX/DLV, ACL, MEMBERS, and\nvalidated durable checkpoints. `registry` is the sole partial artifact. Presence, derived membership\nfeed, leases, native ephemeral/history consumers, credentials, keys, tokens, owner secrets, and actor\nledger files are excluded. `full` means every transferable message and registry stream, not every\nJetStream resource: endpoint submissions/facts/events/timers/workflow state, contract artifacts, and\nthe records/auth/session stores are nonportable control state. Restore recreates those streams empty\nwith their canonical configs before exposing the normal listener, so active endpoint runs,\nlifecycles, and sessions do not cross a backup. Artifacts are exclusively created `0700`;\nsnapshot/checkpoint files and\nthe manifest are `0600`; `manifest.json` is written last with exact sizes and SHA-256 values. The\ndirectory is trusted operator input: hashes detect corruption, not malicious rewriting.\n\nRestore validates and stages the exact allowlisted artifact bytes before moving or creating a store.\nIt requires the same space and existing trust state. The whole pre-commit window holds a journaled\nliveness claim (coordinator, watchdogs, brokers, absolute deadline): ordinary `up` and a repeated\n`up --restore` refuse while the claim is live, and a stale attempt is recovered only after the\ndeadline has elapsed and every recorded owner is proven dead. A retried `up --restore` handles this\nautomatically; an operator can also recover it explicitly with `cotal clean restore-attempt --attempt --force`. Nothing\never rolls back a live attempt. A registry-only artifact restores as registry-only whether or not\n`--restore-only registry` is passed; omitted infrastructure is always created and the exact\npost-restore stream inventory is asserted before commit intent. Ordinary `up` from a preserved cut\nresumes only the exact recorded source store and runtime; a contradicting `--store-dir` or\n`--runtime` fails in preflight. Authenticated restores validate the complete\nspace trust bundle before staging, including nkeys, seed matches, JWTs, signers, and space binding;\nfull restores commit to the validated operator, system-account, data-account, and active-signer root\nchain in addition to the static/user authority fingerprint. Because the system account is part of that\ncommitment, a [`cotal up --rotate-sys`](#up) makes every full artifact taken before it unrestorable\nagainst this root: take a fresh full backup after each rotation. The composed commitment is revalidated\nimmediately before store mutation and never includes secret seeds. Restore never creates fresh auth.\nSame-path restores atomically retain the old\nsource at the journaled fallback path; alternate targets retain it in place; a missing canonical\nsource needs explicit `--accept-missing-source`. Quarantine and target restores use current canonical\nconfigs on isolated random-loopback brokers, never expose native snapshot consumers, and publish a\ncommit-intent immediately before the normal listener starts. Archive bytes never instantiate the real\ntarget: after quarantine validation, every stream is re-snapshotted from the validated quarantine\nstate into attempt-owned sanitized files, and the target is restored solely from those. Before that boundary, failure rolls back\nthe attempt-owned target; after it, ambiguity preserves both stores and records forward-repair\nrecourse. The cooperative maintenance lock excludes Cotal commands, not arbitrary raw NATS processes.\n\nBootstrap brokers in every auth mode, including open, mount the store under a local account with\nrandom operation-specific logins only, each carrying the exact per-phase subject permission matrix;\nnormal static credentials and user-auth sentinel/bearer connections are rejected, and no auth\nservice or callout starts. Open mode differs only in its account label, never in authority. Inventory, each stream snapshot,\nrestore initiation, exact upload id, validation, and each checkpoint recreation use separate exact\nauthorities. Every checkpoint carries the source stream's message/first/last sequence state and must\nmatch its snapshot record before mutation; core then derives and validates the only allowed start\npolicy. TASK is not a CLI exception: the same core checkpoint API recreates its canonical `DeliverAll`\nWorkQueue durable because acknowledged tasks are absent from retention and NATS forbids a\nstart-sequence policy there. Registry-only restore creates every omitted canonical stream and transient\nbucket on the isolated target before the normal listener is exposed. It deliberately does not resume\nretained agents or recreate their DM/DLV/TASK/ACL state; their identity material stays retained and\nstopped rather than being reprovisioned into a partial restore.\n\nAfter listener readiness, the manager starts attempt-bound, validates retained credentials/tokens\nwithout granting or reprovisioning, and resumes the exact persisted principals under cleanup\nsuppression. Registry-only restore uses the same flow with an empty agent set. `commitResume` is an\nidempotent validation barrier only: success must be `awaitingFinalize` with an attempt-bound 64-hex\ncommit token and does not release suppression. Under the workspace lock, the CLI first fsyncs that\nexact evidence as `manager-committed` (restore) or `resume-committed` (ordinary resume), then calls\ntoken-bound `finalizeResume`; only an `active` response for the exact token releases suppression. The\nCLI records the same token in finalization evidence before a restore becomes `active`, or before an\nordinary resume retires and consumes the marker. Re-entry from either committed state skips the prior\nidempotent activation/commit phases, retries finalization with the durable token, and finishes the\nworkspace transition. Failure before finalization preserves the committed state and cleanup\nsuppression; it is not rewritten through a degraded transition. Re-entry between any two earlier\nboundaries reuses the same attempt and may retry the idempotent phases without deleting retained state. A missing or\nchanged per-agent dependency is a named fail-closed result; the journal becomes degraded and remains\navailable for forward repair. A retry from `resume-intent`,\n`resume-active`, or `resume-degraded` reuses the same attempt and inventory after the prior listener is\nproven stopped. Every normal restore listener has an unguessable attempt-bound NATS server name. The\nCLI fsyncs its exact name/nonce, canonical endpoint, process owner, and generation-bound target identity\nimmediately after spawn. Re-entry accepts a surviving listener only when its INFO server name, live PID\nrecord, endpoint, and target identity all match that proof; degraded restore repair then moves through\nthe guarded workspace transition only after manager commit. If an uncommitted bound owner is provably\ndead, recovery retires that exact proof under the maintenance lock and binds a fresh listener for the\nsame attempt, endpoint, and target with a new nonce and server name. A live foreign/mismatched listener\nor ambiguous owner is preserved and refused, never adopted by reachability alone. A reconstructed\ncommit/degraded attempt without either the exact bound proof or a durable dead-listener replacement\nrecord fails closed even when the recorded port is free. A later ordinary startup may pass an `active`\nrestore only when its details prove manager commit and its exact recorded listener is dead.\n\n## Mesh registry\n\n```bash\ncotal meshes\ncotal meshes add # guided, on a terminal\ncotal meshes add --server [--root ] [--mode auth|open|user] [--tls] [--force]\ncotal meshes add --mode user (--user-auth-file | --from )\ncotal meshes rm [ …] [--force]\ncotal use \ncotal status [--space ] [--server ] [--components]\n```\n\n`meshes` lists the meshes this machine knows; a `*` marks the `current` default a bare\n`cotal spawn` joins.\n\nRun on a terminal with the space or `--server` missing, **`meshes add` is guided**: it asks for the\none thing that cannot be derived (the broker URL), probes it, and tells you what answered - open or\nrequiring credentials. It then offers the spaces your `--root` already holds credentials for, states\nthe mode as a fact about that broker rather than asking, and shows the exact record before writing\nanything. A broker that does not answer, or a space name already registered, becomes a choice rather\nthan an error. Anything you pass on the command line is taken as given and not asked again. Without\na terminal - a script, an agent, CI - nothing prompts and the flag form's errors stand\n(`COTAL_NO_PROMPT=1` forces that too).\n\n`cotal up` and `cotal down` maintain their own records. `meshes add` registers a mesh they cannot\nspeak for: one running on another machine, a shared broker, a hosted space. `--root` is the folder\nwhose `.cotal/auth` holds that mesh's credentials and whose `.cotal/agents` holds its personas.\nThe default is the project you run it in. The registry stores that path, never a secret. `--mode`\ndefaults to `auth` when the root holds the space's account record and to `open` otherwise. The\nbroker is probed before anything is recorded, so a wrong address, or credentials that mesh will\nnot accept, fails here instead of at the first `spawn`; `--force` records without verifying (and\nreplaces an existing record).\n\nA hostname or public address is registrable only when the connection will **require TLS**. Pass\n`--tls`, or use a `tls://` URL. The scheme is recorded as enforced intent, so every later dial\nthrough the record demands the handshake (and `meshes add tls://…` against a plaintext broker is\nrefused at registration). Without required TLS the fence admits loopback and private-overlay\nliterals only. RFC1918 addresses are refused in both modes because a cafe LAN is private but does not belong to you.\n\nA **user-auth** mesh registers from supplied pinned trust, never guessed: `--user-auth-file`\ntakes the bundle exported where the mesh runs; `--from` asks before it dials the address at all,\nthen fetches its `/.well-known/cotal-mesh` discovery document (HTTPS only), displays the pins, and\nasks again before adopting them. Neither fetch follows redirects: a 302 can move a pinned fetch\nonto plaintext or onto another host, so it is refused rather than followed, and the pinned\nexchange must itself be an `https://` URL, except for an exchange on this machine, where plain\n`http://` is accepted for a loopback *literal* (`127.0.0.1`, `::1`, any spelling of them) but not\nfor `localhost`, which is a name rather than an address. Registration verifies that the exchange\nanswers `/health` and `/jwks` as the pinned issuer. It also verifies that the broker refuses a bare\nconnect; that auth-required refusal is the pass. The sentinel credentials land in a 0600 file under\nthe entry's root; the registry records only the path.\n\n`meshes rm` drops records. It never stops a mesh. For a mesh running on this machine `cotal down`\nis the right verb, and `rm` says so unless you pass `--force`. A hand-added record is removed by\n`meshes rm`, by an `add --force` replacement, or by a `cotal up` that actually starts the broker for that same space, server and root, which becomes that\nmesh and so takes the record over (a `cotal up` for that space anywhere else refuses instead).\nNothing that merely *infers* a record is stale touches it: an\nunreachable broker is listed `offline` and stays, and `cotal down` / `cotal clean all` leave it\nalone even when it shares a root with the project they are tearing down, because nothing on this\nmachine could write it back.\n\n`use ` sets that default; the selection applies from every directory,\nincluding inside another mesh's project. `status` is a read-only report: machine prerequisites\n(starting with the installed `cotal-ai` version), the installed extensions and their versions, this\nfolder's `.cotal/`, the recorded meshes, and a live snapshot of the selected mesh (roster, channels,\nmembership feed). Stale Claude skills and out-of-date `.agents` skills recommend `cotal setup --skills`,\nnot unscoped `cotal setup`. `status` takes `--space` / `--server` to pick the mesh to inspect; it starts\nnothing.\n\nPersona rows name the catalog they describe. If this folder and the selected mesh use different\ncatalogs, status names both and marks which one spawn launches from. A green `default` means the file\npasses the same agent-file loader spawn uses; a present but invalid file is reported as invalid.\n\n`cotal status --components` adds a fail-loud per-component health pass. It reads **each\ncomponent's own control surface**, rather than treating a PID, a lease, or a successful probe of a\nsibling as proof that the component serves. It prints one of `serving`, `absent`, `not-serving`, or\n`refused` for each component and exits `0`, `1`, `2`, or `3` respectively (the highest observed\nstate wins):\n\n- **manager**: local PID record, its liveness-lease holder and PID, then the manager's own typed\n `status` service reachability from this host. Manager builds that do not report static\n reconciliation say `static reconciliation not reported by this manager build`; the line stays\n visible even when the manager is otherwise `serving`.\n- **delivery**: local PID record, its ready lease (`ready` is the daemon's own bound-control\n signal), and the latest `renewal.json` adoption verdict. A re-signed credential and a\n broker-accepted adoption stay distinct facts.\n- **web**: local PID record and the dashboard's own loopback `/api/meta` response, which must name\n the same PID and its requested port. A different process on the port, an unreadable PID command,\n or an unrecognizable process record is `refused`, not a green default-port guess.\n- **broker**: the registered mesh URL dialed from this host with its recorded TLS requirement.\n\n`absent` means Cotal has no live local component record (or has a stale record); `not-serving`\nmeans the component record is live but its service/readiness surface did not answer or is not ready.\nThose are intentionally separate exit cases. A failed or unreadable probe is `refused`, never an\nabsent component or a clean zero.\n\n## spawn\n\n```bash\ncotal spawn [] [--detach] [--name ] [--agent ] [--model ] [--variant ] [--prompt ] [--cwd ]\ncotal spawn -f [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | resolved mesh | Target space |\n| `--server ` | registry entry | Broker URL override |\n| `--creds ` | none | Control-caller creds for an off-registry manager (`--detach` only) |\n| `--name ` | persona's `name:` | Presence-name override (does not choose the persona) |\n| `--config ` | none | Persona catalog name or file path; wins over the positional |\n| `--agent ` | persona's `agent:`, else `COTAL_DEFAULT_AGENT`, else `claude` | Connector type (`claude`, `opencode`, `jcode`, `hermes`, and so on) |\n| `--role ` | persona's `role:` | Role override |\n| `--model ` | persona's `model:` | Model override |\n| `--variant ` | persona's `variant:` | Model variant override (connector-defined; e.g. OpenCode reasoning tiers) |\n| `--cwd ` | this cwd | Working directory to root the agent at |\n| `--prompt ` | none | Initial prompt auto-submitted at start |\n| `--resume ` | none | Fork an existing session id into the mesh; only connectors that declare resume support accept it (see [the matrix](connectors.md)) |\n| `--events` / `--no-events` | off | Publish the session's structured event plane to its own event channel |\n| `--share-tools ` | none | Share named operator MCP servers with the agent |\n| `--subscribe ` | persona's | Channel read-set override |\n| `--allow-subscribe ` | = subscribe | Read-ACL override |\n| `--allow-publish ` | deny | Post-ACL override |\n| `--detach`, `-d` | off | Launch via the manager into a detached PTY (reattach with `cotal attach`) |\n| `--on ` | class anycast | With `--detach` only: pin the launch to one manager instance id (the whole id, as `ps` prints it). Refused on a foreground spawn (no manager to pin), with `-f` (a manifest deploy launches through the manager class queue), and when empty |\n| `--file `, `-f` | none | Deploy a manifest onto the running mesh |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--allow-stale ` | none | With `-f`: waive named stale agents (apply-only) |\n| `--runtime ` | manifest's | With `-f`: override the manifest's runtime |\n\n`--events` turns on the session's **event plane**: a stream of structured events describing what\nthe agent did, rather than the prose it wrote, on a channel of its own. The channel is named after\nthe agent's principal, `events..`, never after its display name, because two live\nagents are allowed to share a display name and would then share a stream. The launch grants publish\nrights on that channel alone, foreground and detached alike, and a connector that does not\npublish an event plane refuses the flag rather than starting a session whose events have nowhere to\ngo.\n\nThe flag and the grant are separate on purpose. Holding publish rights on a channel is not a request\nto publish to it, so writing an event channel into an agent file's `allowPublish` does not turn the\nplane on: only the launch does.\n\nThe persona (`--config` > positional > `COTAL_DEFAULT_PERSONA` > `default`) is loaded from the\ntarget mesh's `.cotal/agents/`; the launch flags override the file. Foreground runs the agent\nattached to your terminal; `--detach` hands the launch to the running manager. Both modes get the\ndurable backstop on a mesh that runs the delivery daemon; `--live-only` skips it for a foreground\nspawn (messages posted while it is disconnected are then not replayed). A foreground exit retires\nthe agent's creds and broker footprint, like a manager despawn. A `--detach` spawn is an\n**action**: the manager accepts it and returns the allocated identity at once, then the launch\nfollows to a terminal outcome rather than blocking (see [the control surface](control-surface.md)).\nSee [Connect Claude Code](connect-claude.md) and [Agent files](agent-files.md); `-f` is a\n[manifest deploy](#manifest-deploys). (`cotal start` was merged into `cotal spawn --detach`.)\n\n## models\n\n```bash\ncotal models [--agent ] [--refresh]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--agent ` | all registered connectors | Connector whose catalog to list |\n| `--refresh` | off | Ask the connector to refresh its provider cache |\n\nAsks the running manager for each connector's model catalog (model ids plus their variants)\nfor connectors that expose one. OpenCode and Codex query harness/provider surfaces; Jcode reads\nproviders that enable `model_catalog = true` in the operator Jcode `config.toml`. Jcode's listed\neffort tiers render as `variants (declared, not provider-verified)`, and launch can still refuse one.\nA connector without a catalog says so. Pick a result with `cotal spawn --model --variant `,\nwhere `` is the model id as the catalog printed it. OpenCode and Codex ids are the full\n`provider/model`; Jcode ids are bare (`opus-5`, not `cliproxy/opus-5`), because the provider is\nselected by the operator's Jcode config and a prefixed id is refused at launch with the bare form\nnamed.\n\n## endpoints\n\n```bash\ncotal endpoints [--space ] [--server ] [--creds ]\n```\n\nLists the mesh presence roster: agents, the manager, and any other protocol endpoint, with each\nendpoint's role, kind, status, and current activity. Unlike `ps`, this is a read-only presence view;\nit is not limited to child processes owned by the manager.\n\n## Endpoint control\n\n```bash\ncotal describe [--space ]\ncotal invoke [--args ''] [--space ]\ncotal invoke --name [--admin] [--space ]\n```\n\nThe generic v0.4 service surface. `describe` resolves a registered endpoint's command set off the\nwire - the reserved `describe` command answers the registered contract digests, the schemas are\nfetched from the space's content-addressed contract store, recompiled, and verified against those\ndigests - and prints each command with its capability class and targeting shape. `invoke` calls one\ncommand by name: `--args` is a JSON object validated against the fetched input schema *before*\npublish; a targeted command takes `--name ` (resolved to the agent's current principal through\n`inspect`) or `--self`. `--admin` uses the admin instrument credential, whose cross-agent reach rides\nthe operator-only `any` authorization mode. Neither command has compile-time knowledge of any\nendpoint's schemas - this is the same trust chain every built-in control command now uses. Needs an\nauth mesh: the manager registers its service on both static and per-user meshes (a signed-in user\nrides their bearer; each visible or invoked command still requires its existing grant, and cross-agent\nreach needs the `admin` scope). An open mesh has no service registry.\n\n## Managed seats\n\n```bash\ncotal ps [--on ] [--wide | --json] [--space ]\ncotal stop --name [--on ] [--space ]\ncotal attach --name [--on ] [--no-reconnect] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | none | Managed agent to stop / attach (required) |\n| `--on ` | class anycast (`ps`: class scatter) | Pin to one manager instance id (multi-manager space); takes the whole id as `ps` prints it, not a prefix. An empty value (`--on \"\"`, an unset shell variable) is refused, never treated as absent |\n| `--wide` (`ps`) | off | After each seat's compact row, print extra operational facts the manager records: `cwd`, `pid`, spawner, lifecycle uid, and the owning manager's instance id and host. Model and requested variant stay in the identity row rather than printing twice. A fact the manager did not record (for example a runtime with no real process) prints nothing, never a placeholder |\n| `--json` (`ps`) | off | Machine-readable: one JSON object per seat per line, copied unchanged from the manager row. Instance headers and errors go to stderr, so stdout contains only rows. Mutually exclusive with `--wide` |\n| `--no-reconnect` (`attach`) | off | End the attach when its session ends, instead of re-establishing it. For scripts that want one run and one exit code |\n\nThe human `ps` row is presentation text and is not a stable parsing target. Scripts use `--json`,\nwhich is the machine-readable row contract.\n\nThese are operator clients over the running manager's control plane. The default row includes the\nconnector, model pin, optional requested variant, and runtime as operational descriptors for the\nmanaged row. They do not make a shared display name a unique protocol identity; use `--json` when\nunambiguous owner+actor attribution is required. An omitted variant means no override was requested;\nCotal does not invent an effective provider default it cannot observe. `ps` also prints two state\nfacts per managed agent, because they answer different questions: the process fact from the manager's\nown runtime handle (`running` with its uptime, or `exited` with how long it ran), and the mesh fact\nfrom the roster (`idle` / `working` / `waiting` / `mesh offline`, or `not in roster` when the seat has\nno presence row at all: a seat that has not joined yet, or one that never did). A seat can be\n`running` and `mesh offline` at once: the process is alive and its presence has lapsed. On a user-auth\nmesh `ps` also renders each managed agent's last credential-refresh outcome, fail-closed.\n\n**Mode split (chosen up front, never try-scatter-then-degrade):**\n\n- **Static / open mesh.** Bare `ps` is a **class scatter**: it freezes the live manager class from\n the records registry, merges every registered instance's agents grouped and attributed per\n instance, and a non-answering instance is shown as `registered, no answer within the deadline`\n (never silently omitted). That label is the whole claim: the instance is registered and did not\n answer. It does not say the host is down, because a dead host never deregisters itself and a\n live one can be slow; if it is gone, deregister it.\n `--on ` pins the read to one exact instance id instead. A wrong pin fails loud\n rather than falling through: a well-formed id that no live manager carries is reported as\n `manager instance did not answer` (nothing else is asked), and a credential without that\n instance's rail is reported as refused by the broker, not as an unresponsive manager. A manager\n that answers with a refusal is shown with its own cause; \"no manager reachable\" is said only when\n nothing answered at all. If the scatter's own registry read fails (the freeze or the reconcile),\n `ps` says the manager registry could not be read rather than pronouncing on the managers, which\n may all be up.\n\n**`stop` and `attach` route by seat locality.** A seat can only be stopped or attached by the\nmanager actually running it, and the class queue does not know which one that is. So on a\nstatic/open mesh both verbs first ask every registered instance which one hosts the named seat, then\naddress that instance directly. This happens by default; you do not need `--on`.\n\n`--on ` remains the override, for when you already know where the seat lives or the\nlookup itself is degraded. It is also the **only** route on a **user-auth mesh**: a ledger-scoped\nbearer does not hold the registry-read rows the lookup needs, so there the verbs stay on the class\nqueue unless you pin them yourself.\n\nIf no reachable instance has the seat, the error reports how many managers answered and names\nthose that did not. It does not collapse that state into a bare `no agent `. That distinction matters\nbecause a single manager cannot tell \"hosted elsewhere\" from \"does not exist\": it answers\n`not-found` for both.\n- **User-auth mesh.** `cotal ps` reports what **one** manager knows about your agents (an `ep.one`\n read against the manager's in-memory roster, owner-filtered). It does **not** report other\n manager instances. It cannot tell you that one is down: an unreachable manager is absent\n from the list. Completeness across a multi-manager user-auth space is not claimed.\n A manager that does not answer fails the command outright (exit non-zero), rather than printing\n an empty list that could be read as \"no agents\". Your ledger row needs the `admin` scope to\n reach `ps` at all; `spawn` alone is refused by the broker (the ep tier boundary).\n\n`attach` streams and drives an agent's terminal on the `pty` runtime; detach with the escape key\n(Ctrl-] by default; see [`COTAL_DETACH_KEY`](config.md)). It does so over a one-use, holder-bound\nmesh session ([SPEC](../SPEC.md) §13.6): the manager replies with a signed session grant (never a\n`127.0.0.1` URL), the CLI redeems it once over the broker, and the browser console (`cotal console`)\ndrives the same session. `stop` and `attach` need a running manager to talk to. On a static mesh\nthey are cross-agent admin operations. On a user-auth mesh, your own agents (any agent under your\nowner) need only the `spawn` scope; another owner's agent needs `admin` on your ledger row\n([identity & auth](identity-and-auth.md)). Launch detached agents with [`spawn --detach`](#spawn).\n\n**`attach` reconnects when the link dies.** A session lives on a network link, and a laptop that\nsleeps, a VPN that drops or a wifi handover kills it. When that happens `attach` prints\n`[cotal: connection lost, reconnecting]` on stderr and starts asking the manager for a new session:\na fresh grant, a fresh per-session credential, a fresh connection, so every attempt re-runs the same\nauthorization the first attach did. On success it prints `[cotal: reconnected]`, the manager repaints\nthe seat's current screen the way it does for any attach, and you carry on in the same terminal.\nRetries wait 1s, 2s, 5s, 10s, then 30s, for as long as the seat exists. The detach key is read the\nwhole time the loop runs, the waits and the attempts alike, so a reconnect never traps you: press it\nwhile a session is being established and the attach ends there, and a session that lands behind the\npress is handed back to the manager rather than left holding a slot. Everything else you type while\nthere is no session is dropped rather than queued, so keystrokes aimed at a terminal that turned out\nto be frozen, Ctrl-C included, are not delivered to the agent by a reconnect you did not know had\nhappened. That starts before the first session, not at the first reconnect: at a terminal, `attach`\nreads and drops what you type while it is still resolving the mesh, so a key struck at a prompt that\nhas not come up yet does not reach the agent when it does.\n\nA **pipe** carries script input. For example, `printf 'ls\\n' | cotal attach --name web` is\nbuffered until the session opens. Buffering continues across reconnects, so\n`tail -f log | cotal attach --name web` does not lose the part of its feed written while the link was\ndown. Only a terminal gets the reader; `--no-reconnect` keeps the old behaviour on both.\n\nIt stops on its own when reconnecting cannot help, and says why: a manager that refuses the attach\nexits non-zero with the manager's own message, and a reconnect that finds the seat no longer there\n(despawned, or its agent exited while the link was down) exits cleanly with `seat is gone`.\nA refusal that could still pass, such as a manager at its session ceiling, is relayed in the\nmanager's own words while the loop keeps trying, once per refusal rather than once per attempt.\nPressing the detach key, or the agent's process exiting while you are attached, ends the attach as\nit always did. `--no-reconnect` turns all of this off and restores the single-session behaviour,\nwhich is what a script wants.\n\nEach reconnect also hands the abandoned session back to the manager, over the first link that can\ncarry the message, so an attach that flaps does not eat the manager's session slots one outage at a\ntime. If that message never gets a link, the attach says so when it ends.\n\nWhich mesh `attach` resolves also decides **whose trust it redeems with**. Redeeming a session grant\nmeans minting a short-lived, session-scoped credential from the space's seed, and that seed comes\nfrom the root the mesh resolved to, never from a `.cotal` found by walking up from whichever\ndirectory you happen to be standing in. The difference is not hypothetical: `~/.cotal` exists on\nevery install because the mesh registry lives there, so a command run anywhere under your home\ndirectory but outside a project used to mint from your home directory's trust and present it to a\nbroker that trusts a different chain, which surfaced as a bare authorization failure that named\nnothing. A directory that does hold another chain for the same space is now reported on the way\npast, and not obeyed:\n\n```text\n! this directory resolves to /Users/you, whose .cotal/auth holds a DIFFERENT trust chain for space \"team\".\n attach used /Users/you/projects/app, the root this mesh resolved to. The other one is not being used, and is worth a look.\n```\n\nWhen the resolved mesh holds no seed at all, `attach` refuses and names what it resolved, the broker\nand the root, instead of describing a directory it did not use.\n\nTerminal bytes stream over the mesh; the manager's own HTTP/WS face serves the console. That endpoint binds\n**loopback by default**, so nothing is exposed by accident; `cotal up --host ` passes its bind\naddress down, which is what lets you attach to an agent whose manager runs on another machine. A\nbare `cotal supervise` and an embedded manager stay machine-local. Set it directly with\n`supervise --console-host `.\n\nThat address is **recorded on the mesh** and carried forward, because it is a decision rather than\nsomething later commands can work out for themselves (a broker dial address is not a manager bind\naddress). Every later manager launch for the same mesh reuses it, including a same-root `cotal up` repair,\nan adopted preserved or restored listener, and a `spawn -f` manifest deploy. A manager replacement\ndoes not quietly move a reachable attach face back to loopback. Passing `--host` again overrides it,\nso you can widen or narrow exposure whenever you like; a mesh that never asked stays loopback-only\nand records nothing.\n\nBecause that face carries terminal read and write for every managed agent, it is credentialed in two\ntiers. A mesh caller receives a **ticket** bound to the single agent the manager just authorized,\nsingle-use and short-lived, so one authorized attach can never be re-pointed at someone else's\nagent. The **console token** is the operator's own, reaches every agent, and is printed only to the\nmanager's output. The roster, the live feed, and the PTY stream all answer `401` without one; the\nstatic console shell is served openly, since it describes no agent.\n\n## input\n\n```bash\ncotal input --name --text [--no-enter] [--on ] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | | Managed agent to type into (required) |\n| `--text ` | | The text to type, taken verbatim (required) |\n| `--no-enter` | off | Type the text and stop there, without pressing Enter |\n| `--on ` | class anycast | Pin to one manager instance id using the same rules as [`attach`](#managed-seats) |\n\nTypes one line into a running agent's terminal, as if you had typed it there, and returns. This is\nthe half of [`attach`](#managed-seats) that a program wants: `attach` is a live stream that holds a\nsession open and expects a terminal on your side, so a script, a cron job or a web UI cannot use it\nto send a single line. `input` is one authorized call.\n\nWhat it is for is **harness commands**. A line beginning with `/` is not chat and not a message: it\nis something the agent's own harness handles, and the only way in is the keyboard.\n\n```bash\ncotal input --name reviewer --text \"/compact\" # ask the harness to compact its context\ncotal input --name reviewer --text \"/model opus\" # switch its model\ncotal input --name reviewer --text \"hold on that PR\" # ordinary typing works too\n```\n\n**Quoting.** `--text` takes a value, so a payload starting with `/` survives as written. A payload\nstarting with a dash needs the `=` form, because the shell-style `--text --foo` is ambiguous and is\nrefused rather than guessed:\n\n```bash\ncotal input --name reviewer --text=--verbose # dash-leading text: use --text=\n```\n\nEnter is pressed by default, since a command typed but never submitted has not been delivered.\n`--no-enter` types the text and leaves it sitting at the prompt, which is how you stage a line and\nsend it later.\n\nNothing comes back but a delivery receipt (`✓ sent 9 bytes to reviewer`, counting the trailing\ncarriage return). Whatever the agent does next shows up where its output already goes: the mesh, its\ntranscript, or an `attach`.\n\n**This one is operator-only, and more narrowly than `stop` or `attach`.** Those two are granted to\nanything holding `spawn`, so an agent can stop and attach to seats under its own owner. `input` is\nnot: it is granted only to operator credentials, which on a user-auth mesh means your ledger row\nneeds the `admin` scope, the same scope [`ps`](#managed-seats) already needs there. The reason is\nthat a write into a terminal is control of whatever is running in it, and on a user-auth mesh the\nown-owner rule covers every seat under you, not only the ones you launched: a `spawn`-scoped agent\ncould otherwise type into a sibling it never started. Seat locality is still resolved for you.\n\nOnly the `pty` runtime can be typed into. The external terminal runtimes (`tmux`, `cmux`, `orca`,\n`herdr`) attach to a process they do not own, so they have no input stream for it and the command\nrefuses by name rather than dropping the keystroke.\n\n## personas\n\n```bash\ncotal personas list [-v] [--running]\ncotal personas show \ncotal personas edit \ncotal personas new (--prompt | --from ) [--role ] [--model ]\ncotal personas rm --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh's persona catalog |\n| `--role ` | none | `new`: the persona's role |\n| `--model ` | none | `new`: the persona's model |\n| `--prompt ` | none | `new`: the persona's prompt text |\n| `--from ` | none | `new`: seed the prompt from a file |\n| `--verbose`, `-v` | off | `list`: include role / model / description |\n| `--running` | off | `list`: mark personas live on the mesh |\n| `--force` | none | `rm`: required, delete without prompting |\n\nPersonas are the local agent files under the resolved mesh root's `.cotal/agents/`, the same catalog\n`cotal spawn` launches from. `--space` and `--server` therefore move every list, read, write, delete\nand completion operation to the selected mesh. An unresolved target refuses rather than falling back\nto the current directory. See [Agent files](agent-files.md) for the file format.\n\n## supervise\n\n```bash\ncotal supervise [--runtime ] [--space ] [--server ] [--spawn ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space to supervise |\n| `--server ` | hosting mesh, or matching registered mesh | Broker URL. A registered mesh supplies it when omitted; a different explicit value is refused. |\n| `--runtime ` | `pty` | Agent runtime (`pty` built in; extension runtimes are explicit-only) |\n| `--console-port ` | none | Protocol-console port |\n| `--console-host ` | loopback | Bind host for the console + attach endpoint. Loopback keeps it machine-local; `cotal up` passes the address it bound the broker to, which is what lets `cotal attach` reach this manager from another machine |\n| `--roster ` | none | Declarative roster to boot at startup |\n| `--launch ` | none | Resolved manifest launch spec (from `up -f` / `spawn -f`) |\n| `--spawn ` | none | Comma-separated personas to pre-spawn at startup |\n\nThe manager is the agent supervisor and control plane: it answers `spawn --detach`, `stop`, `ps`,\n`attach`, and the `cotal_*` manager tools. `cotal up --detach` starts one for you; run `supervise`\ndirectly to recover a dead manager or drive a custom runtime. Default runtime is `pty`; install an\noptional provider first (`cotal ext add @cotal-ai/orca`, `@cotal-ai/tmux`, `@cotal-ai/cmux`, or `@cotal-ai/herdr`) and\nselect it explicitly. A missing provider or app fails loudly; there is no fallback. See [Deploy](deploy.md).\n\nOn a normal `SIGINT`/`SIGTERM`, the manager stops every seat and requires the selected runtime to\nprove the seat is gone before it releases the manager lease or service registration. A stop that\ncannot prove exit fails loud and keeps manager authority instead of reporting a clean shutdown while\nan orphan still holds broker rails. After an abrupt manager death, the same logical successor\nterminalizes only its own durable static slots, verify-evicts the predecessor's broker principal,\nrecords that result in the lifecycle's caller-readable audit detail, and only then retires the\nlifecycle and frees the alias. Missing or unverified broker evidence keeps the slot terminalizing.\nDelivery-admin does not terminate the orphan OS process; safe successor process reaping requires\ndurable process start-identity pinning and is tracked separately.\n\nA `meshes add --mode user` entry is a **participant** registration, not hosting authority. A\nparticipant may run `supervise` only when the host advertises the remote manager authority service\nand the signed-in actor has the dedicated `supervise` ledger scope. The CLI obtains the closed,\nloopback-only `manager-service` view; `spawn` and `admin` do not substitute for that scope. The\nhost issues the manager's public-nkey JWT material through its lifecycle-bound prepare → activate\n→ renew protocol, never by handing the participant a signer or static provisioner credential.\n\nThe broker URL in the registry entry decides the transport. A remote broker is often published\nover a `wss://` edge rather than a raw `nats://` port, and `supervise` dials whichever scheme the\nrecord holds, starting with the manager-authority registration it runs before the manager exists.\nThe record also decides whether that registration requires TLS, so a participant never downgrades\nthe credential exchange to a plaintext connection the registry did not describe.\n\nWithout that advertised host service or scope, `supervise` refuses before it starts a manager.\nRun `cotal spawn` without `--detach` to launch a foreground agent, or ask the space host to enable\nthe authority service and grant `supervise` for detached agents. If a running remote manager loses\nrenewal, it reports degraded state and refuses unsafe new starts and restarts; live agents are not\nsilently replaced. Do not run `cotal down` or `cotal up` on a participant machine to repair this\ncondition.\n\n## reconcile-gate\n\n```bash\ncotal reconcile-gate [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the frozen gate lives in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint whose gate is frozen |\n| `--instance ` | this folder's persisted manager instance | Instance id |\n\n**When you need this.** A manager restart killed after deregistration begins but before the new\nincarnation finishes leaves the endpoint's issuance gate *frozen*, held by a\nprocess that no longer exists. The freeze is what stops two incarnations serving at once, which is\ncorrect. The successor manager now completes that dead registration itself on boot, using the same\nguard this command uses: it acts only when the freeze-holder is affirmatively gone under a complete\nCONNZ sweep (`gone` and `sweepComplete=true`). If that registration's spec write already committed,\nit finishes the same freeze at the committed registration revision. If the spec did not advance, it\nabort-reopens the gate at generation+1 with processEpoch unchanged and continues the normal takeover.\nLive, unknown, unestablishable, and\nwrong-op-kind still refuse; there is no TTL.\n\nUse this command when the boot path cannot run: the delivery daemon is down, the repair targets a\nnon-manager endpoint, or you want to lift the freeze without starting a manager. It checks that the\nholder really is gone, prints what it found, and then finishes the dead operation the same way as the\ninterrupted restart would have: revoke the old credentials, evict their holders with verification,\nand reopen the gate.\n\nIf verification is interrupted, the command leaves the gate frozen and durably records each holder\nwhose eviction was already verified. A retry still repeats the freeze-holder liveness check, then\nskips only progress bound to the same registration operation, frozen-gate revision, and holder set.\nThe output reports holders completed before this attempt, completed now, and still remaining. A new\nfreeze or changed holder set starts from zero. Cursor cleanup happens only after reopen; a retained\ncursor is harmless because its old gate revision cannot authorize a later freeze.\n\n**It refuses far more often than it acts, on purpose**, and always says which check stopped it:\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `holder-alive` | The freeze-holder still has a live connection: a manager *is* running | Stop that process first. Reconciling would evict a live manager's credentials |\n| `holder-unknown` | The connection sweep could not prove the holder absent | Not safe to proceed: an unprovable holder is treated as a live one. Re-run once the broker answers completely |\n| `liveness-unestablishable` | The delivery daemon could not be asked at all | Start it (`cotal up` runs it) and re-run. Silence is never read as death |\n| `not-frozen` / `no-gate` | The gate is open, or there is no gate at that coordinate | Nothing to repair: check `--endpoint` / `--instance` |\n| `wrong-op-kind` | Frozen under a takeover or retirement, not a registration | Out of scope for this command; it will not reinterpret another operation's intent |\n| `eviction-unverified` | The holder looked gone but eviction could not be verified | The gate is left frozen, unchanged. Investigate the broker before retrying |\n| `raced` | A newer manager moved the gate mid-repair | Re-run `cotal doctor` and look again |\n\nThere is no `--force`, and no path that discards gate state: the only way this reopens a gate is by\nproving the holder is gone and then completing the operation properly.\n\n## deregister-instance\n\n```bash\ncotal deregister-instance [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the instance is registered in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint the instance serves |\n| `--instance ` | this folder's persisted manager instance | Instance id, the whole id as `cotal ps` prints it |\n\n**When you need this.** The service registry records *registration*, not liveness, and nothing in\nthe model expires a row. A manager that stops cleanly removes its own registration. One whose host\ndied without writing anything cannot, so its record goes on claiming a live instance forever: every\nclass scatter in that space freezes the dead slot in, and `cotal ps`, `stop` and `attach` each pay\ntheir whole deadline waiting for a machine that is never coming back. A laptop that was reimaged, a\ncontainer that was deleted, a box that will not be back on the network: those registrations have no\nother exit.\n\nThis command is that exit. It asks the instance first, and it removes a record only when the broker\naffirms the instance's own rail is empty: nothing subscribed there. Then it deletes the\nregistration's two records keys, each pinned to the revision it read, and prints what it removed.\n\n**Silence alone never passes.** An unanswered describe is what a dead host, a wedged process and a\nslow one all look like, and a hung process still holds its subscriptions, so the broker sees\ninterest on its rail. That instance is refused and the observation is printed. A dead process holds\nno connection and therefore no subscription, so a real corpse is still removed.\n\n**Every refusal names the failed check:**\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `instance-answered` | The instance answered a pinned describe. It is alive | Nothing to repair. If it is wedged rather than gone, stop the process first; its own clean stop removes the record |\n| `instance-not-affirmed-gone` | It did not answer, and the broker did not report its rail empty, which is what a held subscription looks like: slow or hung, not affirmed gone | Nothing was removed. Stop the process; its record goes on its own clean stop, or re-run this once it is down |\n| `liveness-unestablishable` | The probe itself failed, so nothing was learned | Fix the probe's path (credential, broker) and re-run. A probe that could not run is never read as death |\n| `not-registered` | No registration at that coordinate | Check `--instance` and `--endpoint`. This takes the whole id, never a prefix |\n| `registration-in-flight` | The instance holds the endpoint governance slot at the live issuance-gate generation, so a registration is still completing | Nothing was removed. Wait for that registration to finish, then re-run |\n| `superseded` | The record moved between the read and the delete | Something is writing to it. Nothing was removed; re-observe before retrying |\n\nThere is no `--force` and no sweep: silence is not death, and a rule that removed rows on silence\nwould eventually remove a live instance that was merely slow. An operator names one instance, the\nbroker's verdict on its rail is what authorizes the removal, and the guard's job is to show them\nthey named a dead one. Removal is not a one way door either. The same instance re-registers over\nthe tombstone on its next start, under the same identity.\n\n## runtimes\n\n```bash\ncotal runtimes\n```\n\nLists every agent runtime the manager can spawn through: the built-in `pty`, the official providers\n(`orca`, `tmux`, `cmux`, `herdr`), and any custom provider installed via `cotal ext add`. Each installed\nprovider is probed so you can see what is actually reachable on this machine before selecting it:\n\n```\npty built in\norca installed · reachable @cotal-ai/orca\ntmux available · cotal ext add @cotal-ai/tmux\ncmux available · cotal ext add @cotal-ai/cmux\nherdr available · cotal ext add @cotal-ai/herdr\n```\n\n`installed · reachable` / `unreachable` is the provider's own `available()` probe; `available` means\nit is a known runtime you can add with the shown command. Selecting an unknown or uninstalled runtime\nvia `up`/`spawn --runtime ` fails loud and, for a known one, points at the exact `cotal ext add`\npackage. There is no silent fallback to `pty`.\n\n## send\n\n```bash\ncotal send dm \"\" [--space ] [--server ] [--creds ]\ncotal send msg \"\"\ncotal send ask \"\"\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and (off-registry) which credential |\n\nOne-shot messaging: connect, send a single direct message (`dm`), channel post (`msg`), or role\nask/anycast (`ask`), then exit. For a running conversation, agents use the mesh tools instead\n([MCP tools](mcp-tools.md)).\n\n`cotal send` requires `COTAL_NAME` plus either `COTAL_ID` or both `COTAL_OWNER` and `COTAL_ACTOR`.\nIf that tuple is missing, `send` refuses before connecting so the recipient never sees a message\nattributed to a nameless command principal. A child that inherited a\nseat's environment is attributed as that seat; this command does not distinguish the two. An operator\nwho is not a live seat can set both variables for the one shot:\n\n```bash\nCOTAL_NAME= COTAL_ID= cotal send ...\n```\n\n## channels\n\n```bash\ncotal channels list\ncotal channels set [--replay | --no-replay] [--window ] [--desc ] [--instructions ]\ncotal channels default --replay | --no-replay\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--replay` / `--no-replay` | none | `set`/`default`: replay history to new joiners, or not |\n| `--window ` | none | `set`: replay window size |\n| `--desc ` | none | `set`: one-line channel description |\n| `--instructions ` | none | `set`: instructions shown to joiners |\n\nInspects and edits the channel registry: replay policy, description, and joiner instructions. ACL\nsemantics (who may read or post) are set at mint / provision time, not here; see\n[Channels and permissions](channels-and-permissions.md). On a user-auth mesh, `list` rides your\nown login as is; `set` and `default` edit the registry over a short-lived\nchannel-writer view, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n\n## history\n\n```bash\ncotal history clear --force [--dms] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--dms` | off | Also clear DM history |\n| `--force` | none | Required: clear without prompting |\n\nPurges retained channel history; `--dms` extends it to direct-message history. An alias of\n[`clean history`](#clean). On a user-auth mesh the purge rides a short-lived purger view over\nyour login, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n## console\n\n```bash\ncotal console [--plain] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to watch |\n| `--plain` | off | Line stream instead of the TUI |\n\nA live protocol view for a space: a lazygit-style TUI, or a plain line stream on `--plain`. On a\nuser-auth mesh it rides the read-only admin view over your login, which needs ledger scope\n`admin`. See [Watch a mesh](watch-a-mesh.md).\n\n## web\n\n```bash\ncotal web [--detach] [--host ] [--port ] [--no-open] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to serve |\n| `--host ` | `127.0.0.1` | Concrete HTTP bind and browser host; wildcard addresses are refused |\n| `--port ` | `7799` | HTTP port |\n| `--detach` | off | Run in the background; stop with `cotal down web` or bare `cotal down` |\n| `--no-open` | off | Don't open the browser |\n\nThe browser observability dashboard: presence, channels, and a live feed. It is **not** part of\n`cotal up`: it ships inside `cotal-ai` as the `@cotal-ai/web` extension, seeded automatically on first\nrun (like the built-in connectors) so it always matches your CLI version. It self-registers `cotal web`\ninto this surface and serves\n`http://cotal.localhost:7799` by default (loopback; `*.localhost` resolves in Chrome/Firefox/Edge; Safari may\nneed `http://127.0.0.1:7799`). On a user-auth mesh the dashboard rides the read-only admin view\nover your login, and a channel purge asks for its own channel-purger view per click; both need\nledger scope `admin`. Detached mode re-execs the current Cotal installation, writes diagnostics to\nthe mesh root's `.cotal/web.log`, and reports success only after the HTTP server answers. It requires\na recorded mesh root, but can be launched from any directory once `cotal up` has recorded the mesh.\nSee [Watch a mesh](watch-a-mesh.md).\n\n## mint\n\n```bash\ncotal mint [--profile ] [--out ] [--signer]\ncotal mint --provision [--role ] [--space ] [--server ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--profile ` | `agent` | Credential profile |\n| `--out ` | `.cotal/auth/creds/space./.creds` | Output path - the default sits under the resolved space's segment (`` is that space's hex encoding, as in [Project files](config.md#project-files)) |\n| `--signer` | off | Emit a stripped account-signing file instead |\n| `--force` | off | With `--signer`: overwrite an existing file |\n| `--allow-subscribe ` | the agent file's, else subscribe | Read-ACL override, **agent profile only**: `observer` and `admin` carry a fixed read set, and `mint` refuses this flag there rather than narrowing nothing |\n| `--allow-publish ` | the agent file's, else deny | Post-ACL override, **agent profile only** |\n| `--role ` | the agent file's | Agent profile: the anycast task queue the identity pulls (`svc_`) |\n| `--provision` | off | Agent profile: also pre-create the identity's bind-only DM/deliver durables (and its role's task queue) on the live mesh, so the credential can consume |\n| `--space `, `--server ` | the resolved mesh | Which root supplies the agent file, static trust and default credential storage; with `--provision`, also which live mesh receives the durables |\n\nMints a NATS creds file for a space in **static** auth mode, scoped to a profile and (optionally)\nexplicit read/post ACLs. `--signer` emits an account-signing file for delegating minting to another\nhost. A per-user-auth space refuses `mint`: agents there join under a logged-in user\n([`login`](#login) + [`actor grant`](#actor)), never via a handed-out creds file. See\n[Identity and auth](identity-and-auth.md).\n\nFor an agent profile, the resolved mesh root supplies the persona ACL, the signing material and the\ndefault credential destination as one authority. If the current folder also holds trust for a\ndifferent space or account, mint refuses before writing and names both roots. It never combines a\npersona from one root with credentials signed or stored under another.\n\nA plain mint is creds only: the identity can publish within its post ACL at once, but on an authed\nmesh its DM inbox and task queue are provisioner-pre-created and bind-only, so a **consuming**\nconnect fails until they exist. `--provision` performs that pre-create in the same command (a\nprovisioner cred is minted from the space's trust material, used, and dropped), so a long-running\nclient you start yourself can receive DMs and role anycasts like a spawned seat. The command prints\nthe identity's principal (its wire id) and lifecycle uid; a consuming client passes that uid as its\n`lifecycleUid`. Agent profile only; an open mesh needs none of this (peers self-create there). The\nThe same resolved authority is used for both the credential and `--provision`, so the broker\nfootprint cannot be created under a different root's trust material.\n\n## Login\n\n```bash\ncotal login --idp [--client-id ]\ncotal logout --idp \n```\n\nSigns you in to a per-user-auth mesh's IdP (device code flow) and caches the session; run it\nonce per machine. It prints your IdP subject, the id the operator grants against. After a\nlogin, every command on that mesh works under your identity: each connect takes a fresh IdP\nproof, exchanges it locally for a short-lived bearer, and is authorized against the actor\nledger at connect time. `logout` revokes the IdP session and clears the cache. See\n[identity & auth](identity-and-auth.md).\n\n## actor\n\n```bash\n# an upsert of the WHOLE row: a flag left off is the WIDE default below, not \"unchanged\"\ncotal actor grant --sub [--scope a,b] [--allow-subscribe a,b] [--allow-publish a,b] [--role ] [--label ]\ncotal actor revoke (--sub | --owner )\ncotal actor list\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | the folder's | Space whose ledger to manage |\n| `--sub ` | none | The IdP subject (shown by `cotal login`) the actor belongs to |\n| `--owner ` | none | The derived owner token (alternative to `--sub`) |\n| `--scope ` | `spawn,role:default` | Capability scope (`''` = none; `spawn` = may run agents; `role:` = may delegate role r; `admin` = cross-agent control; `supervise` = eligible for the closed remote manager-service view when the host enables it) |\n| `--allow-subscribe ` | `>` (all channels) | Channel read ACL; the user's envelope, their agents can never read beyond it |\n| `--allow-publish ` | `>` (all channels) | Channel post ACL; also the envelope for their agents' posting |\n| `--role ` | none | Role (scopes the task-queue consumer) |\n| `--label ` | none | Display label for `actor list` (never the IdP subject) |\n\nThe actor ledger is the single authorization source of a user-auth space: no row, no access.\nA bare `grant` is the **full** envelope (all channels, may spawn); the flags narrow it. A\nre-grant **replaces the whole row**, not the one field you name, so to add a capability spell\nevery field out: the new scope plus the row's current read set, post set, role and label\n(`cotal actor list` shows what a row holds). A field left off does not stay as it was, it\nreverts to the wide default in the table above, which is how a narrow reader becomes a reader\nof every channel. A re-grant retires the current interactive lifecycle through the running auth\nservice before it rotates the row, so copied bearers cannot cross an authorization update. If that\nretirement cannot be confirmed, the row is left unchanged and the command fails with the recovery\naction. `revoke` uses the same retirement before deleting the row, which lets a later grant create a\nreal successor instead of colliding with a live predecessor. `supervise` is separate from `spawn` and `admin`: it only makes a signed-in\nperson eligible for the host-provided closed remote manager-service view; it does not grant\nmanagement of another owner or a general host profile. `revoke` denies the next exchange and\nthe next connect with no restart, and evicts the principal's live connections. Managed-agent rows\n(written by the spawn path) live in a disjoint row space this command never touches. See\n[identity & auth](identity-and-auth.md).\n\n## doctor\n\n```bash\ncotal doctor auth [--fix]\n```\n\nCredential-health diagnosis and repair for this folder's mesh: renders every managed\ncredential as healthy / near-expiry / expired and ends in `healthy` or the exact next\ncommand; `--fix` applies the repairs it can. The one surface every stale-credential error\npoints at.\n\n## join\n\n```bash\ncotal join --space --name [--role ] [--channel ]\ncotal join --link | --token \n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and which credential |\n| `--name ` | none | Your presence name |\n| `--role ` | none | Your role |\n| `--channel ` | none | Channel to join |\n| `--kind ` | `agent` | Endpoint kind |\n| `--link ` | none | Join link (`cotal://…`) |\n| `--token ` | none | Join token |\n| `--lifecycle-uid ` | none | Required with `--creds`: the lifecycle UID minted alongside the credential (`COTAL_LIFECYCLE_UID` works too). A credential's durable grants name exact lifecycle-keyed resources, so `join` refuses to invent one |\n| `--tls` | off | Connect over TLS |\n\nAn interactive presence: join a space under your own name and role, without launching an agent\nharness. A `--link` or `--token` supplies the where and the auth in one value. See\n[Spaces](spaces.md) and [Identity and auth](identity-and-auth.md).\n\n## Manifest deploys\n\nA `cotal.yaml` manifest declares a whole mesh (channels, personas, roles, and ACLs) in one file.\nThree commands consume it, plus a read-only validator:\n\n```bash\ncotal up -f cotal.yaml # boot a fresh mesh from the manifest\ncotal spawn -f cotal.yaml # deploy the manifest additively onto a running mesh\ncotal down -f cotal.yaml # tear that deploy down (or --run for one run)\ncotal topology view -f cotal.yaml # validate + view the access graph, change nothing\n```\n\n`up -f` and `spawn -f` differ in target: `up -f` brings up a new broker and applies the manifest;\n`spawn -f` requires an already-reachable mesh and applies additively (ownership-scoped). On a\nuser-auth mesh, `spawn -f` deploys over your own login (the deployer view, gated on ledger scope\n`spawn`): the manifest's agents land under your owner, a manifest claiming another owner is\nrefused, and seeding new channels additionally needs scope `admin`. Both take\n`--dry-run` to print the plan without mutating anything. `topology` validates the manifest and\nrenders its channel / role / ACL graph. See [Define a team](define-a-team.md) and the\n[manifest reference](manifest.md).\n\n## ext\n\n```bash\ncotal ext # same as `list`\ncotal ext add \ncotal ext remove \ncotal ext list\ncotal ext root # print just the install prefix (scriptable)\ncotal ext seed [--repair|--reset|--force]\n```\n\nOperator-installed extensions: `add` installs an npm package into a cotal-owned prefix and records\nevery registry provider it contributes. Commands appear in help, completion, and dispatch; runtime\nproviders are lazy-loaded by commands such as `supervise`; local process providers participate in\n`status` and selective `down`. `remove` and `list` manage them. The `@cotal-ai/web` dashboard is the\ncanonical command/process example. Installed packages and their location are described in\n[config](config.md).\n\nBare `cotal ext` lists the inventory, headed by the install prefix. That prefix is a cotal-owned npm\nroot kept **separate** from npm's own global tree. These packages never show up in `npm list -g`,\n`cotal ext` (or the Extensions section of `cotal status`) is the canonical inventory. `cotal ext root`\nprints only the path, for scripts. The versions shown are the manifest pin recorded at add time.\n\nRemoving an extension that owns a running local process is refused with the mesh root and its\n`cotal down ` command; stop it first so uninstalling the package never strands a process\nwhose lifecycle provider is gone.\n\n### Built-in connectors are seeded extensions\n\nThe first-party agent connectors (`claude`, `opencode`, `codex`, `hermes`, `jcode`, `pi`) are not compiled into\nthe binary. They are seeded on first run through the **same** `ext add` path a third party uses, and\nappear in `cotal ext list` like any other extension. So you can remove one you do not want\n(`cotal ext remove @cotal-ai/connector-hermes`), and a deliberately-removed connector STAYS removed\nacross upgrades. `cotal ext add ` adds a third-party connector the same way. The web\ndashboard (`@cotal-ai/web`, providing `command:web`) is the seventh built-in seeded on the same path.\n\n`cotal ext seed` is the maintenance entry for that seeding (it runs automatically on the first real\ncommand of each boot, so you rarely call it):\n\n| Flag | Meaning |\n|---|---|\n| (none) | Reconcile: seed any never-seeded built-in, refresh a seeded one whose version the binary bumped, leave a removed one removed. A no-op once current. |\n| `--repair` | Recover after an interrupted seed or a lost authority (rebuilds the interrupted connector; restores the removed-vs-never-seeded record from its durable backup). |\n| `--reset` | Discard the record and re-seed all seven built-ins (the six connectors plus the web dashboard). **Resurrects any you removed.** Rebuilds cleanly over corrupt seed state. |\n| `--force` | Re-seed the built-ins even when the version stamp is current or a downgrade. |\n\nWhen a newer `cotal` advances the operator-global seed store to its generation, it prints one\nmigration line naming the old and new generations, the exact CLI entry that wrote the store, the\ncommit timestamp, and `seed/stamp.json`. That writer and timestamp are kept in the stamp, so a later\nolder CLI refusal can say which executable wrote the generation it will not overwrite and when.\nLegacy generation-only stamps remain readable; their refusal simply has no writer provenance to add.\n\nAn older `cotal` refuses a seed store written by a newer version. When it can verify a sufficient\n`cotal` executable on PATH or at the installer's `~/.local/bin/cotal` location, the refusal names\nthat absolute path so a reduced service PATH does not select the older binary again. Otherwise it\nkeeps the generic newer-version instruction. `--force` rebuilds the store for the running older\nversion without discarding the ever-seeded authority. `--reset` still exists for corrupt state and\nresurrects deliberately-removed connectors.\n\nA source-checkout CLI (`pnpm cotal`, `tsx bin/cotal.ts`, `node bin/cotal.ts`, or a suite child of\nthose) refuses to write or garbage-collect that store. The refusal names the path, the generation\nit declined, and `$XDG_CONFIG_HOME` as the isolation remedy. `COTAL_HOME` does not relocate this\nstore. An entry that cannot be proven as a released install is refused the same way. Isolated\nrelease tests that must seed from a checkout-shaped `bin/` set `COTAL_ALLOW_CHECKOUT_SEED=1` after\npointing `$XDG_CONFIG_HOME` at a scratch dir; that override is documented here, not on the refusal\nline. An opt-in write still records the checkout path in `seed/stamp.json` as `writtenBy`.\n\nThe default connector for a bare `cotal spawn` (no `--agent`) is the persona's `agent:` pin if it\nhas one, else `claude`; set `COTAL_DEFAULT_AGENT` (e.g. `opencode`) to change the fallback. It is\na default, so a persona that pins its harness still wins over it. An `--agent` naming a removed\nconnector fails loud with the exact\n`cotal ext add` to restore it. Set `COTAL_SKIP_CONNECTOR_SEED=1` to turn off the automatic first-run\nseed/refresh entirely (for a controlled or offline setup that manages connectors by hand); `cotal ext\nseed` still runs on request.\n\n## completion\n\n```bash\ncotal completion # print a stub to eval / source\ncotal completion install [shell] # install it persistently\n```\n\nPrints or installs shell completion. Completion candidates come from each command's declared flags\nand, where useful, live mesh state (spaces, personas, managed agents) resolved offline.\n\n## feedback\n\n```bash\ncotal feedback \"\" [--type ] [--email ] [--details ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--type ` | none | `bug` \\| `idea` \\| `friction` \\| `praise` \\| `other` |\n| `--details ` | none | Longer free-form details |\n| `--severity ` | none | `low` \\| `medium` \\| `high` |\n| `--area ` | none | The part of Cotal this concerns |\n| `--email ` | git email | Contact email (required on the keyless public path) |\n| `--name ` | none | Your name (optional) |\n| `--url ` | keyed / public intake | Intake URL override |\n| `--key ` | `COTAL_FEEDBACK_KEY` | Feedback key |\n\nSends feedback to the Cotal developers. With a key (`--key` / `COTAL_FEEDBACK_KEY`) it routes to the\nkeyed beta intake; without one it goes to the public `cotal.ai` intake and requires a contact email\n(`--email` / `COTAL_FEEDBACK_EMAIL`, else your git email). Run a self-hosted intake with\n[`feedback-intake`](#server-daemons).\n\n## run\n\nOperate durable workflow runs (cotal-lang programs) from the terminal.\n\n```bash\ncotal run start --file [--timeout ] [--local]\ncotal run resume [--local --file ]\ncotal run ps [--endpoint ]\ncotal run journal [--endpoint ]\ncotal run answer [--value ] [--artifact ] [--endpoint ] [--local --by ]\n```\n\n`start` hands the program to the mesh's manager, which validates it, mints the run id (the record\nnever takes a caller-supplied one), drives it in its own process, and answers with the id once the\nrun is recorded; a program that does not validate is refused with every problem listed. `resume`\nasks the manager to take an existing run back and continue it from its step journal; the source is\nthe recorded program, so no `--file` is taken. Neither takes `--endpoint`: the manager records\nits runs under its own endpoint, and naming another is refused. `ps` lists the run records and\n`journal` renders one run's durable records; both only inspect. `answer` resolves an open\ncheckpoint through the manager, presenting as the holder that armed it; the manager records the\nanswerer from your credential, so no `--by` is taken there. `--timeout` sets the default\ncheckpoint timeout for a drive (default 1h). `--local` drives in this process instead, over one\nconnection per invocation under the run's own credential minted from the project folder's trust\nmaterial, and is the path on a bare broker with no manager or for a run with no recorded program\n(`cotal run resume --local --file `); `answer --local` takes `--by `. A\nuser-auth mesh runs no programs yet: the manager refuses the family by name, and `--local` has no\ncredential there. The guide is [workflows](workflows.md).\n\n## Server daemons\n\nTwo long-lived infra roles ship with the CLI. They are not part of everyday operation; the delivery\ndaemon comes up automatically with `cotal up --detach` in auth mode.\n\n```bash\ncotal deliver --space [--server ] [--creds ]\ncotal auth-service --space --server [--port ] [--exchange-public-port ] [--exchange-public-url ] [--exchange-trusted-proxy]\ncotal feedback-intake --keys [--port ] [--creds ]\n```\n\n`auth-service` runs a user-auth space's identity plane: the NATS auth callout, the\ncapability-gated local exchange and JWKS, and, when `--exchange-public-port` is set, the closed public\nexchange/discovery face forwarded by an HTTPS reverse proxy. `--exchange-public-url` is the proxy URL\nadvertised to clients; `--exchange-trusted-proxy` opts into last-hop `X-Forwarded-For` attribution.\n`cotal up --user-auth` starts and supervises the service for you, so you run it directly only to\nrecover one by hand.\n\n`deliver` runs the server-side Plane-3 delivery daemon: the durable backstop and membership/ACL\nauthority. It is auth-mode-only and single-instance (`--shard`/`--shards` accept only `N=1`);\n`--dev-mint` mints a scoped cred from the local signer for standalone dev. See the\n[delivery daemon](delivery-daemon.md). `feedback-intake` runs a self-hosted feedback server\n(requires `--keys` and a scoped `--creds`), announcing submissions into a space channel; flags\ninclude `--host`/`--port`, `--store`, `--space`/`--channel`, `--max-bytes`, and `--rate-limit`.\n\n## Plumbing\n\n`cotal __complete ` is the internal entry the shell-completion stubs call to emit candidates\nfor the current command line; you never run it directly. `cotal agent-bearer` is machine-facing\nplumbing on user-auth meshes: spawned agents exec it to print a fresh short-lived bearer from their\nspawn-time secret; you never run it directly either. Its local arm uses `--dir` to discover the\ncapability-gated loopback service. A remotely enrolled, already-granted agent instead receives\n`--exchange-url ` in its launch argv: that arm sends `{owner, actor, actorToken}` to the\npinned public exchange with no local capability, follows no redirects, and refuses every non-HTTPS\nURL because the actor token is the credential in the request body. (`cotal start` is a removed tombstone: it\nerrors and points you to `cotal spawn --detach`.)\n" + "body": "# `cotal` CLI reference\n\n> **Reference**: describes the TypeScript reference implementation (the `cotal` CLI), not the wire contract. · **For:** operators · **Wire contract:** [SPEC](../SPEC.md)\n\n`cotal` is the operator command line for the reference implementation: bring a mesh up, mint\nidentities, launch agents, watch what they do, and tear it all down. It is a thin client over the\nwire contract: the normative subjects and schemas live in the [SPEC](../SPEC.md); this page is\nlookup material for the commands, not a walkthrough; if you are new, start with\n[Getting started](getting-started.md).\n\n## Running it\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH (needs Node 22+)\ncotal --help # every command, grouped\ncotal --version # cotal-ai version + each installed extension's (also `cotal -v`)\ncotal --help # one command's flags and usage\n```\n\n`npx cotal-ai ` runs it without a global install; in a dev clone, `pnpm cotal `\nruns it through `tsx` with no build step. Bare `cotal` prints help. Every command generates its own\n`--help`, usage, and shell completion from its declared flags.\n\nCommands come from the surfaces the binary composes: the base mesh CLI, the manager\n(`supervise`), and the delivery daemon (`deliver`), plus any operator-installed extensions.\n`cotal ext add ` installs any registry providers a package contributes: commands,\nruntimes, and local process lifecycle descriptors. The `web` dashboard and optional manager\nruntimes ship this way.\n\n## Commands\n\n| Area | Command | Purpose |\n|---|---|---|\n| Set up & lifecycle | [`setup`](#setup) | Guided, configure-only setup (installs, seeds personas; launches nothing) |\n| Set up & lifecycle | [`update`](#update) | Reconcile first-party extensions and check or opt into a coherent CLI upgrade |\n| Set up & lifecycle | [`up`](#up) | Start a local mesh (nats-server + JetStream), or boot a whole manifest with `-f` |\n| Set up & lifecycle | [`down`](#down) | Stop the whole stack, selected registered components, or a manifest deploy |\n| Set up & lifecycle | [`backup`](#backups) | Create an offline full-space or registry-only artifact from a preserved cut |\n| Set up & lifecycle | [`clean`](#clean) | Configurable cleanup: purge history (live), or wipe the local store / identity (stopped) |\n| Set up & lifecycle | [`meshes`](#mesh-registry) | List the running meshes on this machine |\n| Set up & lifecycle | [`use`](#mesh-registry) | Set the default mesh a bare `cotal spawn` joins |\n| Set up & lifecycle | [`status`](#mesh-registry) | Read-only diagnostics for setup, processes, and the selected mesh |\n| Agents & personas | [`spawn`](#spawn) | Launch an agent from a persona (foreground, or `--detach` via the manager) |\n| Agents & personas | [`models`](#models) | List connector model catalogs and variants from the manager |\n| Agents & personas | [`ps`](#managed-seats) | List managed agents and their mesh status |\n| Agents & personas | [`stop`](#managed-seats) | Ask the manager to stop a managed agent |\n| Agents & personas | [`attach`](#managed-seats) | Stream and drive a managed agent's terminal (pty runtime) |\n| Agents & personas | [`input`](#input) | Type one line into a managed agent's terminal without attaching |\n| Agents & personas | [`personas`](#personas) | List, show, edit, create, or remove local personas |\n| Agents & personas | [`supervise`](#supervise) | Run a manager daemon (the agent supervisor / control plane) |\n| Agents & personas | [`runtimes`](#runtimes) | List the agent runtimes the manager can spawn through and whether each is reachable |\n| Agents & personas | [`reconcile-gate`](#reconcile-gate) | Unfreeze an issuance gate left frozen by a crashed restart when the successor cannot boot-heal it (holder gone, complete CONNZ sweep) |\n| Messaging & watching | [`endpoints`](#endpoints) | List every endpoint in the live presence roster, including infrastructure |\n| Messaging & watching | [`describe` / `invoke`](#endpoint-control) | Resolve a v0.4 service's command surface off the wire; invoke one command by name |\n| Messaging & watching | [`send`](#send) | Send one message, then exit: DM a peer, post a channel, or ask a role |\n| Messaging & watching | [`channels`](#channels) | Inspect or set the channel registry |\n| Messaging & watching | [`history`](#history) | Clear retained message history |\n| Messaging & watching | [`console`](#console) | Live protocol view for a space (TUI, or `--plain` line stream) |\n| Messaging & watching | [`web`](#web) | Browser dashboard (installed as the `@cotal-ai/web` extension) |\n| Auth & meshes | [`mint`](#mint) | Mint a creds file for a space (static auth mode) |\n| Auth & meshes | [`login`](#login) | Sign in to a per-user-auth mesh's IdP (once per machine) |\n| Auth & meshes | [`logout`](#login) | Revoke the IdP session and clear the cached login |\n| Auth & meshes | [`actor`](#actor) | Manage a user-auth space's actor ledger (grant / revoke / list) |\n| Auth & meshes | [`doctor`](#doctor) | Credential-health diagnosis and repair (`doctor auth`) |\n| Auth & meshes | [`join`](#join) | Join a space as your own presence (interactive) |\n| Manifest | [`topology`](#manifest-deploys) | Validate and view a mesh manifest's access graph (read-only) |\n| Extensions & misc | [`ext`](#ext) | Install / remove operator CLI extensions |\n| Extensions & misc | [`completion`](#completion) | Print or install shell completion |\n| Extensions & misc | [`feedback`](#feedback) | Send feedback to the Cotal developers |\n| Extensions & misc | [`deliver`](#server-daemons) | Run the server-side Plane-3 delivery daemon |\n| Workflow runs | [`run`](#run) | Operate durable workflow runs: start, resume, list, inspect, answer a checkpoint |\n| Extensions & misc | [`feedback-intake`](#server-daemons) | Run a self-hosted feedback intake server |\n\nThe manifest modes of `up`, `spawn`, and `down` (`-f `) plus `topology` are covered\ntogether under [Manifest deploys](#manifest-deploys).\n\n## setup\n\n```bash\ncotal setup [--full] [--demo] [--yes] [--skills]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--full` | off | Redo the full guided flow (implies `--demo`) |\n| `--demo` | off | Also seed the guided expert team (`david`, `sven`, `me`) |\n| `--yes`, `-y` | off | Non-interactive accept-all (for agents / CI) |\n| `--skills` | off | Reconcile Cotal skills only through installed connector providers, plus `~/.agents/skills`. Refused with `--full` or `--demo`. |\n\nGuided setup is **configure-only**: it checks prerequisites, invokes installed connectors' declared setup providers, and\nseeds persona files, and it launches nothing (no mesh, no web, no manager). First run gets the\nnarrated flow; later runs print a status card. By default it seeds one `default` persona; the\n`david`/`sven`/`me` team is opt-in via `--demo`. `cotal status` points stale Claude skills and\nout-of-date `.agents` skills at `cotal setup --skills`, not unscoped `setup`. See [Getting started](getting-started.md) and, for\nmaintainers, [setup internals](setup-internals.md).\n\nWhen a mesh resolves, setup seeds that mesh's recorded `.cotal/agents` catalog, the same catalog a\nfollowing `cotal spawn` reads. It prints the absolute destination. On a fresh machine with no mesh it\nuses this folder and says why; when several meshes are available and none is selected, it refuses\nrather than choosing a catalog.\n\n## update\n\n```bash\ncotal update [--self] [--space ] [--server ] [--creds ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--self` | off | If a newer release exists, install that exact validated `cotal-ai` version globally and reconcile through the newly installed binary |\n| `--space`, `--server`, `--creds` | resolved mesh | Select the running manager whose continuity state is reported |\n\nWithout `--self`, `update` keeps the installed first-party surfaces coherent with the running\nbinary: it force-reconciles the four built-in connectors, then reinstalls other `@cotal-ai/*`\noperator extensions at the binary's exact version. Each extension runs in an isolated child, so one\nfailure cannot poison later replays. It then checks npm; a newer binary is an informational notice\nwith `cotal update --self` as the next command, not an automatic install.\n\nAfter disk reconciliation, `update` reads the selected running manager. A manager without a\ncustody generation is reported as `legacy`: it cannot preserve its manager-owned PTYs, so the\ncommand says that this is not a hot update and prints `exact`, `fork`, `fresh`, or `drain-only`\nfor every seat. This report sends no stop, preservation-commit, or replacement command.\nIt does not preserve a running PTY on a legacy manager. On Linux a detached custodian\nowns each PTY, so a manager-worker death no longer closes the seat and `status` reports\n`custodied`. Other platforms still spawn in-process and report `legacy`. An incompatible native\n`@lydell/node-pty` or ConPTY ABI break remains an explicit per-seat maintenance cut.\n\nWith `--self`, the selected running manager is reported before any global install. When a newer\nrelease exists, Cotal then installs the exact version it validated, resolves and verifies that\npackage in npm's global root, then launches that binary with the same `--space` / `--server` /\n`--creds` selection to reconcile connectors and first-party extensions to the new generation. An npx\nor dev-clone invocation therefore installs and continues through a separate global copy; it never\nclaims the already-running process changed. If the binary is current, `--self` performs the normal\nlocal reconcile without reinstalling it.\n\nThird-party extensions are listed with their installed version and recorded spec but are not\nauto-updated in v1. Floating third-party updates require `@cotal-ai/*` peer-range validation and are\na future follow-up. A failed connector/extension install, npm metadata check, or requested global\ninstall is reported and makes the command exit nonzero. Independent extension attempts continue so\nthe output includes every failure; an unavailable npm registry does not undo a completed local\nreconcile, but the command still exits nonzero because it could not establish that the install is\ncurrent.\n\n## up\n\n```bash\ncotal up [--detach] [--open] [--space ] [--server ] [--channels ] [--runtime ]\ncotal up --user-auth --idp [--exchange-public-port --exchange-public-url [--exchange-trusted-proxy]]\ncotal up --tls-cert --tls-key # serve broker TLS (both, or neither)\ncotal up --restore [--restore-only registry] [--accept-missing-source]\ncotal up -f [--dry-run] [--runtime ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--server ` | auto (free local port) | Listen URL override |\n| `--host ` | none | Bind host override. With no `--server`, the broker URL is derived from it, so `--host ` alone is enough to make a mesh reachable at that address; a `--host`/`--server` pair naming different addresses is refused. A wildcard bind (`0.0.0.0`, `::`) keeps a dialable loopback URL. Recorded on the mesh and reused by every later manager launch, so a repair or resume keeps remote [`attach`](#managed-seats) working |\n| `--space ` | the folder's name | Space name |\n| `--store-dir ` | none | JetStream store directory |\n| `--channels ` | `.cotal/channels.json` if present | Channel-registry seed file (JSON). An explicit path that is missing is an error |\n| `--restore ` | none | Restore a completed offline backup before exposing the normal listener |\n| `--restore-only registry` | artifact selection | Restore only the registry component |\n| `--accept-missing-source` | off | Explicit disaster consent when the inode-bound preserved source is absent |\n| `--open` | off (auth) | Unauthenticated dev mesh: no JWT, no ACLs |\n| `--user-auth` | off | Per-user auth: people `cotal login`; connects are authorized against the actor ledger |\n| `--idp ` | none | With `--user-auth`: the IdP auth base URL to pin on first enable |\n| `--exchange-public-port ` | none | With `--user-auth`: add the public exchange face on this loopback port, for an HTTPS reverse proxy to forward to |\n| `--exchange-public-url ` | none | With `--exchange-public-port`: advertise the reverse proxy's HTTPS URL in discovery |\n| `--exchange-trusted-proxy` | off | With `--exchange-public-port`: attribute public failure buckets to the last `X-Forwarded-For` hop. Enable only when the listener is reachable solely through a trusted proxy; otherwise the socket address is used |\n| `--detach` | off | Run in the background (stop with `cotal down`) |\n| `--tls-cert ` | none | PEM certificate to serve TLS with. Must be given together with `--tls-key`. Before starting the broker, Cotal checks readability, private-key mode, key/certificate match, the validity window, and host coverage. `nats-server` accepts an expired certificate and leaves the failure to clients, so Cotal performs these checks first. The decision is recorded; a later bare `cotal up` keeps serving TLS |\n| `--tls-key ` | none | PEM private key for `--tls-cert`. Refused if group- or other-readable (tighten to `600`) |\n| `--file `, `-f` | none | Launch a whole mesh from a manifest |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--runtime ` | `pty` (or the manifest's, with `-f`) | Agent runtime for the mesh manager (`pty` built in; others are installed extensions, explicit-only). Resolved + probed before the broker starts; an uninstalled/unreachable runtime fails loud. With `-f`, overrides the manifest's runtime |\n| `--rotate-sys` | off | Rotate the space's system account and re-mint its two `$SYS` creds. Needs a stopped mesh; refused with `--open` |\n\n`cotal up` boots a local nats-server with JetStream and, in auth mode (the default), JWT auth and\nper-agent ACLs; `--detach` records the mesh so `cotal spawn` from any directory can find it. With no\n`--server`, it auto-selects a free port if the default address is taken; an explicit `--server`\nstays fail-loud on collision. `--detach` also brings up the control plane (delivery daemon in auth\nmode, then the manager). The `-f` form is a [manifest deploy](#manifest-deploys); see\n[Run a mesh](run-a-mesh.md).\n\n`--user-auth --idp ` starts the space's auth service alongside the broker: the NATS\nauth callout plus its capability-gated local exchange, and optionally the closed public exchange\nface configured by the three `--exchange-*` flags above. The service is torn down with `cotal down`,\nand a re-run of `cotal up` heals a dead service on a running broker. `--user-auth` and `--open`\ncontradict each other and are refused loudly; a running broker cannot change auth mode\nwithout a `cotal down` first. See [identity & auth](identity-and-auth.md).\n\n`--rotate-sys` renews the two `$SYS` credentials (`membership-observer`, `connection-evictor`).\nThey carry a 30-day expiry and nothing re-signs them in place, because the system-account seed is\nnever persisted, so they are renewed by issuing a **new system account** under the same broker\noperator and minting fresh creds against it. A plain re-`up` does **not** do this: it reuses the\nexisting trust record, and its `$SYS` creds along with it.\n\nThe rotation is safe to run on a real space, with one operational cost. The data account, the account\nsigning key, every agent credential minted from it, and the JetStream store are all untouched; what\ndies is the retired system account, and with it any out-of-band copy of the old `$SYS` creds, on every\nbroker that loads the rotated config. The cost is that **earlier full backups stop being restorable**\n(see below), so this is not a no-consequence operation. It needs the broker to restart on the rewritten\nconfig, so it runs as part of a boot:\n\n```bash\ncotal down\ncotal up --rotate-sys --detach # agents reconnect; nothing is re-provisioned\ncotal doctor auth # both $SYS creds healthy again, 30 days out\n```\n\nA rotation is a stopped, fresh boot, and anything that is not one refuses it, all for the same reason\n(the on-disk material and the broker it runs on must never end up on different generations):\n\n- a live mesh, because the running broker would keep serving the retired account;\n- an open mesh, whether that comes from `--open` or from `broker.auth: false` in a manifest, which\n has no system account at all;\n- `--restore`, because reinstating a trust root and superseding it in one command leaves no way to\n say which authority the mesh came up on;\n- an unfinished restore or resume attempt on this root, including one `cotal up` would recover on\n its own, because those paths can adopt a live listener and return without booting a broker;\n- a root that hosts more than one space, because the system account lives in the shared broker\n record and a rotation would retire every tenant's, while the root holds one `$SYS` cred pair\n pinned to one data account.\n\nTwo things to know before you run it:\n\n- **The retirement is config-load-bound.** Old `$SYS` creds are refused by any broker that loads the\n rotated config. A stale `nats-server` still running the *previous* config in memory would keep\n honouring them, so stop every broker for this root first. `--rotate-sys` refuses if this root's\n mesh is recorded as running, if anything unidentified is answering at the address it was given, or\n if the root's pid file names a live (or unreadable) process. Those are Cotal's own ownership\n records, not a scan of the process table: a `nats-server` you started by hand against this root's\n `server.conf` on some other port writes none of them and will not be seen. Do not run one.\n- **It invalidates earlier full backups.** A full artifact binds to the trust chain it was taken\n against, and that commitment covers the operator JWT and the system account. Every full backup\n taken before a rotation refuses to restore afterwards, so take a fresh `cotal backup` once the\n rotated mesh is up. `cotal up --restore` names this case when the data account still matches.\n\nThe commit is not atomic (a trust-record write plus two credential writes), so an interrupted\nrotation leaves the record ahead of the creds. That split is detected rather than silent: every\n`cotal up` on an auth mesh, and every `cotal doctor auth`, compares each `$SYS` cred's issuer against\nthe persisted record and names the retired account. `up` warns rather than refusing, because these\ncreds power the membership graph and live eviction, both of which degrade fail-soft; the mesh is not\nworth taking down over them. Re-running the rotation heals it, at the cost of one generation.\n\nWhile those creds are expired the mesh keeps delivering messages, but the\n[membership feed](delivery-daemon.md) and live connection eviction stay down; `cotal doctor auth`\nand the manager's log both name the credential and this repair.\n\n## down\n\n```bash\ncotal down\ncotal down --with-agents\ncotal down --preserve-state [--store-dir ]\ncotal down manager [delivery auth web nats ...]\ncotal down web [--space ]\ncotal down -f | --run [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--file `, `-f` | none | Tear down this manifest's deploy |\n| `--run ` | none | Tear down one `spawn -f` run by id |\n| `--space ` | current mesh | With components: the mesh whose target-addressed components (e.g. `web`) to stop |\n| `--dry-run` | off | Print the manifest teardown or selected components, mutate nothing |\n| `--preserve-state` | off | Bare whole stack only: fence the manager, retain principals and durable state, stop and prove the stack down, then publish `ready` |\n| `--with-agents` | off | Bare whole stack only: also stop every managed agent (the previous default) |\n| `--store-dir ` | `.cotal/nats` | With `--preserve-state`: the actual store path (required for a custom store) |\n\nBare `cotal down` stops the whole local stack in dependency order and leaves managed agents\nrunning as unmanaged OS processes. If the manager answers `ps`, it prints their names, pids, and\nworking directories, and points at `cotal down --with-agents` to take them with the stack. If the\nmanager cannot be asked, it still signals the stack and says leftovers may remain.\n`--with-agents` in that case still stops the stack, reaps no seats, and exits non-zero\nnaming that the agents are still running unmanaged. A PTY child\nmay still die when the manager process exits. A later manager on the same root may still take\nleftover seats. This listing is `cotal down`. A failed `cotal up` teardown SIGTERMs the manager\nwithout listing leftover seats. An older manager whose `stop()` still reaps will still reap on\nSIGTERM. `--with-agents`\ncannot be combined with `--preserve-state`, component names, `--space`, `--file`, or `--run`.\n`--with-agents --dry-run` prints the seats that would be reaped and mutates nothing. `--with-agents`\nwaits for each seat's runtime to prove exit before signalling the manager. If only some seats stop,\nthe command prints the stopped seats and the still-running seats separately, with each failure, and\nexits non-zero. Positional\ncomponent names stop only those self-registered local processes; for example, `cotal down manager` leaves delivery and\nthe broker running, and `cotal down web` is available when the web extension is installed. A\ncomponent that starts target-resolved (the web dashboard) is stopped the same way: `cotal down web`\nresolves the mesh the same way as `cotal web` (registry current mesh first, `--space` to name one), so\nit works from any directory; the other components always stop under the folder you run it in. The\n`-f` / `--run` forms tear down a [manifest deploy](#manifest-deploys) without stopping the whole mesh\nand cannot be combined with component names. Stopping `nats` alone is refused while an unselected\nregistered daemon is still live; include those components or use bare `cotal down`.\n\n**Teardown verifies pinned process identity before signalling.** PIDs are recycled by every OS,\nso a recorded pid alone is not a durable target identity. `up` records each stack process's\ncreation identity in a sibling `.identity` pin, which holds the pid and the process start\nreported by the OS. Every stop path, including `down` for the broker, web and extension components,\nand the manager, delivery and auth-service stops, applies the same rule. A pin that names a different\nstart means the pid was reused, so teardown refuses and preserves it. A torn or unreadable pin also\nrefuses. Once the recorded process is stopped, rerunning teardown clears the stale record\nautomatically.\n\nThe first teardown after upgrading a running pre-pin stack has a narrower guarantee. A live record\nwith no identity pin is signalled after a loud warning that it predates identity pinning. Restarting\nthe component writes the pin, so later teardowns receive full match and mismatch protection. The\nsame warning applies on platforms where no stable start token is available.\n\nNormal `down` remains destructive at the logical identity/durable layer. `--preserve-state` is a\ndifferent maintenance transition: it suppresses leave/deprovision cleanup, persists the manager's\nsame-principal resume inventory, stops the entire stack without removing run/auth artifacts, and\npublishes a stable inode-bound cut only after every recorded process is proven stopped and the exact\nrecorded NATS endpoint is unreachable. A missing or stale broker pidfile never counts as stopped. The\nattempt is bound durably before the manager is fenced, the resume document and attempt-bound\n`cut-intent` are fsynced before manager commit, and the manager's commitment itself is journaled\n(`cut-committed`) before any process stops. A retry after a crash at any of those boundaries reuses\nthe exact recorded attempt and finishes the remaining stop and endpoint proofs idempotently, without\nneeding the (by then intentionally dead) manager. A partial cut never publishes `ready`. It cannot\nbe combined with component names, manifest teardown, or `--dry-run`.\n\n## clean\n\n```bash\ncotal clean --force\ncotal clean restore-attempt --attempt --force\ncotal clean restore-fallback --attempt --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | `history`: target mesh |\n| `--dms` | off | `history`: also clear DM history |\n| `--store-dir ` | `.cotal/nats` | `store`/`all`: JetStream store directory |\n| `--force` | none | Required: destructive, no prompting |\n| `--attempt ` | none | `restore-attempt`: exact stale pre-commit attempt; `restore-fallback`: matching healthy committed restore |\n\nOne configurable cleanup verb; every target requires `--force`.\n\n- `history` purges the retained message backlog on the **running** broker (channels, plus DMs\n with `--dms`). The same operation as [`history clear`](#history), which stays as an alias.\n- `store` deletes the **stopped** mesh's JetStream store (`.cotal/nats`): streams, durable\n consumers, and messages. This is the reset for stale on-disk broker state, e.g. durables\n minted by an older, incompatible Cotal generation surviving a `down`/`up` cycle.\n- `all` is `store` plus the space identity (`.cotal/auth`), the local creds and markers tied to\n it, any crash residue a normal `down` would have swept (stale pidfiles, `run/`), and the mesh's\n registry entry; the next `cotal up` mints a fresh identity.\n\n`history` needs the mesh up; `store` and `all` refuse while any recorded mesh process is still\nalive or any same-root recorded broker endpoint remains reachable (run `cotal down` first). They\nalso refuse outright on a root that holds accounts for several spaces: the store and the broker\ntrust record are shared by every space on the broker, so both targets would take out all of them\nand no `--space` can narrow that. `down`, `backup` and `up --restore` refuse there for the same\nreason. `cotal status` lists the tenants on such a root. Personas\n(`.cotal/agents`) and logs are never touched. A custom\nstore location is not recorded anywhere, so `--store-dir` must repeat whatever the mesh was\nlaunched with. Custom cleanup targets must contain either the Cotal store-generation marker or a\nreal `jetstream/` store directory; filesystem roots, project roots, and Cotal auth/maintenance trees\nare always refused.\n\n`store` and `all` also refuse every maintenance journal state. After a healthy committed restore,\n`restore-fallback` is the only supported way to remove the recorded unchanged old-store inode; it\nnever deletes the active target, requires both the exact attempt id and `--force`, and retires the\ncompleted restore journal so a later `down --preserve-state` can start a new backup cycle.\n\n## Backups\n\n```bash\ncotal down --preserve-state [--store-dir ]\ncotal backup create [--only full|registry] [--store-dir ]\ncotal up --restore [--restore-only registry] [--accept-missing-source]\n```\n\nBackup is offline-only. It requires the stable `ready` record from `down --preserve-state`, an exact\nstore match, no live recorded process, and an unreachable exact endpoint from the recorded cut.\nThat endpoint is probed immediately before cloning, so a live broker with a missing or stale pidfile\nis still refused. It claims the cut, reflink/copies the stopped source to a\nprivate attempt clone, and opens only that clone on a random loopback bootstrap broker with an\nindependent parent/deadline watchdog. It validates the canonical stream and pull-consumer inventory,\nwrites native snapshots with consumers excluded, and stores conservative contiguous ACK-floor\ncheckpoints separately. The original store is never opened by the backup broker, and the stack is\nnot restarted implicitly. Artifact destinations must not overlap the preserved source or maintenance\nattempt tree. Restore artifacts and targets likewise cannot nest inside or contain each other, the\npreserved source, or the maintenance attempt tree.\n\nStopped client-managed KV ordered consumers are ephemeral read residue, not backup state. Backup\nignores only the pinned client's exact stopped shapes: ordinary last-value watchers and the\nwhole-bucket scanner that uses all-history delivery to collapse concurrent tombstones. A bound\nconsumer or any lookalike with a different filter, inbox, lifetime, or other config is still refused.\n\n`full` is the default and indivisible: channel registry, CHAT/DM/TASK/INBOX/DLV, ACL, MEMBERS, and\nvalidated durable checkpoints. `registry` is the sole partial artifact. Presence, derived membership\nfeed, leases, native ephemeral/history consumers, credentials, keys, tokens, owner secrets, and actor\nledger files are excluded. `full` means every transferable message and registry stream, not every\nJetStream resource: endpoint submissions/facts/events/timers/workflow state, contract artifacts, and\nthe records/auth/session stores are nonportable control state. Restore recreates those streams empty\nwith their canonical configs before exposing the normal listener, so active endpoint runs,\nlifecycles, and sessions do not cross a backup. Artifacts are exclusively created `0700`;\nsnapshot/checkpoint files and\nthe manifest are `0600`; `manifest.json` is written last with exact sizes and SHA-256 values. The\ndirectory is trusted operator input: hashes detect corruption, not malicious rewriting.\n\nRestore validates and stages the exact allowlisted artifact bytes before moving or creating a store.\nIt requires the same space and existing trust state. The whole pre-commit window holds a journaled\nliveness claim (coordinator, watchdogs, brokers, absolute deadline): ordinary `up` and a repeated\n`up --restore` refuse while the claim is live, and a stale attempt is recovered only after the\ndeadline has elapsed and every recorded owner is proven dead. A retried `up --restore` handles this\nautomatically; an operator can also recover it explicitly with `cotal clean restore-attempt --attempt --force`. Nothing\never rolls back a live attempt. A registry-only artifact restores as registry-only whether or not\n`--restore-only registry` is passed; omitted infrastructure is always created and the exact\npost-restore stream inventory is asserted before commit intent. Ordinary `up` from a preserved cut\nresumes only the exact recorded source store and runtime; a contradicting `--store-dir` or\n`--runtime` fails in preflight. Authenticated restores validate the complete\nspace trust bundle before staging, including nkeys, seed matches, JWTs, signers, and space binding;\nfull restores commit to the validated operator, system-account, data-account, and active-signer root\nchain in addition to the static/user authority fingerprint. Because the system account is part of that\ncommitment, a [`cotal up --rotate-sys`](#up) makes every full artifact taken before it unrestorable\nagainst this root: take a fresh full backup after each rotation. The composed commitment is revalidated\nimmediately before store mutation and never includes secret seeds. Restore never creates fresh auth.\nSame-path restores atomically retain the old\nsource at the journaled fallback path; alternate targets retain it in place; a missing canonical\nsource needs explicit `--accept-missing-source`. Quarantine and target restores use current canonical\nconfigs on isolated random-loopback brokers, never expose native snapshot consumers, and publish a\ncommit-intent immediately before the normal listener starts. Archive bytes never instantiate the real\ntarget: after quarantine validation, every stream is re-snapshotted from the validated quarantine\nstate into attempt-owned sanitized files, and the target is restored solely from those. Before that boundary, failure rolls back\nthe attempt-owned target; after it, ambiguity preserves both stores and records forward-repair\nrecourse. The cooperative maintenance lock excludes Cotal commands, not arbitrary raw NATS processes.\n\nBootstrap brokers in every auth mode, including open, mount the store under a local account with\nrandom operation-specific logins only, each carrying the exact per-phase subject permission matrix;\nnormal static credentials and user-auth sentinel/bearer connections are rejected, and no auth\nservice or callout starts. Open mode differs only in its account label, never in authority. Inventory, each stream snapshot,\nrestore initiation, exact upload id, validation, and each checkpoint recreation use separate exact\nauthorities. Every checkpoint carries the source stream's message/first/last sequence state and must\nmatch its snapshot record before mutation; core then derives and validates the only allowed start\npolicy. TASK is not a CLI exception: the same core checkpoint API recreates its canonical `DeliverAll`\nWorkQueue durable because acknowledged tasks are absent from retention and NATS forbids a\nstart-sequence policy there. Registry-only restore creates every omitted canonical stream and transient\nbucket on the isolated target before the normal listener is exposed. It deliberately does not resume\nretained agents or recreate their DM/DLV/TASK/ACL state; their identity material stays retained and\nstopped rather than being reprovisioned into a partial restore.\n\nAfter listener readiness, the manager starts attempt-bound, validates retained credentials/tokens\nwithout granting or reprovisioning, and resumes the exact persisted principals under cleanup\nsuppression. Registry-only restore uses the same flow with an empty agent set. `commitResume` is an\nidempotent validation barrier only: success must be `awaitingFinalize` with an attempt-bound 64-hex\ncommit token and does not release suppression. Under the workspace lock, the CLI first fsyncs that\nexact evidence as `manager-committed` (restore) or `resume-committed` (ordinary resume), then calls\ntoken-bound `finalizeResume`; only an `active` response for the exact token releases suppression. The\nCLI records the same token in finalization evidence before a restore becomes `active`, or before an\nordinary resume retires and consumes the marker. Re-entry from either committed state skips the prior\nidempotent activation/commit phases, retries finalization with the durable token, and finishes the\nworkspace transition. Failure before finalization preserves the committed state and cleanup\nsuppression; it is not rewritten through a degraded transition. Re-entry between any two earlier\nboundaries reuses the same attempt and may retry the idempotent phases without deleting retained state. A missing or\nchanged per-agent dependency is a named fail-closed result; the journal becomes degraded and remains\navailable for forward repair. A retry from `resume-intent`,\n`resume-active`, or `resume-degraded` reuses the same attempt and inventory after the prior listener is\nproven stopped. Every normal restore listener has an unguessable attempt-bound NATS server name. The\nCLI fsyncs its exact name/nonce, canonical endpoint, process owner, and generation-bound target identity\nimmediately after spawn. Re-entry accepts a surviving listener only when its INFO server name, live PID\nrecord, endpoint, and target identity all match that proof; degraded restore repair then moves through\nthe guarded workspace transition only after manager commit. If an uncommitted bound owner is provably\ndead, recovery retires that exact proof under the maintenance lock and binds a fresh listener for the\nsame attempt, endpoint, and target with a new nonce and server name. A live foreign/mismatched listener\nor ambiguous owner is preserved and refused, never adopted by reachability alone. A reconstructed\ncommit/degraded attempt without either the exact bound proof or a durable dead-listener replacement\nrecord fails closed even when the recorded port is free. A later ordinary startup may pass an `active`\nrestore only when its details prove manager commit and its exact recorded listener is dead.\n\n## Mesh registry\n\n```bash\ncotal meshes\ncotal meshes add # guided, on a terminal\ncotal meshes add --server [--root ] [--mode auth|open|user] [--tls] [--force]\ncotal meshes add --mode user (--user-auth-file | --from )\ncotal meshes rm [ …] [--force]\ncotal use \ncotal status [--space ] [--server ] [--components]\n```\n\n`meshes` lists the meshes this machine knows; a `*` marks the `current` default a bare\n`cotal spawn` joins.\n\nRun on a terminal with the space or `--server` missing, **`meshes add` is guided**: it asks for the\none thing that cannot be derived (the broker URL), probes it, and tells you what answered - open or\nrequiring credentials. It then offers the spaces your `--root` already holds credentials for, states\nthe mode as a fact about that broker rather than asking, and shows the exact record before writing\nanything. A broker that does not answer, or a space name already registered, becomes a choice rather\nthan an error. Anything you pass on the command line is taken as given and not asked again. Without\na terminal - a script, an agent, CI - nothing prompts and the flag form's errors stand\n(`COTAL_NO_PROMPT=1` forces that too).\n\n`cotal up` and `cotal down` maintain their own records. `meshes add` registers a mesh they cannot\nspeak for: one running on another machine, a shared broker, a hosted space. `--root` is the folder\nwhose `.cotal/auth` holds that mesh's credentials and whose `.cotal/agents` holds its personas.\nThe default is the project you run it in. The registry stores that path, never a secret. `--mode`\ndefaults to `auth` when the root holds the space's account record and to `open` otherwise. The\nbroker is probed before anything is recorded, so a wrong address, or credentials that mesh will\nnot accept, fails here instead of at the first `spawn`; `--force` records without verifying (and\nreplaces an existing record).\n\nA hostname or public address is registrable only when the connection will **require TLS**. Pass\n`--tls`, or use a `tls://` URL. The scheme is recorded as enforced intent, so every later dial\nthrough the record demands the handshake (and `meshes add tls://…` against a plaintext broker is\nrefused at registration). Without required TLS the fence admits loopback and private-overlay\nliterals only. RFC1918 addresses are refused in both modes because a cafe LAN is private but does not belong to you.\n\nA **user-auth** mesh registers from supplied pinned trust, never guessed: `--user-auth-file`\ntakes the bundle exported where the mesh runs; `--from` asks before it dials the address at all,\nthen fetches its `/.well-known/cotal-mesh` discovery document (HTTPS only), displays the pins, and\nasks again before adopting them. Neither fetch follows redirects: a 302 can move a pinned fetch\nonto plaintext or onto another host, so it is refused rather than followed, and the pinned\nexchange must itself be an `https://` URL, except for an exchange on this machine, where plain\n`http://` is accepted for a loopback *literal* (`127.0.0.1`, `::1`, any spelling of them) but not\nfor `localhost`, which is a name rather than an address. Registration verifies that the exchange\nanswers `/health` and `/jwks` as the pinned issuer. It also verifies that the broker refuses a bare\nconnect; that auth-required refusal is the pass. The sentinel credentials land in a 0600 file under\nthe entry's root; the registry records only the path.\n\n`meshes rm` drops records. It never stops a mesh. For a mesh running on this machine `cotal down`\nis the right verb, and `rm` says so unless you pass `--force`. A hand-added record is removed by\n`meshes rm`, by an `add --force` replacement, or by a `cotal up` that actually starts the broker for that same space, server and root, which becomes that\nmesh and so takes the record over (a `cotal up` for that space anywhere else refuses instead).\nNothing that merely *infers* a record is stale touches it: an\nunreachable broker is listed `offline` and stays, and `cotal down` / `cotal clean all` leave it\nalone even when it shares a root with the project they are tearing down, because nothing on this\nmachine could write it back.\n\n`use ` sets that default; the selection applies from every directory,\nincluding inside another mesh's project. `status` is a read-only report: machine prerequisites\n(starting with the installed `cotal-ai` version), the installed extensions and their versions, this\nfolder's `.cotal/`, the recorded meshes, and a live snapshot of the selected mesh (roster, channels,\nmembership feed). Stale Claude skills and out-of-date `.agents` skills recommend `cotal setup --skills`,\nnot unscoped `cotal setup`. `status` takes `--space` / `--server` to pick the mesh to inspect; it starts\nnothing.\n\nPersona rows name the catalog they describe. If this folder and the selected mesh use different\ncatalogs, status names both and marks which one spawn launches from. A green `default` means the file\npasses the same agent-file loader spawn uses; a present but invalid file is reported as invalid.\n\n`cotal status --components` adds a fail-loud per-component health pass. It reads **each\ncomponent's own control surface**, rather than treating a PID, a lease, or a successful probe of a\nsibling as proof that the component serves. It prints one of `serving`, `absent`, `not-serving`, or\n`refused` for each component and exits `0`, `1`, `2`, or `3` respectively (the highest observed\nstate wins):\n\n- **manager**: local PID record, its liveness-lease holder and PID, then the manager's own typed\n `status` service reachability from this host. Manager builds that do not report static\n reconciliation say `static reconciliation not reported by this manager build`; the line stays\n visible even when the manager is otherwise `serving`.\n- **delivery**: local PID record, its ready lease (`ready` is the daemon's own bound-control\n signal), and the latest `renewal.json` adoption verdict. A re-signed credential and a\n broker-accepted adoption stay distinct facts.\n- **web**: local PID record and the dashboard's own loopback `/api/meta` response, which must name\n the same PID and its requested port. A different process on the port, an unreadable PID command,\n or an unrecognizable process record is `refused`, not a green default-port guess.\n- **broker**: the registered mesh URL dialed from this host with its recorded TLS requirement.\n\n`absent` means Cotal has no live local component record (or has a stale record); `not-serving`\nmeans the component record is live but its service/readiness surface did not answer or is not ready.\nThose are intentionally separate exit cases. A failed or unreadable probe is `refused`, never an\nabsent component or a clean zero.\n\n## spawn\n\n```bash\ncotal spawn [] [--detach] [--name ] [--agent ] [--model ] [--variant ] [--prompt ] [--cwd ]\ncotal spawn -f [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | resolved mesh | Target space |\n| `--server ` | registry entry | Broker URL override |\n| `--creds ` | none | Control-caller creds for an off-registry manager (`--detach` only) |\n| `--name ` | persona's `name:` | Presence-name override (does not choose the persona) |\n| `--config ` | none | Persona catalog name or file path; wins over the positional |\n| `--agent ` | persona's `agent:`, else `COTAL_DEFAULT_AGENT`, else `claude` | Connector type (`claude`, `opencode`, `jcode`, `hermes`, and so on) |\n| `--role ` | persona's `role:` | Role override |\n| `--model ` | persona's `model:` | Model override |\n| `--variant ` | persona's `variant:` | Model variant override (connector-defined; e.g. OpenCode reasoning tiers) |\n| `--cwd ` | this cwd | Working directory to root the agent at |\n| `--prompt ` | none | Initial prompt auto-submitted at start |\n| `--resume ` | none | Fork an existing session id into the mesh; only connectors that declare resume support accept it (see [the matrix](connectors.md)) |\n| `--events` / `--no-events` | off | Publish the session's structured event plane to its own event channel |\n| `--share-tools ` | none | Share named operator MCP servers with the agent |\n| `--subscribe ` | persona's | Channel read-set override |\n| `--allow-subscribe ` | = subscribe | Read-ACL override |\n| `--allow-publish ` | deny | Post-ACL override |\n| `--detach`, `-d` | off | Launch via the manager into a detached PTY (reattach with `cotal attach`) |\n| `--on ` | class anycast | With `--detach` only: pin the launch to one manager instance id (the whole id, as `ps` prints it). Refused on a foreground spawn (no manager to pin), with `-f` (a manifest deploy launches through the manager class queue), and when empty |\n| `--file `, `-f` | none | Deploy a manifest onto the running mesh |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--allow-stale ` | none | With `-f`: waive named stale agents (apply-only) |\n| `--runtime ` | manifest's | With `-f`: override the manifest's runtime |\n\n`--events` turns on the session's **event plane**: a stream of structured events describing what\nthe agent did, rather than the prose it wrote, on a channel of its own. The channel is named after\nthe agent's principal, `events..`, never after its display name, because two live\nagents are allowed to share a display name and would then share a stream. The launch grants publish\nrights on that channel alone, foreground and detached alike, and a connector that does not\npublish an event plane refuses the flag rather than starting a session whose events have nowhere to\ngo.\n\nThe flag and the grant are separate on purpose. Holding publish rights on a channel is not a request\nto publish to it, so writing an event channel into an agent file's `allowPublish` does not turn the\nplane on: only the launch does.\n\nThe persona (`--config` > positional > `COTAL_DEFAULT_PERSONA` > `default`) is loaded from the\ntarget mesh's `.cotal/agents/`; the launch flags override the file. Foreground runs the agent\nattached to your terminal; `--detach` hands the launch to the running manager. Both modes get the\ndurable backstop on a mesh that runs the delivery daemon; `--live-only` skips it for a foreground\nspawn (messages posted while it is disconnected are then not replayed). A foreground exit retires\nthe agent's creds and broker footprint, like a manager despawn. A `--detach` spawn is an\n**action**: the manager accepts it and returns the allocated identity at once, then the launch\nfollows to a terminal outcome rather than blocking (see [the control surface](control-surface.md)).\nSee [Connect Claude Code](connect-claude.md) and [Agent files](agent-files.md); `-f` is a\n[manifest deploy](#manifest-deploys). (`cotal start` was merged into `cotal spawn --detach`.)\n\n## models\n\n```bash\ncotal models [--agent ] [--refresh]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--agent ` | all registered connectors | Connector whose catalog to list |\n| `--refresh` | off | Ask the connector to refresh its provider cache |\n\nAsks the running manager for each connector's model catalog (model ids plus their variants)\nfor connectors that expose one. OpenCode and Codex query harness/provider surfaces; Jcode reads\nproviders that enable `model_catalog = true` in the operator Jcode `config.toml`. Jcode's listed\neffort tiers render as `variants (declared, not provider-verified)`, and launch can still refuse one.\nA connector without a catalog says so. Pick a result with `cotal spawn --model --variant `,\nwhere `` is the model id as the catalog printed it. OpenCode and Codex ids are the full\n`provider/model`; Jcode ids are bare (`opus-5`, not `cliproxy/opus-5`), because the provider is\nselected by the operator's Jcode config and a prefixed id is refused at launch with the bare form\nnamed.\n\n## endpoints\n\n```bash\ncotal endpoints [--space ] [--server ] [--creds ]\n```\n\nLists the mesh presence roster: agents, the manager, and any other protocol endpoint, with each\nendpoint's role, kind, status, and current activity. Unlike `ps`, this is a read-only presence view;\nit is not limited to child processes owned by the manager.\n\n## Endpoint control\n\n```bash\ncotal describe [--space ]\ncotal invoke [--args ''] [--space ]\ncotal invoke --name [--admin] [--space ]\n```\n\nThe generic v0.4 service surface. `describe` resolves a registered endpoint's command set off the\nwire - the reserved `describe` command answers the registered contract digests, the schemas are\nfetched from the space's content-addressed contract store, recompiled, and verified against those\ndigests - and prints each command with its capability class and targeting shape. `invoke` calls one\ncommand by name: `--args` is a JSON object validated against the fetched input schema *before*\npublish; a targeted command takes `--name ` (resolved to the agent's current principal through\n`inspect`) or `--self`. `--admin` uses the admin instrument credential, whose cross-agent reach rides\nthe operator-only `any` authorization mode. Neither command has compile-time knowledge of any\nendpoint's schemas - this is the same trust chain every built-in control command now uses. Needs an\nauth mesh: the manager registers its service on both static and per-user meshes (a signed-in user\nrides their bearer; each visible or invoked command still requires its existing grant, and cross-agent\nreach needs the `admin` scope). An open mesh has no service registry.\n\n## Managed seats\n\n```bash\ncotal ps [--on ] [--wide | --json] [--space ]\ncotal stop --name [--on ] [--space ]\ncotal attach --name [--on ] [--no-reconnect] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | none | Managed agent to stop / attach (required) |\n| `--on ` | class anycast (`ps`: class scatter) | Pin to one manager instance id (multi-manager space); takes the whole id as `ps` prints it, not a prefix. An empty value (`--on \"\"`, an unset shell variable) is refused, never treated as absent |\n| `--wide` (`ps`) | off | After each seat's compact row, print extra operational facts the manager records: `cwd`, `pid`, spawner, lifecycle uid, and the owning manager's instance id and host. Model and requested variant stay in the identity row rather than printing twice. A fact the manager did not record (for example a runtime with no real process) prints nothing, never a placeholder |\n| `--json` (`ps`) | off | Machine-readable: one JSON object per seat per line, copied unchanged from the manager row. Instance headers and errors go to stderr, so stdout contains only rows. Mutually exclusive with `--wide` |\n| `--no-reconnect` (`attach`) | off | End the attach when its session ends, instead of re-establishing it. For scripts that want one run and one exit code |\n\nThe human `ps` row is presentation text and is not a stable parsing target. Scripts use `--json`,\nwhich is the machine-readable row contract.\n\nThese are operator clients over the running manager's control plane. The default row includes the\nconnector, model pin, optional requested variant, and runtime as operational descriptors for the\nmanaged row. They do not make a shared display name a unique protocol identity; use `--json` when\nunambiguous owner+actor attribution is required. An omitted variant means no override was requested;\nCotal does not invent an effective provider default it cannot observe. `ps` also prints two state\nfacts per managed agent, because they answer different questions: the process fact from the manager's\nown runtime handle (`running` with its uptime, or `exited` with how long it ran), and the mesh fact\nfrom the roster (`idle` / `working` / `waiting` / `mesh offline`, or `not in roster` when the seat has\nno presence row at all: a seat that has not joined yet, or one that never did). A seat can be\n`running` and `mesh offline` at once: the process is alive and its presence has lapsed. On a user-auth\nmesh `ps` also renders each managed agent's last credential-refresh outcome, fail-closed.\n\n**Mode split (chosen up front, never try-scatter-then-degrade):**\n\n- **Static / open mesh.** Bare `ps` is a **class scatter**: it freezes the live manager class from\n the records registry, merges every registered instance's agents grouped and attributed per\n instance, and a non-answering instance is shown as `registered, no answer within the deadline`\n (never silently omitted). That label is the whole claim: the instance is registered and did not\n answer. It does not say the host is down, because a dead host never deregisters itself and a\n live one can be slow; if it is gone, deregister it.\n `--on ` pins the read to one exact instance id instead. A wrong pin fails loud\n rather than falling through: a well-formed id that no live manager carries is reported as\n `manager instance did not answer` (nothing else is asked), and a credential without that\n instance's rail is reported as refused by the broker, not as an unresponsive manager. A manager\n that answers with a refusal is shown with its own cause; \"no manager reachable\" is said only when\n nothing answered at all. If the scatter's own registry read fails (the freeze or the reconcile),\n `ps` says the manager registry could not be read rather than pronouncing on the managers, which\n may all be up.\n\n**`stop` and `attach` route by seat locality.** A seat can only be stopped or attached by the\nmanager actually running it, and the class queue does not know which one that is. So on a\nstatic/open mesh both verbs first ask every registered instance which one hosts the named seat, then\naddress that instance directly. This happens by default; you do not need `--on`.\n\n`--on ` remains the override, for when you already know where the seat lives or the\nlookup itself is degraded. It is also the **only** route on a **user-auth mesh**: a ledger-scoped\nbearer does not hold the registry-read rows the lookup needs, so there the verbs stay on the class\nqueue unless you pin them yourself.\n\nIf no reachable instance has the seat, the error reports how many managers answered and names\nthose that did not. It does not collapse that state into a bare `no agent `. That distinction matters\nbecause a single manager cannot tell \"hosted elsewhere\" from \"does not exist\": it answers\n`not-found` for both.\n- **User-auth mesh.** `cotal ps` reports what **one** manager knows about your agents (an `ep.one`\n read against the manager's in-memory roster, owner-filtered). It does **not** report other\n manager instances. It cannot tell you that one is down: an unreachable manager is absent\n from the list. Completeness across a multi-manager user-auth space is not claimed.\n A manager that does not answer fails the command outright (exit non-zero), rather than printing\n an empty list that could be read as \"no agents\". Your ledger row needs the `admin` scope to\n reach `ps` at all; `spawn` alone is refused by the broker (the ep tier boundary).\n\n`attach` streams and drives an agent's terminal on the `pty` runtime; detach with the escape key\n(Ctrl-] by default; see [`COTAL_DETACH_KEY`](config.md)). It does so over a one-use, holder-bound\nmesh session ([SPEC](../SPEC.md) §13.6): the manager replies with a signed session grant (never a\n`127.0.0.1` URL), the CLI redeems it once over the broker, and the browser console (`cotal console`)\ndrives the same session. `stop` and `attach` need a running manager to talk to. On a static mesh\nthey are cross-agent admin operations. On a user-auth mesh, your own agents (any agent under your\nowner) need only the `spawn` scope; another owner's agent needs `admin` on your ledger row\n([identity & auth](identity-and-auth.md)). Launch detached agents with [`spawn --detach`](#spawn).\n\n**`attach` reconnects when the link dies.** A session lives on a network link, and a laptop that\nsleeps, a VPN that drops or a wifi handover kills it. When that happens `attach` prints\n`[cotal: connection lost, reconnecting]` on stderr and starts asking the manager for a new session:\na fresh grant, a fresh per-session credential, a fresh connection, so every attempt re-runs the same\nauthorization the first attach did. On success it prints `[cotal: reconnected]`, the manager repaints\nthe seat's current screen the way it does for any attach, and you carry on in the same terminal.\nRetries wait 1s, 2s, 5s, 10s, then 30s, for as long as the seat exists. The detach key is read the\nwhole time the loop runs, the waits and the attempts alike, so a reconnect never traps you: press it\nwhile a session is being established and the attach ends there, and a session that lands behind the\npress is handed back to the manager rather than left holding a slot. Everything else you type while\nthere is no session is dropped rather than queued, so keystrokes aimed at a terminal that turned out\nto be frozen, Ctrl-C included, are not delivered to the agent by a reconnect you did not know had\nhappened. That starts before the first session, not at the first reconnect: at a terminal, `attach`\nreads and drops what you type while it is still resolving the mesh, so a key struck at a prompt that\nhas not come up yet does not reach the agent when it does.\n\nA **pipe** carries script input. For example, `printf 'ls\\n' | cotal attach --name web` is\nbuffered until the session opens. Buffering continues across reconnects, so\n`tail -f log | cotal attach --name web` does not lose the part of its feed written while the link was\ndown. Only a terminal gets the reader; `--no-reconnect` keeps the old behaviour on both.\n\nIt stops on its own when reconnecting cannot help, and says why: a manager that refuses the attach\nexits non-zero with the manager's own message, and a reconnect that finds the seat no longer there\n(despawned, or its agent exited while the link was down) exits cleanly with `seat is gone`.\nA refusal that could still pass, such as a manager at its session ceiling, is relayed in the\nmanager's own words while the loop keeps trying, once per refusal rather than once per attempt.\nPressing the detach key, or the agent's process exiting while you are attached, ends the attach as\nit always did. `--no-reconnect` turns all of this off and restores the single-session behaviour,\nwhich is what a script wants.\n\nEach reconnect also hands the abandoned session back to the manager, over the first link that can\ncarry the message, so an attach that flaps does not eat the manager's session slots one outage at a\ntime. If that message never gets a link, the attach says so when it ends.\n\nWhich mesh `attach` resolves also decides **whose trust it redeems with**. Redeeming a session grant\nmeans minting a short-lived, session-scoped credential from the space's seed, and that seed comes\nfrom the root the mesh resolved to, never from a `.cotal` found by walking up from whichever\ndirectory you happen to be standing in. The difference is not hypothetical: `~/.cotal` exists on\nevery install because the mesh registry lives there, so a command run anywhere under your home\ndirectory but outside a project used to mint from your home directory's trust and present it to a\nbroker that trusts a different chain, which surfaced as a bare authorization failure that named\nnothing. A directory that does hold another chain for the same space is now reported on the way\npast, and not obeyed:\n\n```text\n! this directory resolves to /Users/you, whose .cotal/auth holds a DIFFERENT trust chain for space \"team\".\n attach used /Users/you/projects/app, the root this mesh resolved to. The other one is not being used, and is worth a look.\n```\n\nWhen the resolved mesh holds no seed at all, `attach` refuses and names what it resolved, the broker\nand the root, instead of describing a directory it did not use.\n\nTerminal bytes stream over the mesh; the manager's own HTTP/WS face serves the console. That endpoint binds\n**loopback by default**, so nothing is exposed by accident; `cotal up --host ` passes its bind\naddress down, which is what lets you attach to an agent whose manager runs on another machine. A\nbare `cotal supervise` and an embedded manager stay machine-local. Set it directly with\n`supervise --console-host `.\n\nThat address is **recorded on the mesh** and carried forward, because it is a decision rather than\nsomething later commands can work out for themselves (a broker dial address is not a manager bind\naddress). Every later manager launch for the same mesh reuses it, including a same-root `cotal up` repair,\nan adopted preserved or restored listener, and a `spawn -f` manifest deploy. A manager replacement\ndoes not quietly move a reachable attach face back to loopback. Passing `--host` again overrides it,\nso you can widen or narrow exposure whenever you like; a mesh that never asked stays loopback-only\nand records nothing.\n\nBecause that face carries terminal read and write for every managed agent, it is credentialed in two\ntiers. A mesh caller receives a **ticket** bound to the single agent the manager just authorized,\nsingle-use and short-lived, so one authorized attach can never be re-pointed at someone else's\nagent. The **console token** is the operator's own, reaches every agent, and is printed only to the\nmanager's output. The roster, the live feed, and the PTY stream all answer `401` without one; the\nstatic console shell is served openly, since it describes no agent.\n\n## input\n\n```bash\ncotal input --name --text [--no-enter] [--on ] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | | Managed agent to type into (required) |\n| `--text ` | | The text to type, taken verbatim (required) |\n| `--no-enter` | off | Type the text and stop there, without pressing Enter |\n| `--on ` | class anycast | Pin to one manager instance id using the same rules as [`attach`](#managed-seats) |\n\nTypes one line into a running agent's terminal, as if you had typed it there, and returns. This is\nthe half of [`attach`](#managed-seats) that a program wants: `attach` is a live stream that holds a\nsession open and expects a terminal on your side, so a script, a cron job or a web UI cannot use it\nto send a single line. `input` is one authorized call.\n\nWhat it is for is **harness commands**. A line beginning with `/` is not chat and not a message: it\nis something the agent's own harness handles, and the only way in is the keyboard.\n\n```bash\ncotal input --name reviewer --text \"/compact\" # ask the harness to compact its context\ncotal input --name reviewer --text \"/model opus\" # switch its model\ncotal input --name reviewer --text \"hold on that PR\" # ordinary typing works too\n```\n\n**Quoting.** `--text` takes a value, so a payload starting with `/` survives as written. A payload\nstarting with a dash needs the `=` form, because the shell-style `--text --foo` is ambiguous and is\nrefused rather than guessed:\n\n```bash\ncotal input --name reviewer --text=--verbose # dash-leading text: use --text=\n```\n\nEnter is pressed by default, since a command typed but never submitted has not been delivered.\n`--no-enter` types the text and leaves it sitting at the prompt, which is how you stage a line and\nsend it later.\n\nNothing comes back but a delivery receipt (`✓ sent 9 bytes to reviewer`, counting the trailing\ncarriage return). Whatever the agent does next shows up where its output already goes: the mesh, its\ntranscript, or an `attach`.\n\n**This one is operator-only, and more narrowly than `stop` or `attach`.** Those two are granted to\nanything holding `spawn`, so an agent can stop and attach to seats under its own owner. `input` is\nnot: it is granted only to operator credentials, which on a user-auth mesh means your ledger row\nneeds the `admin` scope, the same scope [`ps`](#managed-seats) already needs there. The reason is\nthat a write into a terminal is control of whatever is running in it, and on a user-auth mesh the\nown-owner rule covers every seat under you, not only the ones you launched: a `spawn`-scoped agent\ncould otherwise type into a sibling it never started. Seat locality is still resolved for you.\n\nOnly the `pty` runtime can be typed into. The external terminal runtimes (`tmux`, `cmux`, `orca`,\n`herdr`) attach to a process they do not own, so they have no input stream for it and the command\nrefuses by name rather than dropping the keystroke.\n\n## personas\n\n```bash\ncotal personas list [-v] [--running]\ncotal personas show \ncotal personas edit \ncotal personas new (--prompt | --from ) [--role ] [--model ]\ncotal personas rm --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh's persona catalog |\n| `--role ` | none | `new`: the persona's role |\n| `--model ` | none | `new`: the persona's model |\n| `--prompt ` | none | `new`: the persona's prompt text |\n| `--from ` | none | `new`: seed the prompt from a file |\n| `--verbose`, `-v` | off | `list`: include role / model / description |\n| `--running` | off | `list`: mark personas live on the mesh |\n| `--force` | none | `rm`: required, delete without prompting |\n\nPersonas are the local agent files under the resolved mesh root's `.cotal/agents/`, the same catalog\n`cotal spawn` launches from. `--space` and `--server` therefore move every list, read, write, delete\nand completion operation to the selected mesh. An unresolved target refuses rather than falling back\nto the current directory. See [Agent files](agent-files.md) for the file format.\n\n## supervise\n\n```bash\ncotal supervise [--runtime ] [--space ] [--server ] [--spawn ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space to supervise |\n| `--server ` | hosting mesh, or matching registered mesh | Broker URL. A registered mesh supplies it when omitted; a different explicit value is refused. |\n| `--runtime ` | `pty` | Agent runtime (`pty` built in; extension runtimes are explicit-only) |\n| `--console-port ` | none | Protocol-console port |\n| `--console-host ` | loopback | Bind host for the console + attach endpoint. Loopback keeps it machine-local; `cotal up` passes the address it bound the broker to, which is what lets `cotal attach` reach this manager from another machine |\n| `--roster ` | none | Declarative roster to boot at startup |\n| `--launch ` | none | Resolved manifest launch spec (from `up -f` / `spawn -f`) |\n| `--spawn ` | none | Comma-separated personas to pre-spawn at startup |\n\nThe manager is the agent supervisor and control plane: it answers `spawn --detach`, `stop`, `ps`,\n`attach`, and the `cotal_*` manager tools. `cotal up --detach` starts one for you; run `supervise`\ndirectly to recover a dead manager or drive a custom runtime. Default runtime is `pty`; install an\noptional provider first (`cotal ext add @cotal-ai/orca`, `@cotal-ai/tmux`, `@cotal-ai/cmux`, or `@cotal-ai/herdr`) and\nselect it explicitly. A missing provider or app fails loudly; there is no fallback. See [Deploy](deploy.md).\n\nOn a normal `SIGINT`/`SIGTERM`, the manager stops every seat and requires the selected runtime to\nprove the seat is gone before it releases the manager lease or service registration. A stop that\ncannot prove exit fails loud and keeps manager authority instead of reporting a clean shutdown while\nan orphan still holds broker rails. After an abrupt manager death, the same logical successor\nterminalizes only its own durable static slots, verify-evicts the predecessor's broker principal,\nrecords that result in the lifecycle's caller-readable audit detail, and only then retires the\nlifecycle and frees the alias. Missing or unverified broker evidence keeps the slot terminalizing.\nDelivery-admin does not terminate the orphan OS process; safe successor process reaping requires\ndurable process start-identity pinning and is tracked separately.\n\nA `meshes add --mode user` entry is a **participant** registration, not hosting authority. A\nparticipant may run `supervise` only when the host advertises the remote manager authority service\nand the signed-in actor has the dedicated `supervise` ledger scope. The CLI obtains the closed,\nloopback-only `manager-service` view; `spawn` and `admin` do not substitute for that scope. The\nhost issues the manager's public-nkey JWT material through its lifecycle-bound prepare → activate\n→ renew protocol, never by handing the participant a signer or static provisioner credential.\n\nThe broker URL in the registry entry decides the transport. A remote broker is often published\nover a `wss://` edge rather than a raw `nats://` port, and `supervise` dials whichever scheme the\nrecord holds, starting with the manager-authority registration it runs before the manager exists.\nThe record also decides whether that registration requires TLS, so a participant never downgrades\nthe credential exchange to a plaintext connection the registry did not describe.\n\nWithout that advertised host service or scope, `supervise` refuses before it starts a manager.\nRun `cotal spawn` without `--detach` to launch a foreground agent, or ask the space host to enable\nthe authority service and grant `supervise` for detached agents. If a running remote manager loses\nrenewal, it reports degraded state and refuses unsafe new starts and restarts; live agents are not\nsilently replaced. Do not run `cotal down` or `cotal up` on a participant machine to repair this\ncondition.\n\n## reconcile-gate\n\n```bash\ncotal reconcile-gate [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the frozen gate lives in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint whose gate is frozen |\n| `--instance ` | this folder's persisted manager instance | Instance id |\n\n**When you need this.** A manager restart killed after deregistration begins but before the new\nincarnation finishes leaves the endpoint's issuance gate *frozen*, held by a\nprocess that no longer exists. The freeze is what stops two incarnations serving at once, which is\ncorrect. The successor manager now completes that dead registration itself on boot, using the same\nguard this command uses: it acts only when the freeze-holder is affirmatively gone under a complete\nCONNZ sweep (`gone` and `sweepComplete=true`). If that registration's spec write already committed,\nit finishes the same freeze at the committed registration revision. If the spec did not advance, it\nabort-reopens the gate at generation+1 with processEpoch unchanged and continues the normal takeover.\nLive, unknown, unestablishable, and\nwrong-op-kind still refuse; there is no TTL.\n\nUse this command when the boot path cannot run: the delivery daemon is down, the repair targets a\nnon-manager endpoint, or you want to lift the freeze without starting a manager. It checks that the\nholder really is gone, prints what it found, and then finishes the dead operation the same way as the\ninterrupted restart would have: revoke the old credentials, evict their holders with verification,\nand reopen the gate.\n\nIf verification is interrupted, the command leaves the gate frozen and durably records each holder\nwhose eviction was already verified. A retry still repeats the freeze-holder liveness check, then\nskips only progress bound to the same registration operation, frozen-gate revision, and holder set.\nThe output reports holders completed before this attempt, completed now, and still remaining. A new\nfreeze or changed holder set starts from zero. Cursor cleanup happens only after reopen; a retained\ncursor is harmless because its old gate revision cannot authorize a later freeze.\n\n**It refuses far more often than it acts, on purpose**, and always says which check stopped it:\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `holder-alive` | The freeze-holder still has a live connection: a manager *is* running | Stop that process first. Reconciling would evict a live manager's credentials |\n| `holder-unknown` | The connection sweep could not prove the holder absent | Not safe to proceed: an unprovable holder is treated as a live one. Re-run once the broker answers completely |\n| `liveness-unestablishable` | The delivery daemon could not be asked at all | Start it (`cotal up` runs it) and re-run. Silence is never read as death |\n| `not-frozen` / `no-gate` | The gate is open, or there is no gate at that coordinate | Nothing to repair: check `--endpoint` / `--instance` |\n| `wrong-op-kind` | Frozen under a takeover or retirement, not a registration | Out of scope for this command; it will not reinterpret another operation's intent |\n| `eviction-unverified` | The holder looked gone but eviction could not be verified | The gate is left frozen, unchanged. Investigate the broker before retrying |\n| `raced` | A newer manager moved the gate mid-repair | Re-run `cotal doctor` and look again |\n\nThere is no `--force`, and no path that discards gate state: the only way this reopens a gate is by\nproving the holder is gone and then completing the operation properly.\n\n## deregister-instance\n\n```bash\ncotal deregister-instance [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the instance is registered in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint the instance serves |\n| `--instance ` | this folder's persisted manager instance | Instance id, the whole id as `cotal ps` prints it |\n\n**When you need this.** The service registry records *registration*, not liveness, and nothing in\nthe model expires a row. A manager that stops cleanly removes its own registration. One whose host\ndied without writing anything cannot, so its record goes on claiming a live instance forever: every\nclass scatter in that space freezes the dead slot in, and `cotal ps`, `stop` and `attach` each pay\ntheir whole deadline waiting for a machine that is never coming back. A laptop that was reimaged, a\ncontainer that was deleted, a box that will not be back on the network: those registrations have no\nother exit.\n\nThis command is that exit. It asks the instance first, and it removes a record only when the broker\naffirms the instance's own rail is empty: nothing subscribed there. Then it deletes the\nregistration's two records keys, each pinned to the revision it read, and prints what it removed.\n\n**Silence alone never passes.** An unanswered describe is what a dead host, a wedged process and a\nslow one all look like, and a hung process still holds its subscriptions, so the broker sees\ninterest on its rail. That instance is refused and the observation is printed. A dead process holds\nno connection and therefore no subscription, so a real corpse is still removed.\n\n**Every refusal names the failed check:**\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `instance-answered` | The instance answered a pinned describe. It is alive | Nothing to repair. If it is wedged rather than gone, stop the process first; its own clean stop removes the record |\n| `instance-not-affirmed-gone` | It did not answer, and the broker did not report its rail empty, which is what a held subscription looks like: slow or hung, not affirmed gone | Nothing was removed. Stop the process; its record goes on its own clean stop, or re-run this once it is down |\n| `liveness-unestablishable` | The probe itself failed, so nothing was learned | Fix the probe's path (credential, broker) and re-run. A probe that could not run is never read as death |\n| `not-registered` | No registration at that coordinate | Check `--instance` and `--endpoint`. This takes the whole id, never a prefix |\n| `registration-in-flight` | The instance holds the endpoint governance slot at the live issuance-gate generation, so a registration is still completing | Nothing was removed. Wait for that registration to finish, then re-run |\n| `superseded` | The record moved between the read and the delete | Something is writing to it. Nothing was removed; re-observe before retrying |\n\nThere is no `--force` and no sweep: silence is not death, and a rule that removed rows on silence\nwould eventually remove a live instance that was merely slow. An operator names one instance, the\nbroker's verdict on its rail is what authorizes the removal, and the guard's job is to show them\nthey named a dead one. Removal is not a one way door either. The same instance re-registers over\nthe tombstone on its next start, under the same identity.\n\n## runtimes\n\n```bash\ncotal runtimes\n```\n\nLists every agent runtime the manager can spawn through: the built-in `pty`, the official providers\n(`orca`, `tmux`, `cmux`, `herdr`), and any custom provider installed via `cotal ext add`. Each installed\nprovider is probed so you can see what is actually reachable on this machine before selecting it:\n\n```\npty built in\norca installed · reachable @cotal-ai/orca\ntmux available · cotal ext add @cotal-ai/tmux\ncmux available · cotal ext add @cotal-ai/cmux\nherdr available · cotal ext add @cotal-ai/herdr\n```\n\n`installed · reachable` / `unreachable` is the provider's own `available()` probe; `available` means\nit is a known runtime you can add with the shown command. Selecting an unknown or uninstalled runtime\nvia `up`/`spawn --runtime ` fails loud and, for a known one, points at the exact `cotal ext add`\npackage. There is no silent fallback to `pty`.\n\n## send\n\n```bash\ncotal send dm \"\" [--space ] [--server ] [--creds ]\ncotal send msg \"\"\ncotal send ask \"\"\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and (off-registry) which credential |\n\nOne-shot messaging: connect, send a single direct message (`dm`), channel post (`msg`), or role\nask/anycast (`ask`), then exit. For a running conversation, agents use the mesh tools instead\n([MCP tools](mcp-tools.md)).\n\n`cotal send` requires `COTAL_NAME` plus either `COTAL_ID` or both `COTAL_OWNER` and `COTAL_ACTOR`.\nIf that tuple is missing, `send` refuses before connecting so the recipient never sees a message\nattributed to a nameless command principal. A child that inherited a\nseat's environment is attributed as that seat; this command does not distinguish the two. An operator\nwho is not a live seat can set both variables for the one shot:\n\n```bash\nCOTAL_NAME= COTAL_ID= cotal send ...\n```\n\n## channels\n\n```bash\ncotal channels list\ncotal channels set [--replay | --no-replay] [--window ] [--desc ] [--instructions ]\ncotal channels default --replay | --no-replay\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--replay` / `--no-replay` | none | `set`/`default`: replay history to new joiners, or not |\n| `--window ` | none | `set`: replay window size |\n| `--desc ` | none | `set`: one-line channel description |\n| `--instructions ` | none | `set`: instructions shown to joiners |\n\nInspects and edits the channel registry: replay policy, description, and joiner instructions. ACL\nsemantics (who may read or post) are set at mint / provision time, not here; see\n[Channels and permissions](channels-and-permissions.md). On a user-auth mesh, `list` rides your\nown login as is; `set` and `default` edit the registry over a short-lived\nchannel-writer view, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n\n## history\n\n```bash\ncotal history clear --force [--dms] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--dms` | off | Also clear DM history |\n| `--force` | none | Required: clear without prompting |\n\nPurges retained channel history; `--dms` extends it to direct-message history. An alias of\n[`clean history`](#clean). On a user-auth mesh the purge rides a short-lived purger view over\nyour login, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n## console\n\n```bash\ncotal console [--plain] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to watch |\n| `--plain` | off | Line stream instead of the TUI |\n\nA live protocol view for a space: a lazygit-style TUI, or a plain line stream on `--plain`. On a\nuser-auth mesh it rides the read-only admin view over your login, which needs ledger scope\n`admin`. See [Watch a mesh](watch-a-mesh.md).\n\n## web\n\n```bash\ncotal web [--detach] [--host ] [--port ] [--no-open] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to serve |\n| `--host ` | `127.0.0.1` | Concrete HTTP bind and browser host; wildcard addresses are refused |\n| `--port ` | `7799` | HTTP port |\n| `--detach` | off | Run in the background; stop with `cotal down web` or bare `cotal down` |\n| `--no-open` | off | Don't open the browser |\n\nThe browser observability dashboard: presence, channels, and a live feed. It is **not** part of\n`cotal up`: it ships inside `cotal-ai` as the `@cotal-ai/web` extension, seeded automatically on first\nrun (like the built-in connectors) so it always matches your CLI version. It self-registers `cotal web`\ninto this surface and serves\n`http://cotal.localhost:7799` by default (loopback; `*.localhost` resolves in Chrome/Firefox/Edge; Safari may\nneed `http://127.0.0.1:7799`). On a user-auth mesh the dashboard rides the read-only admin view\nover your login, and a channel purge asks for its own channel-purger view per click; both need\nledger scope `admin`. Detached mode re-execs the current Cotal installation, writes diagnostics to\nthe mesh root's `.cotal/web.log`, and reports success only after the HTTP server answers. It requires\na recorded mesh root, but can be launched from any directory once `cotal up` has recorded the mesh.\nSee [Watch a mesh](watch-a-mesh.md).\n\n## mint\n\n```bash\ncotal mint [--profile ] [--out ] [--signer]\ncotal mint --provision [--role ] [--space ] [--server ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--profile ` | `agent` | Credential profile |\n| `--out ` | `.cotal/auth/creds/space./.creds` | Output path - the default sits under the resolved space's segment (`` is that space's hex encoding, as in [Project files](config.md#project-files)) |\n| `--signer` | off | Emit a stripped account-signing file instead |\n| `--force` | off | With `--signer`: overwrite an existing file |\n| `--allow-subscribe ` | the agent file's, else subscribe | Read-ACL override, **agent profile only**: `observer` and `admin` carry a fixed read set, and `mint` refuses this flag there rather than narrowing nothing |\n| `--allow-publish ` | the agent file's, else deny | Post-ACL override, **agent profile only** |\n| `--role ` | the agent file's | Agent profile: the anycast task queue the identity pulls (`svc_`) |\n| `--provision` | off | Agent profile: also pre-create the identity's bind-only DM/deliver durables (and its role's task queue) on the live mesh, so the credential can consume |\n| `--space `, `--server ` | the resolved mesh | Which root supplies the agent file, static trust and default credential storage; with `--provision`, also which live mesh receives the durables |\n\nMints a NATS creds file for a space in **static** auth mode, scoped to a profile and (optionally)\nexplicit read/post ACLs. `--signer` emits an account-signing file for delegating minting to another\nhost. A per-user-auth space refuses `mint`: agents there join under a logged-in user\n([`login`](#login) + [`actor grant`](#actor)), never via a handed-out creds file. See\n[Identity and auth](identity-and-auth.md).\n\nFor an agent profile, the resolved mesh root supplies the persona ACL, the signing material and the\ndefault credential destination as one authority. If the current folder also holds trust for a\ndifferent space or account, mint refuses before writing and names both roots. It never combines a\npersona from one root with credentials signed or stored under another.\n\nA plain mint is creds only: the identity can publish within its post ACL at once, but on an authed\nmesh its DM inbox and task queue are provisioner-pre-created and bind-only, so a **consuming**\nconnect fails until they exist. `--provision` performs that pre-create in the same command (a\nprovisioner cred is minted from the space's trust material, used, and dropped), so a long-running\nclient you start yourself can receive DMs and role anycasts like a spawned seat. The command prints\nthe identity's principal (its wire id) and lifecycle uid; a consuming client passes that uid as its\n`lifecycleUid`. Agent profile only; an open mesh needs none of this (peers self-create there). The\nThe same resolved authority is used for both the credential and `--provision`, so the broker\nfootprint cannot be created under a different root's trust material.\n\n## Login\n\n```bash\ncotal login --idp [--client-id ]\ncotal logout --idp \n```\n\nSigns you in to a per-user-auth mesh's IdP (device code flow) and caches the session; run it\nonce per machine. It prints your IdP subject, the id the operator grants against. After a\nlogin, every command on that mesh works under your identity: each connect takes a fresh IdP\nproof, exchanges it locally for a short-lived bearer, and is authorized against the actor\nledger at connect time. `logout` revokes the IdP session and clears the cache. See\n[identity & auth](identity-and-auth.md).\n\n## actor\n\n```bash\n# an upsert of the WHOLE row: a flag left off is the WIDE default below, not \"unchanged\"\ncotal actor grant --sub [--scope a,b] [--allow-subscribe a,b] [--allow-publish a,b] [--role ] [--label ]\ncotal actor revoke (--sub | --owner )\ncotal actor list\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | the folder's | Space whose ledger to manage |\n| `--sub ` | none | The IdP subject (shown by `cotal login`) the actor belongs to |\n| `--owner ` | none | The derived owner token (alternative to `--sub`) |\n| `--scope ` | `spawn,role:default` | Capability scope (`''` = none; `spawn` = may run agents; `role:` = may delegate role r; `admin` = cross-agent control; `supervise` = eligible for the closed remote manager-service view when the host enables it) |\n| `--allow-subscribe ` | `>` (all channels) | Channel read ACL; the user's envelope, their agents can never read beyond it |\n| `--allow-publish ` | `>` (all channels) | Channel post ACL; also the envelope for their agents' posting |\n| `--role ` | none | Role (scopes the task-queue consumer) |\n| `--label ` | none | Display label for `actor list` (never the IdP subject) |\n\nThe actor ledger is the single authorization source of a user-auth space: no row, no access.\nA bare `grant` is the **full** envelope (all channels, may spawn); the flags narrow it. A\nre-grant **replaces the whole row**, not the one field you name, so to add a capability spell\nevery field out: the new scope plus the row's current read set, post set, role and label\n(`cotal actor list` shows what a row holds). A field left off does not stay as it was, it\nreverts to the wide default in the table above, which is how a narrow reader becomes a reader\nof every channel. A re-grant retires the current interactive lifecycle through the running auth\nservice before it rotates the row, so copied bearers cannot cross an authorization update. If that\nretirement cannot be confirmed, the row is left unchanged and the command fails with the recovery\naction. `revoke` uses the same retirement before deleting the row, which lets a later grant create a\nreal successor instead of colliding with a live predecessor. `supervise` is separate from `spawn` and `admin`: it only makes a signed-in\nperson eligible for the host-provided closed remote manager-service view; it does not grant\nmanagement of another owner or a general host profile. `revoke` denies the next exchange and\nthe next connect with no restart, and evicts the principal's live connections. Managed-agent rows\n(written by the spawn path) live in a disjoint row space this command never touches. See\n[identity & auth](identity-and-auth.md).\n\n## doctor\n\n```bash\ncotal doctor auth [--fix]\n```\n\nCredential-health diagnosis and repair for this folder's mesh: renders every managed\ncredential as healthy / near-expiry / expired and ends in `healthy` or the exact next\ncommand; `--fix` applies the repairs it can. The one surface every stale-credential error\npoints at.\n\n## join\n\n```bash\ncotal join --space --name [--role ] [--channel ]\ncotal join --link | --token \n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and which credential |\n| `--name ` | none | Your presence name |\n| `--role ` | none | Your role |\n| `--channel ` | none | Channel to join |\n| `--kind ` | `agent` | Endpoint kind |\n| `--link ` | none | Join link (`cotal://…`) |\n| `--token ` | none | Join token |\n| `--lifecycle-uid ` | none | Required with `--creds`: the lifecycle UID minted alongside the credential (`COTAL_LIFECYCLE_UID` works too). A credential's durable grants name exact lifecycle-keyed resources, so `join` refuses to invent one |\n| `--tls` | off | Connect over TLS |\n\nAn interactive presence: join a space under your own name and role, without launching an agent\nharness. A `--link` or `--token` supplies the where and the auth in one value. See\n[Spaces](spaces.md) and [Identity and auth](identity-and-auth.md).\n\n## Manifest deploys\n\nA `cotal.yaml` manifest declares a whole mesh (channels, personas, roles, and ACLs) in one file.\nThree commands consume it, plus a read-only validator:\n\n```bash\ncotal up -f cotal.yaml # boot a fresh mesh from the manifest\ncotal spawn -f cotal.yaml # deploy the manifest additively onto a running mesh\ncotal down -f cotal.yaml # tear that deploy down (or --run for one run)\ncotal topology view -f cotal.yaml # validate + view the access graph, change nothing\n```\n\n`up -f` and `spawn -f` differ in target: `up -f` brings up a new broker and applies the manifest;\n`spawn -f` requires an already-reachable mesh and applies additively (ownership-scoped). On a\nuser-auth mesh, `spawn -f` deploys over your own login (the deployer view, gated on ledger scope\n`spawn`): the manifest's agents land under your owner, a manifest claiming another owner is\nrefused, and seeding new channels additionally needs scope `admin`. Both take\n`--dry-run` to print the plan without mutating anything. `topology` validates the manifest and\nrenders its channel / role / ACL graph. See [Define a team](define-a-team.md) and the\n[manifest reference](manifest.md).\n\n## ext\n\n```bash\ncotal ext # same as `list`\ncotal ext add \ncotal ext remove \ncotal ext list\ncotal ext root # print just the install prefix (scriptable)\ncotal ext seed [--repair|--reset|--force]\n```\n\nOperator-installed extensions: `add` installs an npm package into a cotal-owned prefix and records\nevery registry provider it contributes. Commands appear in help, completion, and dispatch; runtime\nproviders are lazy-loaded by commands such as `supervise`; local process providers participate in\n`status` and selective `down`. `remove` and `list` manage them. The `@cotal-ai/web` dashboard is the\ncanonical command/process example. Installed packages and their location are described in\n[config](config.md).\n\nBare `cotal ext` lists the inventory, headed by the install prefix. That prefix is a cotal-owned npm\nroot kept **separate** from npm's own global tree. These packages never show up in `npm list -g`,\n`cotal ext` (or the Extensions section of `cotal status`) is the canonical inventory. `cotal ext root`\nprints only the path, for scripts. The versions shown are the manifest pin recorded at add time.\n\nRemoving an extension that owns a running local process is refused with the mesh root and its\n`cotal down ` command; stop it first so uninstalling the package never strands a process\nwhose lifecycle provider is gone.\n\n### Built-in connectors are seeded extensions\n\nThe first-party agent connectors (`claude`, `opencode`, `codex`, `hermes`, `jcode`, `pi`) are not compiled into\nthe binary. They are seeded on first run through the **same** `ext add` path a third party uses, and\nappear in `cotal ext list` like any other extension. So you can remove one you do not want\n(`cotal ext remove @cotal-ai/connector-hermes`), and a deliberately-removed connector STAYS removed\nacross upgrades. `cotal ext add ` adds a third-party connector the same way. The web\ndashboard (`@cotal-ai/web`, providing `command:web`) is the seventh built-in seeded on the same path.\n\n`cotal ext seed` is the maintenance entry for that seeding (it runs automatically on the first real\ncommand of each boot, so you rarely call it):\n\n| Flag | Meaning |\n|---|---|\n| (none) | Reconcile: seed any never-seeded built-in, refresh a seeded one whose version the binary bumped, leave a removed one removed. A no-op once current. |\n| `--repair` | Recover after an interrupted seed or a lost authority (rebuilds the interrupted connector; restores the removed-vs-never-seeded record from its durable backup). |\n| `--reset` | Discard the record and re-seed all seven built-ins (the six connectors plus the web dashboard). **Resurrects any you removed.** Rebuilds cleanly over corrupt seed state. |\n| `--force` | Re-seed the built-ins even when the version stamp is current or a downgrade. |\n\nWhen a newer `cotal` advances the operator-global seed store to its generation, it prints one\nmigration line naming the old and new generations, the exact CLI entry that wrote the store, the\ncommit timestamp, and `seed/stamp.json`. That writer and timestamp are kept in the stamp, so a later\nolder CLI refusal can say which executable wrote the generation it will not overwrite and when.\nLegacy generation-only stamps remain readable; their refusal simply has no writer provenance to add.\n\nAn older `cotal` refuses a seed store written by a newer version. When it can verify a sufficient\n`cotal` executable on PATH or at the installer's `~/.local/bin/cotal` location, the refusal names\nthat absolute path so a reduced service PATH does not select the older binary again. Otherwise it\nkeeps the generic newer-version instruction. `--force` rebuilds the store for the running older\nversion without discarding the ever-seeded authority. `--reset` still exists for corrupt state and\nresurrects deliberately-removed connectors.\n\nA source-checkout CLI (`pnpm cotal`, `tsx bin/cotal.ts`, `node bin/cotal.ts`, or a suite child of\nthose) refuses to write or garbage-collect that store. The refusal names the path, the generation\nit declined, and `$XDG_CONFIG_HOME` as the isolation remedy. `COTAL_HOME` does not relocate this\nstore. An entry that cannot be proven as a released install is refused the same way. Isolated\nrelease tests that must seed from a checkout-shaped `bin/` set `COTAL_ALLOW_CHECKOUT_SEED=1` after\npointing `$XDG_CONFIG_HOME` at a scratch dir; that override is documented here, not on the refusal\nline. An opt-in write still records the checkout path in `seed/stamp.json` as `writtenBy`.\n\nThe default connector for a bare `cotal spawn` (no `--agent`) is the persona's `agent:` pin if it\nhas one, else `claude`; set `COTAL_DEFAULT_AGENT` (e.g. `opencode`) to change the fallback. It is\na default, so a persona that pins its harness still wins over it. An `--agent` naming a removed\nconnector fails loud with the exact\n`cotal ext add` to restore it. Set `COTAL_SKIP_CONNECTOR_SEED=1` to turn off the automatic first-run\nseed/refresh entirely (for a controlled or offline setup that manages connectors by hand); `cotal ext\nseed` still runs on request.\n\n## completion\n\n```bash\ncotal completion # print a stub to eval / source\ncotal completion install [shell] # install it persistently\n```\n\nPrints or installs shell completion. Completion candidates come from each command's declared flags\nand, where useful, live mesh state (spaces, personas, managed agents) resolved offline.\n\n## feedback\n\n```bash\ncotal feedback \"\" [--type ] [--email ] [--details ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--type ` | none | `bug` \\| `idea` \\| `friction` \\| `praise` \\| `other` |\n| `--details ` | none | Longer free-form details |\n| `--severity ` | none | `low` \\| `medium` \\| `high` |\n| `--area ` | none | The part of Cotal this concerns |\n| `--email ` | git email | Contact email (required on the keyless public path) |\n| `--name ` | none | Your name (optional) |\n| `--url ` | keyed / public intake | Intake URL override |\n| `--key ` | `COTAL_FEEDBACK_KEY` | Feedback key |\n\nSends feedback to the Cotal developers. With a key (`--key` / `COTAL_FEEDBACK_KEY`) it routes to the\nkeyed beta intake; without one it goes to the public `cotal.ai` intake and requires a contact email\n(`--email` / `COTAL_FEEDBACK_EMAIL`, else your git email). Run a self-hosted intake with\n[`feedback-intake`](#server-daemons).\n\n## run\n\nOperate durable workflow runs (cotal-lang programs) from the terminal.\n\n```bash\ncotal run start --file [--timeout ] [--local]\ncotal run resume [--local --file ]\ncotal run ps [--endpoint ]\ncotal run journal [--endpoint ]\ncotal run answer [--value ] [--artifact ] [--endpoint ] [--local --by ]\n```\n\n`start` hands the program to the mesh's manager, which validates it, mints the run id (the record\nnever takes a caller-supplied one), drives it in its own process, and answers with the id once the\nrun is recorded; a program that does not validate is refused with every problem listed. `resume`\nasks the manager to take an existing run back and continue it from its step journal; the source is\nthe recorded program, so no `--file` is taken. Neither takes `--endpoint`: the manager records\nits runs under its own endpoint, and naming another is refused. `ps` lists the run records and\n`journal` renders one run's durable records; both only inspect. `answer` resolves an open\ncheckpoint through the manager, presenting as the holder that armed it; the manager records the\nanswerer from your credential, so no `--by` is taken there. `--timeout` sets the default\ncheckpoint timeout for a drive (default 1h). `--local` drives in this process instead, over one\nconnection per invocation under the run's own credential minted from the project folder's trust\nmaterial, and is the path on a bare broker with no manager or for a run with no recorded program\n(`cotal run resume --local --file `); `answer --local` takes `--by `. A\nuser-auth mesh runs no programs yet: the manager refuses the family by name, and `--local` has no\ncredential there. The guide is [workflows](workflows.md).\n\n## Server daemons\n\nTwo long-lived infra roles ship with the CLI. They are not part of everyday operation; the delivery\ndaemon comes up automatically with `cotal up --detach` in auth mode.\n\n```bash\ncotal deliver --space [--server ] [--creds ]\ncotal auth-service --space --server [--port ] [--exchange-public-port ] [--exchange-public-url ] [--exchange-trusted-proxy]\ncotal feedback-intake --keys [--port ] [--creds ]\n```\n\n`auth-service` runs a user-auth space's identity plane: the NATS auth callout, the\ncapability-gated local exchange and JWKS, and, when `--exchange-public-port` is set, the closed public\nexchange/discovery face forwarded by an HTTPS reverse proxy. `--exchange-public-url` is the proxy URL\nadvertised to clients; `--exchange-trusted-proxy` opts into last-hop `X-Forwarded-For` attribution.\n`cotal up --user-auth` starts and supervises the service for you, so you run it directly only to\nrecover one by hand.\n\n`deliver` runs the server-side Plane-3 delivery daemon: the durable backstop and membership/ACL\nauthority. It is auth-mode-only and single-instance (`--shard`/`--shards` accept only `N=1`);\n`--dev-mint` mints a scoped cred from the local signer for standalone dev. See the\n[delivery daemon](delivery-daemon.md). `feedback-intake` runs a self-hosted feedback server\n(requires `--keys` and a scoped `--creds`), announcing submissions into a space channel; flags\ninclude `--host`/`--port`, `--store`, `--space`/`--channel`, `--max-bytes`, and `--rate-limit`.\n\n## Plumbing\n\n`cotal __complete ` is the internal entry the shell-completion stubs call to emit candidates\nfor the current command line; you never run it directly. `cotal agent-bearer` is machine-facing\nplumbing on user-auth meshes: spawned agents exec it to print a fresh short-lived bearer from their\nspawn-time secret; you never run it directly either. Its local arm uses `--dir` to discover the\ncapability-gated loopback service. A remotely enrolled, already-granted agent instead receives\n`--exchange-url ` in its launch argv: that arm sends `{owner, actor, actorToken}` to the\npinned public exchange with no local capability, follows no redirects, and refuses every non-HTTPS\nURL because the actor token is the credential in the request body. (`cotal start` is a removed tombstone: it\nerrors and points you to `cotal spawn --detach`.)\n" }, { "slug": "config", @@ -145,7 +145,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "Define a team", "kind": "Guide (informative)", "summary": "The Quickstart gives you one agent. To run a specific team (your own channels, your own agents, and the channel access for each agent), describe it once in a cotal.yaml and launch it with a single…", - "body": "# Define a team\n\n> **Guide** (informative) · **For:** operators · **Prereqs:** [Quickstart](getting-started.md)\n\nThe [Quickstart](getting-started.md) gives you one agent. To run a **specific team** (your own\nchannels, your own agents, and the channel access for each agent), describe it once in a\n`cotal.yaml` and launch it with a single command.\n\n## What a manifest is\n\nA manifest (`cotal.yaml`, `kind: Mesh`) is the declarative form of what you'd otherwise do by\nhand: start a broker, seed channels, spawn agents, and mint each agent creds scoped to the channels\nit may use. It is a convenience over the CLI and adds no wire concepts. Today it is **single-space**\n(one `space:` per file).\n\nIt is **channel-centric**: you list the channels, and under each one name the agents that may read\nand post. Cotal inverts that into one least-privilege credential per agent, so the file reads the\nway you think about a team (\"who's in #review?\"), while each agent only gets the access you granted.\n\n## Quickstart\n\nA complete, runnable manifest (two agents, two channels, no separate files):\n\n```yaml\napiVersion: cotal/v1\nkind: Mesh\nspace: main # the default space, runnable fresh or right after `cotal up`\nagent: claude # the harness that runs each agent\n\nagents: # inline personas (no external files needed)\n planner:\n instructions: Break the work into steps and post the plan.\n builder:\n instructions: Implement the smallest change that works.\n\nchannels:\n general:\n subscribe: [planner, builder] # auto-listen at boot\n allowPublish: [planner, builder] # may post: default-deny, so list everyone who posts\n review:\n subscribe: [planner] # only planner auto-listens\n allowSubscribe: [planner, builder] # builder MAY read #review, but isn't auto-subscribed\n allowPublish: [planner, builder]\n```\n\nSave it as `cotal.yaml` and launch:\n\n```bash\ncotal topology view -f cotal.yaml # validate + render the access graph (no broker needed)\ncotal up -f cotal.yaml # broker + channels + agents, all fresh\ncotal ps --space main # see the agents the manager booted\ncotal web --space main # ...or watch it in the browser\ncotal down # stop the whole mesh\n```\n\nThe manifest introduces no access model of its own; the three verbs are the same ones\nCotal uses everywhere: `subscribe` (auto-listen at boot, and implicitly may read),\n`allowSubscribe` (**read**; defaults to `subscribe`, must be a superset of it), and\n`allowPublish` (**post**; default-deny: an empty or omitted list means nobody posts).\nAbove, `builder` *may read* #review but doesn't *auto-listen* to it. Every top-level key,\nthe three `agents:` forms, channel cards, and the resolution rules are in the\n[manifest reference](manifest.md).\n\n## The command lifecycle\n\n| Command | What it does |\n|---|---|\n| `cotal topology view -f ` | Validate the file and render its access graph. Read-only: needs no broker, mutates nothing. Run it before you launch. |\n| `cotal up -f ` | Bring up a **fresh** mesh: broker + seeded channels + booted agents. |\n| `cotal spawn -f ` | Deploy a manifest **additively** onto a mesh that is already running. |\n| `cotal down [-f ]` | Tear down (see \"Tearing down\" below). |\n\n`up -f` and `spawn -f` accept `--dry-run` (preview the plan, change nothing). `up -f` also takes\n`--server` / `--host` / `--space` / `--runtime` / `--open` to override the file for one run.\n\n> If a Cotal mesh is already running at the manifest's broker address (e.g. the default\n> `127.0.0.1:4222` from `cotal up`), `up -f` **refuses**; it never re-seeds a live broker. The\n> check is on the *address*, not the `space:` name. Run `cotal down` first, point the manifest at\n> another address (`broker: { servers: nats://127.0.0.1:14999 }`, or `--server`), or use\n> `cotal spawn -f` to deploy onto the running mesh.\n\n**Tearing down.** A fresh mesh from `up -f` is torn down with plain **`cotal down`**: it owns the\nwhole space. An additive deploy from `spawn -f` is torn down with **`cotal down -f `** (or\n`cotal down -f --run `), which removes *only* that run's agents and channels.\n\n## Manifest ownership\n\nThe rule: **`up -f` owns the whole space; `spawn -f` owns only what it created.** Cotal only ever\ntears down what it owns; foreign actors on a shared mesh are never touched.\n\n- A fresh mesh from `up -f` → `cotal down` stops all of it.\n- An additive deploy from `spawn -f` records a creation-only **ledger**\n (`.cotal/manifests/.json`) of the channels and agents it added; `cotal down -f`\n removes only those. The **run id** is printed by `spawn -f` and is the filename under\n `.cotal/manifests/`; pass it to `down -f --run ` when the file has changed since the deploy\n (an edited file no longer matches its ledger) or to finish a teardown that was retained.\n\n`down -f` is deliberately conservative; it treats the ledger as untrusted and validates before\ndeleting: an owned agent is stopped only when the live agent's recorded name *and* id match; an\nowned channel is removed only when no other members remain; and if the broker is unreachable or\nanything is uncertain, nothing remote is removed and the ledger is **retained** for a later\n`down -f --run `. It is local-only: run it from the checkout that created the run.\n\n(`.cotal/` holds creds, the ledger, and runtime artifacts: add it to your `.gitignore`; commit\nyour `cotal.yaml` and persona files, not what's under it.)\n\n## Deploying onto a shared mesh (`spawn -f`)\n\n`spawn -f` is additive and never adopts or mutates anything it didn't create. It classifies each\ndeclared item against the live mesh:\n\n| Item | Classification | Behaviour |\n|---|---|---|\n| Channel, brand-new | created + owned | Seeded and recorded in the ledger. |\n| Channel, already present | `exists-unmanaged` | Left untouched: card not mutated; the desired card is shown against the live one. |\n| Agent, not yet created | will-create | Booted and recorded. |\n| Agent, already created, unchanged | already-owned | No-op. |\n| Agent, already created, policy changed | `stale` | Exits non-zero unless `--allow-stale ` (then it restarts). |\n\n> **Security.** If an **unmanaged** actor already has read access to a channel you declare,\n> `spawn -f` prints a warning: an isolation conflict on a shared mesh. It is an explicit *lower\n> bound* (presence plus the broker membership feed), not a guarantee that no other access exists.\n\n**Deploying to a remote manager.** The mesh's manager may live on another machine (or another\ncheckout): `spawn -f` detects that from the manager lease and pushes the resolved launch spec\ninline over the control plane. The manager validates it as untrusted input and persists it under\nits own `.cotal/run/` before launching, so nothing changes downstream. Run the deploy from the\ncheckout the mesh is **registered** to on your machine (that's where the ledger lands), and run\n`down -f` from that same checkout; it stops remote agents over the control plane and treats a\nlocally-absent cred file as proven-absent. One residual: the agents' cred files minted on the\nmanager's host stay there until the mesh's own cleanup, the same way it would after a crash.\n\n## Operating a manifest mesh\n\nEvery mesh-touching command resolves the broker from the mesh registry, so `--space ` is\nenough; `send`, `channels`, `console`, `web`, the manifest verbs, and the manager control commands\n(`cotal ps` / `stop` / `attach`, plus `cotal spawn --detach`) all reach a manifest mesh on any port\nwith no `--server`:\n\n```bash\ncotal ps --space research-team # finds research-team's broker via the registry\n```\n\n`--server` remains an explicit override for an off-registry broker.\n\n---\n\nSee **[manifest.md](manifest.md)** for the complete field reference and the resolution rules,\n[channels and permissions](channels-and-permissions.md) for the access model, and\n[agent files](agent-files.md) for the persona format the `agents:` entries point at.\n" + "body": "# Define a team\n\n> **Guide** (informative) · **For:** operators · **Prereqs:** [Quickstart](getting-started.md)\n\nThe [Quickstart](getting-started.md) gives you one agent. To run a **specific team** (your own\nchannels, your own agents, and the channel access for each agent), describe it once in a\n`cotal.yaml` and launch it with a single command.\n\n## What a manifest is\n\nA manifest (`cotal.yaml`, `kind: Mesh`) is the declarative form of what you'd otherwise do by\nhand: start a broker, seed channels, spawn agents, and mint each agent creds scoped to the channels\nit may use. It is a convenience over the CLI and adds no wire concepts. Today it is **single-space**\n(one `space:` per file).\n\nIt is **channel-centric**: you list the channels, and under each one name the agents that may read\nand post. Cotal inverts that into one least-privilege credential per agent, so the file reads the\nway you think about a team (\"who's in #review?\"), while each agent only gets the access you granted.\n\n## Quickstart\n\nA complete, runnable manifest (two agents, two channels, no separate files):\n\n```yaml\napiVersion: cotal/v1\nkind: Mesh\nspace: main # the default space, runnable fresh or right after `cotal up`\nagent: claude # the harness that runs each agent\n\nagents: # inline personas (no external files needed)\n planner:\n instructions: Break the work into steps and post the plan.\n builder:\n instructions: Implement the smallest change that works.\n\nchannels:\n general:\n subscribe: [planner, builder] # auto-listen at boot\n allowPublish: [planner, builder] # may post: default-deny, so list everyone who posts\n review:\n subscribe: [planner] # only planner auto-listens\n allowSubscribe: [planner, builder] # builder MAY read #review, but isn't auto-subscribed\n allowPublish: [planner, builder]\n```\n\nSave it as `cotal.yaml` and launch:\n\n```bash\ncotal topology view -f cotal.yaml # validate + render the access graph (no broker needed)\ncotal up -f cotal.yaml # broker + channels + agents, all fresh\ncotal ps --space main # see the agents the manager booted\ncotal web --space main # ...or watch it in the browser\ncotal down # stop the stack; managed agents stay running unless --with-agents\n```\n\nA PTY child may still die when the manager process exits. A later manager on the same\nroot may still take leftover seats.\n\nThe manifest introduces no access model of its own; the three verbs are the same ones\nCotal uses everywhere: `subscribe` (auto-listen at boot, and implicitly may read),\n`allowSubscribe` (**read**; defaults to `subscribe`, must be a superset of it), and\n`allowPublish` (**post**; default-deny: an empty or omitted list means nobody posts).\nAbove, `builder` *may read* #review but doesn't *auto-listen* to it. Every top-level key,\nthe three `agents:` forms, channel cards, and the resolution rules are in the\n[manifest reference](manifest.md).\n\n## The command lifecycle\n\n| Command | What it does |\n|---|---|\n| `cotal topology view -f ` | Validate the file and render its access graph. Read-only: needs no broker, mutates nothing. Run it before you launch. |\n| `cotal up -f ` | Bring up a **fresh** mesh: broker + seeded channels + booted agents. |\n| `cotal spawn -f ` | Deploy a manifest **additively** onto a mesh that is already running. |\n| `cotal down [-f ]` | Tear down (see \"Tearing down\" below). |\n\n`up -f` and `spawn -f` accept `--dry-run` (preview the plan, change nothing). `up -f` also takes\n`--server` / `--host` / `--space` / `--runtime` / `--open` to override the file for one run.\n\n> If a Cotal mesh is already running at the manifest's broker address (e.g. the default\n> `127.0.0.1:4222` from `cotal up`), `up -f` **refuses**; it never re-seeds a live broker. The\n> check is on the *address*, not the `space:` name. Run `cotal down` first, point the manifest at\n> another address (`broker: { servers: nats://127.0.0.1:14999 }`, or `--server`), or use\n> `cotal spawn -f` to deploy onto the running mesh.\n\n**Tearing down.** A fresh mesh from `up -f` is torn down with plain **`cotal down`**: it owns the\nwhole space. An additive deploy from `spawn -f` is torn down with **`cotal down -f `** (or\n`cotal down -f --run `), which removes *only* that run's agents and channels.\n\n## Manifest ownership\n\nThe rule: **`up -f` owns the whole space; `spawn -f` owns only what it created.** Cotal only ever\ntears down what it owns; foreign actors on a shared mesh are never touched.\n\n- A fresh mesh from `up -f` → `cotal down` stops all of it.\n- An additive deploy from `spawn -f` records a creation-only **ledger**\n (`.cotal/manifests/.json`) of the channels and agents it added; `cotal down -f`\n removes only those. The **run id** is printed by `spawn -f` and is the filename under\n `.cotal/manifests/`; pass it to `down -f --run ` when the file has changed since the deploy\n (an edited file no longer matches its ledger) or to finish a teardown that was retained.\n\n`down -f` is deliberately conservative; it treats the ledger as untrusted and validates before\ndeleting: an owned agent is stopped only when the live agent's recorded name *and* id match; an\nowned channel is removed only when no other members remain; and if the broker is unreachable or\nanything is uncertain, nothing remote is removed and the ledger is **retained** for a later\n`down -f --run `. It is local-only: run it from the checkout that created the run.\n\n(`.cotal/` holds creds, the ledger, and runtime artifacts: add it to your `.gitignore`; commit\nyour `cotal.yaml` and persona files, not what's under it.)\n\n## Deploying onto a shared mesh (`spawn -f`)\n\n`spawn -f` is additive and never adopts or mutates anything it didn't create. It classifies each\ndeclared item against the live mesh:\n\n| Item | Classification | Behaviour |\n|---|---|---|\n| Channel, brand-new | created + owned | Seeded and recorded in the ledger. |\n| Channel, already present | `exists-unmanaged` | Left untouched: card not mutated; the desired card is shown against the live one. |\n| Agent, not yet created | will-create | Booted and recorded. |\n| Agent, already created, unchanged | already-owned | No-op. |\n| Agent, already created, policy changed | `stale` | Exits non-zero unless `--allow-stale ` (then it restarts). |\n\n> **Security.** If an **unmanaged** actor already has read access to a channel you declare,\n> `spawn -f` prints a warning: an isolation conflict on a shared mesh. It is an explicit *lower\n> bound* (presence plus the broker membership feed), not a guarantee that no other access exists.\n\n**Deploying to a remote manager.** The mesh's manager may live on another machine (or another\ncheckout): `spawn -f` detects that from the manager lease and pushes the resolved launch spec\ninline over the control plane. The manager validates it as untrusted input and persists it under\nits own `.cotal/run/` before launching, so nothing changes downstream. Run the deploy from the\ncheckout the mesh is **registered** to on your machine (that's where the ledger lands), and run\n`down -f` from that same checkout; it stops remote agents over the control plane and treats a\nlocally-absent cred file as proven-absent. One residual: the agents' cred files minted on the\nmanager's host stay there until the mesh's own cleanup, the same way it would after a crash.\n\n## Operating a manifest mesh\n\nEvery mesh-touching command resolves the broker from the mesh registry, so `--space ` is\nenough; `send`, `channels`, `console`, `web`, the manifest verbs, and the manager control commands\n(`cotal ps` / `stop` / `attach`, plus `cotal spawn --detach`) all reach a manifest mesh on any port\nwith no `--server`:\n\n```bash\ncotal ps --space research-team # finds research-team's broker via the registry\n```\n\n`--server` remains an explicit override for an off-registry broker.\n\n---\n\nSee **[manifest.md](manifest.md)** for the complete field reference and the resolution rules,\n[channels and permissions](channels-and-permissions.md) for the access model, and\n[agent files](agent-files.md) for the persona format the `agents:` entries point at.\n" }, { "slug": "delivery-daemon", @@ -166,7 +166,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "Embedding Cotal", "kind": "Guide (informative)", "summary": "The cotal binary in this repo is one composition root: an operator CLI.", - "body": "# Embedding Cotal\n\n> **Guide** (informative) · **For:** implementers building a service on top of Cotal · **Prereqs:** [Architecture](architecture.md), [Identity and auth](identity-and-auth.md), [Delivery daemon](delivery-daemon.md)\n\nThe `cotal` binary in this repo is one composition root: an operator CLI. A separate service\n(for example a hosted, multi-tenant Cotal) does not fork this repo. It writes its **own**\ncomposition root that depends on the published `@cotal-ai/*` packages and imports the surfaces it\nwants. `bin/cotal.ts` uses the same composition pattern. This page is the contract for that: what is a real library\nexport you can build against, how to boot the server-side daemons from those exports, and where the\ncurrent export surface stops short of a fully hosted composition.\n\nThis is the \"guarded substrate\" boundary in practice. Nothing here reveals or assumes a specific\nhost; it documents the public seams any embedder composes.\n\n## What you embed\n\nThe supported reference shape here is **one broker operator serving one space** (one tenant: a\ndedicated data account, under an operator that also holds the system account and a quarantined\nauth-callout account) plus three standalone processes. The trust layer itself composes many spaces\nunder one broker operator today (`createBrokerAuth` + `createSpaceAccountAuth` + N-space\n`serverConfig`); what does not exist yet is the per-space **lifecycle** on a shared broker (see\n[Known gaps](#hosted-composition-gaps)). The three processes:\n\n| daemon | package | what it is |\n|---|---|---|\n| auth-service | `@cotal-ai/auth` | the NATS auth callout, the IdP token exchange, and JWKS. Plane 1 to Plane 2. |\n| delivery | `@cotal-ai/delivery` | the Plane-3 durable backstop: fan-out writer plus trusted reader, per space. |\n| supervise | `@cotal-ai/manager` | the per-machine agent lifecycle (spawn/despawn/attach), per space. |\n\n`mint`, `deliver`, and `auth-service` expose their behavior as direct library primitives, and the\nsupported one-space bootstrap below re-composes from exported low-level primitives. `supervise` and\nthe full `up` orchestration are **not** public runners: `up` also does broker bring-up, restore,\nprocess and registry management, and lifecycle work, and `supervise`'s orchestration is private (see\n[Supervisor signing authority](#supervisor-signing-authority)).\n\n## The export surface\n\nEverything below is a real export of a published package, reachable from the package root (each\npackage publishes only `.` via `dist/index.{js,d.ts}` and ships `files: [\"dist\"]`). Type-only names\nare marked; import them with `import type`.\n\n**Daemon runners and lifecycle**\n\n| symbol | package | purpose |\n|---|---|---|\n| `runAuthService(args, store?)` | `@cotal-ai/auth` | boot the auth-service daemon; `store` injects the secret material. |\n| `runDelivery(args, store?)` | `@cotal-ai/delivery` | boot the delivery daemon; `store` injects the scoped `delivery` cred. |\n| `deliveryCredsKey(space, composition)`, `membershipRwCredsKey(space, composition)` | `@cotal-ai/workspace` | build the secret-store keys the delivery cred and the membership feed's rw cred are read/re-signed under. Keys are **per-space**: `space./`. A hosted composition passes `{ injected: true }`. |\n| `DELIVERY_CREDS_KIND`, `MEMBERSHIP_RW_CREDS_KIND` | `@cotal-ai/workspace` | the operator-facing KIND names (`delivery.creds`, `membership-rw.creds`) those keys are built from, and what renewal results report. A kind is **not** a key: putting a cred under the bare kind writes the pre-0.4 flat location, which nothing reads. |\n| `Manager`, `ManagerOptions` *(type)* | `@cotal-ai/manager` | construct and run a supervisor in-process; `ManagerOptions.secretStore` injects the one store it reads/writes every secret through. |\n| `createRuntime`, `Runtime` *(type)* | `@cotal-ai/manager` | resolve the spawn backend (pty built in). |\n\n**Provisioning and minting** (all `@cotal-ai/core`)\n\n| symbol | purpose |\n|---|---|\n| `createBrokerAuth(label)` | mint BROKER trust: the operator and system account one nats-server trusts. One per broker, shared by every space on it. |\n| `createSpaceAccountAuth(broker, space)` | mint one space's own data account, signed by that broker's operator: the add-a-tenant primitive. |\n| `createSpaceAuth(space)` | the one-space convenience: broker trust + one account in a single composed bundle. |\n| `setupSpaceStreams({ servers, space, creds })` | create the space's JetStream streams. |\n| `ensureDefaultDeliveryClass({ servers, space, creds?, deliveryClass })` | write the space's default delivery class at creation so it is wire-discoverable (SPEC section 4). |\n| `serverConfig(broker, spaces, { storeDir, extraAccounts?, port?, host? })` | render the broker config: one operator, N space accounts. `storeDir` is required and `extraAccounts` preloads the auth-callout account. |\n| `mintCreds(auth, identity, profile, opts?)` | mint a scoped cred for any `Profile`. |\n| `mintMembershipObserverCreds`, `mintConnectionEvictorCreds` | mint the membership/eviction scoped creds. |\n| `provisionAgent`, `provisionAgentDurables` | create a principal's bind-only durables. |\n| `newIdentity`, `stripSpaceAuth` | a fresh nkey identity; a stripped signer bundle (data signing seed only). |\n| `Profile`, `CredentialKind`, `MintOpts`, `SpaceAuth` *(types)*, `CREDENTIAL_LIFETIMES` | the profile matrix and cred lifetime policy. |\n\n**Auth building blocks** (all `@cotal-ai/auth`)\n\n| symbol | purpose |\n|---|---|\n| `createCalloutAuth`, `startAuthCallout` | the NATS auth-callout responder. |\n| `createUserTokenIssuer`, `pinnedJwksResolver` | mint and verify the Cotal user bearer. |\n| `createIdpBridge` | exchange a verified IdP JWT for a Cotal bearer (see [the callout contract](identity-and-auth.md#the-idp-callout-contract)). |\n| `deriveOwnerToken`, `validateUserToken` | owner derivation; strict bearer validation. |\n| `cotalAuthProvider` | the self-registering `auth-provider` extension. |\n| `ensureCalloutAuth`/`loadCalloutAuth`, `ensureIssuer`/`loadIssuer`, `ensureOwnerSecret`/`loadOwnerSecret` | read/write the auth secret kinds through a `SecretStore`. |\n\n**Seams and the wire** (all `@cotal-ai/core` unless noted)\n\n| symbol | purpose |\n|---|---|\n| `SecretStore` *(type)* | the durable hosted-secret seam (get/put/delete); `get()` returns raw seeds/keys into process memory, so it is a blob seam, not HSM/KMS signing. |\n| `FsSecretStore`, `workspaceSecretStore(root)` | the filesystem default. **These live in `@cotal-ai/workspace`, not core.** |\n| `AuthProvider` *(type)*, `Connector` *(type)*, `Runtime` *(type)*, `Command` *(type)* | the extension contracts; implementations self-register on import. |\n| `registry` | the shared registry a composition root pulls surfaces into. |\n| `CotalEndpoint`, subjects, message types | the wire client and shapes. |\n| `ParsedArgs` *(type)* | the shape the daemon runners take (see below). |\n\nThe runners take a CLI-shaped `ParsedArgs`, not a typed options object, so a host fabricates one:\n\n```ts\nconst args: ParsedArgs = { values: { space, server, port: \"0\" }, positionals: [], raw: [] };\n```\n\n### Long-lived endpoints take a bearer function\n\n`EndpointOptions.bearer` accepts either a string or a function, and the difference is not stylistic.\nA string is minted once, so when it expires (which it will: callout bearers live minutes) the\nendpoint has nothing to renew with. It will not present the dead token to the broker, since that is\na guaranteed denial that still costs a full auth-callout round trip. It refuses to reconnect, emits\n`warning` saying which case it is in, and retries on a widening backoff until the process\nre-authenticates and rebuilds it. Retry notices use `warning` rather than `error` because Node\nrethrows an unhandled `error` event and would kill a host the endpoint is still trying to recover.\n\nPass a **function** for anything that outlives one bearer. That is a renewal source: it is called\nahead of each expiry and again whenever a reconnect finds the cached bearer dead, and it requires\nexplicit `card.owner` and `card.actor`. The first-party surfaces already do this\n(`UserViewAuth.source`, the connector's `agentBearerCommand`). A string bearer is for a short\none-shot connection.\n\nLong-lived hosts must also subscribe to the endpoint's `warning` event. It carries conditions the\nendpoint is surviving, including failed credential renewal and reconnect retries. A host may choose\nto ignore warnings for a one-shot endpoint whose awaited operation owns the verdict, but that choice\nshould be explicit. An unhandled warning is nonfatal and silent.\n\n## Booting the daemons\n\n### auth-service\n\n`runAuthService(args, store?)` reads its provisioned long-lived secret kinds (service keys, callout\naccount, issuer keys, owner secret) through the injected `SecretStore`; a host provisions those into\nthe store first. It is a **signer and identity authority**, not a scoped daemon: at runtime it holds\nthe data-account and callout-account signing seeds, the issuer's private JWKs, and the\nowner-derivation secret in process memory (`SecretStore.get` exports raw values). The IdP pin and the\nactor ledger are **not** store-injected: `runAuthService` resolves them under\n`userAuthStateDir(findCotalRoot(), space)`, a path relative to the process working directory, so a\nhost provisions those into that exact directory (neither `store` nor `COTAL_HOME` selects it). It\nalso writes an ephemeral `auth-service.json` discovery file there that carries the live exchange\ncapability.\n\n```ts\nimport { runAuthService } from \"@cotal-ai/auth\";\n// store implements SecretStore over your secret backend; get() returns raw seeds into memory.\n// Provision the auth secret kinds into the store, AND the IdP pin + actor ledger under\n// userAuthStateDir(findCotalRoot(), space), before this call.\nawait runAuthService(\n { values: { space, server: brokerUrl, port: \"8081\" }, positionals: [], raw: [] },\n store,\n);\n```\n\n### delivery\n\n`runDelivery(args, store?)` runs from a **pre-minted scoped `delivery` cred** and never loads the\nsigner. Provide the cred either through the injected store (under\n`deliveryCredsKey(space, { injected: true })`) or with a\n`--creds` file; the two are mutually exclusive. The daemon re-fetches the cred from the store at 75%\nof its JWT lifetime and fails loud rather than riding to expiry, so **something must re-sign a fresh\ncred into that same store**.\n\n```ts\nimport { runDelivery } from \"@cotal-ai/delivery\";\nawait runDelivery({ values: { space, server: brokerUrl }, positionals: [], raw: [] }, store);\n```\n\nThat renewal is a **signer** operation, not the delivery daemon's:\n`remintDaemonCreds(root, space, store?, { preflight? })` (`@cotal-ai/workspace`) reads the `SpaceAuth`\nsigner **through the same resolved `store`** (`getSpaceAuth(store ?? workspaceSecretStore(root), space)`,\nkeys `auth/broker.json` + `auth/account..json`; the pre-split `auth/auth.json` monolith is\nmigration input and the container signer mount only) and re-signs the daemon creds (`delivery.creds` and the membership feed's\n`membership-rw.creds`) back into that store. The injected `store` is both the signer source and the\ncredential destination, never a split. `space` is **required** and validated against the store's signer, so a\nstore swapped to a different space cannot re-sign over the wrong broker's creds. `preflight` is a\ncaller-supplied proof that the broker accepts the credential. The reference `Manager` passes a\n`probeConnect` over its `servers`. It gates **every** candidate before overwriting the last-good,\nwhether the signer is a full bundle or a stripped projection: a bundle's JWT chain proves only that\nit is self-consistent and\nnamed the space, NOT that its account is the broker's *current* account for that space (two\n`createSpaceAuth(space)` calls yield same-named, different-account chains), so a same-label alternate\nsigner would otherwise mint a broker-dead cred and clobber the good one. The offline local repair (`doctor auth --fix`) has no preflight. It permits the overwrite only\nunder **authority continuity**: the candidate must be signed by the same account signing key (`iss`) as the current\n(already broker-accepted) cred. A same-label alternate account breaks continuity and is refused, full or\nstripped; a legitimate local re-sign is continuous and proceeds without a network. The reference\n`Manager` runs it on a schedule against its **own**\n`secretStore` (see below), so passing the manager and the delivery daemon the *same* store closes the\nrenewal loop end-to-end on an injected backend: the manager reads the signer from the store, re-signs\ninto it, and the daemon adopts each generation on a preflight-proven 75% timer. It never throws: it\nreturns per-file results (`skipped: \"no-auth\"` when the store holds no signer records),\nso the caller must check them or the cred still rides to expiry. A composition whose signer lives in\nKMS/Vault simply injects that store; no bespoke renewal is needed. A `--creds` file path must be\nreplaced atomically before the 75% read. The signer can now be injected behind the store seam, which\nresolves custody. The remaining hosted gap is signer **isolation**. The seed is still decrypted\nin-process at the manager's uid, so it needs an OS sandbox or remote signer.\n\n### Supervisor signing authority\n\n`@cotal-ai/manager` exports the `Manager` class; there is **no** `runSupervise(opts)` runner. The\nprivate CLI `runManager` also does broker-reachability checks, space/default resolution,\nroster/launch parsing and materialization, installed-extension resolution, signal handling, staged\npre-spawn, and the forever wait. A host composes that lifecycle itself around `Manager`:\n\n```ts\nimport { Manager } from \"@cotal-ai/manager\";\nconst mgr = new Manager({ space, servers: brokerUrl, workspaceRoot });\nawait mgr.start(); // then wire your own SIGINT/SIGTERM -> mgr.stop()\n```\n\nUnlike delivery, the manager is **not** a pre-minted-scoped-cred daemon (auth-service is also a\nsigner: it holds fewer artifacts than the full trust bundle, but its data-account signing seed still\ngrants complete data-account mint authority on compromise, so this is not least-privilege). On `start()`\nthe manager reads its space's full trust chain **through its `secretStore`** (`getSpaceAuth(this.secrets,\nthis.space)`, composed from `auth/broker.json` + `auth/account..json`; a container may instead\nmount a stripped signer bundle at the legacy `auth/auth.json` key) and **self-mints** its supervisor cred and renewals from the\ndata-account signing seed. In static mode it also mints every per-agent cred from that seed; in user\nmode agents instead receive callout-minted bearers, but the manager still holds the signing seed for\nits own creds and renewal. So a hosted supervisor is a **trusted per-tenant account-signer process**,\nnot a least-privilege connect client. It additionally requires a `~/.cotal/meshes/space..json`\nregistry record and the workspace user-auth marker to start in user mode. `ManagerOptions.secretStore`\ninjects the one `SecretStore` the manager uses for **the signer itself (the split trust\nrecords)**, daemon-credential renewal (`remintDaemonCreds`), and per-agent secrets,\ndefaulting to the workspace filesystem store; pass the delivery daemon the *same* store for end-to-end\nhosted renewal. The signer IS now injectable: a hosted composition injects a KMS/Vault store and no\nsigning seed lands on the hosted disk. What remains is signer **isolation**. The seed is decrypted\nin-process at the manager's uid. That issue needs an OS sandbox or remote signer; it is no longer a\ncustody problem. The other knobs are `workspaceRoot` and the process-global `COTAL_HOME`.\n\n> Scope note: the **static-auth** operator paths (`cotal spawn`/`join`/`status`/`web`, via\n> `mesh-target` → `connect`/`preflight`) still read the signer from the local split records (sync\n> `loadSpaceAuth`). That is the single-machine composition, where the signer is on local disk by the\n> static-auth model; multi-tenant hosting runs **user mode**, which never mints from on-disk trust. The\n> store-injectable signer path is the hosted-server set: the manager, `remintDaemonCreds`, and delivery.\n\n**Signer isolation needs an OS sandbox.** The default pty runtime\nruns agent children under the *same* OS uid and the *same* `workspaceRoot`, so mode-0600 on\nthe trust records does not stop a hostile same-uid agent from reading their absolute paths. The reference\n[deploy](deploy.md) tree does not solve this: it mounts the signer into the agent's own container, so\nits phase-1 boundary isolates agents from each other, not the signer from the agent. A hosted\ncomposition must run the manager/minter that holds the signer in a different uid, container, or mount\nnamespace from the agent children, which mount no signer at all; that split is future\nhosted-composition work, so until it (or a remote/injected minter) exists, do not run untrusted\nagents under this manager.\n\n## Provisioning a space (one-space reference shape)\n\n```ts\nimport { createSpaceAuth, setupSpaceStreams, ensureDefaultDeliveryClass, mintCreds, newIdentity } from \"@cotal-ai/core\";\nconst auth = await createSpaceAuth(space); // trust bundle (in-memory seeds)\nconst provisionerCreds = await mintCreds(auth, newIdentity(), \"provisioner\");\nawait setupSpaceStreams({ servers: brokerUrl, space, creds: provisionerCreds });\n// SPEC section 4: write the default delivery class at space creation so it is wire-discoverable,\n// never inferred from the resolution fallback. A daemon-backed space is \"durable\".\nawait ensureDefaultDeliveryClass({ servers: brokerUrl, space, creds: provisionerCreds, deliveryClass: \"durable\" });\nconst deliveryCreds = await mintCreds(auth, newIdentity(), \"delivery\");\n// put deliveryCreds into your SecretStore under deliveryCredsKey(space, { injected: true })\n// (@cotal-ai/workspace) before booting delivery — the key is per-space, not the bare kind.\n```\n\nRendering the broker config for a user-auth space is `serverConfig(broker, spaces, { storeDir,\nextraAccounts })`, where `extraAccounts` must include the callout account from `createCalloutAuth` so\nthe auth-service has a broker account to answer on. That account never shares the data account.\n\nBroker trust and space accounts are separate authorities: `createBrokerAuth` mints the one\noperator + system account a broker trusts, `createSpaceAccountAuth(broker, space)` signs each\ntenant's data account under it, and `serverConfig(broker, spaces, opts)` renders them all into one\nconfig. A host composition can therefore provision several spaces on one broker today. `cotal up`\nrenders that config from every tenant the root's auth directory holds, so booting one space keeps\nthe broker trusting its siblings, and it refuses to render at all while any account record is\nunreadable. The rest of the CLI lifecycle is still broker-wide: `down`, `clean` and `backup` refuse\non a multi-space root rather than scoping to one tenant, and the per-space lifecycle is the\nremaining multi-space operator layer. See\n[Known gaps](#hosted-composition-gaps).\n\n## Hazardous provisioning primitives\n\n`mintCreds`, the full `Profile`/`CredentialKind` matrix, `createSpaceAuth`, and `stripSpaceAuth` are\nlow-level operator primitives. Handle them as account-authority material:\n\n- A holder of a `SpaceAuth` (or a `stripSpaceAuth` bundle, which **keeps** the data signing seed) is\n a fully-trusted tenant-account authority: it can mint `admin`, `provisioner`, and destructive\n profiles, not merely `supervisor`, and mint a DM-reading identity. `createSpaceAuth`'s full result\n holds operator, system, and account seeds in memory.\n- Choose `profile` and `MintOpts` from **server-side constants**, never from tenant input. `MintOpts`\n can widen the bounded TTL defaults; cap it at your boundary. `CREDENTIAL_LIFETIMES` is a policy\n record, not an authorization boundary.\n- Never log signer material or export it into env. Do not co-locate signer access with an untrusted\n connector/runtime process at the same OS uid (file permissions do not contain a same-uid reader;\n see the manager's isolation note). Segregate per tenant; rotate on compromise\n (`rotateDataAccountSigningKey`).\n\n## Hosted composition gaps\n\nThe primitives above are present as exports, but three capabilities are **not** cleanly composable\nfrom the public contract today. Each is tied to work in flight; a host either waits for the seam or\nscopes the capability out. None is a wire concern.\n\n1. **Delivery immediate live eviction and a fully-hosted membership feed.** The renewable\n `membership-rw.creds` is now a `SecretStore` kind. `startMembership` reads it through the\n injected store, and the manager re-signs it there. The graph-feed writer therefore renews on a hosted\n backend (its data connection adopts each generation on a preflight-proven 75% timer). What still\n reads from a fixed on-disk path are the *static* `membership-observer.creds` and\n `connection-evictor.creds` ($SYS creds, minted at the `up` that provisions the account and renewed by `up --rotate-sys`) and `membership.json`\n (`{accountId}`, non-secret config); those, plus the private provisioning wrapper, keep immediate\n live eviction and a fully-hosted feed a partial gap. Missing files degrade membership to\n traffic-only and make live eviction refuse (loudly). The supported delivery contract here is the\n Plane-3 durable backstop.\n2. **Supervisor signer isolation.** `ManagerOptions.secretStore` now injects the one `SecretStore` the\n manager reads/writes every secret through, including the composed `SpaceAuth`\n signer (the split trust records), its daemon-cred renewal, and its per-agent kinds. What remains is process\n isolation: the manager still decrypts the signer in-process at its uid, so untrusted agent children\n must run under a different uid/container/mount namespace or behind a future remote signer.\n3. **Per-space lifecycle on a shared broker.** The trust layer is multi-space\n (`createBrokerAuth` + `createSpaceAccountAuth` + N-space `serverConfig`, persisted as\n `broker.json` + `account..json`) and `cotal up` renders the whole tenant list, but there is\n no per-space provisioning verb and no per-space teardown/backup/restore: the CLI's broker-wide\n lifecycle verbs refuse on a multi-space root, naming the tenants.\n This is the remaining multi-space operator layer.\n4. **A non-Better-Auth production IdP.** The exchange core (`createIdpBridge`) is EdDSA-generic, but\n the stock provider and login client are Better-Auth-endpoint-shaped, `cotalAuthProvider`\n self-registers on import (colliding with a host-owned provider under `resolveAuthProvider`), and\n the login flow speaks Better Auth's device-code endpoints. A different IdP is a host-built auth\n composition on the low-level primitives, not a configuration change (see\n [the IdP callout contract](identity-and-auth.md#the-idp-callout-contract)).\n\n## Hosted durability\n\nSpace-durable **coordination** state (chat/DM/task history, live presence, membership runtime, the\ndurable ACL registry, leases) lives in **JetStream**, written by the delivery daemon and the\nendpoints. It is broker-resident and needs no host-side durable path.\n\nWhat is **not** in JetStream, and is hosting-critical, is trust and authorization state a host must\nplace and keep:\n\n| state | class | where today | hosted injection |\n|---|---|---|---|\n| full `SpaceAuth` trust chain (`auth/broker.json` + `auth/account..json`, composed; a stripped signer bundle may instead be mounted at the legacy `auth/auth.json` key) | signing authority | `SecretStore` | `SecretStore` (manager + renewal) |\n| auth kinds: callout account/creds/xkey, issuer private keys, owner-derivation secret, data-signer projection | signing/identity authority | four `SecretStore` kinds | `SecretStore` (auth-service) |\n| `delivery.creds` | standing scoped cred | `SecretStore` or `--creds` | `SecretStore` (delivery) |\n| actor ledger, IdP pin | authorization + trust config | ambient `userAuthStateDir(findCotalRoot(), space)` | none (root-relative; not `store`/`COTAL_HOME`) |\n| `membership-rw.creds` | standing scoped cred | `SecretStore` | `SecretStore` (delivery + manager renewal) |\n| membership-observer / connection-evictor creds + `membership.json` | scoped $SYS creds / config | workspace filesystem | none (see gap 1) |\n| manager agent creds, actor tokens, sentinel creds | lifecycle authority | `SecretStore` | `SecretStore` (manager `secretStore`) |\n| `~/.cotal/meshes/space..json` record (holds IdP trust pins/root pointers) | non-secret, integrity-critical | machine home | process-global `COTAL_HOME` only |\n| auth-health, renewal records | non-secret diagnostics | workspace filesystem | `workspaceRoot` |\n\nThe `SpaceAuth` trust chain and the auth-service store kinds are **separate** identities/projections,\nnever parts of one document. `auth-service.json` (the live exchange capability) is ephemeral runtime\nstate, not durable, but is sensitive while the daemon runs. `@cotal-ai/workspace` is machine-local\noperator tooling by design; personas, PID files, and the `current-mesh` pointer are truly local and\nmust **not** sit on a hosted durable path. Everything classed above as an authority is what a hosted\ncomposition must provision and persist: signer-bearing server secrets now have `SecretStore` seams;\nthe remaining non-injectable rows are the explicit ambient `workspaceRoot`/cwd paths above.\n\n## See also\n\n- [Substrate stability](stability.md): what v0.3 and the 0.x packages guarantee, and the projected v0.4 break.\n- [Identity and auth](identity-and-auth.md): the profile matrix, the signer, and the IdP callout contract.\n- [Delivery daemon](delivery-daemon.md): the Plane-3 durable backstop.\n- [Deploy](deploy.md): the reference container against an external broker.\n" + "body": "# Embedding Cotal\n\n> **Guide** (informative) · **For:** implementers building a service on top of Cotal · **Prereqs:** [Architecture](architecture.md), [Identity and auth](identity-and-auth.md), [Delivery daemon](delivery-daemon.md)\n\nThe `cotal` binary in this repo is one composition root: an operator CLI. A separate service\n(for example a hosted, multi-tenant Cotal) does not fork this repo. It writes its **own**\ncomposition root that depends on the published `@cotal-ai/*` packages and imports the surfaces it\nwants. `bin/cotal.ts` uses the same composition pattern. This page is the contract for that: what is a real library\nexport you can build against, how to boot the server-side daemons from those exports, and where the\ncurrent export surface stops short of a fully hosted composition.\n\nThis is the \"guarded substrate\" boundary in practice. Nothing here reveals or assumes a specific\nhost; it documents the public seams any embedder composes.\n\n## What you embed\n\nThe supported reference shape here is **one broker operator serving one space** (one tenant: a\ndedicated data account, under an operator that also holds the system account and a quarantined\nauth-callout account) plus three standalone processes. The trust layer itself composes many spaces\nunder one broker operator today (`createBrokerAuth` + `createSpaceAccountAuth` + N-space\n`serverConfig`); what does not exist yet is the per-space **lifecycle** on a shared broker (see\n[Known gaps](#hosted-composition-gaps)). The three processes:\n\n| daemon | package | what it is |\n|---|---|---|\n| auth-service | `@cotal-ai/auth` | the NATS auth callout, the IdP token exchange, and JWKS. Plane 1 to Plane 2. |\n| delivery | `@cotal-ai/delivery` | the Plane-3 durable backstop: fan-out writer plus trusted reader, per space. |\n| supervise | `@cotal-ai/manager` | the per-machine agent lifecycle (spawn/despawn/attach), per space. |\n\n`mint`, `deliver`, and `auth-service` expose their behavior as direct library primitives, and the\nsupported one-space bootstrap below re-composes from exported low-level primitives. `supervise` and\nthe full `up` orchestration are **not** public runners: `up` also does broker bring-up, restore,\nprocess and registry management, and lifecycle work, and `supervise`'s orchestration is private (see\n[Supervisor signing authority](#supervisor-signing-authority)).\n\n## The export surface\n\nEverything below is a real export of a published package, reachable from the package root (each\npackage publishes only `.` via `dist/index.{js,d.ts}` and ships `files: [\"dist\"]`). Type-only names\nare marked; import them with `import type`.\n\n**Daemon runners and lifecycle**\n\n| symbol | package | purpose |\n|---|---|---|\n| `runAuthService(args, store?)` | `@cotal-ai/auth` | boot the auth-service daemon; `store` injects the secret material. |\n| `runDelivery(args, store?)` | `@cotal-ai/delivery` | boot the delivery daemon; `store` injects the scoped `delivery` cred. |\n| `deliveryCredsKey(space, composition)`, `membershipRwCredsKey(space, composition)` | `@cotal-ai/workspace` | build the secret-store keys the delivery cred and the membership feed's rw cred are read/re-signed under. Keys are **per-space**: `space./`. A hosted composition passes `{ injected: true }`. |\n| `DELIVERY_CREDS_KIND`, `MEMBERSHIP_RW_CREDS_KIND` | `@cotal-ai/workspace` | the operator-facing KIND names (`delivery.creds`, `membership-rw.creds`) those keys are built from, and what renewal results report. A kind is **not** a key: putting a cred under the bare kind writes the pre-0.4 flat location, which nothing reads. |\n| `Manager`, `ManagerOptions` *(type)* | `@cotal-ai/manager` | construct and run a supervisor in-process; `ManagerOptions.secretStore` injects the one store it reads/writes every secret through. |\n| `createRuntime`, `Runtime` *(type)* | `@cotal-ai/manager` | resolve the spawn backend (pty built in). |\n\n**Provisioning and minting** (all `@cotal-ai/core`)\n\n| symbol | purpose |\n|---|---|\n| `createBrokerAuth(label)` | mint BROKER trust: the operator and system account one nats-server trusts. One per broker, shared by every space on it. |\n| `createSpaceAccountAuth(broker, space)` | mint one space's own data account, signed by that broker's operator: the add-a-tenant primitive. |\n| `createSpaceAuth(space)` | the one-space convenience: broker trust + one account in a single composed bundle. |\n| `setupSpaceStreams({ servers, space, creds })` | create the space's JetStream streams. |\n| `ensureDefaultDeliveryClass({ servers, space, creds?, deliveryClass })` | write the space's default delivery class at creation so it is wire-discoverable (SPEC section 4). |\n| `serverConfig(broker, spaces, { storeDir, extraAccounts?, port?, host? })` | render the broker config: one operator, N space accounts. `storeDir` is required and `extraAccounts` preloads the auth-callout account. |\n| `mintCreds(auth, identity, profile, opts?)` | mint a scoped cred for any `Profile`. |\n| `mintMembershipObserverCreds`, `mintConnectionEvictorCreds` | mint the membership/eviction scoped creds. |\n| `provisionAgent`, `provisionAgentDurables` | create a principal's bind-only durables. |\n| `newIdentity`, `stripSpaceAuth` | a fresh nkey identity; a stripped signer bundle (data signing seed only). |\n| `Profile`, `CredentialKind`, `MintOpts`, `SpaceAuth` *(types)*, `CREDENTIAL_LIFETIMES` | the profile matrix and cred lifetime policy. |\n\n**Auth building blocks** (all `@cotal-ai/auth`)\n\n| symbol | purpose |\n|---|---|\n| `createCalloutAuth`, `startAuthCallout` | the NATS auth-callout responder. |\n| `createUserTokenIssuer`, `pinnedJwksResolver` | mint and verify the Cotal user bearer. |\n| `createIdpBridge` | exchange a verified IdP JWT for a Cotal bearer (see [the callout contract](identity-and-auth.md#the-idp-callout-contract)). |\n| `deriveOwnerToken`, `validateUserToken` | owner derivation; strict bearer validation. |\n| `cotalAuthProvider` | the self-registering `auth-provider` extension. |\n| `ensureCalloutAuth`/`loadCalloutAuth`, `ensureIssuer`/`loadIssuer`, `ensureOwnerSecret`/`loadOwnerSecret` | read/write the auth secret kinds through a `SecretStore`. |\n\n**Seams and the wire** (all `@cotal-ai/core` unless noted)\n\n| symbol | purpose |\n|---|---|\n| `SecretStore` *(type)* | the durable hosted-secret seam (get/put/delete); `get()` returns raw seeds/keys into process memory, so it is a blob seam, not HSM/KMS signing. |\n| `FsSecretStore`, `workspaceSecretStore(root)` | the filesystem default. **These live in `@cotal-ai/workspace`, not core.** |\n| `AuthProvider` *(type)*, `Connector` *(type)*, `Runtime` *(type)*, `Command` *(type)* | the extension contracts; implementations self-register on import. |\n| `registry` | the shared registry a composition root pulls surfaces into. |\n| `CotalEndpoint`, subjects, message types | the wire client and shapes. |\n| `ParsedArgs` *(type)* | the shape the daemon runners take (see below). |\n\nThe runners take a CLI-shaped `ParsedArgs`, not a typed options object, so a host fabricates one:\n\n```ts\nconst args: ParsedArgs = { values: { space, server, port: \"0\" }, positionals: [], raw: [] };\n```\n\n### Long-lived endpoints take a bearer function\n\n`EndpointOptions.bearer` accepts either a string or a function, and the difference is not stylistic.\nA string is minted once, so when it expires (which it will: callout bearers live minutes) the\nendpoint has nothing to renew with. It will not present the dead token to the broker, since that is\na guaranteed denial that still costs a full auth-callout round trip. It refuses to reconnect, emits\n`warning` saying which case it is in, and retries on a widening backoff until the process\nre-authenticates and rebuilds it. Retry notices use `warning` rather than `error` because Node\nrethrows an unhandled `error` event and would kill a host the endpoint is still trying to recover.\n\nPass a **function** for anything that outlives one bearer. That is a renewal source: it is called\nahead of each expiry and again whenever a reconnect finds the cached bearer dead, and it requires\nexplicit `card.owner` and `card.actor`. The first-party surfaces already do this\n(`UserViewAuth.source`, the connector's `agentBearerCommand`). A string bearer is for a short\none-shot connection.\n\nLong-lived hosts must also subscribe to the endpoint's `warning` event. It carries conditions the\nendpoint is surviving, including failed credential renewal and reconnect retries. A host may choose\nto ignore warnings for a one-shot endpoint whose awaited operation owns the verdict, but that choice\nshould be explicit. An unhandled warning is nonfatal and silent.\n\n## Booting the daemons\n\n### auth-service\n\n`runAuthService(args, store?)` reads its provisioned long-lived secret kinds (service keys, callout\naccount, issuer keys, owner secret) through the injected `SecretStore`; a host provisions those into\nthe store first. It is a **signer and identity authority**, not a scoped daemon: at runtime it holds\nthe data-account and callout-account signing seeds, the issuer's private JWKs, and the\nowner-derivation secret in process memory (`SecretStore.get` exports raw values). The IdP pin and the\nactor ledger are **not** store-injected: `runAuthService` resolves them under\n`userAuthStateDir(findCotalRoot(), space)`, a path relative to the process working directory, so a\nhost provisions those into that exact directory (neither `store` nor `COTAL_HOME` selects it). It\nalso writes an ephemeral `auth-service.json` discovery file there that carries the live exchange\ncapability.\n\n```ts\nimport { runAuthService } from \"@cotal-ai/auth\";\n// store implements SecretStore over your secret backend; get() returns raw seeds into memory.\n// Provision the auth secret kinds into the store, AND the IdP pin + actor ledger under\n// userAuthStateDir(findCotalRoot(), space), before this call.\nawait runAuthService(\n { values: { space, server: brokerUrl, port: \"8081\" }, positionals: [], raw: [] },\n store,\n);\n```\n\n### delivery\n\n`runDelivery(args, store?)` runs from a **pre-minted scoped `delivery` cred** and never loads the\nsigner. Provide the cred either through the injected store (under\n`deliveryCredsKey(space, { injected: true })`) or with a\n`--creds` file; the two are mutually exclusive. The daemon re-fetches the cred from the store at 75%\nof its JWT lifetime and fails loud rather than riding to expiry, so **something must re-sign a fresh\ncred into that same store**.\n\n```ts\nimport { runDelivery } from \"@cotal-ai/delivery\";\nawait runDelivery({ values: { space, server: brokerUrl }, positionals: [], raw: [] }, store);\n```\n\nThat renewal is a **signer** operation, not the delivery daemon's:\n`remintDaemonCreds(root, space, store?, { preflight? })` (`@cotal-ai/workspace`) reads the `SpaceAuth`\nsigner **through the same resolved `store`** (`getSpaceAuth(store ?? workspaceSecretStore(root), space)`,\nkeys `auth/broker.json` + `auth/account..json`; the pre-split `auth/auth.json` monolith is\nmigration input and the container signer mount only) and re-signs the daemon creds (`delivery.creds` and the membership feed's\n`membership-rw.creds`) back into that store. The injected `store` is both the signer source and the\ncredential destination, never a split. `space` is **required** and validated against the store's signer, so a\nstore swapped to a different space cannot re-sign over the wrong broker's creds. `preflight` is a\ncaller-supplied proof that the broker accepts the credential. The reference `Manager` passes a\n`probeConnect` over its `servers`. It gates **every** candidate before overwriting the last-good,\nwhether the signer is a full bundle or a stripped projection: a bundle's JWT chain proves only that\nit is self-consistent and\nnamed the space, NOT that its account is the broker's *current* account for that space (two\n`createSpaceAuth(space)` calls yield same-named, different-account chains), so a same-label alternate\nsigner would otherwise mint a broker-dead cred and clobber the good one. The offline local repair (`doctor auth --fix`) has no preflight. It permits the overwrite only\nunder **authority continuity**: the candidate must be signed by the same account signing key (`iss`) as the current\n(already broker-accepted) cred. A same-label alternate account breaks continuity and is refused, full or\nstripped; a legitimate local re-sign is continuous and proceeds without a network. The reference\n`Manager` runs it on a schedule against its **own**\n`secretStore` (see below), so passing the manager and the delivery daemon the *same* store closes the\nrenewal loop end-to-end on an injected backend: the manager reads the signer from the store, re-signs\ninto it, and the daemon adopts each generation on a preflight-proven 75% timer. It never throws: it\nreturns per-file results (`skipped: \"no-auth\"` when the store holds no signer records),\nso the caller must check them or the cred still rides to expiry. A composition whose signer lives in\nKMS/Vault simply injects that store; no bespoke renewal is needed. A `--creds` file path must be\nreplaced atomically before the 75% read. The signer can now be injected behind the store seam, which\nresolves custody. The remaining hosted gap is signer **isolation**. The seed is still decrypted\nin-process at the manager's uid, so it needs an OS sandbox or remote signer.\n\n### Supervisor signing authority\n\n`@cotal-ai/manager` exports the `Manager` class; there is **no** `runSupervise(opts)` runner. The\nprivate CLI `runManager` also does broker-reachability checks, space/default resolution,\nroster/launch parsing and materialization, installed-extension resolution, signal handling, staged\npre-spawn, and the forever wait. A host composes that lifecycle itself around `Manager`:\n\n```ts\nimport { Manager } from \"@cotal-ai/manager\";\nconst mgr = new Manager({ space, servers: brokerUrl, workspaceRoot });\nawait mgr.start(); // then wire SIGINT/SIGTERM -> mgr.stop() (Linux pty leaves seats; pass { withAgents: true } to reap; in-process node-pty cannot spare)\n```\n\nUnlike delivery, the manager is **not** a pre-minted-scoped-cred daemon (auth-service is also a\nsigner: it holds fewer artifacts than the full trust bundle, but its data-account signing seed still\ngrants complete data-account mint authority on compromise, so this is not least-privilege). On `start()`\nthe manager reads its space's full trust chain **through its `secretStore`** (`getSpaceAuth(this.secrets,\nthis.space)`, composed from `auth/broker.json` + `auth/account..json`; a container may instead\nmount a stripped signer bundle at the legacy `auth/auth.json` key) and **self-mints** its supervisor cred and renewals from the\ndata-account signing seed. In static mode it also mints every per-agent cred from that seed; in user\nmode agents instead receive callout-minted bearers, but the manager still holds the signing seed for\nits own creds and renewal. So a hosted supervisor is a **trusted per-tenant account-signer process**,\nnot a least-privilege connect client. It additionally requires a `~/.cotal/meshes/space..json`\nregistry record and the workspace user-auth marker to start in user mode. `ManagerOptions.secretStore`\ninjects the one `SecretStore` the manager uses for **the signer itself (the split trust\nrecords)**, daemon-credential renewal (`remintDaemonCreds`), and per-agent secrets,\ndefaulting to the workspace filesystem store; pass the delivery daemon the *same* store for end-to-end\nhosted renewal. The signer IS now injectable: a hosted composition injects a KMS/Vault store and no\nsigning seed lands on the hosted disk. What remains is signer **isolation**. The seed is decrypted\nin-process at the manager's uid. That issue needs an OS sandbox or remote signer; it is no longer a\ncustody problem. The other knobs are `workspaceRoot` and the process-global `COTAL_HOME`.\n\n> Scope note: the **static-auth** operator paths (`cotal spawn`/`join`/`status`/`web`, via\n> `mesh-target` → `connect`/`preflight`) still read the signer from the local split records (sync\n> `loadSpaceAuth`). That is the single-machine composition, where the signer is on local disk by the\n> static-auth model; multi-tenant hosting runs **user mode**, which never mints from on-disk trust. The\n> store-injectable signer path is the hosted-server set: the manager, `remintDaemonCreds`, and delivery.\n\n**Signer isolation needs an OS sandbox.** The default pty runtime\nruns agent children under the *same* OS uid and the *same* `workspaceRoot`, so mode-0600 on\nthe trust records does not stop a hostile same-uid agent from reading their absolute paths. The reference\n[deploy](deploy.md) tree does not solve this: it mounts the signer into the agent's own container, so\nits phase-1 boundary isolates agents from each other, not the signer from the agent. A hosted\ncomposition must run the manager/minter that holds the signer in a different uid, container, or mount\nnamespace from the agent children, which mount no signer at all; that split is future\nhosted-composition work, so until it (or a remote/injected minter) exists, do not run untrusted\nagents under this manager.\n\n## Provisioning a space (one-space reference shape)\n\n```ts\nimport { createSpaceAuth, setupSpaceStreams, ensureDefaultDeliveryClass, mintCreds, newIdentity } from \"@cotal-ai/core\";\nconst auth = await createSpaceAuth(space); // trust bundle (in-memory seeds)\nconst provisionerCreds = await mintCreds(auth, newIdentity(), \"provisioner\");\nawait setupSpaceStreams({ servers: brokerUrl, space, creds: provisionerCreds });\n// SPEC section 4: write the default delivery class at space creation so it is wire-discoverable,\n// never inferred from the resolution fallback. A daemon-backed space is \"durable\".\nawait ensureDefaultDeliveryClass({ servers: brokerUrl, space, creds: provisionerCreds, deliveryClass: \"durable\" });\nconst deliveryCreds = await mintCreds(auth, newIdentity(), \"delivery\");\n// put deliveryCreds into your SecretStore under deliveryCredsKey(space, { injected: true })\n// (@cotal-ai/workspace) before booting delivery — the key is per-space, not the bare kind.\n```\n\nRendering the broker config for a user-auth space is `serverConfig(broker, spaces, { storeDir,\nextraAccounts })`, where `extraAccounts` must include the callout account from `createCalloutAuth` so\nthe auth-service has a broker account to answer on. That account never shares the data account.\n\nBroker trust and space accounts are separate authorities: `createBrokerAuth` mints the one\noperator + system account a broker trusts, `createSpaceAccountAuth(broker, space)` signs each\ntenant's data account under it, and `serverConfig(broker, spaces, opts)` renders them all into one\nconfig. A host composition can therefore provision several spaces on one broker today. `cotal up`\nrenders that config from every tenant the root's auth directory holds, so booting one space keeps\nthe broker trusting its siblings, and it refuses to render at all while any account record is\nunreadable. The rest of the CLI lifecycle is still broker-wide: `down`, `clean` and `backup` refuse\non a multi-space root rather than scoping to one tenant, and the per-space lifecycle is the\nremaining multi-space operator layer. See\n[Known gaps](#hosted-composition-gaps).\n\n## Hazardous provisioning primitives\n\n`mintCreds`, the full `Profile`/`CredentialKind` matrix, `createSpaceAuth`, and `stripSpaceAuth` are\nlow-level operator primitives. Handle them as account-authority material:\n\n- A holder of a `SpaceAuth` (or a `stripSpaceAuth` bundle, which **keeps** the data signing seed) is\n a fully-trusted tenant-account authority: it can mint `admin`, `provisioner`, and destructive\n profiles, not merely `supervisor`, and mint a DM-reading identity. `createSpaceAuth`'s full result\n holds operator, system, and account seeds in memory.\n- Choose `profile` and `MintOpts` from **server-side constants**, never from tenant input. `MintOpts`\n can widen the bounded TTL defaults; cap it at your boundary. `CREDENTIAL_LIFETIMES` is a policy\n record, not an authorization boundary.\n- Never log signer material or export it into env. Do not co-locate signer access with an untrusted\n connector/runtime process at the same OS uid (file permissions do not contain a same-uid reader;\n see the manager's isolation note). Segregate per tenant; rotate on compromise\n (`rotateDataAccountSigningKey`).\n\n## Hosted composition gaps\n\nThe primitives above are present as exports, but three capabilities are **not** cleanly composable\nfrom the public contract today. Each is tied to work in flight; a host either waits for the seam or\nscopes the capability out. None is a wire concern.\n\n1. **Delivery immediate live eviction and a fully-hosted membership feed.** The renewable\n `membership-rw.creds` is now a `SecretStore` kind. `startMembership` reads it through the\n injected store, and the manager re-signs it there. The graph-feed writer therefore renews on a hosted\n backend (its data connection adopts each generation on a preflight-proven 75% timer). What still\n reads from a fixed on-disk path are the *static* `membership-observer.creds` and\n `connection-evictor.creds` ($SYS creds, minted at the `up` that provisions the account and renewed by `up --rotate-sys`) and `membership.json`\n (`{accountId}`, non-secret config); those, plus the private provisioning wrapper, keep immediate\n live eviction and a fully-hosted feed a partial gap. Missing files degrade membership to\n traffic-only and make live eviction refuse (loudly). The supported delivery contract here is the\n Plane-3 durable backstop.\n2. **Supervisor signer isolation.** `ManagerOptions.secretStore` now injects the one `SecretStore` the\n manager reads/writes every secret through, including the composed `SpaceAuth`\n signer (the split trust records), its daemon-cred renewal, and its per-agent kinds. What remains is process\n isolation: the manager still decrypts the signer in-process at its uid, so untrusted agent children\n must run under a different uid/container/mount namespace or behind a future remote signer.\n3. **Per-space lifecycle on a shared broker.** The trust layer is multi-space\n (`createBrokerAuth` + `createSpaceAccountAuth` + N-space `serverConfig`, persisted as\n `broker.json` + `account..json`) and `cotal up` renders the whole tenant list, but there is\n no per-space provisioning verb and no per-space teardown/backup/restore: the CLI's broker-wide\n lifecycle verbs refuse on a multi-space root, naming the tenants.\n This is the remaining multi-space operator layer.\n4. **A non-Better-Auth production IdP.** The exchange core (`createIdpBridge`) is EdDSA-generic, but\n the stock provider and login client are Better-Auth-endpoint-shaped, `cotalAuthProvider`\n self-registers on import (colliding with a host-owned provider under `resolveAuthProvider`), and\n the login flow speaks Better Auth's device-code endpoints. A different IdP is a host-built auth\n composition on the low-level primitives, not a configuration change (see\n [the IdP callout contract](identity-and-auth.md#the-idp-callout-contract)).\n\n## Hosted durability\n\nSpace-durable **coordination** state (chat/DM/task history, live presence, membership runtime, the\ndurable ACL registry, leases) lives in **JetStream**, written by the delivery daemon and the\nendpoints. It is broker-resident and needs no host-side durable path.\n\nWhat is **not** in JetStream, and is hosting-critical, is trust and authorization state a host must\nplace and keep:\n\n| state | class | where today | hosted injection |\n|---|---|---|---|\n| full `SpaceAuth` trust chain (`auth/broker.json` + `auth/account..json`, composed; a stripped signer bundle may instead be mounted at the legacy `auth/auth.json` key) | signing authority | `SecretStore` | `SecretStore` (manager + renewal) |\n| auth kinds: callout account/creds/xkey, issuer private keys, owner-derivation secret, data-signer projection | signing/identity authority | four `SecretStore` kinds | `SecretStore` (auth-service) |\n| `delivery.creds` | standing scoped cred | `SecretStore` or `--creds` | `SecretStore` (delivery) |\n| actor ledger, IdP pin | authorization + trust config | ambient `userAuthStateDir(findCotalRoot(), space)` | none (root-relative; not `store`/`COTAL_HOME`) |\n| `membership-rw.creds` | standing scoped cred | `SecretStore` | `SecretStore` (delivery + manager renewal) |\n| membership-observer / connection-evictor creds + `membership.json` | scoped $SYS creds / config | workspace filesystem | none (see gap 1) |\n| manager agent creds, actor tokens, sentinel creds | lifecycle authority | `SecretStore` | `SecretStore` (manager `secretStore`) |\n| `~/.cotal/meshes/space..json` record (holds IdP trust pins/root pointers) | non-secret, integrity-critical | machine home | process-global `COTAL_HOME` only |\n| auth-health, renewal records | non-secret diagnostics | workspace filesystem | `workspaceRoot` |\n\nThe `SpaceAuth` trust chain and the auth-service store kinds are **separate** identities/projections,\nnever parts of one document. `auth-service.json` (the live exchange capability) is ephemeral runtime\nstate, not durable, but is sensitive while the daemon runs. `@cotal-ai/workspace` is machine-local\noperator tooling by design; personas, PID files, and the `current-mesh` pointer are truly local and\nmust **not** sit on a hosted durable path. Everything classed above as an authority is what a hosted\ncomposition must provision and persist: signer-bearing server secrets now have `SecretStore` seams;\nthe remaining non-injectable rows are the explicit ambient `workspaceRoot`/cwd paths above.\n\n## See also\n\n- [Substrate stability](stability.md): what v0.3 and the 0.x packages guarantee, and the projected v0.4 break.\n- [Identity and auth](identity-and-auth.md): the profile matrix, the signer, and the IdP callout contract.\n- [Delivery daemon](delivery-daemon.md): the Plane-3 durable backstop.\n- [Deploy](deploy.md): the reference container against an external broker.\n" }, { "slug": "examples", @@ -229,7 +229,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "Run a mesh", "kind": "Guide (informative)", "summary": "Day-to-day operation of a local mesh: what cotal up actually runs, how spawning resolves personas, harnesses, and models, how to reach a mesh from any directory, and the operator-only maintenance v…", - "body": "# Run a mesh\n\n> **Guide** (informative) · **For:** operators · **Prereqs:** [Quickstart](getting-started.md)\n\nDay-to-day operation of a local mesh: what `cotal up` actually runs, how spawning\nresolves personas, harnesses, and models, how to reach a mesh from any directory, and the\noperator-only maintenance verbs. Every command's full flag set is in the\n[CLI reference](cli.md).\n\n## The stack\n\n`cotal up` brings up the whole local stack and bare `cotal down` stops it:\n\n- **Broker**: a local `nats-server` (logs to `.cotal/nats.log`).\n- **Delivery daemon**: the durable backstop, auth mode only\n ([what it does](delivery-daemon.md)).\n- **Manager**: a detached supervisor answering the control plane, so\n `cotal spawn --detach` and the `cotal_spawn` tool work right after `up`.\n\nThree modes:\n\n- **Default (static auth).** JWT-authed, on by default: sender authenticity and per-agent\n ACLs, enforced by the broker ([how](identity-and-auth.md)).\n- **`--user-auth --idp `.** Per-user auth: people `cotal login` once, the operator\n grants their agents on the actor ledger, and every connect is authorized live against\n that grant. Starts the space's auth service alongside the broker\n ([how](identity-and-auth.md)).\n- **`--open`.** An unauthenticated, live-only dev mesh (no auth, no delivery daemon). For\n quick local experiments.\n\nThe broker and local services bind **loopback** by default. `--host 0.0.0.0` widens the broker\nbind independently of the auth mode, so \"network-reachable\" never silently means\n\"unauthenticated\". With no explicit `--server`, `cotal up` auto-selects a free local port when\nthe default address is already held by another project; an explicit `--server` fails loud on\ncollision.\n\nA user-auth mesh can expose only its credential exchange through an operator-owned HTTPS reverse\nproxy while leaving the existing local exchange untouched:\n\n```bash\ncotal up --user-auth --idp https://idp.example/api/auth \\\n --exchange-public-port 7443 \\\n --exchange-public-url https://auth.example\n```\n\nThe public listener itself still binds `127.0.0.1:7443`; configure the proxy to terminate TLS and\nforward to it. It serves only `/health`, `/jwks`, `/exchange`, and `/.well-known/cotal-mesh` with\nthe documented methods. It needs no local file capability: the signed IdP JWT or managed-agent\nactor token is the proof, while the original loopback listener remains capability-gated. Add\n`--exchange-trusted-proxy` only when that listener is reachable exclusively through your trusted\nproxy; it keys failure throttling by the last `X-Forwarded-For` hop instead of the socket address.\nThe well-known bundle includes IdP pins and a deny-all sentinel credential, so fetch it only from\nthe configured HTTPS origin. To change these listener flags, stop and restart the mesh; a refresh\nof an already-running service does not replace its bind or proxy policy. See\n[Identity & auth](identity-and-auth.md#per-user-authentication) for the trust boundary.\n\n`cotal status` prints the detailed setup, process, registry, and live mesh status. Its Machine\nsection names the running CLI's source checkout, installed package root, or npx package root beside\nthe version. A stale Claude skills row names the installed and CLI versions it compared. `cotal\nsetup` (after the first run) prints the compact card.\n\nBefore reporting ready, the manager resolves every installed connector's declared harness\nbinaries against its own environment. A missing binary does not stop unrelated manager work: boot\ncontinues, but prints a named `connector unavailable` line and records that reason in the\nmanager's `status` response. Available connector rows record the absolute paths boot resolved.\nSpawn keeps the same pre-mint check as a backstop for connectors registered after boot.\n\nOn an authenticated manager start, unfinished static lifecycle rows reconcile while the control\nendpoint is already serving. The manager `status` response reports\nthe `staticReconciliation` state, the last sweep counts, and each failed alias with its durable\nphase and literal disposition. `cotal status --components` reports the state and per-alias failure\ndetails. A failed exact terminal is retried in the same process after 1, 5,\nand 30 seconds. Each attempt re-reads the durable slot and re-enters the same deterministic terminal\noperation; the delays only schedule work and never release the lifecycle fence.\n\nOn shutdown, the manager fences new reconciliation work and waits for an exact terminal that already\nstarted. The current serial sweep stops before its next alias, and startup cannot publish the manager\nservice after `stop()` completes.\n\nThe four-attempt budget is per manager process. An exhausted row stays held and reports\n`retry-exhausted` with the remedy to restart the manager. The next process derives a fresh budget\nfrom the still-authoritative durable row. A `recovered` row remains visible until the next static\nreconciliation sweep, then clears. This component reports reconciliation outcomes. It does not say\nwhether footprint cleanup completed independently of the terminal result; that separate durable\nprojection remains tracked by #1274.\n\nThere is no supported `cotal service install` command yet. Running the manager as a launchd agent or\nsystemd user service remains operator-managed; service installation is separate from this boot-time\ndetection behavior.\n\nStop one part without tearing down the mesh by naming its registered component: `cotal down\nmanager`, `cotal down delivery`, or `cotal down web`. Component names from installed extensions\njoin the same surface; `cotal down` with no names retains whole-stack behavior.\n\n## Remote supervised agents\n\nOn a remote user-auth mesh, foreground `cotal spawn` remains the default participant path. A\nparticipant can run detached agents only after the host advertises and operates the remote manager\nauthority service, and the participant's actor-ledger row includes `supervise`. This is not implied\nby `spawn` or `admin`.\n\nThe participant's loopback/operator exchange obtains one closed `manager-service` view for its\nordinary derived owner, a fixed server-selected manager actor, and one opaque manager instance.\nThe host, not the participant, issues the public-nkey JWT material via the replay-safe,\nlifecycle-bound prepare → activate → renew exchange. It never exports the space signer, a static\nprovisioner credential, or generic storage authority. The manager may provision only descendants\nof that same owner, with host validation at each provision.\n\nThe registry entry decides the broker URL `supervise` dials, so a mesh published over `wss://` is\ndialed as a websocket. The manager-authority registration it runs first also takes its TLS\nrequirement from that entry, so the prepare credential is not exchanged over a plaintext\nconnection the record did not describe. `cotal meshes add` records both.\n\nWhen the authority service, login, or renewal is unavailable, the remote manager degrades\nfail-closed: it refuses new agents, restarts, and credential replacement rather than pretending\nlocal authority exists. Existing agents remain live only while their independent credentials are\nvalid. Restore service and renew successfully before asking it to recover an agent. See\n[Identity & auth](identity-and-auth.md#remote-manager-authority) and the [CLI\nreference](cli.md#supervise).\n\n## Spawning agents\n\n```bash\ncotal spawn # foreground: your default agent, in this terminal\ncotal spawn reviewer --detach # supervised: the manager runs it in a PTY\ncotal attach --name reviewer # watch/type into a detached agent (Ctrl-] detaches)\ncotal ps # what the manager is running\ncotal stop --name reviewer # stop one\n```\n\nHow a spawn resolves:\n\n- **Persona.** A bare `cotal spawn` uses `.cotal/agents/default.md`; a positional name\n picks `.cotal/agents/.md`; `--config` takes an explicit ref or path. Set\n `COTAL_DEFAULT_PERSONA=` to change the fallback. Fields and format:\n [agent files](agent-files.md).\n- **Harness.** Resolution order is an explicit `--agent` or `cotal_spawn` `agent` argument,\n then the persona file's `agent:` pin, then the invoking caller's `COTAL_DEFAULT_AGENT`,\n then the manager's `COTAL_DEFAULT_AGENT`, then the product default (Claude). Compared in\n [Connectors](connectors.md); per-connector guides:\n [Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n [Hermes](connect-hermes.md) · [pi](connect-pi.md).\n- **Model.** `--model` overrides the persona file's `model:` (Claude: `opus` / `sonnet` or\n a full id; OpenCode: `provider/model`). Connectors that expose a catalog report it via\n `cotal models --agent opencode`: model ids plus available variants; pick one with\n `--model provider/model --variant high`.\n- **Tools.** A spawned agent gets only the cotal tools by default; share your own MCP\n servers deliberately with `--share-tools` ([config](config.md)).\n- **Launch options.** `--opt key=value` (repeatable) passes a native harness flag straight\n through; a persona or manifest `launchOptions:` mapping does the same declaratively (a\n `--opt` wins per key). It is a **raw passthrough**, with no allow/deny list: Claude renders\n each as `--key value` (a bare `--key` for an empty value), OpenCode merges them into its\n agent config, and Hermes has no option surface so it fails loud. The trust boundary is the\n `spawn` capability itself, not the flag set, so granting `spawn` is host-launch authority\n ([security](security.md)). A key must be a plain flag name; malformed or prototype-polluting\n keys are refused.\n\nDetach from an attached PTY with **Ctrl-]** (the agent keeps running); rebind it with\n`COTAL_DETACH_KEY=ctrl-` when it clashes with a keybinding inside the agent's TUI.\n\n**Runtimes.** The manager spawns into a **pty** by default. On Linux a detached per-seat\ncustodian owns that PTY, so replacing the manager worker does not close the seat. Other\nplatforms still spawn the PTY in-process; `adopt` throws until their transport lands. Optional runtimes are installed\nthrough the extension surface, for example `cotal ext add @cotal-ai/orca`, then selected with\n`--runtime orca` (similarly `@cotal-ai/tmux`, `@cotal-ai/cmux`, and `@cotal-ai/herdr`). They put teammates in native\nterminal surfaces rather than manager-owned PTYs. Runtime names are open-ended and resolved from\nthe registry; a missing provider or app throws, never silently falls back\n([architecture](architecture.md)).\n\n## Mesh registry\n\n`cotal up` records each running mesh in a machine-local registry\n(`~/.cotal/meshes/space..json`, named by a case-safe hex encoding of the space: broker URL, the project root holding its creds and\npersonas, and its mode). So a bare `cotal spawn ` from *any* directory joins the\nrunning mesh with the right credentials instead of mistaking the cwd for a space:\n\n- `cotal use ` sets the default from every directory, including inside another mesh's\n project. `--space ` overrides it for one command.\n- When one broker has records for several spaces, `cotal up --space ` refreshes that named\n space.\n- With no live selected default, a project with its own `.cotal/` resolves to that project's\n mesh; otherwise one running mesh is used automatically and several are an error.\n- `cotal meshes` lists them (a `*` marks the default); `cotal down` removes the entry.\n\nThe registry stores a *path*, never a secret; trust material stays in each project's\n`.cotal/auth`. If the mesh is down or won't take your creds, spawn fails with one\nsentence, never a raw NATS trace.\n\n### Meshes you did not start here\n\nA mesh running on another machine has no `cotal up` on this one, so register it by hand:\n\n```bash\ncotal meshes add # guided: asks for the broker, probes it, offers what it finds\ncotal meshes add optiplex --server nats://100.90.12.34:4222 --root ~/meshes/optiplex \\\n --allow-unencrypted-overlay # see below: an overlay address needs this\ncotal meshes rm optiplex\n```\n\nOn a terminal, a bare `cotal meshes add` walks you through it: it probes the broker you name and\nreports whether it is open or requires credentials, offers the spaces the folder already holds\ncredentials for, and shows the record before writing it. Scripts and agents keep the flag form -\nwithout a terminal nothing prompts.\n\n`--root` is the local folder holding that mesh's `.cotal/auth` and `.cotal/agents` (its personas);\nthe mode is inferred from what that folder holds.\n\n**Know what you are copying.** For an authenticated mesh that folder carries the space's account\n**signing seed**, which is the authority to mint any identity in the space. A machine holding it\nis a certificate authority for the mesh rather than a client of it: anyone who reads it can\nimpersonate any agent, read every retained channel and DM, change ACLs, and keep issuing\nthemselves credentials. There is no per-machine revocation; undoing it means rotating the signing\nkey and re-minting every credential in the space. Copy it only to machines you would trust with\nthe whole mesh. `cotal mint` on its own does not substitute here: registering an `auth` mesh needs\nsigning material that composes, which a minted user credential is not. The\nbroker is probed before the record is written, so a bad address or a credential that mesh will not\naccept fails at registration rather than at your first `spawn` (`--force` records it without verifying,\nuseful when the mesh is simply down right now).\n\n#### Which addresses you may register\n\nRegistering a mesh is how this machine starts sending agent credentials to a broker it does not\nrun. NATS announces itself in plaintext before anyone authenticates, so an attacker on the path\ncan pose as the broker and read the credential out of the connect unless the connection\n**requires TLS**, which is recorded on the entry and enforced on every dial through it.\n\nWhat the record will require decides what you may register:\n\n- **Without required TLS**, the address is the gate: **loopback** (`127.0.0.0/8`, `::1`), or\n **your private overlay** (`100.64.0.0/10`, `fd7a:115c:a1e0::/48`) with\n `--allow-unencrypted-overlay`. The tunnel provides the protection, and this command cannot check\n its state. Hostnames are refused because the lookup would choose which machine receives your\n credentials.\n- **With required TLS**, set `--tls` or use a `tls://` URL. The recorded scheme enforces the TLS\n requirement. A **hostname or public address** is accepted because the certificate chain and\n hostname check identify the peer. A registration whose broker cannot complete the handshake\n fails unless you pass `--force`, which records the entry without verification.\n\nOrdinary private ranges like `10.x` and `192.168.x` are refused in **both** modes. A café's wifi\nis private but does not belong to you, and no public CA issues certificates for those ranges. An\naddress spelling changes nothing: `[::ffff:192.168.1.10]`, `3232235786`, `0300.0250.01.012`, and\n`192.168.257` all resolve to private addresses and receive the same refusal as the dotted form.\n`--force` exists for a mesh that is down. It never permits an unsafe credential destination.\n\n#### Registering a hosted user-auth mesh\n\nA user-auth space's IdP pins are established where the mesh runs and are never guessed. Register\none from **supplied** trust: `--user-auth-file bundle.json` (exported on the mesh's machine), or\n`--from https://…/.well-known/cotal-mesh`, which asks before it contacts the address at all,\nfetches the discovery document over HTTPS, shows you the pins, and asks again before adopting\nthem. Redirects are refused because a 302 can walk a pinned fetch down to\nplaintext or onto another host, and the pinned exchange must be an `https://` URL too. The one\nexception is an exchange on **this machine**, where nothing leaves the box: plain `http://` is\naccepted for a loopback *literal* (`127.0.0.1`, `::1`, and any spelling of them), but **not** for\n`localhost`, which a hosts entry or poisoned lookup could point elsewhere. Use the\nliteral. Registration checks that the pinned exchange\nanswers `/health` and `/jwks` as the pinned issuer. It also checks that the broker refuses a\nbare connect; that refusal is the pass. The bundle's sentinel credentials are written to a private (0600) file\nunder the entry's root; the registry itself never carries the secret.\n\n**Without required TLS**, an overlay address is **refused unless you accept the dependency\nexplicitly**, with `--allow-unencrypted-overlay`. The address is not the guarantee: it is protected\nwhile the tunnel is up, and if the tunnel is down that range is ordinary carrier-grade NAT and\nwhoever answers the dial receives your credentials. Only you can know which it is, so the command\nasks you to say so. Your acceptance is recorded on the mesh entry rather than printed and\nforgotten, and the guided form asks the same question instead of taking the flag.\n\n**With required TLS** (`--tls`, or a `tls://` URL) that consent is no longer asked for, and the\nflag is not needed: the handshake is what protects the connection, so the acceptance it stood in\nfor has been replaced by proof rather than promise. `cotal meshes add --server\nnats://100.64.0.1 --tls` registers an overlay address with no prompt, no flag and no recorded\nacceptance. This is the \"the flag disappears once the broker can be served over TLS\" case, and it\nhas now arrived.\n\nThis gate is on **registration**. `cotal join --creds --server ` deliberately takes an\nexplicit connection at face value and does not consult the registry, so it is not covered. Join\nthat way only to an address you would have registered.\n\nRecords added this way are removed only by something that names them. A mesh this machine started\ncan be dropped on a hunch, such as a failed liveness probe or a `cotal down` in its project, because\n`cotal up` writes the record straight back. One you registered by hand cannot be reconstructed, so\nnothing removes it by inference: an unreachable broker is shown as `offline` in `cotal meshes`, and\n`cotal down` / `cotal clean all` leave it alone even when `--root` pointed at the project they are\ntearing down. A `cotal up` for that space refuses outright (naming `cotal meshes rm`) unless it is\nthat same endpoint: finding a broker already answering there is a refresh that starts nothing and\nleaves the record's provenance alone, while actually starting the broker for that space, server and\nroot makes this machine the one running it, so the record becomes an ordinary local one that\n`cotal down` clears. `cotal meshes rm` drops it and re-registering with `--force` replaces it. `rm`\nonly forgets a mesh. To stop one running here, use `cotal down`.\n\n## Watching\n\n`cotal console` is the terminal view (TUI on a real terminal, plain line stream when\npiped); `cotal web` is the browser dashboard. Both are read-only observers; the\nwalkthrough is [Watch a mesh](watch-a-mesh.md).\n\n## History\n\nRetained history is operator-owned. `cotal clean history --force` purges a space's\nretained channel history; `--dms` also purges DMs (`cotal history clear` is an alias).\nIt is deliberately **not** an agent tool: agents cannot wipe the record\n([identity & auth](identity-and-auth.md)). For a **stopped** mesh, `cotal clean store\n--force` deletes the on-disk JetStream store outright, and `cotal clean all --force`\nalso resets the space identity ([CLI reference](cli.md#clean)).\n\n## Offline backup\n\nFor a coherent durable cut, preserve the whole stack first, then create the artifact while it stays\ndown:\n\n```bash\ncotal down --preserve-state\ncotal backup create ./space-backup # full by default\n# later: deliberately resume the unchanged source\ncotal up --detach\n# or, from another preserved cut, restore before the normal listener opens\ncotal up --restore ./space-backup --detach\n```\n\nUse `--store-dir` on both preservation and backup for a custom JetStream store. `registry` is the\nonly partial selection (`backup create ... --only registry`; `up --restore ... --restore-only\nregistry`). Backup never stops or restarts a mesh implicitly, never opens the original store, and\ndoes not contain credentials or trust secrets. Backup/restore in every auth mode, open included,\nuses isolated, operation-specific maintenance logins; normal agent credentials cannot enter that\nlistener. Full\nrestore requires the same space and exact current local trust continuity, recreates conservative\nconsumer checkpoints bound to their snapshot stream sequence state, and resumes retained agents under\ntheir original principals. The trust commitment includes the cryptographically validated full\noperator/system/data-account root chain as well as static/user authority state. A registry-only\nrestore completes canonical empty infrastructure but leaves retained agents stopped because their\nDM/DLV/TASK/ACL state is outside that selection. Authenticated restore validates the complete space\ntrust bundle before staging or changing the preserved store. Interrupted ordinary resume retries the\nsame durable attempt after its prior listener is stopped. Restore re-entry can recover a surviving normal listener\nonly when its attempt nonce, NATS server name, process owner, endpoint, and target-store identity all\nmatch the fsynced proof. A provably dead uncommitted owner is retired under lock and replaced with a\nfresh attempt-bound listener; an occupied foreign listener or ambiguous owner is never adopted. The\nmanager commit validates while retained cleanup is still suppressed; the CLI durably records its\nattempt-bound 64-hex token in `manager-committed` / `resume-committed` before `finalizeResume` can\nrelease suppression. A retry from either committed state goes straight to exact-token finalization;\nfailure preserves the committed gate and retained cleanup suppression. Missing commit evidence,\ninterrupted finalization, a live recorded endpoint despite missing pidfiles, or ambiguous proof fails closed. See the [CLI\nbackup and restore contract](cli.md#backups) for artifact, checkpoint, fallback,\ndisaster-consent, and degraded-recovery details.\n\n## Personas from the CLI\n\n`cotal personas` manages the local catalog offline: `list` (`--running` overlays live\nmarkers), `show `, `edit ` (re-validates on save), `new `, `rm \n--force`. The runtime write is `cotal_persona`; the runtime read is `cotal_personas`\n(list / show), both over the wire with the manager's ownership checks. Fields: [agent files](agent-files.md).\n\n## Gate recovery\n\nA manager that dies mid-registration leaves its issuance gate *frozen* under that registration\nop. The freeze is correct: it stops two incarnations serving at once. The successor now completes\nthat dead op on boot, using the same guard as [`cotal reconcile-gate`](cli.md#reconcile-gate): it\nacts only when the freeze-holder is affirmatively gone under a complete CONNZ sweep (`gone` and\n`sweepComplete=true`). If the dead op's spec write committed, it finishes that same freeze\n(promote and reopen at the committed registration revision). If the spec did not advance, it\nabort-reopens the gate (generation+1, processEpoch unchanged) and continues the normal takeover.\nA live holder, an incomplete sweep, or an unreachable delivery daemon still\nrefuses. Silence is never evidence of death, and there is no TTL. If holder verification is\ninterrupted, the frozen operation resumes from its durable, operation-and-gate-revision-bound\nprogress after liveness is checked again. A later freeze cannot reuse that progress: the cursor\nbinds the exact op, gate revision, and holder set. Use `cotal reconcile-gate` when the boot path cannot run\n(daemon down, a non-manager endpoint, or you want to lift the freeze without starting a manager). A spawn that hits the same frozen gate names that verb in the refusal\n(`blockedOp=registration`, the holding `opId`, `remedy=cotal reconcile-gate`) instead of a\nwait-timeout: the facts were always in the manager log; they now reach the spawn caller too.\n\n## When something looks absent\n\nPermission denials are **loud, never silent**: an over-tight ACL rejects the endpoint call and\nalso shows up as a logged denial, instead of returning an empty or incomplete result that looks\nsuccessful. Check\n`.cotal/manager..log`, `.cotal/delivery..log` (one pair per space, keyed as\n[Config](config.md#project-files) describes), and `.cotal/nats.log`; `cotal status` shows\nwhat is actually running. The access rules are collected in\n[Channels & permissions](channels-and-permissions.md).\n" + "body": "# Run a mesh\n\n> **Guide** (informative) · **For:** operators · **Prereqs:** [Quickstart](getting-started.md)\n\nDay-to-day operation of a local mesh: what `cotal up` actually runs, how spawning\nresolves personas, harnesses, and models, how to reach a mesh from any directory, and the\noperator-only maintenance verbs. Every command's full flag set is in the\n[CLI reference](cli.md).\n\n## The stack\n\n`cotal up` brings up the whole local stack and bare `cotal down` stops it. Managed agents\nstay running as unmanaged OS processes; pass `--with-agents` to take them with the stack.\nA PTY child may still die when the manager process exits. A later manager on the same root\nmay still take leftover seats. The stack is:\n\n- **Broker**: a local `nats-server` (logs to `.cotal/nats.log`).\n- **Delivery daemon**: the durable backstop, auth mode only\n ([what it does](delivery-daemon.md)).\n- **Manager**: a detached supervisor answering the control plane, so\n `cotal spawn --detach` and the `cotal_spawn` tool work right after `up`.\n\nThree modes:\n\n- **Default (static auth).** JWT-authed, on by default: sender authenticity and per-agent\n ACLs, enforced by the broker ([how](identity-and-auth.md)).\n- **`--user-auth --idp `.** Per-user auth: people `cotal login` once, the operator\n grants their agents on the actor ledger, and every connect is authorized live against\n that grant. Starts the space's auth service alongside the broker\n ([how](identity-and-auth.md)).\n- **`--open`.** An unauthenticated, live-only dev mesh (no auth, no delivery daemon). For\n quick local experiments.\n\nThe broker and local services bind **loopback** by default. `--host 0.0.0.0` widens the broker\nbind independently of the auth mode, so \"network-reachable\" never silently means\n\"unauthenticated\". With no explicit `--server`, `cotal up` auto-selects a free local port when\nthe default address is already held by another project; an explicit `--server` fails loud on\ncollision.\n\nA user-auth mesh can expose only its credential exchange through an operator-owned HTTPS reverse\nproxy while leaving the existing local exchange untouched:\n\n```bash\ncotal up --user-auth --idp https://idp.example/api/auth \\\n --exchange-public-port 7443 \\\n --exchange-public-url https://auth.example\n```\n\nThe public listener itself still binds `127.0.0.1:7443`; configure the proxy to terminate TLS and\nforward to it. It serves only `/health`, `/jwks`, `/exchange`, and `/.well-known/cotal-mesh` with\nthe documented methods. It needs no local file capability: the signed IdP JWT or managed-agent\nactor token is the proof, while the original loopback listener remains capability-gated. Add\n`--exchange-trusted-proxy` only when that listener is reachable exclusively through your trusted\nproxy; it keys failure throttling by the last `X-Forwarded-For` hop instead of the socket address.\nThe well-known bundle includes IdP pins and a deny-all sentinel credential, so fetch it only from\nthe configured HTTPS origin. To change these listener flags, stop and restart the mesh; a refresh\nof an already-running service does not replace its bind or proxy policy. See\n[Identity & auth](identity-and-auth.md#per-user-authentication) for the trust boundary.\n\n`cotal status` prints the detailed setup, process, registry, and live mesh status. Its Machine\nsection names the running CLI's source checkout, installed package root, or npx package root beside\nthe version. A stale Claude skills row names the installed and CLI versions it compared. `cotal\nsetup` (after the first run) prints the compact card.\n\nBefore reporting ready, the manager resolves every installed connector's declared harness\nbinaries against its own environment. A missing binary does not stop unrelated manager work: boot\ncontinues, but prints a named `connector unavailable` line and records that reason in the\nmanager's `status` response. Available connector rows record the absolute paths boot resolved.\nSpawn keeps the same pre-mint check as a backstop for connectors registered after boot.\n\nOn an authenticated manager start, unfinished static lifecycle rows reconcile while the control\nendpoint is already serving. The manager `status` response reports\nthe `staticReconciliation` state, the last sweep counts, and each failed alias with its durable\nphase and literal disposition. `cotal status --components` reports the state and per-alias failure\ndetails. A failed exact terminal is retried in the same process after 1, 5,\nand 30 seconds. Each attempt re-reads the durable slot and re-enters the same deterministic terminal\noperation; the delays only schedule work and never release the lifecycle fence.\n\nOn shutdown, the manager fences new reconciliation work and waits for an exact terminal that already\nstarted. The current serial sweep stops before its next alias, and startup cannot publish the manager\nservice after `stop()` completes.\n\nThe four-attempt budget is per manager process. An exhausted row stays held and reports\n`retry-exhausted` with the remedy to restart the manager. The next process derives a fresh budget\nfrom the still-authoritative durable row. A `recovered` row remains visible until the next static\nreconciliation sweep, then clears. This component reports reconciliation outcomes. It does not say\nwhether footprint cleanup completed independently of the terminal result; that separate durable\nprojection remains tracked by #1274.\n\nThere is no supported `cotal service install` command yet. Running the manager as a launchd agent or\nsystemd user service remains operator-managed; service installation is separate from this boot-time\ndetection behavior.\n\nStop one part without tearing down the mesh by naming its registered component: `cotal down\nmanager`, `cotal down delivery`, or `cotal down web`. Component names from installed extensions\njoin the same surface; `cotal down` with no names retains whole-stack behavior and leaves\nmanaged agents running as unmanaged OS processes. `cotal down --with-agents` is the previous\nreap. A PTY child may still die when the manager process exits. A later manager on the same\nroot may still take leftover seats.\n\n## Remote supervised agents\n\nOn a remote user-auth mesh, foreground `cotal spawn` remains the default participant path. A\nparticipant can run detached agents only after the host advertises and operates the remote manager\nauthority service, and the participant's actor-ledger row includes `supervise`. This is not implied\nby `spawn` or `admin`.\n\nThe participant's loopback/operator exchange obtains one closed `manager-service` view for its\nordinary derived owner, a fixed server-selected manager actor, and one opaque manager instance.\nThe host, not the participant, issues the public-nkey JWT material via the replay-safe,\nlifecycle-bound prepare → activate → renew exchange. It never exports the space signer, a static\nprovisioner credential, or generic storage authority. The manager may provision only descendants\nof that same owner, with host validation at each provision.\n\nThe registry entry decides the broker URL `supervise` dials, so a mesh published over `wss://` is\ndialed as a websocket. The manager-authority registration it runs first also takes its TLS\nrequirement from that entry, so the prepare credential is not exchanged over a plaintext\nconnection the record did not describe. `cotal meshes add` records both.\n\nWhen the authority service, login, or renewal is unavailable, the remote manager degrades\nfail-closed: it refuses new agents, restarts, and credential replacement rather than pretending\nlocal authority exists. Existing agents remain live only while their independent credentials are\nvalid. Restore service and renew successfully before asking it to recover an agent. See\n[Identity & auth](identity-and-auth.md#remote-manager-authority) and the [CLI\nreference](cli.md#supervise).\n\n## Spawning agents\n\n```bash\ncotal spawn # foreground: your default agent, in this terminal\ncotal spawn reviewer --detach # supervised: the manager runs it in a PTY\ncotal attach --name reviewer # watch/type into a detached agent (Ctrl-] detaches)\ncotal ps # what the manager is running\ncotal stop --name reviewer # stop one\n```\n\nHow a spawn resolves:\n\n- **Persona.** A bare `cotal spawn` uses `.cotal/agents/default.md`; a positional name\n picks `.cotal/agents/.md`; `--config` takes an explicit ref or path. Set\n `COTAL_DEFAULT_PERSONA=` to change the fallback. Fields and format:\n [agent files](agent-files.md).\n- **Harness.** Resolution order is an explicit `--agent` or `cotal_spawn` `agent` argument,\n then the persona file's `agent:` pin, then the invoking caller's `COTAL_DEFAULT_AGENT`,\n then the manager's `COTAL_DEFAULT_AGENT`, then the product default (Claude). Compared in\n [Connectors](connectors.md); per-connector guides:\n [Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n [Hermes](connect-hermes.md) · [pi](connect-pi.md).\n- **Model.** `--model` overrides the persona file's `model:` (Claude: `opus` / `sonnet` or\n a full id; OpenCode: `provider/model`). Connectors that expose a catalog report it via\n `cotal models --agent opencode`: model ids plus available variants; pick one with\n `--model provider/model --variant high`.\n- **Tools.** A spawned agent gets only the cotal tools by default; share your own MCP\n servers deliberately with `--share-tools` ([config](config.md)).\n- **Launch options.** `--opt key=value` (repeatable) passes a native harness flag straight\n through; a persona or manifest `launchOptions:` mapping does the same declaratively (a\n `--opt` wins per key). It is a **raw passthrough**, with no allow/deny list: Claude renders\n each as `--key value` (a bare `--key` for an empty value), OpenCode merges them into its\n agent config, and Hermes has no option surface so it fails loud. The trust boundary is the\n `spawn` capability itself, not the flag set, so granting `spawn` is host-launch authority\n ([security](security.md)). A key must be a plain flag name; malformed or prototype-polluting\n keys are refused.\n\nDetach from an attached PTY with **Ctrl-]** (the agent keeps running); rebind it with\n`COTAL_DETACH_KEY=ctrl-` when it clashes with a keybinding inside the agent's TUI.\n\n**Runtimes.** The manager spawns into a **pty** by default. On Linux a detached per-seat\ncustodian owns that PTY, so replacing the manager worker does not close the seat. Other\nplatforms still spawn the PTY in-process; `adopt` throws until their transport lands. Optional runtimes are installed\nthrough the extension surface, for example `cotal ext add @cotal-ai/orca`, then selected with\n`--runtime orca` (similarly `@cotal-ai/tmux`, `@cotal-ai/cmux`, and `@cotal-ai/herdr`). They put teammates in native\nterminal surfaces rather than manager-owned PTYs. Runtime names are open-ended and resolved from\nthe registry; a missing provider or app throws, never silently falls back\n([architecture](architecture.md)).\n\n## Mesh registry\n\n`cotal up` records each running mesh in a machine-local registry\n(`~/.cotal/meshes/space..json`, named by a case-safe hex encoding of the space: broker URL, the project root holding its creds and\npersonas, and its mode). So a bare `cotal spawn ` from *any* directory joins the\nrunning mesh with the right credentials instead of mistaking the cwd for a space:\n\n- `cotal use ` sets the default from every directory, including inside another mesh's\n project. `--space ` overrides it for one command.\n- When one broker has records for several spaces, `cotal up --space ` refreshes that named\n space.\n- With no live selected default, a project with its own `.cotal/` resolves to that project's\n mesh; otherwise one running mesh is used automatically and several are an error.\n- `cotal meshes` lists them (a `*` marks the default); `cotal down` removes the entry.\n\nThe registry stores a *path*, never a secret; trust material stays in each project's\n`.cotal/auth`. If the mesh is down or won't take your creds, spawn fails with one\nsentence, never a raw NATS trace.\n\n### Meshes you did not start here\n\nA mesh running on another machine has no `cotal up` on this one, so register it by hand:\n\n```bash\ncotal meshes add # guided: asks for the broker, probes it, offers what it finds\ncotal meshes add optiplex --server nats://100.90.12.34:4222 --root ~/meshes/optiplex \\\n --allow-unencrypted-overlay # see below: an overlay address needs this\ncotal meshes rm optiplex\n```\n\nOn a terminal, a bare `cotal meshes add` walks you through it: it probes the broker you name and\nreports whether it is open or requires credentials, offers the spaces the folder already holds\ncredentials for, and shows the record before writing it. Scripts and agents keep the flag form -\nwithout a terminal nothing prompts.\n\n`--root` is the local folder holding that mesh's `.cotal/auth` and `.cotal/agents` (its personas);\nthe mode is inferred from what that folder holds.\n\n**Know what you are copying.** For an authenticated mesh that folder carries the space's account\n**signing seed**, which is the authority to mint any identity in the space. A machine holding it\nis a certificate authority for the mesh rather than a client of it: anyone who reads it can\nimpersonate any agent, read every retained channel and DM, change ACLs, and keep issuing\nthemselves credentials. There is no per-machine revocation; undoing it means rotating the signing\nkey and re-minting every credential in the space. Copy it only to machines you would trust with\nthe whole mesh. `cotal mint` on its own does not substitute here: registering an `auth` mesh needs\nsigning material that composes, which a minted user credential is not. The\nbroker is probed before the record is written, so a bad address or a credential that mesh will not\naccept fails at registration rather than at your first `spawn` (`--force` records it without verifying,\nuseful when the mesh is simply down right now).\n\n#### Which addresses you may register\n\nRegistering a mesh is how this machine starts sending agent credentials to a broker it does not\nrun. NATS announces itself in plaintext before anyone authenticates, so an attacker on the path\ncan pose as the broker and read the credential out of the connect unless the connection\n**requires TLS**, which is recorded on the entry and enforced on every dial through it.\n\nWhat the record will require decides what you may register:\n\n- **Without required TLS**, the address is the gate: **loopback** (`127.0.0.0/8`, `::1`), or\n **your private overlay** (`100.64.0.0/10`, `fd7a:115c:a1e0::/48`) with\n `--allow-unencrypted-overlay`. The tunnel provides the protection, and this command cannot check\n its state. Hostnames are refused because the lookup would choose which machine receives your\n credentials.\n- **With required TLS**, set `--tls` or use a `tls://` URL. The recorded scheme enforces the TLS\n requirement. A **hostname or public address** is accepted because the certificate chain and\n hostname check identify the peer. A registration whose broker cannot complete the handshake\n fails unless you pass `--force`, which records the entry without verification.\n\nOrdinary private ranges like `10.x` and `192.168.x` are refused in **both** modes. A café's wifi\nis private but does not belong to you, and no public CA issues certificates for those ranges. An\naddress spelling changes nothing: `[::ffff:192.168.1.10]`, `3232235786`, `0300.0250.01.012`, and\n`192.168.257` all resolve to private addresses and receive the same refusal as the dotted form.\n`--force` exists for a mesh that is down. It never permits an unsafe credential destination.\n\n#### Registering a hosted user-auth mesh\n\nA user-auth space's IdP pins are established where the mesh runs and are never guessed. Register\none from **supplied** trust: `--user-auth-file bundle.json` (exported on the mesh's machine), or\n`--from https://…/.well-known/cotal-mesh`, which asks before it contacts the address at all,\nfetches the discovery document over HTTPS, shows you the pins, and asks again before adopting\nthem. Redirects are refused because a 302 can walk a pinned fetch down to\nplaintext or onto another host, and the pinned exchange must be an `https://` URL too. The one\nexception is an exchange on **this machine**, where nothing leaves the box: plain `http://` is\naccepted for a loopback *literal* (`127.0.0.1`, `::1`, and any spelling of them), but **not** for\n`localhost`, which a hosts entry or poisoned lookup could point elsewhere. Use the\nliteral. Registration checks that the pinned exchange\nanswers `/health` and `/jwks` as the pinned issuer. It also checks that the broker refuses a\nbare connect; that refusal is the pass. The bundle's sentinel credentials are written to a private (0600) file\nunder the entry's root; the registry itself never carries the secret.\n\n**Without required TLS**, an overlay address is **refused unless you accept the dependency\nexplicitly**, with `--allow-unencrypted-overlay`. The address is not the guarantee: it is protected\nwhile the tunnel is up, and if the tunnel is down that range is ordinary carrier-grade NAT and\nwhoever answers the dial receives your credentials. Only you can know which it is, so the command\nasks you to say so. Your acceptance is recorded on the mesh entry rather than printed and\nforgotten, and the guided form asks the same question instead of taking the flag.\n\n**With required TLS** (`--tls`, or a `tls://` URL) that consent is no longer asked for, and the\nflag is not needed: the handshake is what protects the connection, so the acceptance it stood in\nfor has been replaced by proof rather than promise. `cotal meshes add --server\nnats://100.64.0.1 --tls` registers an overlay address with no prompt, no flag and no recorded\nacceptance. This is the \"the flag disappears once the broker can be served over TLS\" case, and it\nhas now arrived.\n\nThis gate is on **registration**. `cotal join --creds --server ` deliberately takes an\nexplicit connection at face value and does not consult the registry, so it is not covered. Join\nthat way only to an address you would have registered.\n\nRecords added this way are removed only by something that names them. A mesh this machine started\ncan be dropped on a hunch, such as a failed liveness probe or a `cotal down` in its project, because\n`cotal up` writes the record straight back. One you registered by hand cannot be reconstructed, so\nnothing removes it by inference: an unreachable broker is shown as `offline` in `cotal meshes`, and\n`cotal down` / `cotal clean all` leave it alone even when `--root` pointed at the project they are\ntearing down. A `cotal up` for that space refuses outright (naming `cotal meshes rm`) unless it is\nthat same endpoint: finding a broker already answering there is a refresh that starts nothing and\nleaves the record's provenance alone, while actually starting the broker for that space, server and\nroot makes this machine the one running it, so the record becomes an ordinary local one that\n`cotal down` clears. `cotal meshes rm` drops it and re-registering with `--force` replaces it. `rm`\nonly forgets a mesh. To stop one running here, use `cotal down`.\n\n## Watching\n\n`cotal console` is the terminal view (TUI on a real terminal, plain line stream when\npiped); `cotal web` is the browser dashboard. Both are read-only observers; the\nwalkthrough is [Watch a mesh](watch-a-mesh.md).\n\n## History\n\nRetained history is operator-owned. `cotal clean history --force` purges a space's\nretained channel history; `--dms` also purges DMs (`cotal history clear` is an alias).\nIt is deliberately **not** an agent tool: agents cannot wipe the record\n([identity & auth](identity-and-auth.md)). For a **stopped** mesh, `cotal clean store\n--force` deletes the on-disk JetStream store outright, and `cotal clean all --force`\nalso resets the space identity ([CLI reference](cli.md#clean)).\n\n## Offline backup\n\nFor a coherent durable cut, preserve the whole stack first, then create the artifact while it stays\ndown:\n\n```bash\ncotal down --preserve-state\ncotal backup create ./space-backup # full by default\n# later: deliberately resume the unchanged source\ncotal up --detach\n# or, from another preserved cut, restore before the normal listener opens\ncotal up --restore ./space-backup --detach\n```\n\nUse `--store-dir` on both preservation and backup for a custom JetStream store. `registry` is the\nonly partial selection (`backup create ... --only registry`; `up --restore ... --restore-only\nregistry`). Backup never stops or restarts a mesh implicitly, never opens the original store, and\ndoes not contain credentials or trust secrets. Backup/restore in every auth mode, open included,\nuses isolated, operation-specific maintenance logins; normal agent credentials cannot enter that\nlistener. Full\nrestore requires the same space and exact current local trust continuity, recreates conservative\nconsumer checkpoints bound to their snapshot stream sequence state, and resumes retained agents under\ntheir original principals. The trust commitment includes the cryptographically validated full\noperator/system/data-account root chain as well as static/user authority state. A registry-only\nrestore completes canonical empty infrastructure but leaves retained agents stopped because their\nDM/DLV/TASK/ACL state is outside that selection. Authenticated restore validates the complete space\ntrust bundle before staging or changing the preserved store. Interrupted ordinary resume retries the\nsame durable attempt after its prior listener is stopped. Restore re-entry can recover a surviving normal listener\nonly when its attempt nonce, NATS server name, process owner, endpoint, and target-store identity all\nmatch the fsynced proof. A provably dead uncommitted owner is retired under lock and replaced with a\nfresh attempt-bound listener; an occupied foreign listener or ambiguous owner is never adopted. The\nmanager commit validates while retained cleanup is still suppressed; the CLI durably records its\nattempt-bound 64-hex token in `manager-committed` / `resume-committed` before `finalizeResume` can\nrelease suppression. A retry from either committed state goes straight to exact-token finalization;\nfailure preserves the committed gate and retained cleanup suppression. Missing commit evidence,\ninterrupted finalization, a live recorded endpoint despite missing pidfiles, or ambiguous proof fails closed. See the [CLI\nbackup and restore contract](cli.md#backups) for artifact, checkpoint, fallback,\ndisaster-consent, and degraded-recovery details.\n\n## Personas from the CLI\n\n`cotal personas` manages the local catalog offline: `list` (`--running` overlays live\nmarkers), `show `, `edit ` (re-validates on save), `new `, `rm \n--force`. The runtime write is `cotal_persona`; the runtime read is `cotal_personas`\n(list / show), both over the wire with the manager's ownership checks. Fields: [agent files](agent-files.md).\n\n## Gate recovery\n\nA manager that dies mid-registration leaves its issuance gate *frozen* under that registration\nop. The freeze is correct: it stops two incarnations serving at once. The successor now completes\nthat dead op on boot, using the same guard as [`cotal reconcile-gate`](cli.md#reconcile-gate): it\nacts only when the freeze-holder is affirmatively gone under a complete CONNZ sweep (`gone` and\n`sweepComplete=true`). If the dead op's spec write committed, it finishes that same freeze\n(promote and reopen at the committed registration revision). If the spec did not advance, it\nabort-reopens the gate (generation+1, processEpoch unchanged) and continues the normal takeover.\nA live holder, an incomplete sweep, or an unreachable delivery daemon still\nrefuses. Silence is never evidence of death, and there is no TTL. If holder verification is\ninterrupted, the frozen operation resumes from its durable, operation-and-gate-revision-bound\nprogress after liveness is checked again. A later freeze cannot reuse that progress: the cursor\nbinds the exact op, gate revision, and holder set. Use `cotal reconcile-gate` when the boot path cannot run\n(daemon down, a non-manager endpoint, or you want to lift the freeze without starting a manager). A spawn that hits the same frozen gate names that verb in the refusal\n(`blockedOp=registration`, the holding `opId`, `remedy=cotal reconcile-gate`) instead of a\nwait-timeout: the facts were always in the manager log; they now reach the spawn caller too.\n\n## When something looks absent\n\nPermission denials are **loud, never silent**: an over-tight ACL rejects the endpoint call and\nalso shows up as a logged denial, instead of returning an empty or incomplete result that looks\nsuccessful. Check\n`.cotal/manager..log`, `.cotal/delivery..log` (one pair per space, keyed as\n[Config](config.md#project-files) describes), and `.cotal/nats.log`; `cotal status` shows\nwhat is actually running. The access rules are collected in\n[Channels & permissions](channels-and-permissions.md).\n" }, { "slug": "security", @@ -243,7 +243,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "Setup internals (maintainer notes)", "kind": "Project (non-normative maintainer notes)", "summary": "cotal setup (implementations/cli/src/commands/setup.ts) is configure-only: it checks prerequisites, installs the Claude Code plugin, and seeds persona files, and it launches nothing: no mesh, no we…", - "body": "# Setup internals (maintainer notes)\n\n> **Project** (non-normative maintainer notes) · **For:** maintainers changing how setup works\n>\n> How `cotal setup` works, and the cross-repo couplings it depends on. If you change one of\n> the things in the **Invariants** table, update the listed siblings in the same change, or\n> setup silently breaks for npx users.\n\n## The flow\n\n`cotal setup`\n([`implementations/cli/src/commands/setup.ts`](../implementations/cli/src/commands/setup.ts))\nis **configure-only**: it checks prerequisites, installs the Claude Code\nplugin, and seeds persona files, and it **launches nothing**: no mesh, no web dashboard, no\nmanager, no delivery daemon, no cmux/tmux session, no demo. Starting the stack is `cotal up`; the\ndashboard is `cotal web`. Every file it writes is announced (`→ wrote …` via `provenance.wrote`).\nIt is two-tier, gated on a machine marker. Persona seeding resolves the selected mesh root first,\nthen uses the same `.cotal/agents` catalog as spawn. With no mesh it names a cwd fallback; an\nambiguous or broken target refuses rather than choosing a root.\n\n**First run** (no `~/.cotal/onboarded.json`, or `--full`, or `--yes`) runs `runFirstRun(yes)`:\n\n- splash → intro → core **checks** (Node >= 22; **locate** `nats-server`: located, never\n started) → **connector picker** → resolve and announce the persona destination → seed the generic\n `default` and optional demo personas (david/sven/me) there → **offer a global install**\n (`offerGlobalInstall`) → onboarded marker → a finale that\n lists the commands to start things (`cotal up --detach`, `cotal web`, `cotal spawn …`,\n `cotal console`, `cotal down`). Nothing is running when it returns.\n- The old `--auth` / `--open` flags are **gone**: they set the mesh MODE at launch time, and setup\n no longer launches; mode is now `cotal up [--open]`'s concern (an unknown-option error names\n them, no silent no-op).\n\n**Later runs** run `runEnsure`: resolve and announce the same destination, re-seed the `default`\npersona if it's missing,\nre-offer the **global install** (`offerGlobalInstall`, same `isNpx()` + PATH-scan gate as first\nrun, so a repeat `npx cotal-ai setup` on a machine that still lacks a durable `cotal` finally\ninstalls it), then print the **status card** (`readyCard`). The card is **read-only probes** (`machineStatus`/`meshStatus`/`webUp`/`managerUp` for NATS, the plugin, the mesh, the web\ndashboard, and the manager) and for anything down it prints the exact command to start it\n(`cotal up --detach`, `cotal web`, `cotal supervise`). Displaying state never depends on it; setup\nstill launches nothing.\n\n**`--skills`** is the status-card write: it asks installed connectors with a declared skills setup\nhook to reconcile their own harness, then reconciles `~/.agents/skills`. The base CLI passes only the\nvendor-neutral skills directory, version, and state directory; connector packages own native assets\nand commands. It does not seed personas, install the mesh\nconnector, offer a global install, or write the onboarded stamp. Combined with `--full` or\n`--demo` it is refused.\n\nThe seeded `default` persona has an empty active `subscribe` set and wildcard\n`allowSubscribe`/`allowPublish` ACLs. A fresh agent receives no channel traffic until it joins a\nchannel, but can join, create, read, and post to channels on demand. The guided demo personas keep\ntheir existing `welcome` read and post scope. Repeat setup replaces the prior default template only\nwhen its bytes still match the shipped legacy body with `allowPublish: []`; any user edit makes the\nfile ineligible and leaves it byte-identical.\n\nSteps run in-process via `runSteps`\n([`lib/steps.ts`](../implementations/cli/src/lib/steps.ts)). A step can be `optional` (asked\nY/n), carry a `confirm` consent prompt, or be `live` (it draws its own pane via\n[`lib/live-window.ts`](../implementations/cli/src/lib/live-window.ts)). On failure, an\ninteractive run offers a Claude handoff\n([`lib/assist.ts`](../implementations/cli/src/lib/assist.ts)).\n\nThe **connector picker** (`pickConnectors`) multiselects the **setup connector surface**\n(`setupConnectorSurface`): every connector name the live registry or the installed extension\nmanifest advertises, materialized through the same loader the rest of the CLI uses. No connector\nname is written into `setup.ts`. `setupConnectorCandidates` turns that surface into choices and\nreads each hint off the connector's own declarations: `requires` names the executables a candidate\nstill needs on PATH, `setup` says whether it owns setup actions at all, and `pluginRoot` says\nwhether those actions install plugin assets. A selected candidate runs its connector-owned\n`connector` action through `connectorSetupStep`, which receives the `Connector` itself; a candidate\nthat declares no provider is simply marked ready (OpenCode auto-wires at spawn, injecting its\nplugin via `buildLaunch` and never writing the user's config). The `skills` action runs for every\npresent connector that declares one, selected or not, because Cotal's authored skills are\nindependent of mesh membership. Two experts (david, the engineer; sven, the guide) plus the\noperator's own driving session (`me`) are written by default, and `me` is the persona\n`cotal spawn me` drives.\n\n**`--yes`** forces non-interactive accept-all even on a TTY: optional plus `confirm` steps run\n(so the demo personas are written), the global install takes its default, and a failure aborts\nwith the log path and a non-zero exit. It still launches nothing. The control plane comes up with\n`cotal up --detach`. This is the agent/CI contract; keep it working.\n\n## Invariants\n\n| Thing | Must stay in sync across | Why |\n|---|---|---|\n| Marketplace name **`cotal-mesh`** | `setup.ts` (materialized `marketplace.json`), `CHANNEL_REF` in [`extensions/connector-claude-code/src/extension.ts`](../extensions/connector-claude-code/src/extension.ts), repo [`.claude-plugin/marketplace.json`](../.claude-plugin/marketplace.json) | The wake channel ref `plugin:cotal@cotal-mesh` binds by this name |\n| Plugin assets | the claude connector's own [`src/setup.ts`](../extensions/connector-claude-code/src/setup.ts) copy list (`dist/mcp.cjs`, `dist/hook.cjs`, `.claude-plugin/plugin.json`, `.mcp.json`, `hooks/hooks.json`) and its `package.json` `files` field | The connector materializes its own plugin; missing or renamed assets break the install, and the base CLI has no copy of the list |\n| `Connector.pluginRoot` | [`packages/core/src/connector.ts`](../packages/core/src/connector.ts) (contract) plus set in the claude connector's `extension.ts` | A connector's declaration that it ships installable plugin assets; the picker phrases its hint from it |\n| `BUNDLED_PKG_PREFIX` | [`lib/nats-bin.ts`](../implementations/cli/src/lib/nats-bin.ts) ↔ the `@eplightning/nats-server-*` `optionalDependencies` in [`implementations/cli/package.json`](../implementations/cli/package.json) | The bundled NATS binary is resolved by `${prefix}-${platform}-${arch}`. (Future: swap the prefix to our own `@cotal-ai/nats-server-*`.) |\n| Onboard marker plus `ONBOARD_VERSION` | `~/.cotal/onboarded.json` in [`lib/onboard.ts`](../implementations/cli/src/lib/onboard.ts); version const in `setup.ts` | Flips first-run vs ensure |\n| Demo-agent format | `DEMO_AGENTS` in `setup.ts` matches the frontmatter shape read by [`packages/core/src/agent-file.ts`](../packages/core/src/agent-file.ts) (same as `examples/01-lateral-coordination/agents/`) | `cotal spawn ` loads these |\n| Managed personas | each `DEMO_AGENTS` body carries a `# managed by cotal-setup` frontmatter marker; `writeDemoAgent` refreshes the file when the body changes, backing a marker-less (user-edited) file up to `.md.bak` first | Edit `DEMO_AGENTS` plus re-run setup to update david/sven/me; delete the marker line to take ownership |\n| `DEFAULT_SERVER` | [`packages/core/src/endpoint.ts`](../packages/core/src/endpoint.ts) | The address `cotal up` starts and the status card probes |\n\n## Background processes (`cotal up`)\n\n`cotal up` brings up the whole local stack in one place; since setup became configure-only\n(stage 2b), this is where the mesh and control plane start, so `cotal spawn --detach` /\n`cotal_spawn` find a manager right after `up`. The control plane comes up in cutover order:\nold-manager preflight → **delivery daemon** (auth mode only) → **manager**, via\n`ensureControlPlane`\n([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)). The detached\nprocesses, all stopped by `cotal down`:\n\nWith no explicit `--server`, `cotal up` auto-selects a free local port when the default broker\naddress is already held by another root or an unrecorded broker; an explicit `--server` remains\nfail-loud on collision.\n\n- **Mesh:** `startMeshDetached`\n ([`commands/up.ts`](../implementations/cli/src/commands/up.ts)) is the one place that boots a\n background nats-server (foreground `up` and `up --detach` both route through it). Writes\n `.cotal/nats.pid` and tails `.cotal/nats.log`.\n- **Delivery daemon:** `startDeliveryDetached` / `ensureDelivery`\n ([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)) re-execs `cotal\n deliver` detached with a pre-minted scoped `delivery.creds` (auth mode only, the durable\n backstop; open mode has none). Writes `.cotal/delivery..pid` and `.cotal/delivery..log`,\n where `` is the space key ([Config](config.md#project-files)), so a root can serve a space per\n daemon.\n- **Manager:** `startManagerDetached` / `ensureManager`\n ([`lib/manager-proc.ts`](../implementations/cli/src/lib/manager-proc.ts)) re-execs `cotal\n supervise` detached (pty runtime); it answers the control plane\n (`cotal_spawn` / `cotal_despawn` / `cotal_persona`). Writes `.cotal/manager..log`;\n `managerUp(space)` checks that space's pid record for setup's status card. The **manager itself**\n writes `.cotal/manager..pid`, so a supervisor started by a container entrypoint, by cron, or\n by hand is recorded the same way a detached `cotal up` is. Readers verify the recorded pid is alive and is a\n supervisor before trusting it ([Config](config.md#project-files)).\n\nThe **web dashboard** is *not* part of `cotal up`. It ships inside `cotal-ai` as the `@cotal-ai/web`\nextension and is seeded automatically by the boot reconcile, the same durable, version-locked path as\nthe built-in connectors (`SEEDED_EXTENSIONS`), so it always matches the CLI version and needs no\nseparate install. Start it with `cotal web`; it records\n`.cotal/web.pid`, self-registers that process with `down`, and is addressed as\n`http://cotal.localhost:7799` (binds loopback; `*.localhost` resolves in Chrome/Firefox/Edge,\nSafari may need plain `127.0.0.1`). `webUp()` probes the port for setup's status card.\n\nAll recorded local processes self-register `local-process` descriptors. Bare `cotal down` resolves\nthe full set and stops it in dependency order; `cotal down manager` (or another component name)\nselects only that descriptor. Installed extensions cache their contributed registry keys, so the\nbase CLI does not hardcode optional package pidfiles.\n\nEach recorded pidfile also carries a sibling `.identity` pin (pid plus the process's\nstart, where the OS reports one). The pin proves that the live process is still the recorded one.\nA reused pid and a torn or unreadable pin are refused and preserved. A pre-pin record warns and is\nsignalled so an upgraded CLI can stop a stack launched by the previous version; the next launch\nwrites a pin. A record clears only once death is confirmed.\n\nAll re-execs resolve this CLI via `selfArgv()` / `selfCotal()`\n([`lib/self-exec.ts`](../implementations/cli/src/lib/self-exec.ts)) = `[node, ...loaderFlags,\nentry]` (tsx loader in dev, compiled JS in prod), so they never need `cotal` on PATH; the stack\ncomes up identically via `npx`, `npm i -g`, and a dev clone.\n\nFor ergonomics only, an npx run with no global `cotal` offers to `npm i -g cotal-ai`\n(`offerGlobalInstall`, pinned to the running version): gated on `isNpx()` plus a PATH scan\n(`cotalOnPath()`, not `onPath(\"cotal\")`, since `cotal --version` is not a real command). The\ninteractive prompt defaults to yes, the non-interactive path (`--yes` or no TTY) takes the\ndefault, and a failed install is non-fatal (warn plus manual command). The same `self-exec.ts`\nexposes `displayCmd()`, the prefix (`cotal` / `npx cotal-ai` / `pnpm cotal`) used in the\nstatus-card hints so they match how you ran it.\n\n## Built-in connectors are seeded extensions\n\nThe first-party connectors (`claude`, `opencode`, `codex`, `hermes`, `pi`) are **not** static-imported by\nthe binary. The composition root (`bin/cotal.ts`) registers no connector; they self-register only when\nimported, and they are imported only once installed. On the first real command of each boot the CLI\n**seeds** them through the same `cotal ext add` path a third party uses, so they are ordinary\nextensions you can `cotal ext remove`. Code lives in [`implementations/cli/src/seed/`](../implementations/cli/src/seed/);\nthe entry is `reconcileSeededConnectors()`, gated in `runCli` before the manifest overlay so\n`ext seed --repair` survives a corrupt manifest.\n\n**What ships where.** The connectors are `devDependencies` of `cotal-ai` (not runtime deps), and a\n`prepack` step ([`bin/scripts/copy-seeded-connectors.mjs`](../bin/scripts/copy-seeded-connectors.mjs))\n`npm pack`s each into `bin/seeded-connectors//` (honoring each connector's own `files`), added to\nthe package `files`. `SEEDED_EXTENSIONS` (`@cotal-ai/workspace`) is the shared list: the\nconnectors plus `web`. The prepack asserts that every bundled payload's `name` and `version` match\nthe umbrella (the\n`fixed` changeset group keeps them lockstep), so a version-skewed payload can never be published; `web`\nalso emits `dist/web/vendor/vendor-manifest.json` (name/version/license/sha512) as the auditable\ninventory of its vendored browser libs (marked/DOMPurify ship as opaque `dist` bytes, not runtime deps).\n`seed/paths.ts:shippedSourceDir` resolves the live `extensions/` dir in a\nsource checkout and `/seeded-connectors/` in a published install. The reconcile copies\nthat payload into the durable store `seed/store//` and `ext add --install-links` reifies\nthe `file:` dep from THAT stable path (a volatile source would fail to re-reify); `ext add` then\njunction-links each `@cotal-ai/*` peer to the binary's own copy. Before the first lazy import in each\nprocess, materialization rechecks those links by realpath and rebinds stale links under the extension\nlock. This lets the registry-facing imports of a global install, npx, and source worktrees share the\nmachine prefix while each process still gets its host's single `@cotal-ai/core` registry instance;\nlauncher artifacts are self-contained and do not resolve those mutable links later.\n\n**Reconcile policy** (generation = the `cotal-ai` version): a never-seeded built-in is seeded; a\nstill-installed one WE seeded (`source: \"seeded\"`) is refreshed only when the version bumps (semver\ncompare) or under `--force`; an operator-managed official entry (a manual `ext add` at a chosen\nversion, no seeded marker) is left untouched on upgrade; a deliberately-removed one stays removed. The\n`ever-seeded` **authority** (`seed/authority.json`, mirrored to a monotonic `.bak`) is the sole arbiter\nof removed-vs-never-seeded and is unioned with its backup on read, so a truncated authority never\nresurrects a removal. Before writing the generation stamp, setup verifies that every\n(re)installed extension is recorded in the manifest, present on disk with a resolvable entry file,\nand at the generation version. A version-skewed payload fails loud (`ext seed --repair`) rather than being stamped as current. A cotal\n**older** than the store's stamped generation refuses before writing anything, rather than stamping the\nstore back down to its own version while refreshing nothing: run the newer cotal, or `ext seed --force`\nto rebuild the store for the version you are running. `--reset` is not that recovery: it discards the\never-seeded authority and resurrects deliberately-removed connectors. The refusal names a concrete cotal executable\nonly after a bounded `--version` probe proves that executable is at least the store generation;\notherwise it retains the generic instruction. A generation advance records the exact\nrealpath-resolved CLI entry and an ISO timestamp in `seed/stamp.json`, then announces the migration\nafter that stamp commits. An older CLI includes those fields in its refusal when present; legacy\ngeneration-only stamps stay valid and retain the shorter refusal. A CLI whose package root is the\nrepo `bin/` (a source checkout, including a suite child of `bin/cotal.ts`) refuses that write, stamp,\nand generation GC rather than migrating the operator-global store. The refusal names\n`$XDG_CONFIG_HOME` as the isolation remedy; `COTAL_HOME` does not relocate this store. Isolated\nin-tree seed smokes set `COTAL_ALLOW_CHECKOUT_SEED=1` after pointing `$XDG_CONFIG_HOME` at a scratch\ndir. An unproven entry is refused the same way: a missing identity answer is not treated as a\nreleased install.\n\n**Crash safety.** One shared advisory lock ([`packages/workspace/src/advisory-lock.ts`](../packages/workspace/src/advisory-lock.ts):\natomic hard-link publish, PID + process-start liveness, bounded wait, dead-owner reclaim) guards the\nwhole reconcile and every `cotal ext` mutation; a live reconcile is waited on, not mistaken for a crash.\nA crash **cursor** is journaled before each connector mutation and cleared only at the final commit, so\na SIGKILL mid-run is detected on the next boot (fail loud → `ext seed --repair` re-installs the\ninterrupted connector before it clears the evidence). Seed children are authenticated (they carry the\nlive lock's nonce + parent PID, not a bare env flag) and record a liveness marker so a post-crash repair\nrefuses to race an orphaned installer. `ext seed --reset` quarantines corrupt manifest/authority state\naside and rebuilds. See [cli.md `ext`](cli.md#ext) for the operator-facing flags.\n" + "body": "# Setup internals (maintainer notes)\n\n> **Project** (non-normative maintainer notes) · **For:** maintainers changing how setup works\n>\n> How `cotal setup` works, and the cross-repo couplings it depends on. If you change one of\n> the things in the **Invariants** table, update the listed siblings in the same change, or\n> setup silently breaks for npx users.\n\n## The flow\n\n`cotal setup`\n([`implementations/cli/src/commands/setup.ts`](../implementations/cli/src/commands/setup.ts))\nis **configure-only**: it checks prerequisites, installs the Claude Code\nplugin, and seeds persona files, and it **launches nothing**: no mesh, no web dashboard, no\nmanager, no delivery daemon, no cmux/tmux session, no demo. Starting the stack is `cotal up`; the\ndashboard is `cotal web`. Every file it writes is announced (`→ wrote …` via `provenance.wrote`).\nIt is two-tier, gated on a machine marker. Persona seeding resolves the selected mesh root first,\nthen uses the same `.cotal/agents` catalog as spawn. With no mesh it names a cwd fallback; an\nambiguous or broken target refuses rather than choosing a root.\n\n**First run** (no `~/.cotal/onboarded.json`, or `--full`, or `--yes`) runs `runFirstRun(yes)`:\n\n- splash → intro → core **checks** (Node >= 22; **locate** `nats-server`: located, never\n started) → **connector picker** → resolve and announce the persona destination → seed the generic\n `default` and optional demo personas (david/sven/me) there → **offer a global install**\n (`offerGlobalInstall`) → onboarded marker → a finale that\n lists the commands to start things (`cotal up --detach`, `cotal web`, `cotal spawn …`,\n `cotal console`, `cotal down`). Nothing is running when it returns.\n- The old `--auth` / `--open` flags are **gone**: they set the mesh MODE at launch time, and setup\n no longer launches; mode is now `cotal up [--open]`'s concern (an unknown-option error names\n them, no silent no-op).\n\n**Later runs** run `runEnsure`: resolve and announce the same destination, re-seed the `default`\npersona if it's missing,\nre-offer the **global install** (`offerGlobalInstall`, same `isNpx()` + PATH-scan gate as first\nrun, so a repeat `npx cotal-ai setup` on a machine that still lacks a durable `cotal` finally\ninstalls it), then print the **status card** (`readyCard`). The card is **read-only probes** (`machineStatus`/`meshStatus`/`webUp`/`managerUp` for NATS, the plugin, the mesh, the web\ndashboard, and the manager) and for anything down it prints the exact command to start it\n(`cotal up --detach`, `cotal web`, `cotal supervise`). Displaying state never depends on it; setup\nstill launches nothing.\n\n**`--skills`** is the status-card write: it asks installed connectors with a declared skills setup\nhook to reconcile their own harness, then reconciles `~/.agents/skills`. The base CLI passes only the\nvendor-neutral skills directory, version, and state directory; connector packages own native assets\nand commands. It does not seed personas, install the mesh\nconnector, offer a global install, or write the onboarded stamp. Combined with `--full` or\n`--demo` it is refused.\n\nThe seeded `default` persona has an empty active `subscribe` set and wildcard\n`allowSubscribe`/`allowPublish` ACLs. A fresh agent receives no channel traffic until it joins a\nchannel, but can join, create, read, and post to channels on demand. The guided demo personas keep\ntheir existing `welcome` read and post scope. Repeat setup replaces the prior default template only\nwhen its bytes still match the shipped legacy body with `allowPublish: []`; any user edit makes the\nfile ineligible and leaves it byte-identical.\n\nSteps run in-process via `runSteps`\n([`lib/steps.ts`](../implementations/cli/src/lib/steps.ts)). A step can be `optional` (asked\nY/n), carry a `confirm` consent prompt, or be `live` (it draws its own pane via\n[`lib/live-window.ts`](../implementations/cli/src/lib/live-window.ts)). On failure, an\ninteractive run offers a Claude handoff\n([`lib/assist.ts`](../implementations/cli/src/lib/assist.ts)).\n\nThe **connector picker** (`pickConnectors`) multiselects the **setup connector surface**\n(`setupConnectorSurface`): every connector name the live registry or the installed extension\nmanifest advertises, materialized through the same loader the rest of the CLI uses. No connector\nname is written into `setup.ts`. `setupConnectorCandidates` turns that surface into choices and\nreads each hint off the connector's own declarations: `requires` names the executables a candidate\nstill needs on PATH, `setup` says whether it owns setup actions at all, and `pluginRoot` says\nwhether those actions install plugin assets. A selected candidate runs its connector-owned\n`connector` action through `connectorSetupStep`, which receives the `Connector` itself; a candidate\nthat declares no provider is simply marked ready (OpenCode auto-wires at spawn, injecting its\nplugin via `buildLaunch` and never writing the user's config). The `skills` action runs for every\npresent connector that declares one, selected or not, because Cotal's authored skills are\nindependent of mesh membership. Two experts (david, the engineer; sven, the guide) plus the\noperator's own driving session (`me`) are written by default, and `me` is the persona\n`cotal spawn me` drives.\n\n**`--yes`** forces non-interactive accept-all even on a TTY: optional plus `confirm` steps run\n(so the demo personas are written), the global install takes its default, and a failure aborts\nwith the log path and a non-zero exit. It still launches nothing. The control plane comes up with\n`cotal up --detach`. This is the agent/CI contract; keep it working.\n\n## Invariants\n\n| Thing | Must stay in sync across | Why |\n|---|---|---|\n| Marketplace name **`cotal-mesh`** | `setup.ts` (materialized `marketplace.json`), `CHANNEL_REF` in [`extensions/connector-claude-code/src/extension.ts`](../extensions/connector-claude-code/src/extension.ts), repo [`.claude-plugin/marketplace.json`](../.claude-plugin/marketplace.json) | The wake channel ref `plugin:cotal@cotal-mesh` binds by this name |\n| Plugin assets | the claude connector's own [`src/setup.ts`](../extensions/connector-claude-code/src/setup.ts) copy list (`dist/mcp.cjs`, `dist/hook.cjs`, `.claude-plugin/plugin.json`, `.mcp.json`, `hooks/hooks.json`) and its `package.json` `files` field | The connector materializes its own plugin; missing or renamed assets break the install, and the base CLI has no copy of the list |\n| `Connector.pluginRoot` | [`packages/core/src/connector.ts`](../packages/core/src/connector.ts) (contract) plus set in the claude connector's `extension.ts` | A connector's declaration that it ships installable plugin assets; the picker phrases its hint from it |\n| `BUNDLED_PKG_PREFIX` | [`lib/nats-bin.ts`](../implementations/cli/src/lib/nats-bin.ts) ↔ the `@eplightning/nats-server-*` `optionalDependencies` in [`implementations/cli/package.json`](../implementations/cli/package.json) | The bundled NATS binary is resolved by `${prefix}-${platform}-${arch}`. (Future: swap the prefix to our own `@cotal-ai/nats-server-*`.) |\n| Onboard marker plus `ONBOARD_VERSION` | `~/.cotal/onboarded.json` in [`lib/onboard.ts`](../implementations/cli/src/lib/onboard.ts); version const in `setup.ts` | Flips first-run vs ensure |\n| Demo-agent format | `DEMO_AGENTS` in `setup.ts` matches the frontmatter shape read by [`packages/core/src/agent-file.ts`](../packages/core/src/agent-file.ts) (same as `examples/01-lateral-coordination/agents/`) | `cotal spawn ` loads these |\n| Managed personas | each `DEMO_AGENTS` body carries a `# managed by cotal-setup` frontmatter marker; `writeDemoAgent` refreshes the file when the body changes, backing a marker-less (user-edited) file up to `.md.bak` first | Edit `DEMO_AGENTS` plus re-run setup to update david/sven/me; delete the marker line to take ownership |\n| `DEFAULT_SERVER` | [`packages/core/src/endpoint.ts`](../packages/core/src/endpoint.ts) | The address `cotal up` starts and the status card probes |\n\n## Background processes (`cotal up`)\n\n`cotal up` brings up the whole local stack in one place; since setup became configure-only\n(stage 2b), this is where the mesh and control plane start, so `cotal spawn --detach` /\n`cotal_spawn` find a manager right after `up`. The control plane comes up in cutover order:\nold-manager preflight → **delivery daemon** (auth mode only) → **manager**, via\n`ensureControlPlane`\n([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)). The detached\nprocesses, all stopped by `cotal down` (managed agents stay running unless\n`cotal down --with-agents`). A PTY child may still die when the manager process\nexits. A later manager on the same root may still take leftover seats.\n\nWith no explicit `--server`, `cotal up` auto-selects a free local port when the default broker\naddress is already held by another root or an unrecorded broker; an explicit `--server` remains\nfail-loud on collision.\n\n- **Mesh:** `startMeshDetached`\n ([`commands/up.ts`](../implementations/cli/src/commands/up.ts)) is the one place that boots a\n background nats-server (foreground `up` and `up --detach` both route through it). Writes\n `.cotal/nats.pid` and tails `.cotal/nats.log`.\n- **Delivery daemon:** `startDeliveryDetached` / `ensureDelivery`\n ([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)) re-execs `cotal\n deliver` detached with a pre-minted scoped `delivery.creds` (auth mode only, the durable\n backstop; open mode has none). Writes `.cotal/delivery..pid` and `.cotal/delivery..log`,\n where `` is the space key ([Config](config.md#project-files)), so a root can serve a space per\n daemon.\n- **Manager:** `startManagerDetached` / `ensureManager`\n ([`lib/manager-proc.ts`](../implementations/cli/src/lib/manager-proc.ts)) re-execs `cotal\n supervise` detached (pty runtime); it answers the control plane\n (`cotal_spawn` / `cotal_despawn` / `cotal_persona`). Writes `.cotal/manager..log`;\n `managerUp(space)` checks that space's pid record for setup's status card. The **manager itself**\n writes `.cotal/manager..pid`, so a supervisor started by a container entrypoint, by cron, or\n by hand is recorded the same way a detached `cotal up` is. Readers verify the recorded pid is alive and is a\n supervisor before trusting it ([Config](config.md#project-files)).\n\nThe **web dashboard** is *not* part of `cotal up`. It ships inside `cotal-ai` as the `@cotal-ai/web`\nextension and is seeded automatically by the boot reconcile, the same durable, version-locked path as\nthe built-in connectors (`SEEDED_EXTENSIONS`), so it always matches the CLI version and needs no\nseparate install. Start it with `cotal web`; it records\n`.cotal/web.pid`, self-registers that process with `down`, and is addressed as\n`http://cotal.localhost:7799` (binds loopback; `*.localhost` resolves in Chrome/Firefox/Edge,\nSafari may need plain `127.0.0.1`). `webUp()` probes the port for setup's status card.\n\nAll recorded local processes self-register `local-process` descriptors. Bare `cotal down` resolves\nthe full set and stops it in dependency order, leaving managed agents running; `cotal down\n--with-agents` also stops those seats. `cotal down manager` (or another component name)\nselects only that descriptor. Installed extensions cache their contributed registry keys, so the\nbase CLI does not hardcode optional package pidfiles.\n\nEach recorded pidfile also carries a sibling `.identity` pin (pid plus the process's\nstart, where the OS reports one). The pin proves that the live process is still the recorded one.\nA reused pid and a torn or unreadable pin are refused and preserved. A pre-pin record warns and is\nsignalled so an upgraded CLI can stop a stack launched by the previous version; the next launch\nwrites a pin. A record clears only once death is confirmed.\n\nAll re-execs resolve this CLI via `selfArgv()` / `selfCotal()`\n([`lib/self-exec.ts`](../implementations/cli/src/lib/self-exec.ts)) = `[node, ...loaderFlags,\nentry]` (tsx loader in dev, compiled JS in prod), so they never need `cotal` on PATH; the stack\ncomes up identically via `npx`, `npm i -g`, and a dev clone.\n\nFor ergonomics only, an npx run with no global `cotal` offers to `npm i -g cotal-ai`\n(`offerGlobalInstall`, pinned to the running version): gated on `isNpx()` plus a PATH scan\n(`cotalOnPath()`, not `onPath(\"cotal\")`, since `cotal --version` is not a real command). The\ninteractive prompt defaults to yes, the non-interactive path (`--yes` or no TTY) takes the\ndefault, and a failed install is non-fatal (warn plus manual command). The same `self-exec.ts`\nexposes `displayCmd()`, the prefix (`cotal` / `npx cotal-ai` / `pnpm cotal`) used in the\nstatus-card hints so they match how you ran it.\n\n## Built-in connectors are seeded extensions\n\nThe first-party connectors (`claude`, `opencode`, `codex`, `hermes`, `pi`) are **not** static-imported by\nthe binary. The composition root (`bin/cotal.ts`) registers no connector; they self-register only when\nimported, and they are imported only once installed. On the first real command of each boot the CLI\n**seeds** them through the same `cotal ext add` path a third party uses, so they are ordinary\nextensions you can `cotal ext remove`. Code lives in [`implementations/cli/src/seed/`](../implementations/cli/src/seed/);\nthe entry is `reconcileSeededConnectors()`, gated in `runCli` before the manifest overlay so\n`ext seed --repair` survives a corrupt manifest.\n\n**What ships where.** The connectors are `devDependencies` of `cotal-ai` (not runtime deps), and a\n`prepack` step ([`bin/scripts/copy-seeded-connectors.mjs`](../bin/scripts/copy-seeded-connectors.mjs))\n`npm pack`s each into `bin/seeded-connectors//` (honoring each connector's own `files`), added to\nthe package `files`. `SEEDED_EXTENSIONS` (`@cotal-ai/workspace`) is the shared list: the\nconnectors plus `web`. The prepack asserts that every bundled payload's `name` and `version` match\nthe umbrella (the\n`fixed` changeset group keeps them lockstep), so a version-skewed payload can never be published; `web`\nalso emits `dist/web/vendor/vendor-manifest.json` (name/version/license/sha512) as the auditable\ninventory of its vendored browser libs (marked/DOMPurify ship as opaque `dist` bytes, not runtime deps).\n`seed/paths.ts:shippedSourceDir` resolves the live `extensions/` dir in a\nsource checkout and `/seeded-connectors/` in a published install. The reconcile copies\nthat payload into the durable store `seed/store//` and `ext add --install-links` reifies\nthe `file:` dep from THAT stable path (a volatile source would fail to re-reify); `ext add` then\njunction-links each `@cotal-ai/*` peer to the binary's own copy. Before the first lazy import in each\nprocess, materialization rechecks those links by realpath and rebinds stale links under the extension\nlock. This lets the registry-facing imports of a global install, npx, and source worktrees share the\nmachine prefix while each process still gets its host's single `@cotal-ai/core` registry instance;\nlauncher artifacts are self-contained and do not resolve those mutable links later.\n\n**Reconcile policy** (generation = the `cotal-ai` version): a never-seeded built-in is seeded; a\nstill-installed one WE seeded (`source: \"seeded\"`) is refreshed only when the version bumps (semver\ncompare) or under `--force`; an operator-managed official entry (a manual `ext add` at a chosen\nversion, no seeded marker) is left untouched on upgrade; a deliberately-removed one stays removed. The\n`ever-seeded` **authority** (`seed/authority.json`, mirrored to a monotonic `.bak`) is the sole arbiter\nof removed-vs-never-seeded and is unioned with its backup on read, so a truncated authority never\nresurrects a removal. Before writing the generation stamp, setup verifies that every\n(re)installed extension is recorded in the manifest, present on disk with a resolvable entry file,\nand at the generation version. A version-skewed payload fails loud (`ext seed --repair`) rather than being stamped as current. A cotal\n**older** than the store's stamped generation refuses before writing anything, rather than stamping the\nstore back down to its own version while refreshing nothing: run the newer cotal, or `ext seed --force`\nto rebuild the store for the version you are running. `--reset` is not that recovery: it discards the\never-seeded authority and resurrects deliberately-removed connectors. The refusal names a concrete cotal executable\nonly after a bounded `--version` probe proves that executable is at least the store generation;\notherwise it retains the generic instruction. A generation advance records the exact\nrealpath-resolved CLI entry and an ISO timestamp in `seed/stamp.json`, then announces the migration\nafter that stamp commits. An older CLI includes those fields in its refusal when present; legacy\ngeneration-only stamps stay valid and retain the shorter refusal. A CLI whose package root is the\nrepo `bin/` (a source checkout, including a suite child of `bin/cotal.ts`) refuses that write, stamp,\nand generation GC rather than migrating the operator-global store. The refusal names\n`$XDG_CONFIG_HOME` as the isolation remedy; `COTAL_HOME` does not relocate this store. Isolated\nin-tree seed smokes set `COTAL_ALLOW_CHECKOUT_SEED=1` after pointing `$XDG_CONFIG_HOME` at a scratch\ndir. An unproven entry is refused the same way: a missing identity answer is not treated as a\nreleased install.\n\n**Crash safety.** One shared advisory lock ([`packages/workspace/src/advisory-lock.ts`](../packages/workspace/src/advisory-lock.ts):\natomic hard-link publish, PID + process-start liveness, bounded wait, dead-owner reclaim) guards the\nwhole reconcile and every `cotal ext` mutation; a live reconcile is waited on, not mistaken for a crash.\nA crash **cursor** is journaled before each connector mutation and cleared only at the final commit, so\na SIGKILL mid-run is detected on the next boot (fail loud → `ext seed --repair` re-installs the\ninterrupted connector before it clears the evidence). Seed children are authenticated (they carry the\nlive lock's nonce + parent PID, not a bare env flag) and record a liveness marker so a post-crash repair\nrefuses to race an orphaned installer. `ext seed --reset` quarantines corrupt manifest/authority state\naside and rebuilds. See [cli.md `ext`](cli.md#ext) for the operator-facing flags.\n" }, { "slug": "spaces", diff --git a/implementations/auth/smoke/freeslot-respawn-barrier.smoke.ts b/implementations/auth/smoke/freeslot-respawn-barrier.smoke.ts index d950aa1cd..f2ac901b5 100644 --- a/implementations/auth/smoke/freeslot-respawn-barrier.smoke.ts +++ b/implementations/auth/smoke/freeslot-respawn-barrier.smoke.ts @@ -656,7 +656,7 @@ try { console.error(" ✗ scenario threw:", (e as Error).stack ?? (e as Error).message); process.exitCode = 1; } finally { - try { await manager?.stop(); } catch { /* already stopped */ } + try { await manager?.stop({ withAgents: true }); } catch { /* already stopped */ } try { await delivery?.stop(); } catch { /* already stopped */ } if (authChild?.pid) { try { process.kill(authChild.pid, "SIGKILL"); } catch { /* gone */ } } if (broker?.pid) { try { process.kill(broker.pid, "SIGKILL"); } catch { /* gone */ } } diff --git a/implementations/auth/smoke/int2-revoke-hold.smoke.ts b/implementations/auth/smoke/int2-revoke-hold.smoke.ts index e929e035d..ee52a435b 100644 --- a/implementations/auth/smoke/int2-revoke-hold.smoke.ts +++ b/implementations/auth/smoke/int2-revoke-hold.smoke.ts @@ -574,7 +574,7 @@ try { // the fault arms above, and the `rmSync` below could not clear it. Imported at the top now, with // the same best-effort intent. try { chmodSync(managedActorLedgerDir(dir), 0o700); } catch { /* best-effort restore before rm */ } - try { await manager?.stop(); } catch { /* already stopped */ } + try { await manager?.stop({ withAgents: true }); } catch { /* already stopped */ } try { await delivery?.stop(); } catch { /* already stopped */ } if (authChild?.pid) { try { process.kill(authChild.pid, "SIGKILL"); } catch { /* gone */ } } if (broker?.pid) { try { process.kill(broker.pid, "SIGKILL"); } catch { /* gone */ } } diff --git a/implementations/auth/smoke/user-spawn.smoke.ts b/implementations/auth/smoke/user-spawn.smoke.ts index 1327e3957..8e6c4bada 100644 --- a/implementations/auth/smoke/user-spawn.smoke.ts +++ b/implementations/auth/smoke/user-spawn.smoke.ts @@ -1505,7 +1505,7 @@ try { // ---------- F. revocation ---------- console.log("F) manager teardown revokes the managed row + shreds files; the old token is uniformly denied"); const alphaFamily = incFiles("alpha"); // resolve the incarnation paths BEFORE teardown shreds them - await manager.stop(); // teardown deprovisions alpha: user-mode revoke (row delete) + token/sentinel/health shred + await manager.stop({ withAgents: true }); // teardown deprovisions alpha: user-mode revoke (row delete) + token/sentinel/health shred managerStopped = true; const rowGone = !existsSync(managedRowPath); const filesGone = [alphaFamily.actorToken, alphaFamily.sentinelCreds, alphaFamily.health].every((f) => !existsSync(f)) && noIncFiles("alpha"); @@ -1529,7 +1529,7 @@ try { try { await observer?.stop(); } catch { /* */ } try { await shortEp?.stop(); } catch { /* */ } for (const e of ctlEps) { try { await e.stop(); } catch { /* */ } } - if (manager && !managerStopped) await manager.stop().catch(() => {}); + if (manager && !managerStopped) await manager.stop({ withAgents: true }).catch(() => {}); await killPid(authChild?.pid); broker?.kill("SIGKILL"); idpSrv.close(); diff --git a/implementations/cli/smoke/clean.smoke.ts b/implementations/cli/smoke/clean.smoke.ts index 1f2ecea58..0fff2830a 100644 --- a/implementations/cli/smoke/clean.smoke.ts +++ b/implementations/cli/smoke/clean.smoke.ts @@ -334,19 +334,33 @@ try { rmSync(downRoot, { recursive: true, force: true }); const blockedRoot = meshRoot(); + const blockedCanonical = realpathSync.native(blockedRoot); + // A different spelling of the same directory (`path.join` would collapse `/.`). Listing must + // match canonical-root-wise; a raw `===` misses this record and names "no recorded mesh". + const blockedRecorded = `${blockedCanonical}/.`; const blockedBroker = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60_000)"], { stdio: "ignore" }); writeFileSync(join(blockedRoot, ".cotal", "manager.pid"), "1"); writeFileSync(join(blockedRoot, ".cotal", "manager.pid.identity"), `1 ${defaultStartToken(1)}`); writeFileSync(join(blockedRoot, ".cotal", "nats.pid"), String(blockedBroker.pid)); - recordMesh(entry("blocked-down", blockedRoot)); + recordMesh(entry("blocked-down", blockedRecorded)); process.exitCode = 0; + const blockedErr: string[] = []; + const realBlockedErr = console.error; + console.error = ((...args: unknown[]) => { blockedErr.push(args.map(String).join(" ")); }) as typeof console.error; process.chdir(blockedRoot); try { await down({ positionals: [], values: {}, raw: [] }); } finally { + console.error = realBlockedErr; process.chdir(cwd2); } check("down: a failed dependent prevents the broker stop", blockedBroker.exitCode === null); + check( + "down: listing a failed dependent names the control plane, not a missing record", + blockedErr.some((line) => /manager control plane could not be reached/.test(line)) && + !blockedErr.some((line) => /no recorded mesh/.test(line)), + blockedErr, + ); check("down: a failed dependent preserves the mesh registry", loadMeshes().some((m) => m.space === "blocked-down")); blockedBroker.kill("SIGKILL"); await new Promise((resolve) => blockedBroker.once("exit", resolve)); diff --git a/implementations/cli/smoke/down-partial-reap.smoke.ts b/implementations/cli/smoke/down-partial-reap.smoke.ts new file mode 100644 index 000000000..fc2e2e91f --- /dev/null +++ b/implementations/cli/smoke/down-partial-reap.smoke.ts @@ -0,0 +1,138 @@ +/** + * A partial `cotal down --with-agents` reap reports the seats it stopped separately from the seats + * still running. The old renderer said "no seats were reaped" whenever any one stop failed, even + * after an earlier seat had stopped successfully. + * + * Representative: a real open NATS broker, a real Manager endpoint, the real generic control rail, + * and the real `down()` command. Only the two runtime handles are fakes, so one exit can be proven + * while the other reports a failed proof deterministically. + * + * Run: pnpm smoke:down-partial-reap + */ +import { strict as assert } from "node:assert"; +import { spawn, type ChildProcess } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pickFreePort } from "../../../packages/core/smoke/_free-port.js"; +import { SMOKE_BROKER_TOKEN, teardownOnSignal } from "@cotal-ai/smoke-kit"; +import type { AgentHandle, AttachSession } from "@cotal-ai/core"; + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const scratch = mkdtempSync(join(tmpdir(), SMOKE_BROKER_TOKEN)); +const home = join(scratch, "home"); +const root = join(scratch, "root"); +mkdirSync(join(home, ".cotal"), { recursive: true }); +mkdirSync(join(root, ".cotal", "agents"), { recursive: true }); +process.env.COTAL_HOME = home; + +let manager: InstanceType | undefined; +let stackChild: ChildProcess | undefined; +let releaseBroker: (() => void) | undefined; +let failedMayExit = false; +let passed = false; + +try { + const core = await import("@cotal-ai/core"); + const workspace = await import("@cotal-ai/workspace"); + const { Manager } = await import("../../manager/src/manager.js"); + const { down } = await import("../src/commands/down.js"); + await import("@cotal-ai/cli"); + + const port = await pickFreePort(); + const server = `nats://127.0.0.1:${port}`; + const space = "down-partial-reap"; + const store = join(scratch, "js"); + const broker = spawn("nats-server", ["-p", String(port), "-js", "-sd", store], { stdio: "ignore" }); + releaseBroker = teardownOnSignal(broker, scratch); + let up = false; + for (let i = 0; i < 100; i++) { + if (await core.isReachable(server)) { up = true; break; } + await wait(50); + } + if (!up) throw new Error(`fixture broker never came up on ${server}`); + + workspace.recordMesh({ space, server, root, mode: "open", ts: new Date().toISOString() }); + manager = new Manager({ space, servers: server, runtime: "pty", workspaceRoot: root, preserveStopTimeoutMs: 100 }); + await manager.start(); + + const session: AttachSession = { + cols: 80, rows: 24, backlog: () => Buffer.alloc(0), onData: () => () => {}, + onExit: () => () => {}, write: () => {}, resize: () => {}, + }; + const handle = (name: string, fails: boolean): AgentHandle => { + let exited = false; + return { + name, kind: "fixture", status: () => exited ? "exited" : "running", + stop: () => { if (!fails || failedMayExit) exited = true; }, + waitForExit: async () => { + if (fails && !failedMayExit) throw new Error("fixture exit proof failed"); + exited = true; + }, + interrupt: () => {}, attach: () => session, + }; + }; + const agents = (manager as unknown as { agents: Map }).agents; + const managed = (name: string, fails: boolean) => ({ + name, role: "worker", agent: "fixture", id: core.newIdentity().id, + lifecycleUid: core.mintLifecycleUid(), spawner: "local.manager", startedAt: Date.now() - 60_000, + handle: handle(name, fails), suppressCleanup: false, terminalizing: false, + launch: { + source: { kind: "persona", ref: name, configPath: join(root, ".cotal", "agents", `${name}.md`), configSha256: "fixture" }, + cwd: root, subscribe: [], allowSubscribe: [], allowPublish: [], capabilities: [], + }, + }); + for (const name of ["seat-a", "seat-b"]) + writeFileSync(join(root, ".cotal", "agents", `${name}.md`), `---\nname: ${name}\nrole: worker\n---\n`); + agents.set("seat-a", managed("seat-a", false)); + agents.set("seat-b", managed("seat-b", true)); + + stackChild = spawn(process.execPath, ["-e", "setInterval(()=>{},1000)"], { detached: true, stdio: "ignore" }); + stackChild.unref(); + const pidPath = workspace.localProcessPath(workspace.MANAGER_PIDFILE, { root, space }); + writeFileSync(pidPath, String(stackChild.pid), { mode: 0o600 }); + writeFileSync(`${pidPath}.identity`, `${stackChild.pid} ${workspace.defaultStartToken(stackChild.pid ?? 0)}`, { mode: 0o600 }); + + const out: string[] = []; + const err: string[] = []; + const realLog = console.log; + const realError = console.error; + const beforeCwd = process.cwd(); + const beforeExitCode = process.exitCode; + process.exitCode = 0; + try { + console.log = (...args: unknown[]) => { out.push(args.map(String).join(" ")); }; + console.error = (...args: unknown[]) => { err.push(args.map(String).join(" ")); }; + process.chdir(root); + await down({ values: { "with-agents": true }, positionals: [], raw: [] }); + } finally { + process.chdir(beforeCwd); + console.log = realLog; + console.error = realError; + } + + const stdout = out.join("\n"); + const stderr = err.join("\n"); + assert.match(stdout, /stopped 1 managed agent/); + assert.match(stdout, /seat-a/); + assert.match(stderr, /could not stop 1 managed agent/); + assert.match(stderr, /seat-b/); + assert.match(stderr, /fixture exit proof failed/); + assert.doesNotMatch(`${stdout}\n${stderr}`, /no seats were reaped/); + assert.equal(process.exitCode, 1); + assert.equal(agents.has("seat-a"), false); + assert.equal(agents.has("seat-b"), true); + process.exitCode = beforeExitCode; + + console.log("down partial reap smoke: 9 checks passed"); + passed = true; +} finally { + failedMayExit = true; + await manager?.stop({ withAgents: true }).catch(() => {}); + if (stackChild?.pid) { + try { process.kill(stackChild.pid, "SIGKILL"); } catch { /* already stopped by down */ } + } + releaseBroker?.(); + rmSync(scratch, { recursive: true, force: true }); + if (passed) process.exit(0); +} diff --git a/implementations/cli/smoke/down-target.smoke.ts b/implementations/cli/smoke/down-target.smoke.ts index 04683453b..fcb52e729 100644 --- a/implementations/cli/smoke/down-target.smoke.ts +++ b/implementations/cli/smoke/down-target.smoke.ts @@ -5,9 +5,10 @@ * `cotal web` (target-resolved) claimed `/.cotal/web.pid` but `cotal down web` only * looked under the folder it ran in and reported "Nothing running for web". * - * Hermetic (no broker): COTAL_HOME and the temp root are sandboxed, meshes are recorded straight - * into the registry, and the dashboard is a real SIGTERM-able child whose pid sits in the mesh - * root's web.pid. Run: pnpm smoke:down-target + * Hermetic: COTAL_HOME and the temp root are sandboxed, meshes are recorded straight into the + * registry, and the dashboard is a real SIGTERM-able child whose pid sits in the mesh root's + * web.pid. The unreaped `--with-agents` cell boots its own nats-server (needs it on PATH) with + * no manager responder. Run: pnpm smoke:down-target */ import { strict as assert } from "node:assert"; import { spawn, type ChildProcess } from "node:child_process"; @@ -15,7 +16,9 @@ import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node: import { tmpdir } from "node:os"; import { join } from "node:path"; import { makeScratch } from "../../../bin/smoke/_scratch.js"; -import { probeLiveness, defaultStartToken, type LocalProcess } from "@cotal-ai/workspace"; +import { isReachable } from "@cotal-ai/core"; +import { probeLiveness, defaultStartToken, MANAGER_PIDFILE, localProcessPath, type LocalProcess } from "@cotal-ai/workspace"; +import { pickFreePort } from "../../../packages/core/smoke/_free-port.js"; // Isolate BOTH the machine-home AND the temp root. `findCotalRoot` walks to `/` with no boundary, // so a `.cotal` above the temp base (observed: `/tmp/.cotal` on CI; a home-dir `.cotal` when the @@ -43,6 +46,7 @@ try { ({ cacheLocalProcess, extensionLocalProcesses, findCotalRoot, recordMesh, setCurrent } = await import("@cotal-ai/workspace")); ({ down } = await import("../src/commands/down.js")); ({ webProcess } = await import("../../web/src/web.js")); + await import("@cotal-ai/cli"); // registers manager/delivery/nats so a bare down can stop a planted manager } catch (e) { cleanScratch(e); } let pass = 0; @@ -148,8 +152,127 @@ try { await assert.rejects(run(["web"], { space: "nosuch" }), /no mesh named/); check("--space with an unknown mesh fails loud", true); - console.log(`\ndown target-addressed smoke: ${pass} checks passed`); + // Guardrails: --with-agents is bare whole-stack only and cannot mix with preserve-state. + await assert.rejects(run([], { "preserve-state": true, "with-agents": true }), /--preserve-state cannot be combined with --with-agents/); + check("--preserve-state with --with-agents is refused", true); + await assert.rejects(run(["web"], { "with-agents": true }), /--with-agents is bare-whole-stack only/); + check("--with-agents with a component is refused", true); + await assert.rejects(run([], { "with-agents": true, file: "cotal.yaml" }), /--with-agents is bare-whole-stack only/); + check("--with-agents with --file is refused", true); + await assert.rejects(run([], { "with-agents": true, run: "abc" }), /--with-agents is bare-whole-stack only/); + check("--with-agents with --run is refused", true); + await assert.rejects(run([], { "with-agents": true, space: "teamA" }), /--with-agents is bare-whole-stack only/); + check("--with-agents with --space is refused", true); + const realExit = process.exit; + let dryCombo: string | undefined; + (process as { exit: (code?: number) => never }).exit = ((code?: number) => { + throw new Error(`exit ${code ?? 0}`); + }) as typeof process.exit; + try { + await run([], { "with-agents": true, "dry-run": true }); + } catch (e) { + dryCombo = (e as Error).message; + } finally { + process.exit = realExit; + } + check( + "--with-agents with --dry-run is allowed (preview, not a combination refusal)", + dryCombo !== undefined && !/--with-agents is bare-whole-stack only/.test(dryCombo), + dryCombo, + ); + + // `--with-agents` against a live broker with no manager responder: still stop the stack, do not + // claim the reap. Listing must THROW (onRefusal) rather than process.exit, or this cell never + // reaches the stop loop. Both halves live in one assertion: a mutant that restores + // printCouldNotList without the exit code reddens here, and so does a mutant that re-exits. + { + const folder = mkdtempSync(join(scratch, "with-agents-unreaped-")); + mkdirSync(join(folder, ".cotal"), { recursive: true }); + const store = mkdtempSync(join(scratch, "with-agents-js-")); + const port = await pickFreePort(); + const server = `nats://127.0.0.1:${port}`; + const broker = spawn("nats-server", ["-p", String(port), "-js", "-sd", store], { stdio: "ignore" }); + spawnedChildren.push(broker); + let up = false; + for (let i = 0; i < 100 && !up; i++) { + if (await isReachable(server)) { up = true; break; } + await sleep(50); + } + if (!up) throw new Error(`fixture broker never came up on ${server}`); + recordMesh({ space: "main", server, root: folder, mode: "open", ts: "2026-07-27T00:00:00.000Z" }); + const child = spawn(process.execPath, ["-e", "setInterval(()=>{}, 1000);"], { detached: true, stdio: "ignore" }); + spawnedChildren.push(child); + child.unref(); + const pidPath = localProcessPath(MANAGER_PIDFILE, { root: folder, space: "main" }); + writeFileSync(pidPath, String(child.pid), { mode: 0o600 }); + writeFileSync(`${pidPath}.identity`, `${child.pid} ${defaultStartToken(child.pid ?? 0)}`, { mode: 0o600 }); + const prevCode = process.exitCode; + process.exitCode = 0; + const err: string[] = []; + const realErr = console.error; + console.error = ((...args: unknown[]) => { err.push(args.map(String).join(" ")); }) as typeof console.error; + const prevDir = process.cwd(); + const realExit = process.exit; + (process as { exit: (code?: number) => never }).exit = ((code?: number) => { + throw new Error(`process.exit(${code ?? 0})`); + }) as typeof process.exit; + try { + process.chdir(folder); + await run([], { "with-agents": true }); + } finally { + process.exit = realExit; + console.error = realErr; + process.chdir(prevDir); + try { broker.kill("SIGKILL"); } catch { /* gone */ } + } + for (let i = 0; i < 100 && alive(child.pid!); i++) await sleep(50); + const loud = err.some((line) => /no seats were reaped/.test(line) && /--with-agents/.test(line)); + check( + "--with-agents against an unreachable manager still stops the stack AND exits non-zero naming that no seats were reaped", + !alive(child.pid!) && !existsSync(pidPath) && process.exitCode === 1 && loud, + { alive: alive(child.pid!), pidfile: existsSync(pidPath), exitCode: process.exitCode, err }, + ); + process.exitCode = prevCode; + } + + // `cotal down manager` against a recorded mesh whose broker is down must still SIGTERM the + // manager. Listing throws; the catch is a dim leftover warning; the stop loop still runs. + { + const folder = mkdtempSync(join(scratch, "down-manager-dead-broker-")); + mkdirSync(join(folder, ".cotal"), { recursive: true }); + recordMesh({ space: "main", server: "nats://127.0.0.1:1", root: folder, mode: "open", ts: "2026-07-27T00:00:00.000Z" }); + const child = spawn(process.execPath, ["-e", "setInterval(()=>{}, 1000);"], { detached: true, stdio: "ignore" }); + spawnedChildren.push(child); + child.unref(); + const pidPath = localProcessPath(MANAGER_PIDFILE, { root: folder, space: "main" }); + writeFileSync(pidPath, String(child.pid), { mode: 0o600 }); + writeFileSync(`${pidPath}.identity`, `${child.pid} ${defaultStartToken(child.pid ?? 0)}`, { mode: 0o600 }); + const prevCode = process.exitCode; + process.exitCode = 0; + const realExit = process.exit; + let exitCalled: number | undefined; + (process as { exit: (code?: number) => never }).exit = ((code?: number) => { + exitCalled = code ?? 0; + throw new Error(`process.exit(${code ?? 0})`); + }) as typeof process.exit; + const prevDir = process.cwd(); + try { + process.chdir(folder); + await run(["manager"]); + } finally { + process.exit = realExit; + process.chdir(prevDir); + } + for (let i = 0; i < 100 && alive(child.pid!); i++) await sleep(50); + check( + "down manager against a recorded unreachable broker still stops the manager and does not process.exit", + !alive(child.pid!) && !existsSync(pidPath) && (process.exitCode ?? 0) === 0 && exitCalled === undefined, + { alive: alive(child.pid!), pidfile: existsSync(pidPath), exitCode: process.exitCode, exitCalled }, + ); + process.exitCode = prevCode; + } } finally { + console.log(`\ndown target-addressed smoke: ${pass} checks passed`); process.chdir(prevCwd); // Only the ones that were actually created — a throw partway through leaves the rest undefined, // and `finally` has to cope with a half-built fixture rather than assume a complete one. diff --git a/implementations/cli/smoke/mutations/down-failed-dependent-preserves-mesh.json b/implementations/cli/smoke/mutations/down-failed-dependent-preserves-mesh.json new file mode 100644 index 000000000..dcc2b8687 --- /dev/null +++ b/implementations/cli/smoke/mutations/down-failed-dependent-preserves-mesh.json @@ -0,0 +1,30 @@ +{ + "suite": "implementations/cli/smoke/clean.smoke.ts", + "guard": "listing seats on a failed down must restore a mesh record that preflight pruned", + "command": "pnpm smoke:clean", + "proveWith": "pnpm mutation-proof --config implementations/cli/smoke/mutations/down-failed-dependent-preserves-mesh.json", + "why": [ + "Bare down lists seats for honesty. resolveControlTarget preflight deletes an unreachable broker's registry entry.", + "A failed dependent (unsignalable manager, live broker pid) must keep that record, or later clean can proceed under a live process.", + "MUTATION 1 restores raw root === matching. Listing then misses a canonically recorded mesh and names a missing record instead of the control plane.", + "MUTATION 2 drops the restore. The preserve-registry cell goes red only if listing actually connected and pruned." + ], + "mutations": [ + { + "name": "listing matches recorded roots by raw string again", + "file": "implementations/cli/src/commands/down.ts", + "find": "function meshForContext(context: LocalProcessContext) {\n const matching = meshesForRoot(context.root).filter((mesh) => mesh.space === context.space);\n return matching[0] ?? meshesForRoot(context.root)[0];\n}", + "replace": "function meshForContext(context: LocalProcessContext) {\n const matching = loadMeshes().filter((mesh) => mesh.root === context.root && mesh.space === context.space);\n return matching[0] ?? loadMeshes().find((mesh) => mesh.root === context.root);\n}", + "expectRed": "down: listing a failed dependent names the control plane, not a missing record", + "cell": "down: listing a failed dependent names the control plane, not a missing record" + }, + { + "name": "listing no longer restores a mesh record that preflight pruned", + "file": "implementations/cli/src/commands/down.ts", + "find": " if (!findMesh(mesh.space)) recordMesh(mesh);\n return { ok: false, reason: `the manager control plane could not be reached (${(e as Error).message})` };", + "replace": " return { ok: false, reason: `the manager control plane could not be reached (${(e as Error).message})` };", + "expectRed": "down: a failed dependent preserves the mesh registry", + "cell": "down: a failed dependent preserves the mesh registry" + } + ] +} diff --git a/implementations/cli/smoke/mutations/down-partial-reap.json b/implementations/cli/smoke/mutations/down-partial-reap.json new file mode 100644 index 000000000..3c925eb51 --- /dev/null +++ b/implementations/cli/smoke/mutations/down-partial-reap.json @@ -0,0 +1,20 @@ +{ + "suite": "implementations/cli/smoke/down-partial-reap.smoke.ts", + "command": "pnpm smoke:down-partial-reap", + "proveWith": "node scripts/mutation-proof.mjs --config implementations/cli/smoke/mutations/down-partial-reap.json", + "why": [ + "The live smoke drives two real manager control calls: one seat stops and one seat cannot prove", + "exit. M1 removes the successful outcome from the summary, restoring the old false all-or-nothing", + "claim in effect. The named assertion must fail because seat-a disappears from stdout." + ], + "mutations": [ + { + "name": "M1 omit successful seats from the partial-reap summary", + "file": "implementations/cli/src/commands/down.ts", + "find": " if (stopped.ok) outcomes.stopped.push(row);", + "replace": " if (stopped.ok) void row;", + "expectRed": "stopped 1 managed agent", + "cell": "stopped 1 managed agent" + } + ] +} diff --git a/implementations/cli/smoke/mutations/down-with-agents-unreaped.json b/implementations/cli/smoke/mutations/down-with-agents-unreaped.json new file mode 100644 index 000000000..b77d9fab9 --- /dev/null +++ b/implementations/cli/smoke/mutations/down-with-agents-unreaped.json @@ -0,0 +1,30 @@ +{ + "suite": "implementations/cli/smoke/down-target.smoke.ts", + "guard": "--with-agents against an unlistable manager still stops the stack, reaps none, and exits non-zero; listing never process.exit", + "command": "pnpm smoke:down-target", + "completionMarker": "down target-addressed smoke:", + "proveWith": "pnpm mutation-proof --config implementations/cli/smoke/mutations/down-with-agents-unreaped.json", + "why": [ + "The suite imports down.ts from source, so a mutation on the command is live with no dist in the path.", + "MUTATION 1 restores the F4 false-success: --with-agents that cannot list seats prints a dim leftover warning and exits 0 after stopping the stack.", + "MUTATION 2 restores connectOrExit on listing. The down-manager cell stubs process.exit so an uncatchable refusal cannot hide: the planted manager stays running." + ], + "mutations": [ + { + "name": "the defect restored: --with-agents that cannot list still claims success", + "file": "implementations/cli/src/commands/down.ts", + "find": " } else if (withAgents) {\n printWithAgentsUnreaped(listed.reason);\n withAgentsUnreaped = true;\n } else {", + "replace": " } else if (withAgents) {\n printCouldNotList(listed.reason);\n } else {", + "expectRed": "--with-agents against an unreachable manager still stops the stack AND exits non-zero naming that no seats were reaped", + "cell": "--with-agents against an unreachable manager still stops the stack AND exits non-zero naming that no seats were reaped" + }, + { + "name": "listing again process.exit on an unreachable broker", + "file": "implementations/cli/src/commands/down.ts", + "find": " target = await resolveControlTarget({ space: mesh.space, server: mesh.server }, \"control-caller-privileged\", undefined, { onRefusal: \"throw\" });\n } catch (e) {\n if (!findMesh(mesh.space)) recordMesh(mesh);\n return { ok: false, reason: `the manager control plane could not be reached (${(e as Error).message})` };", + "replace": " target = await resolveControlTarget({ space: mesh.space, server: mesh.server }, \"control-caller-privileged\");\n } catch (e) {\n if (!findMesh(mesh.space)) recordMesh(mesh);\n return { ok: false, reason: `the manager control plane could not be reached (${(e as Error).message})` };", + "expectRed": "down manager against a recorded unreachable broker still stops the manager and does not process.exit", + "cell": "down manager against a recorded unreachable broker still stops the manager and does not process.exit" + } + ] +} diff --git a/implementations/cli/src/commands/down.ts b/implementations/cli/src/commands/down.ts index 682b00012..65e25fec7 100644 --- a/implementations/cli/src/commands/down.ts +++ b/implementations/cli/src/commands/down.ts @@ -11,7 +11,10 @@ import { clearPreservationCommitIntent, clearPreservationPrepareIntent, completeMaintenanceCut, + findMesh, loadMeshes, + meshesForRoot, + recordMesh, localProcessPath, localProcessPathCandidates, readMaintenanceJournal, @@ -78,8 +81,12 @@ export function downComplete(argv: string[]): CompletionResult { /** Stop the whole local stack by default, or only named self-registered process components. The * manifest forms remain ownership-scoped deploy teardown and cannot be mixed with components. */ export async function down(args: ParsedArgs): Promise { - const values = args.values as { file?: string; run?: string; "dry-run"?: boolean; "preserve-state"?: boolean; "store-dir"?: string; space?: string }; + const values = args.values as { file?: string; run?: string; "dry-run"?: boolean; "preserve-state"?: boolean; "with-agents"?: boolean; "store-dir"?: string; space?: string }; const requested = [...new Set(args.positionals)]; + if (values["preserve-state"] && values["with-agents"]) + throw new Error("--preserve-state cannot be combined with --with-agents"); + if (values["with-agents"] && (requested.length || values.file || values.run || values.space)) + throw new Error("--with-agents is bare-whole-stack only and cannot be combined with components, --space, --file, or --run"); if (values["preserve-state"]) { if (requested.length || values.file || values.run || values["dry-run"] || values.space) throw new Error("--preserve-state is bare-whole-stack only and cannot be combined with components, --space, --file, --run, or --dry-run"); @@ -150,9 +157,21 @@ export async function down(args: ParsedArgs): Promise { } } + const managerComp = selected.find((component) => component.name === "manager"); + const managerLive = Boolean(managerComp && mayBeRunning(managerComp, contextFor(managerComp))); + const withAgents = Boolean(values["with-agents"]); + if (values["dry-run"]) { const recorded = selected.filter((component) => processRecorded(component, contextFor(component))); for (const component of recorded) console.log(c.dim(`would stop ${component.label}`)); + if (withAgents && managerComp && managerLive) { + const listed = await listManagerSeats(contextFor(managerComp)); + if (listed.ok) printWouldReap(listed.rows); + else { + printWithAgentsUnreaped(listed.reason); + process.exitCode = 1; + } + } if (!recorded.length) { const target = requested.length ? requested.join(", ") : "the local stack"; console.error(c.red(`Nothing running for ${target} (no recorded pidfiles).`)); @@ -162,6 +181,37 @@ export async function down(args: ParsedArgs): Promise { return; } + // Spare down signals the manager; `Manager.stop()` detaches internally. Listing is honesty, never a + // refuse-to-signal: an unreachable manager must stay stoppable (a broker that is down cannot be + // asked, and a safety check that fails closed on unreachability leaves an unstoppable process). + // `--with-agents` still reaps through the existing `ps` + per-seat `stop` ops when the manager + // answers. There is no detach control op. An older manager whose `stop()` still reaps will reap + // on SIGTERM; that version-skew hole is named, not closed. + let leftover: DownSeatRow[] | undefined; + let withAgentsUnreaped = false; + if (managerComp && managerLive) { + const listed = await listManagerSeats(contextFor(managerComp)); + if (listed.ok) { + leftover = listed.rows; + if (withAgents && listed.rows.length) { + try { + const reaped = await reapListedSeats(contextFor(managerComp), listed.rows); + printReapOutcomes(reaped); + if (reaped.failed.length) withAgentsUnreaped = true; + } catch (e) { + printWithAgentsUnreaped((e as Error).message); + withAgentsUnreaped = true; + } + } + // Still stop the stack. Unreachability must never strand an operator. Do not claim the reap. + } else if (withAgents) { + printWithAgentsUnreaped(listed.reason); + withAgentsUnreaped = true; + } else { + printCouldNotList(listed.reason); + } + } + let any = false; let allStopped = true; for (const component of selected) { @@ -198,6 +248,8 @@ export async function down(args: ParsedArgs): Promise { console.error(c.red(`Nothing running for ${target} (no recorded pidfiles).`)); process.exit(1); } + if (leftover?.length && !withAgents) printLeftRunning(leftover); + if (withAgentsUnreaped) process.exitCode = 1; } const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -208,6 +260,117 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); // stop. `isAlive` = the probe says the process EXISTS; every caller below feeds it a parsed pid. export const isAlive = (pid: number): boolean => probeLiveness(pid) === "alive"; +type DownSeatRow = { + name: string; + mode?: string; + pid?: number; + agent?: string; + cwd?: string; + status?: string; +}; + +function meshForContext(context: LocalProcessContext) { + const matching = meshesForRoot(context.root).filter((mesh) => mesh.space === context.space); + return matching[0] ?? meshesForRoot(context.root)[0]; +} + +type SeatList = { ok: true; rows: DownSeatRow[] } | { ok: false; reason: string }; +type SeatReapOutcomes = { + stopped: DownSeatRow[]; + failed: Array<{ row: DownSeatRow; reason: string }>; +}; + +/** Best-effort `ps`. Failure is a named leftover, never a refuse-to-signal. */ +async function listManagerSeats(context: LocalProcessContext): Promise { + const mesh = meshForContext(context); + if (!mesh) + return { ok: false, reason: "this folder has no recorded mesh, so leftover seats cannot be listed" }; + // Catchable: the default `connectOrExit` would `process.exit(1)` and strand a live manager + // whose broker is down. Listing is honesty, never a refuse-to-signal. Honesty only: preflight + // treats an unreachable broker as a stale registry entry and deletes it. A failed dependent + // must still keep that record, so restore if the connect pruned it. + let target; + try { + target = await resolveControlTarget({ space: mesh.space, server: mesh.server }, "control-caller-privileged", undefined, { onRefusal: "throw" }); + } catch (e) { + if (!findMesh(mesh.space)) recordMesh(mesh); + return { ok: false, reason: `the manager control plane could not be reached (${(e as Error).message})` }; + } + const reply = await askManager(target.space, target.server, "ps", undefined, target.auth, "any"); + if (!reply.ok) + return { ok: false, reason: `the manager seat list could not be read (${reply.error ?? "ps failed"})` }; + return { ok: true, rows: Array.isArray(reply.data) ? (reply.data as DownSeatRow[]) : [] }; +} + +async function reapListedSeats(context: LocalProcessContext, rows: DownSeatRow[]): Promise { + const mesh = meshForContext(context); + if (!mesh) throw new Error("--with-agents could not stop every managed seat: this folder has no recorded mesh"); + // Same class as listing: a broker that dies between ps and the reap must not `process.exit` + // before the stack stops. Failures join the per-seat list and still let down signal. + let target; + try { + target = await resolveControlTarget({ space: mesh.space, server: mesh.server }, "control-caller-privileged", undefined, { onRefusal: "throw" }); + } catch (e) { + if (!findMesh(mesh.space)) recordMesh(mesh); + throw new Error(`--with-agents could not stop every managed seat: the manager control plane could not be reached (${(e as Error).message})`); + } + const outcomes: SeatReapOutcomes = { stopped: [], failed: [] }; + for (const row of rows) { + const stopped = await askManager(target.space, target.server, "stop", { name: row.name, graceful: false, waitForExit: true }, target.auth, "any", 30_000); + if (stopped.ok) outcomes.stopped.push(row); + else outcomes.failed.push({ row, reason: stopped.error ?? "stop failed" }); + } + return outcomes; +} + +function printSeatRow(row: DownSeatRow): void { + console.log(seatRowText(row)); +} + +function seatRowText(row: DownSeatRow): string { + const bits = [row.name, row.mode, row.pid !== undefined ? `pid ${row.pid}` : undefined, row.agent, row.cwd, row.status].filter(Boolean); + return ` ${bits.join(" · ")}`; +} + +function printLeftRunning(rows: DownSeatRow[]): void { + console.log(c.dim(`left ${rows.length} managed agent${rows.length === 1 ? "" : "s"} running (no longer managed):`)); + for (const row of rows) printSeatRow(row); + console.log(c.dim("these are unmanaged OS processes. `cotal stop --name ` needs a manager.")); + console.log(c.dim("to stop them with the stack: cotal down --with-agents")); +} + +function printWouldReap(rows: DownSeatRow[]): void { + if (!rows.length) { + console.log(c.dim("would reap 0 managed agents")); + return; + } + console.log(c.dim(`would reap ${rows.length} managed agent${rows.length === 1 ? "" : "s"}:`)); + for (const row of rows) printSeatRow(row); +} + +function printCouldNotList(reason: string): void { + console.error(c.dim(`could not list managed seats (${reason}); leftovers may remain after the stack stops`)); +} + +function printReapOutcomes(outcomes: SeatReapOutcomes): void { + if (outcomes.stopped.length) { + console.log(c.dim(`stopped ${outcomes.stopped.length} managed agent${outcomes.stopped.length === 1 ? "" : "s"}:`)); + for (const row of outcomes.stopped) printSeatRow(row); + } + if (outcomes.failed.length) { + console.error(c.red(`✗ could not stop ${outcomes.failed.length} managed agent${outcomes.failed.length === 1 ? "" : "s"}; ${outcomes.failed.length === 1 ? "it is" : "they are"} still running unmanaged:`)); + for (const { row, reason } of outcomes.failed) { + console.error(c.red(seatRowText(row))); + console.error(c.red(` ${reason}`)); + } + } +} + +/** `--with-agents` could not begin a reap. The stack still stops; success is not claimed. */ +function printWithAgentsUnreaped(reason: string): void { + console.error(c.red(`✗ --with-agents could not stop every managed seat (${reason}); no seats were reaped and they are still running unmanaged`)); +} + export function processRecorded(component: LocalProcess, context: LocalProcessContext): boolean { return existsSync(localProcessPath(component.pidFile, context)) || (component.artifacts ?? []).map((artifact) => localProcessPath(artifact, context)).some(existsSync); } @@ -443,6 +606,7 @@ async function preserveStateDown(storeOverride?: string): Promise { managerCommitJournaled = true; } if (!managerCommitJournaled) { + // Preserve-state stays a hard exit on an unreachable broker: a half-cut must not continue. const retryTarget = await resolveControlTarget({ space: mesh.space, server: mesh.server }, "control-caller-admin"); // Plane-3 fence precedes the re-prepared inventory, exactly as on the fresh path. const retryDelivery = all.find((component) => component.name === "delivery"); @@ -483,6 +647,7 @@ async function preserveStateDown(storeOverride?: string): Promise { attemptId, space: mesh.space, mode: mesh.mode, server: mesh.server, storeDir, }); } + // Preserve-state stays a hard exit on an unreachable broker: a half-cut must not continue. const target = await resolveControlTarget({ space: mesh.space, server: mesh.server }, "control-caller-admin"); // Fence Plane 3 BEFORE any inventory work: with the delivery daemon stopped, no durable // join/leave can mutate MEMBERS at or after the moment the inventory is taken. @@ -552,6 +717,7 @@ async function preserveStateDown(storeOverride?: string): Promise { // intent). A crash here is genuinely pre-commit — recovery MUST still abort, not finish forward. if (process.env.COTAL_SMOKE_EXIT_AFTER_CUT_INTENT_BEFORE_COMMIT === "1") process.exit(93); writePreservationCommitIntent(lock, { attemptId }); + // Preserve-state stays a hard exit on an unreachable broker: a half-cut must not continue. const target = await resolveControlTarget({ space: mesh.space, server: mesh.server }, "control-caller-admin"); const commit = await askManager( target.space, diff --git a/implementations/cli/src/commands/setup.ts b/implementations/cli/src/commands/setup.ts index f39990fe7..91535f743 100644 --- a/implementations/cli/src/commands/setup.ts +++ b/implementations/cli/src/commands/setup.ts @@ -176,13 +176,13 @@ async function runFirstRun(yes: boolean, demo: boolean): Promise { `${ok("✓")} drive a session ${dim(`${cmd} spawn me`)}`, `${ok("✓")} ask the experts ${dim(`${cmd} spawn david · ${cmd} spawn sven`)}`, `${ok("✓")} watch the mesh ${dim(`${cmd} console`)}`, - `${ok("✓")} stop everything ${dim(`${cmd} down`)}`, + `${ok("✓")} stop the stack ${dim(`${cmd} down`)} ${dim("(agents stay unless --with-agents)")}`, ] : [ `${ok("✓")} start the mesh ${dim(`${cmd} up --detach`)}`, `${ok("✓")} talk to your agent ${dim(`${cmd} spawn`)}`, `${ok("✓")} watch the mesh ${dim(`${cmd} console`)}`, - `${ok("✓")} stop everything ${dim(`${cmd} down`)}`, + `${ok("✓")} stop the stack ${dim(`${cmd} down`)} ${dim("(agents stay unless --with-agents)")}`, ]; const tail = demo ? [dim(`Visual dashboard: ${cmd} web`)] diff --git a/implementations/cli/src/index.ts b/implementations/cli/src/index.ts index c06adfdf9..5fab96756 100644 --- a/implementations/cli/src/index.ts +++ b/implementations/cli/src/index.ts @@ -112,7 +112,7 @@ const baseCommands: Command[] = [ kind: "command", name: "down", group: "Mesh", - summary: "stop the whole local stack, or name only the components to stop", + summary: "stop the whole local stack (managed agents stay running unless --with-agents), or name only the components to stop", positionals: "[ …]", flags: [ { name: "file", type: "string", short: "f", value: "", description: "tear down this manifest's deploy" }, @@ -120,6 +120,7 @@ const baseCommands: Command[] = [ { name: "space", type: "string", value: "", description: "with components: the mesh whose target-addressed components (e.g. web) to stop" }, { name: "dry-run", type: "boolean", description: "print what would stop, mutate nothing" }, { name: "preserve-state", type: "boolean", description: "bare whole stack: stop without logical teardown and publish an offline backup cut" }, + { name: "with-agents", type: "boolean", description: "bare whole stack: also stop every managed agent (the previous default)" }, { name: "store-dir", type: "string", value: "", description: "with --preserve-state: actual JetStream store (default .cotal/nats)" }, ], run: down, diff --git a/implementations/manager/README.md b/implementations/manager/README.md index 035b667a5..75a237d72 100644 --- a/implementations/manager/README.md +++ b/implementations/manager/README.md @@ -44,7 +44,12 @@ Library composition roots can call `Manager.preserveState({ attemptId, persistIn the first child launches. Static and open entries reuse and validate the exact retained principal. User-auth entries are validated internally through `resolveAuthProvider().validateRetainedAgent()`; the manager never calls `grantAgent` or provisions a replacement identity. Ordinary -`Manager.stop()` remains destructive while the manager is active. +`Manager.stop()` leaves managed agents running. Pass `{ withAgents: true }` to reap them. +Linux custodial pty releases the unix socket on a plain stop and leaves the child. In-process +node-pty (`LegacyPtyRuntime`, used off Linux) cannot drop the master without killing the child, +so a plain stop of those seats throws rather than spare. A leftover PTY master may still kill +a child when the manager process exits. A later manager on the same root may still take leftover +seats. Preservation still always stops retained children. After restore, start the manager with `supervise --resume-attempt `, wait for normal manager readiness, then send the admin control request: diff --git a/implementations/manager/smoke/_probe-attach-reconnect.ts b/implementations/manager/smoke/_probe-attach-reconnect.ts index 82c8c6797..16e6def6b 100644 --- a/implementations/manager/smoke/_probe-attach-reconnect.ts +++ b/implementations/manager/smoke/_probe-attach-reconnect.ts @@ -223,7 +223,7 @@ try { } finally { child?.kill("SIGKILL"); await sever(); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); srv.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); releaseBroker(); diff --git a/implementations/manager/smoke/_probe-cellj-timing.ts b/implementations/manager/smoke/_probe-cellj-timing.ts index 8169ebed8..cb1b178e7 100644 --- a/implementations/manager/smoke/_probe-cellj-timing.ts +++ b/implementations/manager/smoke/_probe-cellj-timing.ts @@ -360,7 +360,7 @@ try { att?.kill(); onKnock = undefined; await closeLink(); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); // Order is the whole of it, and the suite states the rule: kill and remove FIRST, release LAST. // `releaseBroker` does not stop anything, it hands ownership BACK (`owned.delete(entry)`), and // the reap that runs on exit only touches what it still owns. Releasing before exiting therefore diff --git a/implementations/manager/smoke/_probe-late-delivery.ts b/implementations/manager/smoke/_probe-late-delivery.ts index dae717864..aa0ca6dfa 100644 --- a/implementations/manager/smoke/_probe-late-delivery.ts +++ b/implementations/manager/smoke/_probe-late-delivery.ts @@ -227,7 +227,7 @@ try { } finally { try { child?.kill("SIGKILL"); } catch { /* gone */ } await closeLink(); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); srv.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); releaseBroker(); diff --git a/implementations/manager/smoke/_probe-live-socket-gap.ts b/implementations/manager/smoke/_probe-live-socket-gap.ts index b8a0aa705..40de968ba 100644 --- a/implementations/manager/smoke/_probe-live-socket-gap.ts +++ b/implementations/manager/smoke/_probe-live-socket-gap.ts @@ -371,7 +371,7 @@ try { } finally { for (const x of started) x.kill(); for (const p of pipedChildren) p.kill(); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); await closeLink(); srv.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); diff --git a/implementations/manager/smoke/_probe-missed-handoff.ts b/implementations/manager/smoke/_probe-missed-handoff.ts index dad725cc8..fc38a9e09 100644 --- a/implementations/manager/smoke/_probe-missed-handoff.ts +++ b/implementations/manager/smoke/_probe-missed-handoff.ts @@ -222,7 +222,7 @@ try { } finally { try { child?.kill("SIGKILL"); } catch { /* gone */ } await closeLink(); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); srv.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); releaseBroker(); diff --git a/implementations/manager/smoke/_probe-pipe-oneshot-exit.ts b/implementations/manager/smoke/_probe-pipe-oneshot-exit.ts index afe3c7c0e..c88b19863 100644 --- a/implementations/manager/smoke/_probe-pipe-oneshot-exit.ts +++ b/implementations/manager/smoke/_probe-pipe-oneshot-exit.ts @@ -156,7 +156,7 @@ try { console.log(`transcript tail: ${JSON.stringify(buf.slice(-400))}`); } finally { try { child?.kill("SIGKILL"); } catch { /* already gone */ } - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); srv.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); releaseBroker(); diff --git a/implementations/manager/smoke/_probe-session-leak.ts b/implementations/manager/smoke/_probe-session-leak.ts index 505fd4cb2..221f40365 100644 --- a/implementations/manager/smoke/_probe-session-leak.ts +++ b/implementations/manager/smoke/_probe-session-leak.ts @@ -203,7 +203,7 @@ try { } finally { for (const k of kids) { try { k.kill("SIGKILL"); } catch { /* already gone */ } } await sever(); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); srv.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); releaseBroker(); diff --git a/implementations/manager/smoke/_probe-stdin-window.ts b/implementations/manager/smoke/_probe-stdin-window.ts index aa5d3a39c..8bcbd25fe 100644 --- a/implementations/manager/smoke/_probe-stdin-window.ts +++ b/implementations/manager/smoke/_probe-stdin-window.ts @@ -305,7 +305,7 @@ try { att?.kill(); onKnock = undefined; await closeLink(); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); // Kill and remove FIRST, release LAST: `releaseBroker` hands the kill duty back rather than doing // it, so releasing before an explicit exit leaves a live broker and its store dir behind (#587). srv.kill("SIGKILL"); diff --git a/implementations/manager/smoke/attach-reconnect.smoke.ts b/implementations/manager/smoke/attach-reconnect.smoke.ts index d632db7df..143d4167f 100644 --- a/implementations/manager/smoke/attach-reconnect.smoke.ts +++ b/implementations/manager/smoke/attach-reconnect.smoke.ts @@ -655,7 +655,7 @@ try { } finally { for (const a of started) a.kill(); await sever(); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); srv.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); releaseBroker(); // last: ownership is held until this teardown has actually finished diff --git a/implementations/manager/smoke/attach-stdin.smoke.ts b/implementations/manager/smoke/attach-stdin.smoke.ts index b1f65d910..76045137a 100644 --- a/implementations/manager/smoke/attach-stdin.smoke.ts +++ b/implementations/manager/smoke/attach-stdin.smoke.ts @@ -1099,7 +1099,7 @@ try { onKnock = undefined; sawClientPing = undefined; await closeLink(); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); // Kill and remove FIRST, release LAST: `releaseBroker` hands the kill duty back rather than doing // it, so releasing before the kill leaves a window where nobody owns this broker. srv.kill("SIGKILL"); diff --git a/implementations/manager/smoke/cli-seat-locality.smoke.ts b/implementations/manager/smoke/cli-seat-locality.smoke.ts index 1300030ac..b99f500e0 100644 --- a/implementations/manager/smoke/cli-seat-locality.smoke.ts +++ b/implementations/manager/smoke/cli-seat-locality.smoke.ts @@ -227,8 +227,8 @@ try { console.log(`\n${fail === 0 ? "PASS" : "FAIL"} — ${pass} passed, ${fail} failed`); } finally { - await m1?.stop().catch(() => {}); - await m2?.stop().catch(() => {}); + await m1?.stop({ withAgents: true }).catch(() => {}); + await m2?.stop({ withAgents: true }).catch(() => {}); srv.kill("SIGKILL"); rmSync(dir, { recursive: true, force: true }); releaseBroker(); // last: ownership is held until this teardown has actually finished diff --git a/implementations/manager/smoke/console-ws-duplex.smoke.ts b/implementations/manager/smoke/console-ws-duplex.smoke.ts index 2503f0503..b8361c007 100644 --- a/implementations/manager/smoke/console-ws-duplex.smoke.ts +++ b/implementations/manager/smoke/console-ws-duplex.smoke.ts @@ -96,7 +96,7 @@ try { await nc.close(); console.log(`\nconsole-ws-duplex: ${pass} passed, ${fail} failed`); } finally { - await mgr?.stop().catch(() => {}); + await mgr?.stop({ withAgents: true }).catch(() => {}); for (const k of kids) k.kill("SIGKILL"); await wait(200); } diff --git a/implementations/manager/smoke/custodial-pty.smoke.ts b/implementations/manager/smoke/custodial-pty.smoke.ts index 65675c72f..956db1b68 100644 --- a/implementations/manager/smoke/custodial-pty.smoke.ts +++ b/implementations/manager/smoke/custodial-pty.smoke.ts @@ -108,6 +108,10 @@ if (process.platform !== "linux") { check("spawned handle exposes a durable reference", h.reference !== undefined && h.reference.kind === "pty", h.reference); const adopted = requireRuntimeAdopt(rt, h.reference!); check("production adopt returns a live proxy", typeof adopted.attach === "function" && adopted.pid === h.pid); + check( + "production adopt exposes release", + typeof (adopted as { release?: unknown }).release === "function", + ); h.stop({ graceful: false }); await h.waitForExit?.(); drop(adopted); diff --git a/implementations/manager/smoke/fixtures/shutdown-seat-exit.mutations.json b/implementations/manager/smoke/fixtures/shutdown-seat-exit.mutations.json index 73e527d0c..d3bd05951 100644 --- a/implementations/manager/smoke/fixtures/shutdown-seat-exit.mutations.json +++ b/implementations/manager/smoke/fixtures/shutdown-seat-exit.mutations.json @@ -1,11 +1,12 @@ { "suite": "implementations/manager/smoke/start-model-preflight.smoke.ts", - "guard": "normal Manager.stop does not release manager authority until every managed AgentHandle has authoritatively exited", + "guard": "Manager.stop with withAgents true waits for every seat to exit; a plain stop leaves those seats running", "command": "pnpm smoke:start-model", "proveWith": "node scripts/mutation-proof.mjs --config implementations/manager/smoke/fixtures/shutdown-seat-exit.mutations.json", "why": [ "The deterministic shutdown cell gives both target seats a runtime exit witness and asserts stop proves both exits before returning.", - "A second cell makes one runtime stop throw and its wait report still-running; shutdown must fail loud rather than release authority into an orphan window." + "A second cell makes one runtime stop throw and its wait report still-running; shutdown must fail loud rather than release authority into an orphan window.", + "A third cell is the #964 spare path: a plain stop must not call handle.stop. Restoring the old always-reap line reddens only that cell." ], "mutations": [ { @@ -15,6 +16,15 @@ "replace": " void failures;", "expectRed": "shutdown: stop() proves every managed child exited before releasing manager authority", "cell": "shutdown: stop() proves every managed child exited before releasing manager authority" + }, + { + "name": "the defect restored: a plain stop still reaps every managed agent", + "file": "implementations/manager/src/manager.ts", + "find": " if (opts?.withAgents === true) await this.teardownManagedAgents();\n else {\n try {\n this.detachManagedAgents();\n } catch (e) {\n spareError = e;\n }\n }", + "replace": " await this.teardownManagedAgents();", + "expectRed": "spare: a plain stop does not hard-stop the child", + "cell": "spare: a plain stop does not hard-stop the child", + "note": "Issue #964 restored: stop() on the active path always tears down children. The withAgents:true cells stay green; only the spare cell reddens." } ] } diff --git a/implementations/manager/smoke/lease-loss-keeps-serving.smoke.ts b/implementations/manager/smoke/lease-loss-keeps-serving.smoke.ts index ea1914250..f7100a4ae 100644 --- a/implementations/manager/smoke/lease-loss-keeps-serving.smoke.ts +++ b/implementations/manager/smoke/lease-loss-keeps-serving.smoke.ts @@ -18,8 +18,8 @@ * per tick. `gone` must also put the key back. * * THE POSITIVE CONTROL IS NOT OPTIONAL. "stops === 0" is also what a broken counter reports. Cell 0 - * drives the ORDINARY shutdown path, which stays destructive, and requires the same counter to reach 1. - * Without that, every zero below is unearned. + * drives `stop({ withAgents: true })`, the explicit reap, and requires the same counter to reach 1. + * Without that, every zero below is unearned. Ordinary `stop()` leaves agents running (#964). * * NOT GRADED: the wire. Whether the broker actually answers `gone` after an expiry, and whether a real * manager survives a real blackout, is `smoke:lease-renew` (a real broker behind a relay that can stall @@ -167,14 +167,14 @@ const timeout = async (): Promise => { throw new Error("timeout"); }; const other: ManagerLeaseInfo = { holder: "local.other", instanceId: "smoke-instance", runtime: "pty", root, pid: process.pid + 1, since: 0 }; // ── Cell 0 — POSITIVE CONTROL ──────────────────────────────────────────────────────────────── -// The ordinary shutdown path is deliberately destructive and must stay so: `cotal down` and Ctrl-C -// mean shut the mesh down. If this cell does not see a stop, the counter is broken and every zero +// The explicit reap (`stop({ withAgents: true })`) must still stop children. Ordinary `stop()` +// leaves them (#964). If this cell does not see a stop, the counter is broken and every zero // below is worthless rather than reassuring. { const h = fakeHandle("worker"); const { manager } = managerWith([h], { renew: timeout, read: timeout }); - await manager.stop(); - check("CONTROL: the ordinary stop path stops the child (instrument fires)", h.stops === 1, `stops=${h.stops}`); + await manager.stop({ withAgents: true }); + check("CONTROL: the explicit reap path stops the child (instrument fires)", h.stops === 1, `stops=${h.stops}`); } // ── Cell 1 — unknown: the broker cannot be asked, for as long as that lasts ────────────────── diff --git a/implementations/manager/smoke/lifecycle-e2e.smoke.ts b/implementations/manager/smoke/lifecycle-e2e.smoke.ts index d5a862dc8..19abcde15 100644 --- a/implementations/manager/smoke/lifecycle-e2e.smoke.ts +++ b/implementations/manager/smoke/lifecycle-e2e.smoke.ts @@ -13,7 +13,7 @@ * KEPT (not deprovisioned — it may still be booting), distinct from both started and failed. * 3c. FAIL BEFORE PRESENCE — a launch missing the launcher uid, or lying a different one while * consuming, dies with NO roster ghost (SPEC 13.1 fail-before-presence). - * 4. SHUTDOWN teardown — Manager.stop() deprovisions every still-managed agent's footprint. + * 4. SHUTDOWN teardown — Manager.stop({ withAgents: true }) deprovisions every still-managed agent's footprint. * * Run: pnpm smoke:lifecycle-e2e (needs nats-server + node on PATH) */ @@ -270,13 +270,13 @@ try { } // 4 — SHUTDOWN teardown: stop() deprovisions the still-managed agents (w2 + the kept idle1). - console.log("4. manager stop() → still-managed footprint torn down:"); + console.log("4. manager stop({ withAgents: true }) → still-managed footprint torn down:"); const r4 = await mgr.startAgent({ name: "w2", agent: "e2e-stub", cwd: repoRoot }); check("second agent started", r4.ok === true, r4); const id2 = (r4.data as { id?: string } | undefined)?.id ?? ""; const uid2 = uidOf("w2"); // capture before stop() clears the managed set check("w2 footprint exists before stop", (await footprint(id2, uid2, "w2")).dm, await footprint(id2, uid2, "w2")); - await mgr.stop(); // awaits teardownManagedAgents → deprovision + await mgr.stop({ withAgents: true }); // awaits teardownManagedAgents → deprovision const fp2 = await footprint(id2, uid2, "w2"); check("w2 dm_ durable gone after stop()", !fp2.dm, fp2); check("w2 dlv_ durable gone after stop()", !fp2.dlv, fp2); @@ -289,7 +289,7 @@ try { console.error(" ✗ scenario threw:", (e as Error).stack ?? (e as Error).message); process.exitCode = 1; } finally { - try { await mgr.stop(); } catch { /* already stopped */ } + try { await mgr.stop({ withAgents: true }); } catch { /* already stopped */ } await delivery?.stop().catch(() => {}); srv.kill("SIGKILL"); await wait(300); diff --git a/implementations/manager/smoke/manager-multi-live.smoke.ts b/implementations/manager/smoke/manager-multi-live.smoke.ts index d5a12eb36..50a11a89f 100644 --- a/implementations/manager/smoke/manager-multi-live.smoke.ts +++ b/implementations/manager/smoke/manager-multi-live.smoke.ts @@ -139,8 +139,8 @@ try { scatter2.missing.includes(IID2) && scatter2.complete === false, { missing: scatter2.missing, complete: scatter2.complete }); } finally { try { await nc?.drain(); } catch { /* ignore */ } - await m2?.stop().catch(() => {}); - await m1?.stop().catch(() => {}); + await m2?.stop({ withAgents: true }).catch(() => {}); + await m1?.stop({ withAgents: true }).catch(() => {}); for (const k of kids) { try { k.kill("SIGKILL"); } catch { /* best effort */ } } // The scratch tree goes too. Its absence here is the defect: this suite passed, said so, and // left one directory behind on every green run — reproduced by count, not inferred. The pause diff --git a/implementations/manager/smoke/manager-reconcile-redrive.smoke.ts b/implementations/manager/smoke/manager-reconcile-redrive.smoke.ts index ca6742b9b..0c4896dd2 100644 --- a/implementations/manager/smoke/manager-reconcile-redrive.smoke.ts +++ b/implementations/manager/smoke/manager-reconcile-redrive.smoke.ts @@ -336,7 +336,7 @@ try { check("shutdown control reached its first accepted exact terminal", await until(() => shutdownFirstEntered, 20_000)); check("shutdown control reached service registration after the startup fence", await until(() => shutdownRegistrationEntered, 20_000)); let shutdownSettled = false; - const shutdownStopping = shutdownManager.stop().then(() => { shutdownSettled = true; }); + const shutdownStopping = shutdownManager.stop({ withAgents: true }).then(() => { shutdownSettled = true; }); await wait(150); check("stop waits for an accepted startup reconciliation terminal", shutdownSettled === false); releaseShutdownFirst(); @@ -350,8 +350,8 @@ try { shutdownManager = undefined; } finally { console.error = realError; - await shutdownManager?.stop().catch(() => {}); - await manager?.stop().catch(() => {}); + await shutdownManager?.stop({ withAgents: true }).catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); await callerNc?.drain().catch(() => callerNc?.close()); await observer?.drain().catch(() => observer?.close()); broker.kill("SIGTERM"); diff --git a/implementations/manager/smoke/manager-reconcile-startup.smoke.ts b/implementations/manager/smoke/manager-reconcile-startup.smoke.ts index 3bd57355e..0863ef207 100644 --- a/implementations/manager/smoke/manager-reconcile-startup.smoke.ts +++ b/implementations/manager/smoke/manager-reconcile-startup.smoke.ts @@ -578,14 +578,14 @@ try { (await readGoalResult(actx, ref))?.state === "failed", await readGoalResult(actx, ref)); } - await adopting.stop().catch(() => {}); + await adopting.stop({ withAgents: true }).catch(() => {}); } } finally { await callerNc?.drain().catch(() => callerNc?.close()); await observerNc?.drain().catch(() => observerNc?.close()); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); await delivery?.stop().catch(() => {}); await broker.stop().catch(() => {}); } diff --git a/implementations/manager/smoke/manager-service-invoke.smoke.ts b/implementations/manager/smoke/manager-service-invoke.smoke.ts index 369852ec1..b8bda95c1 100644 --- a/implementations/manager/smoke/manager-service-invoke.smoke.ts +++ b/implementations/manager/smoke/manager-service-invoke.smoke.ts @@ -110,7 +110,7 @@ try { console.log("1. resolveService: describe + fetch + recompile the full surface (no hand-imported schemas)"); const service = await resolveService(nc, space, MANAGER_ENDPOINT, caller, { deadlineMs: 10_000 }); - check("static reconciliation status advances the manager cluster revision", shipped.revision === 14, shipped); + check("static reconciliation status advances the manager cluster revision", shipped.revision === 15, shipped); check("the resolved surface matches the shipped cluster document", service.commands.size === shipped.commandCount && shipped.names.every((n) => service.commands.has(n)) && service.commands.has("status") && service.commands.has("spawn") && service.commands.has("despawn"), [...service.commands.keys()].sort()); const statusCmd = service.commands.get("status")!; check("status resolved: untargeted, read capability, recompiled contracts carry closure digests", diff --git a/implementations/manager/smoke/mutations/custodial-pty.json b/implementations/manager/smoke/mutations/custodial-pty.json index adbbfce43..dd0a40a0d 100644 --- a/implementations/manager/smoke/mutations/custodial-pty.json +++ b/implementations/manager/smoke/mutations/custodial-pty.json @@ -1,10 +1,10 @@ { "suite": "implementations/manager/smoke/custodial-pty.smoke.ts", - "guard": "the production pty runtime adopts a live proxy and a runtime without adopt still refuses by name", + "guard": "the production pty runtime adopts a live proxy that exposes release, and a runtime without adopt still refuses by name", "command": "pnpm --filter @cotal-ai/seat --filter @cotal-ai/core --filter @cotal-ai/workspace --filter @cotal-ai/manager build && pnpm smoke:custodial-pty", "proveWith": "node scripts/mutation-proof.mjs --config implementations/manager/smoke/mutations/custodial-pty.json", "why": [ - "The refusal cell is the earliest expectRed and is reached through production requireRuntimeAdopt. The production adopt cell is a later, separate observation of a live proxy from CustodialPtyRuntime.adopt." + "The refusal cell is the earliest expectRed and is reached through production requireRuntimeAdopt. The production adopt cell is a later observation of a live successor from CustodialPtyRuntime.adopt. A separate cell then requires that successor to expose release, which is the property proxy() adds." ], "mutations": [ { @@ -19,7 +19,7 @@ { "name": "M2 return a dead handle from production adopt", "file": "implementations/manager/src/runtime/custodial-pty.ts", - "find": "return adoptSeatSync(loadSeat(this.root, reference.id)) as unknown as AgentHandle;", + "find": "return this.proxy(adoptSeatSync(loadSeat(this.root, reference.id)));", "replace": "return { name: \"dead\", kind: \"pty\", pid: -1, status: () => \"exited\", stop: () => {}, interrupt: () => {}, attach: () => { throw new Error(\"no\"); } } as unknown as AgentHandle;", "expectRed": "production adopt returns a live proxy", "cell": "production adopt returns a live proxy", @@ -42,6 +42,15 @@ "expectRed": "status output accepts custodied", "cell": "status output accepts custodied", "note": "The compiled status output is the required enum pin. Dropping custodied must redden that cell." + }, + { + "name": "M5 return the unproxied seat from production adopt", + "file": "implementations/manager/src/runtime/custodial-pty.ts", + "find": "return this.proxy(adoptSeatSync(loadSeat(this.root, reference.id)));", + "replace": "return adoptSeatSync(loadSeat(this.root, reference.id)) as unknown as AgentHandle;", + "expectRed": "production adopt exposes release", + "cell": "production adopt exposes release", + "note": "The mutation keeps the live seat and drops only proxy(). pid and attach still match, so the live-proxy cell stays green; the release cell is what must red." } ] } diff --git a/implementations/manager/smoke/mutations/preserve-state-spare.json b/implementations/manager/smoke/mutations/preserve-state-spare.json new file mode 100644 index 000000000..3d7fa6bb1 --- /dev/null +++ b/implementations/manager/smoke/mutations/preserve-state-spare.json @@ -0,0 +1,31 @@ +{ + "suite": "implementations/manager/smoke/preserve-state.smoke.ts", + "guard": "plain Manager.stop leaves managed agents running; stop({ withAgents: true }) still reaps", + "command": "pnpm smoke:preserve-state", + "completionMarker": "PRESERVE-STATE SMOKE", + "proveWith": "node scripts/mutation-proof.mjs --config implementations/manager/smoke/mutations/preserve-state-spare.json", + "why": [ + "The suite imports Manager relatively (`../src/manager.js`), so a mutation on manager.ts is live with no dist in the path.", + "MUTATION 1 restores the #964 defect: a plain stop always tears down children. The named spare cells redden; the withAgents counterpart pair stays green, so the first red is the spare contract, not an earlier short-circuit." + ], + "mutations": [ + { + "name": "the defect restored: a plain stop still reaps every managed agent", + "file": "implementations/manager/src/manager.ts", + "find": " if (opts?.withAgents === true) await this.teardownManagedAgents();\n else {\n try {\n this.detachManagedAgents();\n } catch (e) {\n spareError = e;\n }\n }", + "replace": " await this.teardownManagedAgents();", + "expectRed": "normal stop leaves managed agents running", + "cell": "normal stop leaves managed agents running", + "note": "Issue #964 restored on the active path. The stop({ withAgents: true }) counterpart pair stays green; only the spare cells redden." + }, + { + "name": "the hang restored: pty spare optional-chains close and never calls release", + "file": "implementations/manager/src/manager.ts", + "find": " if (a.handle.kind === \"pty\") {\n const release = (a.handle as { release?: () => void }).release;\n if (typeof release !== \"function\") {\n throw new Error(\n `runtime \"${a.handle.kind}\" cannot spare agent \"${a.name}\": handle has no release()`,\n );\n }\n release.call(a.handle);\n }", + "replace": " (a.handle as Partial<{ close(): void }>).close?.();", + "expectRed": "plain stop of a pty seat calls release() and does not stop the child", + "cell": "plain stop of a pty seat calls release() and does not stop the child", + "note": "Restores the silent no-op. Fake spare cells stay green (they have no close). The pty spy cell reddens because release is never invoked. The live LegacyPtyRuntime cell also reddens because spare no longer throws." + } + ] +} diff --git a/implementations/manager/smoke/mutations/shipped-surface-pin.json b/implementations/manager/smoke/mutations/shipped-surface-pin.json index eccf44629..d50bf2125 100644 --- a/implementations/manager/smoke/mutations/shipped-surface-pin.json +++ b/implementations/manager/smoke/mutations/shipped-surface-pin.json @@ -25,10 +25,10 @@ "cell": "the resolved surface matches the shipped cluster document" }, { - "name": "M2 keep revision 13 after the fold adds static reconciliation on top of custody", + "name": "M2 keep revision 14 after the fold combines waitForExit with static reconciliation", "file": "implementations/manager/src/manager-service-contract.ts", - "find": " revision: 14,", - "replace": " revision: 13,", + "find": " revision: 15,", + "replace": " revision: 14,", "expectRed": "static reconciliation status advances the manager cluster revision", "cell": "static reconciliation status advances the manager cluster revision" } diff --git a/implementations/manager/smoke/persona-role-capability.smoke.ts b/implementations/manager/smoke/persona-role-capability.smoke.ts index 061bcc123..461439563 100644 --- a/implementations/manager/smoke/persona-role-capability.smoke.ts +++ b/implementations/manager/smoke/persona-role-capability.smoke.ts @@ -314,7 +314,9 @@ try { (async () => { await definer?.stop().catch(() => {}); await provisioner?.stop().catch(() => {}); - await mgr.stop().catch(() => {}); + // Reap leftover PTY seats. A plain stop leaves node-pty's waitpid worker holding this + // process open after the banner (Linux CI shard 2 hung that way after this suite). + await mgr.stop({ withAgents: true }).catch(() => {}); })(), sleep(10_000), ]); @@ -324,3 +326,4 @@ try { await Promise.all([drain(process.stdout), drain(process.stderr)]); process.exit(code); } +process.exit(code); diff --git a/implementations/manager/smoke/preserve-state.smoke.ts b/implementations/manager/smoke/preserve-state.smoke.ts index 8e54731f4..59dd482c7 100644 --- a/implementations/manager/smoke/preserve-state.smoke.ts +++ b/implementations/manager/smoke/preserve-state.smoke.ts @@ -19,6 +19,7 @@ import { import { agentCredsDir, agentLifecycleSecretFilePaths } from "@cotal-ai/workspace"; import { Manager, type ManagerResumeIdentity, type ManagerResumeAgent, type ManagerResumeInventory } from "../src/manager.js"; import { MAX_RESUME_CONTROL_BYTES } from "../src/resume.js"; +import { LegacyPtyRuntime } from "../src/runtime/pty.js"; let failures = 0; function check(label: string, condition: boolean, extra?: unknown): void { @@ -1010,7 +1011,9 @@ let openInventory: ManagerResumeAgent; retainedAuthority = { ...retainedAuthority, allowSubscribe: ["general"] }; } -// Regression: active-mode stop remains the existing destructive shutdown path. +// Regression: active-mode stop leaves managed agents running (#964). The previous reap is +// `stop({ withAgents: true })`. Both halves stay covered: a mutant that restores always-reap +// reddens the spare cells; a mutant that drops the explicit reap reddens the counterpart pair. { const manager = managerWith((name) => fakeHandle(name)); const handle = fakeHandle("normal"); @@ -1019,8 +1022,115 @@ let openInventory: ManagerResumeAgent; let deprovisions = 0; (manager as unknown as { deprovision: () => Promise }).deprovision = async () => { deprovisions++; }; await manager.stop(); - check("normal stop still hard-stops managed agents", handle.stops === 1, handle.stops); - check("normal stop still deprovisions managed agents", deprovisions === 1, deprovisions); + check("normal stop leaves managed agents running", handle.stops === 0, handle.stops); + check("normal stop does not deprovision managed agents", deprovisions === 0, deprovisions); +} + +{ + let releases = 0; + let stops = 0; + let alive = true; + const exits = new Set<() => void>(); + const handle: AgentHandle = { + name: "pty-spare", + kind: "pty", + status: () => (alive ? "running" : "exited"), + stop: () => { + stops++; + alive = false; + for (const fn of exits) fn(); + }, + waitForExit: () => alive + ? new Promise((resolve) => { + const done = (): void => { exits.delete(done); resolve(); }; + exits.add(done); + }) + : Promise.resolve(), + interrupt: () => {}, + attach: () => ({ + cols: 80, + rows: 24, + backlog: () => Buffer.alloc(0), + onData: () => () => {}, + onExit: (fn) => { exits.add(fn); return () => exits.delete(fn); }, + write: () => {}, + resize: () => {}, + }), + release: () => { releases++; }, + } as AgentHandle; + const manager = managerWith((name) => fakeHandle(name)); + const map = (manager as unknown as { agents: Map }).agents; + map.set("pty-spare", managed("pty-spare", "pty_spare_id", handle, "persona")); + await manager.stop(); + check("plain stop of a pty seat calls release() and does not stop the child", releases === 1 && stops === 0, { releases, stops }); +} + +{ + // Linux can still construct LegacyPtyRuntime (its own M1 residual). Injecting it measures + // the spare refusal; it does not exercise the macOS or Windows node-pty backend. + const rt = new LegacyPtyRuntime(); + const handle = rt.spawn( + "legacy-spare", + { command: process.execPath, args: ["-e", "setInterval(()=>{},1000)"], env: { PATH: process.env.PATH ?? "" } }, + process.cwd(), + ); + try { + const manager = managerWith((name) => fakeHandle(name)); + let leases = 0; + const ep = (manager as unknown as { ep: { releaseManagerLease: () => Promise } }).ep; + const previousRelease = ep.releaseManagerLease.bind(ep); + ep.releaseManagerLease = async () => { + leases++; + await previousRelease(); + }; + const sibling = fakeHandle("sibling"); + const agents = (manager as unknown as { agents: Map }).agents; + agents.set("legacy-spare", managed("legacy-spare", "legacy_spare_id", handle, "persona")); + agents.set("sibling", managed("sibling", "sibling_id", sibling, "persona")); + let threw = ""; + try { + await manager.stop(); + } catch (e) { + threw = (e as Error).message; + } + const childAlive = (() => { + try { + if (handle.pid === undefined) return false; + process.kill(handle.pid, 0); + return true; + } catch { + return false; + } + })(); + check( + "legacy pty spare throws rather than no-op or kill", + /cannot spare agent "legacy-spare"/.test(threw) && /in-process node-pty cannot release/.test(threw), + threw, + ); + check("legacy pty spare left the child running", handle.status() === "running" && childAlive, { status: handle.status(), pid: handle.pid, childAlive }); + check( + "a pty spare refusal does not suppressCleanup on the refused seat", + (agents.get("legacy-spare") as { suppressCleanup?: boolean } | undefined)?.suppressCleanup !== true, + (agents.get("legacy-spare") as { suppressCleanup?: boolean } | undefined)?.suppressCleanup, + ); + check("a later seat still spares after a pty refusal", !agents.has("sibling") && sibling.stops === 0, { remaining: [...agents.keys()], siblingStops: sibling.stops }); + check("a pty spare refusal still releases the manager lease", leases === 1, leases); + } finally { + handle.stop({ graceful: false }); + await handle.waitForExit?.(); + } +} + +{ + const manager = managerWith((name) => fakeHandle(name)); + const handle = fakeHandle("reap"); + const map = (manager as unknown as { agents: Map }).agents; + map.set("reap", managed("reap", "reap_id", handle, "persona")); + let deprovisions = 0; + (manager as unknown as { deprovision: () => Promise }).deprovision = async () => { deprovisions++; }; + await manager.stop({ withAgents: true }); + check("stop({ withAgents: true }) hard-stops managed agents", handle.stops === 1, handle.stops); + check("stop({ withAgents: true }) deprovisions managed agents", deprovisions === 1, deprovisions); } // Regression: an accepted control stop frees the slot AT ONCE — `stop` replying ✓ has to mean `ps` diff --git a/implementations/manager/smoke/renewal-terminal-race.smoke.ts b/implementations/manager/smoke/renewal-terminal-race.smoke.ts index a6f13c37c..f3a4ef79f 100644 --- a/implementations/manager/smoke/renewal-terminal-race.smoke.ts +++ b/implementations/manager/smoke/renewal-terminal-race.smoke.ts @@ -208,7 +208,7 @@ const M = mgr as unknown as { retiring: Map; renewManagedStaticCred(agent: Agent): Promise; renewDaemonCreds(): Promise; - despawnAuthorized(agent: Agent, graceful: boolean, trackNonAdmin: boolean): { ok: boolean }; + despawnAuthorized(agent: Agent, graceful: boolean, trackNonAdmin: boolean, requireAuthoritativeExit?: boolean): Promise<{ ok: boolean }>; }; async function openLifecycleView(alias: string, actor: string, uid: string): Promise<{ @@ -302,7 +302,7 @@ try { let terminalEntered = false; if (scenario.terminal === "despawn") - terminalEntered = M.despawnAuthorized(agent, false, true).ok; + terminalEntered = (await M.despawnAuthorized(agent, false, true)).ok; else terminalEntered = handles.get(name)?.exitNaturally() === true && !M.agents.has(name); check(`${name}: ${scenario.terminal} enters the terminal path`, terminalEntered); diff --git a/implementations/manager/smoke/start-model-preflight.smoke.ts b/implementations/manager/smoke/start-model-preflight.smoke.ts index ea171b25c..f26507471 100644 --- a/implementations/manager/smoke/start-model-preflight.smoke.ts +++ b/implementations/manager/smoke/start-model-preflight.smoke.ts @@ -26,10 +26,9 @@ * 8. MISSED-EXIT REAP (issue #159 B1, review hardening) — an agent that joins presence (→ started) then * dies just as watchExit subscribes (onExit never fires for a late subscriber) is still reaped: * watchExit re-checks status() right after subscribing and removes the leaked agent. - * 9. SHUTDOWN TEARDOWN (issue #159 B2, review blocker) — Manager.stop() reaps EVERY managed agent (hard- - * stops the child + clears the map), not just the lease/endpoints, so a manager shutdown doesn't - * orphan their footprints. (The broker-side deprovision is a no-op in open mode — proven under auth in - * deprovision-agent-auth.smoke — so this covers the stop() wiring.) + * 9. SHUTDOWN TEARDOWN (issue #159 B2 / #964) — Manager.stop({ withAgents: true }) reaps EVERY managed agent (hard- + * stops the child + clears the map), not just the lease/endpoints. Bare stop() leaves agents (#964); + * cell 9b is the spare path on the same fake runtime. * 10. BEST-EFFORT TEARDOWN (issue #159 B2, review re-check) — teardownManagedAgents() attempts every * child and empties the map even when one hard-stop throws, so stop() can't exit leaving footprints. * 11. BEST-EFFORT STOP ON REAP (issue #159 B2, review round 5) — stopHandle() (the single stop chokepoint) @@ -70,7 +69,7 @@ const connectors = onWin ? [claudeConnector, opencodeConnector] : [claudeConnect const workspaceRoot = mkdtempSync(join(tmpdir(), "cotal-start-ws-")); const agentsDir = join(workspaceRoot, ".cotal", "agents"); mkdirSync(agentsDir, { recursive: true }); -for (const n of ["reject1", "rec1", "rec2", "rrec1", "rrec2", "norsm1", "norsm2", "dead1", "dead2", "missed1", "shut1", "shut2", "lease1", "lease2", "unc1"]) writeFileSync(join(agentsDir, `${n}.md`), `---\nname: ${n}\n---\n`); +for (const n of ["reject1", "rec1", "rec2", "rrec1", "rrec2", "norsm1", "norsm2", "dead1", "dead2", "missed1", "shut1", "shut2", "spare1", "lease1", "lease2", "unc1"]) writeFileSync(join(agentsDir, `${n}.md`), `---\nname: ${n}\n---\n`); // rec3 carries an explicit access policy — its frontmatter ACL must thread through to LaunchOpts. writeFileSync(join(agentsDir, "rec3.md"), `---\nname: rec3\nsubscribe: [team]\nallowSubscribe: [team, team.>]\nallowPublish: [team]\n---\n`); const mgr = new Manager({ space: "smoke", servers: undefined, runtime: "pty", workspaceRoot }); @@ -441,11 +440,11 @@ registry.register(recNoResumeCon); check("missed-exit: watchExit status-check reaps the leaked agent (not left in the map)", agentCount() === before, agentCount()); } -// 9 — SHUTDOWN TEARDOWN (#159 B2, review blocker): Manager.stop() must reap every managed agent — not just -// release the lease + stop endpoints — or a manager Ctrl-C/SIGTERM orphans their footprints (creds/durables/ -// ACL). Spawn two survivors, then stop(): both children must be hard-stopped and the managed-agents map -// emptied. Broker deprovision is a no-op in open mode (its footprint teardown is proven under auth in -// deprovision-agent-auth.smoke); this proves the stop() wiring. Stub ep/attach so stop() has no live mesh. +// 9 — SHUTDOWN TEARDOWN (#159 B2 / #964): Manager.stop({ withAgents: true }) must reap every managed +// agent — not just release the lease + stop endpoints. Spawn two survivors, then stop with the +// explicit reap: both children must be hard-stopped and the managed-agents map emptied. Bare stop() +// is the #964 spare path, proven in bin/smoke/manager-stop-reaps-agents.smoke.ts. Broker deprovision +// is a no-op in open mode. Stub ep/attach so stop() has no live mesh. { agentsMap().clear(); const stopped: string[] = []; @@ -465,12 +464,36 @@ registry.register(recNoResumeCon); await mgr.startAgent({ name: "shut1", agent: "smoke-rec" }); await mgr.startAgent({ name: "shut2", agent: "smoke-rec" }); check("shutdown: two managed agents present before stop", agentCount() >= 2, agentCount()); - await mgr.stop(); + await mgr.stop({ withAgents: true }); check("shutdown: stop() hard-stops every managed child", stopped.includes("shut1") && stopped.includes("shut2"), stopped); check("shutdown: stop() proves every managed child exited before releasing manager authority", exitProofs.has("shut1") && exitProofs.has("shut2"), [...exitProofs]); check("shutdown: stop() empties the managed-agents map (no orphaned footprint)", agentCount() === 0, agentCount()); } +// 9b — SPARE PATH (#964): a plain Manager.stop() empties the table and does not hard-stop the child. +{ + agentsMap().clear(); + const stopped: string[] = []; + const exited = new Set(); + const liveHandle = (name: string): AgentHandle => ({ + name, kind: "fake", status: () => exited.has(name) ? "exited" : "running", + stop: () => { stopped.push(name); exited.add(name); }, + waitForExit: async () => { if (!exited.has(name)) throw new Error("still running"); }, + interrupt: () => {}, attach: () => fakeSession, + }); + (mgr as unknown as { runtime: { kind: string; spawn: (n: string, s: LaunchSpec) => AgentHandle } }).runtime = { + kind: "fake", + spawn: (name) => liveHandle(name), + }; + (mgr as unknown as { ep: unknown }).ep = fakeEp({ releaseManagerLease: async () => {}, stop: async () => {} }); + (mgr as unknown as { attach: { stop: () => Promise } }).attach = { stop: async () => {} }; + await mgr.startAgent({ name: "spare1", agent: "smoke-rec" }); + check("spare: one managed agent present before stop", agentCount() === 1, agentCount()); + await mgr.stop(); + check("spare: a plain stop does not hard-stop the child", !stopped.includes("spare1"), stopped); + check("spare: a plain stop empties the managed-agents map", agentCount() === 0, agentCount()); +} + // 10 — BEST-EFFORT TEARDOWN (#159 B2, review re-check): `teardownManagedAgents()` is the helper stop() // reaps through. Assert it directly: it hard-stops every child + empties the map even when one stop // throws, touching no lease/endpoint. diff --git a/implementations/manager/smoke/supervise-restart.smoke.ts b/implementations/manager/smoke/supervise-restart.smoke.ts index d4e1329cd..a16e30889 100644 --- a/implementations/manager/smoke/supervise-restart.smoke.ts +++ b/implementations/manager/smoke/supervise-restart.smoke.ts @@ -274,7 +274,7 @@ try { await seatNc?.drain().catch(() => seatNc?.close()); await runnerNc?.drain().catch(() => runnerNc?.close()); await userMgr?.stop().catch(() => {}); - await manager?.stop().catch(() => {}); + await manager?.stop({ withAgents: true }).catch(() => {}); await delivery?.stop().catch(() => {}); await broker.stop().catch(() => {}); if (prevHome === undefined) delete process.env.COTAL_HOME; diff --git a/implementations/manager/src/commands.ts b/implementations/manager/src/commands.ts index 0d64be34c..def41665d 100644 --- a/implementations/manager/src/commands.ts +++ b/implementations/manager/src/commands.ts @@ -317,8 +317,9 @@ async function runManager(args: ParsedArgs, defaultRuntime: RuntimeMode): Promis `\n console: ${mgr.consoleUrl}` + c.dim("\n spawn: cotal spawn --detach · stop: cotal stop --name (Ctrl-C to shut down)"), ); - // Register shutdown handlers before any spawning, so a Ctrl-C during the (possibly slow, - // staggered) boot tears the manager and its spawned teammates down rather than orphaning them. + // Register shutdown handlers before any spawning. SIGTERM/SIGINT spare managed agents (#964); + // `cotal down --with-agents` is the reap. A Ctrl-C during boot therefore leaves already-started + // seats running rather than taking them with the supervisor. const shutdown = () => void mgr.stop() .then(() => { releasePidRecord(); diff --git a/implementations/manager/src/index.ts b/implementations/manager/src/index.ts index f286d82f4..a6c00ba7a 100644 --- a/implementations/manager/src/index.ts +++ b/implementations/manager/src/index.ts @@ -13,6 +13,7 @@ export { type ManagerPreservationPlan, type ManagerPreserveOptions, type ManagerPreserveResult, + type ManagerStopOptions, } from "./manager.js"; export { parseResumeControlArgs, diff --git a/implementations/manager/src/manager-service-contract.ts b/implementations/manager/src/manager-service-contract.ts index 51d9cb13a..e63c4a875 100644 --- a/implementations/manager/src/manager-service-contract.ts +++ b/implementations/manager/src/manager-service-contract.ts @@ -290,7 +290,7 @@ const SPAWN_OUTPUT_SCHEMA = { const GRACEFUL_INPUT_SCHEMA = { type: "object", additionalProperties: false, - properties: { graceful: { type: "boolean" } }, + properties: { graceful: { type: "boolean" }, waitForExit: { type: "boolean" } }, } as const; const STOP_OUTPUT_SCHEMA = { @@ -769,10 +769,14 @@ export const MANAGER_STATUS_CONTRACT: { input: CompiledContract; output: Compile * alongside `legacy`. A changed output contract is a changed described surface even though * the command name is unchanged. * - * 14 = manager `status` adds static reconciliation state. Its output digest changed again, so - * cached revision-13 descriptions cannot name the new required output contract. This is a + * 14 = `despawn` / `stop` accept optional `waitForExit`. The mass-reap caller passes it so + * proof of exit is not derived from `graceful`. A changed input contract is a changed described + * surface even though the command names are unchanged. + * + * 15 = manager `status` adds static reconciliation state. Its output digest changed again, so + * cached revision-14 descriptions cannot name the new required output contract. This is a * second, independent output change landing on the same command as 13, so it cannot fold into - * it: a caller holding a revision-13 descriptor would be told the surface it already knows. */ + * it: a caller holding a revision-14 descriptor would be told the surface it already knows. */ export function managerClusterDocument(): { urn: string; revision: number; @@ -790,7 +794,7 @@ export function managerClusterDocument(): { } { return { urn: MANAGER_CLUSTER_URN, - revision: 14, + revision: 15, attributes: [], events: [], commands: ROWS.map((r) => ({ diff --git a/implementations/manager/src/manager.ts b/implementations/manager/src/manager.ts index 0c35b13db..937d5c157 100644 --- a/implementations/manager/src/manager.ts +++ b/implementations/manager/src/manager.ts @@ -472,6 +472,14 @@ export interface ManagerPreserveOptions { persistInventory(inventory: ManagerResumeInventory): Promise; } +/** Options for {@link Manager.stop}. Preservation still always stops retained children. */ +export interface ManagerStopOptions { + /** Reap every managed agent (hard-stop + deprovision). Default false leaves them running on + * runtimes that can drop a retaining handle without killing the child (Linux custodial pty, + * tmux/cmux/orca/herdr). In-process node-pty cannot, and a plain stop of those seats throws. */ + withAgents?: boolean; +} + export interface ManagerResumeResult { ok: boolean; agents: Array<{ name: string; reply: ControlReply }>; @@ -845,6 +853,9 @@ export class Manager { private staticLifecycleEvict?: (principal: string) => Promise; private readonly preserveStopTimeoutMs: number; private readonly agents = new Map(); + /** Seats released by a sparing {@link stop} so their runtime handles are not GC'd while this + * process is still exiting. Not listed by `ps`. Not stopped. Not deprovisioned. */ + private readonly detached: ManagedAgent[] = []; /** Names whose spawn is in flight (reserved synchronously before the provision await) — counted * toward the ceiling so two concurrent same-name spawns can't both pass the gate (P4a). */ private readonly reserved = new Set(); @@ -1907,8 +1918,42 @@ export class Manager { }; } - /** Tear down every managed agent's footprint on a graceful {@link stop} (#159 B2). A manager exit is - * a mass agent-exit, and without this its agents' footprints (creds files + `dm_`/`dlv_` durables + ACL + /** Drop every managed seat from the table without stopping or deprovisioning it (#964). A later + * `ps` is empty; the OS processes and their footprints stay. A pty handle must implement + * `release()`: Linux custodial pty closes the unix socket and leaves the child; in-process + * node-pty (`LegacyPtyRuntime`) throws because dropping the master would kill the child. + * tmux/cmux/orca/herdr retain no fd, so they spare without a release. `suppressCleanup` is set + * only after that release succeeds, so a refusal stays deprovisionable. One refusal does not + * skip later seats. Handles stay on {@link detached} so a still-running manager does not GC + * them. When this process itself exits, a leftover PTY master close may still SIGHUP those + * children. */ + private detachManagedAgents(): void { + const managed = [...this.agents.values()]; + const failures: string[] = []; + for (const a of managed) { + try { + if (a.handle.kind === "pty") { + const release = (a.handle as { release?: () => void }).release; + if (typeof release !== "function") { + throw new Error( + `runtime "${a.handle.kind}" cannot spare agent "${a.name}": handle has no release()`, + ); + } + release.call(a.handle); + } + a.suppressCleanup = true; + this.agents.delete(a.name); + this.detached.push(a); + } catch (e) { + failures.push(`${a.name}: ${(e as Error).message}`); + } + } + if (failures.length) + throw new Error(`manager stop could not spare every seat: ${failures.join("; ")}`); + } + + /** Tear down every managed agent's footprint on {@link stop} with `withAgents: true` (#159 B2). A + * manager exit is a mass agent-exit, and without this its agents' footprints (creds files + `dm_`/`dlv_` durables + ACL * rows) would orphan exactly as the per-agent exit path prevents. Hard-stop each child (an exit has no * time for the graceful grace window) and AWAIT its deprovision — bounded per agent (`withTimeout`) and * best-effort (`allSettled` + a loud log), so one slow/failed teardown can neither hang nor abort exit. @@ -1965,7 +2010,7 @@ export class Manager { throw new Error(`manager preservation shutdown incomplete: ${failures.join("; ")}`); } - async stop(): Promise { + async stop(opts?: ManagerStopOptions): Promise { this.staticReconcileStopping = true; const starting = this.startTask; for (const item of this.staticReconcileItems.values()) { @@ -1978,8 +2023,16 @@ export class Manager { if (this.leaseTimer) clearInterval(this.leaseTimer); if (this.credRenewTimer) clearInterval(this.credRenewTimer); if (this.sessionKeyRenewTimer) clearInterval(this.sessionKeyRenewTimer); + let spareError: unknown; if (this.maintenanceState === "active" && !this.resumeRequired) { - await this.teardownManagedAgents(); // normal shutdown stays destructive (#159 B2) + if (opts?.withAgents === true) await this.teardownManagedAgents(); + else { + try { + this.detachManagedAgents(); + } catch (e) { + spareError = e; + } + } } else { // A signal after a partial preservation must never fall back into destructive teardown. await this.stopRetainedAgentsOnExit(); @@ -2000,6 +2053,7 @@ export class Manager { await this.stopSessionPlane(); await this.ep.stop(); await this.attach.stop(); + if (spareError) throw spareError; } /** @@ -2501,7 +2555,8 @@ export class Manager { const a = targetAgent(ctx); const denied = await this.authorizeNamed(a, callerOf(ctx), await this.epAnyModeAdmin(ctx)); if (denied) throw new EpEnvelopeError("permission-denied", denied); - return unwrap(this.despawnAuthorized(a, args(ctx).graceful !== false, true)); + const graceful = args(ctx).graceful !== false; + return unwrap(await this.despawnAuthorized(a, graceful, true, args(ctx).waitForExit === true)); }), attach: (ctx) => this.serveGated(ctx, async () => { const a = targetAgent(ctx); @@ -5103,7 +5158,7 @@ export class Manager { const name = String(args.name ?? "").trim(); const a = this.agents.get(name); if (!a) return { ok: false, error: `no agent "${name}"` }; - return this.despawnCore(a, caller, admin, args.graceful !== false); + return this.despawnCore(a, caller, admin, args.graceful !== false, args.waitForExit === true); } /** The ONE named-terminal core both doors share (P2 item 1, checklist 8): the ctl named `stop` @@ -5111,18 +5166,27 @@ export class Manager { * ({@link authorizeNamed}: own-child / owner-domain on privileged, any on admin), stop, track. * The ep door runs the SAME two pieces separately so a policy denial surfaces as the §13.3 * `permission-denied` (never a generic failure). */ - private async despawnCore(a: ManagedAgent, caller: string, admin: boolean, graceful: boolean): Promise { + private async despawnCore(a: ManagedAgent, caller: string, admin: boolean, graceful: boolean, requireAuthoritativeExit = false): Promise { const denied = await this.authorizeNamed(a, caller, admin); if (denied) return { ok: false, error: denied }; - return this.despawnAuthorized(a, graceful, !admin); + return this.despawnAuthorized(a, graceful, !admin, requireAuthoritativeExit); } /** The post-authorization terminal effect (both doors). `trackNonAdmin` mirrors the ctl door's - * `trackStoppedHandle(a, !admin)` disposition. */ - private despawnAuthorized(a: ManagedAgent, graceful: boolean, trackNonAdmin: boolean): ControlReply { + * `trackStoppedHandle(a, !admin)` disposition. `requireAuthoritativeExit` is passed by the + * mass-reap caller (`cotal down --with-agents` sends `waitForExit: true`); it is not derived + * from `graceful`. Ordinary `cotal stop` / `cotal_despawn` stay acceptance-not-exit. */ + private async despawnAuthorized(a: ManagedAgent, graceful: boolean, trackNonAdmin: boolean, requireAuthoritativeExit = false): Promise { this.stopHandle(a, graceful); - this.trackStoppedHandle(a, trackNonAdmin); + this.trackStoppedHandle(a, trackNonAdmin, requireAuthoritativeExit); void this.cancelAgentGoal(a.name, graceful ? "graceful" : "terminate"); // M4: cancel a live spawn goal + if (requireAuthoritativeExit) { + try { + await this.awaitHandleExit(a.handle); + } catch (e) { + return { ok: false, error: `stop requested but exit was not proven: ${(e as Error).message}` }; + } + } return { ok: true, data: { name: a.name, stopped: true, graceful } }; } diff --git a/implementations/manager/src/runtime/custodial-pty.ts b/implementations/manager/src/runtime/custodial-pty.ts index 26235833a..8e6ebda3c 100644 --- a/implementations/manager/src/runtime/custodial-pty.ts +++ b/implementations/manager/src/runtime/custodial-pty.ts @@ -29,13 +29,23 @@ export class CustodialPtyRuntime implements Runtime { spec: { command: spec.command, args: spec.args, env: spec.env ?? {}, confirm: Boolean(spec.confirm) }, cwd, }); - return adoptSeatSync(rec) as unknown as AgentHandle; + return this.proxy(adoptSeatSync(rec)); } adopt(reference: RuntimeReference): AgentHandle { if (process.platform !== "linux") throw unsupportedTransport(); if (reference.kind !== "pty") throw new Error(`cannot adopt runtime kind "${reference.kind}" with pty`); - return adoptSeatSync(loadSeat(this.root, reference.id)) as unknown as AgentHandle; + return this.proxy(adoptSeatSync(loadSeat(this.root, reference.id))); + } + + /** SeatHandle.close() is erased by the AgentHandle cast. Expose it as release() so a spare + * stop can drop the unix socket without optional-chaining a missing method. */ + private proxy(seat: ReturnType): AgentHandle { + return Object.assign(seat as unknown as AgentHandle, { + release: () => { + seat.close(); + }, + }); } } diff --git a/implementations/manager/src/runtime/pty.ts b/implementations/manager/src/runtime/pty.ts index d18729733..5256bb8e2 100644 --- a/implementations/manager/src/runtime/pty.ts +++ b/implementations/manager/src/runtime/pty.ts @@ -103,7 +103,7 @@ export class LegacyPtyRuntime implements Runtime { for (const fn of exitSubs) fn(); }); - return { + const handle: AgentHandle = { name, kind: "pty", pid: proc.pid, @@ -207,6 +207,16 @@ export class LegacyPtyRuntime implements Runtime { }, }), }; + return Object.assign(handle, { + // node-pty owns the master fd in this process. There is no API that drops that + // fd without closing the session, and closing it kills the child. Spare-stop + // therefore refuses rather than no-op or kill. + release: () => { + throw new Error( + `runtime "pty" cannot spare agent "${name}": in-process node-pty cannot release the master without killing the child`, + ); + }, + }); } adopt(_reference: RuntimeReference): AgentHandle { diff --git a/package.json b/package.json index 1db8edee6..11763c078 100644 --- a/package.json +++ b/package.json @@ -219,6 +219,7 @@ "smoke:attach-auth-root": "tsx bin/smoke/attach-auth-root.smoke.ts", "smoke:manager-two-root-renewal": "pnpm --filter cotal-ai... build && tsx bin/smoke/manager-two-root-renewal.smoke.ts", "smoke:manager-stop-reap": "pnpm --filter cotal-ai... build && tsx bin/smoke/manager-stop-reaps-agents.smoke.ts", + "smoke:manager-stop-spare-guard": "tsx bin/smoke/manager-stop-spare-guard.smoke.ts", "smoke:control-transport-dial": "tsx bin/smoke/control-transport-dial.smoke.ts && pnpm smoke:control-transport-cleanup", "smoke:supervise-remote-dial": "tsx bin/smoke/supervise-remote-dial.smoke.ts", "smoke:control-transport-cleanup": "tsx bin/smoke/fixtures/control-transport-cleanup.mts", @@ -494,6 +495,7 @@ "smoke:mint-provision:auth": "tsx implementations/cli/smoke/mint-provision.smoke.ts", "smoke:down-lease-tombstone": "tsx implementations/cli/smoke/down-lease-tombstone.smoke.ts", "smoke:down-target": "tsx implementations/cli/smoke/down-target.smoke.ts", + "smoke:down-partial-reap": "tsx implementations/cli/smoke/down-partial-reap.smoke.ts", "smoke:flag-inventory": "tsx bin/smoke/flag-inventory.smoke.ts", "smoke:persona-agent": "tsx bin/smoke/persona-agent.smoke.ts", "smoke:start-overrides": "tsx implementations/manager/smoke/start-overrides.smoke.ts", diff --git a/packages/workspace/smoke/fixtures/presence-render-sinks.json b/packages/workspace/smoke/fixtures/presence-render-sinks.json index 41e889b81..1c30ecf3a 100644 --- a/packages/workspace/smoke/fixtures/presence-render-sinks.json +++ b/packages/workspace/smoke/fixtures/presence-render-sinks.json @@ -1,10 +1,10 @@ { "expected": { - "total": 312, + "total": 315, "honest-text": 32, "presence-only-glyph/count": 41, "command-ack": 10, - "non-render/control": 229 + "non-render/control": 232 }, "renderers": [ { @@ -2332,6 +2332,27 @@ "anchor": "`${v.status.state}, holder ${v.status.holder}, epoch ${v.status.epoch}`", "class": "non-render/control", "rationale": "Workflow-run status view: the run record's own state (running, completed, released), holder and epoch, echoed from the run-status reply; not an agent's presence." + }, + { + "path": "implementations/cli/src/commands/down.ts", + "kind": "derived-output", + "anchor": "`the manager seat list could not be read (${reply.error ?? \"ps failed\"})`", + "class": "non-render/control", + "rationale": "Manager ps control-path failure when the seat list cannot be read, not a rendered presence status." + }, + { + "path": "implementations/cli/src/commands/down.ts", + "kind": "derived-output", + "anchor": "`${row.name}: ${stopped.error ?? \"stop failed\"}`", + "class": "non-render/control", + "rationale": "Aggregated per-seat lifecycle-stop failure from the manager stop command, not an agent progress claim." + }, + { + "path": "implementations/cli/src/commands/down.ts", + "kind": "derived-output", + "anchor": "` ${bits.join(\" \u00b7 \")}`", + "class": "non-render/control", + "rationale": "Leftover/would-reap seat inventory row: name, mode, pid, agent, cwd, and optional manager-reported process status, not an observed work-progress claim." } ] } diff --git a/packages/workspace/smoke/mutations/presence-render-census.json b/packages/workspace/smoke/mutations/presence-render-census.json index be6938f9b..4b7877969 100644 --- a/packages/workspace/smoke/mutations/presence-render-census.json +++ b/packages/workspace/smoke/mutations/presence-render-census.json @@ -77,8 +77,8 @@ { "name": "M8 expected AST candidate total drifts by one", "file": "packages/workspace/smoke/fixtures/presence-render-sinks.json", - "find": " \"total\": 312,", - "replace": " \"total\": 309,", + "find": " \"total\": 315,", + "replace": " \"total\": 314,", "expectRed": "presence render census count drifted" } ]