diff --git a/.changeset/tidy-tigers-detach.md b/.changeset/tidy-tigers-detach.md new file mode 100644 index 000000000..63bcf6003 --- /dev/null +++ b/.changeset/tidy-tigers-detach.md @@ -0,0 +1,8 @@ +--- +"cotal-ai": minor +"@cotal-ai/cli": minor +"@cotal-ai/connector-core": minor +"@cotal-ai/workspace": minor +--- + +Make `cotal up` detach by default and keep a durable self-hosted mesh record so stopped stacks remain discoverable and restartable from their recorded root. diff --git a/README.md b/README.md index ece548b09..d44b58265 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Prefer your agent to do it? Point it at . Setup gets your machine ready and **starts nothing**. Then: ```bash -cotal up --detach # start the mesh +cotal up # start the detached mesh cotal spawn # put your agent on it and talk to it (Ctrl-C to leave) cotal web # watch it in the browser cotal down # stop everything diff --git a/bin/smoke/ci-suites.txt b/bin/smoke/ci-suites.txt index 8a9679b04..f77f27789 100644 --- a/bin/smoke/ci-suites.txt +++ b/bin/smoke/ci-suites.txt @@ -530,6 +530,8 @@ smoke:claude-boot-wake # Control commands resolve ws:// / wss:// meshes too. Exercise the real CLI against one broker's # websocket and TCP listeners so raw node-transport regressions name their `wsconnect` refusal. smoke:control-transport-dial +# Windows detached stacks must either escape the invoking job with CREATE_BREAKAWAY_FROM_JOB or +# fail before launch and point to --foreground. Pure branch coverage runs on every CI platform. # A manager restart killed mid-barrier left the issuance gate frozen, and the successor refused # SPEC 13.8 until an operator ran reconcile-gate (#783 item 3 / #871). Boot now completes that # same dead registration when the freeze-holder is gone under a complete CONNZ sweep. Appended @@ -543,3 +545,4 @@ smoke:web-console-auth smoke:no-implicit-general smoke:session-channels smoke:jcode-private-lifecycle +smoke:windows-detached-spawn diff --git a/bin/smoke/flag-inventory.smoke.ts b/bin/smoke/flag-inventory.smoke.ts index 2ccdd6c72..0dce5f9d4 100644 --- a/bin/smoke/flag-inventory.smoke.ts +++ b/bin/smoke/flag-inventory.smoke.ts @@ -30,7 +30,7 @@ const GOLDEN: Record {\n await stop(child);", + "replace": " return rethrowAfterDetachedCleanup(new Error(\"cleanup replaced primary\"), async () => {\n await stop(child);", + "expectRed": "C09 an unbound-listener teardown failure preserves the primary error and attaches the cleanup failure", + "cell": "C09 an unbound-listener teardown failure preserves the primary error and attaches the cleanup failure", + "note": "IN C09: EPERM is attached to a replacement error instead of the original bind error. IN C10: the successful ESRCH path throws the replacement error instead of the original bind error. OUT C08: pid retention on EPERM is unchanged. OUT C01-C07 and C11-C16: they do not read the unbound wrapper's primary error." + }, + { + "name": "not-ready cleanup removes pid on non-ESRCH signal failure", + "file": "implementations/cli/src/commands/up.ts", + "find": " child.kill(\"SIGTERM\");\n removePid?.();", + "replace": " try { child.kill(\"SIGTERM\"); }\n finally { removePid?.(); }", + "expectRed": "C11 a not-ready non-ESRCH kill failure keeps the bound-listener pidfile", + "cell": "C11 a not-ready non-ESRCH kill failure keeps the bound-listener pidfile", + "note": "IN C11: EPERM now runs pid removal in finally. OUT C12: rethrowAfterDetachedCleanup still attaches EPERM to the original readiness error. OUT C13: ESRCH returns normally and both forms remove stale pid state. OUT C01-C10 and C14-C16: they do not execute the not-ready wrapper's pid-removal ordering." + }, + { + "name": "not-ready cleanup replaces the readiness failure", + "file": "implementations/cli/src/commands/up.ts", + "find": "async function rethrowNotReadyListenerFailure(\n child: ChildProcess,\n primary: unknown,\n removePid?: () => void,\n): Promise {\n return rethrowAfterDetachedCleanup(primary, () => {", + "replace": "async function rethrowNotReadyListenerFailure(\n child: ChildProcess,\n primary: unknown,\n removePid?: () => void,\n): Promise {\n return rethrowAfterDetachedCleanup(new Error(\"cleanup replaced primary\"), () => {", + "expectRed": "C12 a not-ready kill failure preserves the readiness error and attaches the signal failure", + "cell": "C12 a not-ready kill failure preserves the readiness error and attaches the signal failure", + "note": "IN C12: EPERM is attached to a replacement error instead of the original readiness error. IN C13: the successful ESRCH path throws the replacement error instead of the original readiness error. OUT C11: pid retention on EPERM is unchanged. OUT C01-C10 and C14-C16: they do not read the not-ready wrapper's primary error." + }, + { + "name": "postStart cleanup swallows a non-ESRCH signal failure and removes the pidfile", + "file": "implementations/cli/src/commands/up.ts", + "find": " return rethrowAfterDetachedCleanup(primary, () => {\n child.kill(\"SIGTERM\");\n removePid();\n });", + "replace": " try { child.kill(\"SIGTERM\"); } catch {}\n removePid();\n throw primary;", + "expectRed": "C14 a postStart non-ESRCH signal failure keeps the pidfile", + "cell": "C14 a postStart non-ESRCH signal failure keeps the pidfile", + "note": "IN C14: EPERM is swallowed and pid removal runs. IN C15: the original postStart error has no attached signal failure. OUT C16: false returns normally, pid removal runs, and the original error is preserved in both forms. OUT C01-C13: they do not execute postStart cleanup." + }, + { + "name": "postStart cleanup replaces the postStart failure", + "file": "implementations/cli/src/commands/up.ts", + "find": "async function rethrowPostStartListenerFailure(\n child: ChildProcess,\n primary: unknown,\n removePid: () => void,\n): Promise {\n return rethrowAfterDetachedCleanup(primary, () => {", + "replace": "async function rethrowPostStartListenerFailure(\n child: ChildProcess,\n primary: unknown,\n removePid: () => void,\n): Promise {\n return rethrowAfterDetachedCleanup(new Error(\"cleanup replaced primary\"), () => {", + "expectRed": "C15 a postStart signal failure preserves the postStart error and attaches the signal failure", + "cell": "C15 a postStart signal failure preserves the postStart error and attaches the signal failure", + "note": "IN C15: EPERM is attached to a replacement error instead of the original postStart error. IN C16: the successful false path throws the replacement error instead of the original postStart error. OUT C14: pid retention on EPERM is unchanged. OUT C01-C13: they do not read the postStart wrapper's primary error." + }, + { + "name": "postStart already-gone cleanup leaves a stale pidfile", + "file": "implementations/cli/src/commands/up.ts", + "find": " child.kill(\"SIGTERM\");\n removePid();", + "replace": " if (child.kill(\"SIGTERM\")) removePid();", + "expectRed": "C16 a postStart ESRCH result removes the stale pidfile and preserves the postStart error", + "cell": "C16 a postStart ESRCH result removes the stale pidfile and preserves the postStart error", + "note": "IN C16: false now skips stale pid removal. OUT C14-C15: EPERM throws before the conditional result is read, so pid retention and cause attachment are unchanged. OUT C01-C13: they do not execute postStart cleanup." + } + ] +} diff --git a/bin/smoke/up-stack-live.smoke.ts b/bin/smoke/up-stack-live.smoke.ts index 7b4d558ba..574f9f9f0 100644 --- a/bin/smoke/up-stack-live.smoke.ts +++ b/bin/smoke/up-stack-live.smoke.ts @@ -1,9 +1,9 @@ /** - * LIVE e2e for `cotal up --detach` — the stage-2b claim the docs make ("start the mesh + delivery + * LIVE e2e for detached-by-default `cotal up` — the stage-2b claim the docs make ("start the mesh + delivery * daemon + manager") exercised as REAL usage: the actual binary as subprocesses, a real JWT-authed * broker on an isolated port, and the control plane answering a real `cotal ps`. * - * 1. `up --detach` (auth default) brings up ALL THREE: nats-server, delivery daemon, manager — + * 1. bare `up` (auth default) exits while bringing up ALL THREE: nats-server, delivery daemon, manager — * pid files written, processes alive, the delivery-aware marker bound to the manager pid. * 2. a real `cotal ps` is ANSWERED by the detached manager (control plane reachable, creds minted * from this folder's auth — the exact "spawn --detach works right after up" promise). @@ -15,9 +15,9 @@ * never pkill, so a co-running broker on :4222 is untouched. Needs `nats-server` on PATH. * Run: pnpm smoke:up-stack:live */ -import { spawnSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import { createConnection, createServer, type AddressInfo } from "node:net"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { renderDetachedSummary } from "../../implementations/cli/src/lib/up-report.js"; @@ -31,6 +31,7 @@ const freePort = (): Promise => }); const PORT = await freePort(); const SERVER = `nats://127.0.0.1:${PORT}`; +const SPACE = "up-stack-live"; const DEFAULT_SERVER = "nats://127.0.0.1:4222"; const WT = resolve(import.meta.dirname, "..", ".."); const CLI = join(WT, "bin", "cotal.ts"); @@ -68,6 +69,16 @@ const portOpenAt = (port: number) => }); const portOpen = () => portOpenAt(PORT); const cliIn = (cwd: string, ...args: string[]) => spawnSync(TSX, [CLI, ...args], { cwd, env, encoding: "utf8", timeout: 120_000 }); +const matchingPids = () => { + if (process.platform === "win32") { + const out = execFileSync("wmic", ["process", "get", "ProcessId,CommandLine", "/format:csv"], { encoding: "utf8" }); + return out.split(/\r?\n/).filter((line) => line.includes(root) || line.includes(autoRoot) || line.includes(occupantRoot)) + .map((line) => Number(line.trim().split(",").at(-1))).filter(Number.isFinite); + } + const out = execFileSync("ps", ["-axo", "pid=,command="], { encoding: "utf8" }); + return out.split("\n").filter((line) => line.includes(root) || line.includes(autoRoot) || line.includes(occupantRoot)) + .map((line) => Number(/^\s*(\d+)/.exec(line)?.[1])).filter(Number.isFinite); +}; const pids: number[] = []; let startedOccupant = false; @@ -85,25 +96,27 @@ try { manager: false, }) === "✓ running in the background: nats-server (pid 42), delivery daemon - stop with: cotal down"); - // Default-port collision: `up` without an explicit `--server` should allocate a free port and - // record it, not fail with "use --server ...:". If the developer already has a real :4222 - // broker, leave it alone; otherwise start a sandbox occupant and tear it down below. - if (!(await portOpenAt(4222))) { - const occupant = cliIn(occupantRoot, "up", "--detach", "--open"); - ok("default-port occupant starts for auto-port regression", occupant.status === 0, occupant.stdout + occupant.stderr); - startedOccupant = true; - } - const auto = cliIn(autoRoot, "up", "--detach", "--open", "--space", "auto"); - ok("up --detach auto-selects a free port when :4222 is occupied", auto.status === 0, auto.stdout + auto.stderr); - const autoEntry = JSON.parse(readFileSync(join(home, "meshes", "space.6175746f.json"), "utf8")) as { server: string }; - ok("auto-port mesh is not recorded on the default server", autoEntry.server !== DEFAULT_SERVER, autoEntry); - ok("auto-port mesh broker is reachable", await portOpenAt(Number(new URL(autoEntry.server).port)), autoEntry); - cliIn(autoRoot, "down"); - if (startedOccupant) cliIn(occupantRoot, "down"); - + // Named cells for the #864 mutations must be the FIRST record/detach-dependent checks. An earlier + // `readFileSync` of a mesh record throws ENOENT (WRONG-RED) if persist is skipped, and a status-only + // detach cell stays green when spawnSync SIGTERM's a foreground `up` whose exit handler still exits 0. // 1) the full stack comes up from ONE command, JWT-authed by default. - const up = cli("up", "--detach", "--server", SERVER); - ok("up --detach exits 0", up.status === 0, up.stdout + up.stderr); + const recPath = join(home, "meshes", `space.${Buffer.from(SPACE).toString("hex")}.json`); + const up = cli("up", "--space", SPACE, "--server", SERVER); + const recorded = existsSync(recPath) + ? JSON.parse(readFileSync(recPath, "utf8")) as { origin?: string; root?: string } + : undefined; + ok( + "up persists a self-hosted mesh record at provision time", + recorded?.origin === "self-hosted" && recorded.root !== undefined && realpathSync(recorded.root) === realpathSync(root), + recorded ?? recPath, + ); + ok( + "bare up exits 0 while the stack remains alive (detached by default)", + up.status === 0 + && /running in the background: nats-server \(pid \d+\)/.test(plain(up.stdout)) + && !/Press Ctrl-C to stop/.test(plain(up.stdout + up.stderr)), + up.stdout + up.stderr, + ); ok( "auth up reports the exact running component set", /^✓ running in the background: nats-server \(pid \d+\), delivery daemon, manager - stop with: cotal down$/m.test(plain(up.stdout)), @@ -117,6 +130,37 @@ try { } ok("delivery-aware marker is bound to the manager pid", pidOf("manager.delivery-aware") === pidOf("manager.pid")); ok("auth material was provisioned (.cotal/auth)", existsSync(join(root, ".cotal", "auth"))); + // After spawnSync returns, the invoker is gone. If the stack were still that process's children, + // they would be dead. Also pin parentage: a detached nats-server is not a child of this suite. + const natsPid = pidOf("nats.pid"); + if (process.platform === "win32") { + const csv = execFileSync("wmic", ["process", "where", `ProcessId=${natsPid}`, "get", "ParentProcessId", "/format:csv"], { encoding: "utf8" }); + const ppid = Number(csv.trim().split(/\r?\n/).at(-1)?.split(",").at(-1)); + ok("detached nats-server is not a child of this suite", Number.isFinite(ppid) && ppid !== process.pid, { natsPid, ppid, suite: process.pid }); + } else { + const ppid = Number(execFileSync("ps", ["-o", "ppid=", "-p", String(natsPid)], { encoding: "utf8" }).trim()); + ok("detached nats-server is not a child of this suite", Number.isFinite(ppid) && ppid !== process.pid, { natsPid, ppid, suite: process.pid }); + } + + // Default-port collision: `up` without an explicit `--server` should allocate a free port and + // record it, not fail with "use --server ...:". After the named persist/detach cells so a + // skipped record cannot throw ENOENT here and grade WRONG-RED. If the developer already has a + // real :4222 broker, leave it alone; otherwise start a sandbox occupant and tear it down below. + if (!(await portOpenAt(4222))) { + const occupant = cliIn(occupantRoot, "up", "--detach", "--open"); + ok("default-port occupant starts for auto-port regression", occupant.status === 0, occupant.stdout + occupant.stderr); + startedOccupant = true; + } + const auto = cliIn(autoRoot, "up", "--detach", "--open", "--space", "auto"); + ok("up --detach auto-selects a free port when :4222 is occupied", auto.status === 0, auto.stdout + auto.stderr); + const autoRecPath = join(home, "meshes", "space.6175746f.json"); + const autoEntry = existsSync(autoRecPath) + ? JSON.parse(readFileSync(autoRecPath, "utf8")) as { server: string } + : undefined; + ok("auto-port mesh is not recorded on the default server", autoEntry !== undefined && autoEntry.server !== DEFAULT_SERVER, autoEntry ?? autoRecPath); + ok("auto-port mesh broker is reachable", autoEntry !== undefined && await portOpenAt(Number(new URL(autoEntry.server).port)), autoEntry); + cliIn(autoRoot, "down"); + if (startedOccupant) cliIn(occupantRoot, "down"); // 2) the manager ANSWERS a real `cotal ps` — no pre-arranged creds, resolved from the folder's // auth + the sandboxed mesh registry, exactly as an operator would run it. Retried while the @@ -124,7 +168,7 @@ try { let answered = false; let last = { stdout: "", stderr: "" }; for (let i = 0; i < 15 && !answered; i++) { - const r = cli("ps"); + const r = cli("ps", "--space", SPACE); last = { stdout: r.stdout, stderr: r.stderr }; answered = r.status === 0 && /no managed agents/.test(r.stdout); if (!answered) await sleep(2000); @@ -141,7 +185,7 @@ try { const deliveryPidFile = join(root, ".cotal", "delivery.pid"); const liveDelivery = readFileSync(deliveryPidFile, "utf8"); rmSync(deliveryPidFile); - const lost = cli("up", "--server", SERVER); + const lost = cli("up", "--space", SPACE, "--server", SERVER); const lostOut = plain(lost.stdout + lost.stderr); ok("a refresh whose delivery launch loses the single-flight lease exits non-zero", lost.status !== 0, lostOut); ok("the refresh says the daemon it started exited without becoming ready", /exited without becoming ready/.test(lostOut), lostOut); @@ -161,12 +205,16 @@ try { } ok("all pid files removed by down", (["nats.pid", "delivery.pid", "manager.pid"] as const).every((f) => !existsSync(join(root, ".cotal", f)))); ok("all three processes are dead + broker port closed", dead, pids.filter(alive)); + const offlineList = cli("meshes"); + ok("a stopped self-hosted mesh remains listed offline", offlineList.status === 0 && new RegExp(`${SPACE}.*self-hosted.*offline`).test(plain(offlineList.stdout)), offlineList.stdout + offlineList.stderr); + const offlineUse = cli("ps", "--space", SPACE); + ok("dead self-hosted target names its root and restart command", offlineUse.status !== 0 && plain(offlineUse.stdout + offlineUse.stderr).includes(`mesh "${SPACE}" is recorded at ${realpathSync(root)} but not running - run \`cotal up\` there to restart`), offlineUse.stdout + offlineUse.stderr); // Open mode in the SAME root retains static-auth files from the prior boot. Reporting follows the // effective live mode, not stale on-disk auth material: broker + manager, never delivery/auth-service. ok("static auth material remains before the open-mode reporting check", existsSync(join(root, ".cotal", "auth"))); - const open = cli("up", "--detach", "--open", "--server", SERVER); - ok("open up --detach exits 0", open.status === 0, open.stdout + open.stderr); + const open = cli("up", "--open", "--space", SPACE, "--server", SERVER); + ok("open bare up exits 0", open.status === 0, open.stdout + open.stderr); ok( "open up reports the exact running component set", /^✓ running in the background: nats-server \(pid \d+\), manager - stop with: cotal down$/m.test(plain(open.stdout)), @@ -182,6 +230,10 @@ try { spawnSync(TSX, [CLI, "down"], { cwd: autoRoot, env, encoding: "utf8" }); if (startedOccupant) spawnSync(TSX, [CLI, "down"], { cwd: occupantRoot, env, encoding: "utf8" }); for (const p of pids) if (alive(p)) { try { process.kill(p, "SIGTERM"); } catch { /* gone */ } } + for (const p of matchingPids()) { try { process.kill(p, "SIGTERM"); } catch { /* gone */ } } + await sleep(500); + const survivors = matchingPids(); + if (survivors.length > 0) throw new Error(`FAIL: up-stack teardown left matching processes — ${survivors.join(", ")}`); rmSync(home, { recursive: true, force: true }); for (const d of [root, autoRoot, occupantRoot]) rmSync(d, { recursive: true, force: true }); } diff --git a/bin/smoke/windows-detached-spawn.smoke.ts b/bin/smoke/windows-detached-spawn.smoke.ts new file mode 100644 index 000000000..d55535a33 --- /dev/null +++ b/bin/smoke/windows-detached-spawn.smoke.ts @@ -0,0 +1,190 @@ +import type { ChildProcess } from "node:child_process"; +import { + assertWindowsDetachAllowed, + assertDetachedChildExitObservable, + windowsDetachedChild, + WINDOWS_JOB_REFUSAL, +} from "../../implementations/cli/src/lib/detached-spawn.js"; +import { + rethrowNotReadyListenerFailureForTest, + rethrowPostStartListenerFailureForTest, + rethrowUnboundListenerFailureForTest, +} from "../../implementations/cli/src/commands/up.js"; + +let failures = 0; +function check(label: string, condition: boolean, extra?: unknown): void { + console.log(`${condition ? "✓" : "✗"} ${label}${condition ? "" : ` — ${String(extra)}`}`); + if (!condition) failures++; +} + +check("C01 Windows process outside a job detaches without breakaway", assertWindowsDetachAllowed({ inJob: false, breakawayAllowed: false }) === false); +check("C02 Windows job with breakaway permission requests CREATE_BREAKAWAY_FROM_JOB", assertWindowsDetachAllowed({ inJob: true, breakawayAllowed: true }) === true); +try { + assertWindowsDetachAllowed({ inJob: true, breakawayAllowed: false }); + check("C03 Windows job without breakaway permission refuses detached up and names --foreground", false, "did not throw"); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + check( + "C03 Windows job without breakaway permission refuses detached up and names --foreground", + message === WINDOWS_JOB_REFUSAL && message.includes("--foreground") && message.includes("cannot host a detached stack"), + message, + ); +} + +const signals: Array<{ pid: number; signal?: NodeJS.Signals | number }> = []; +const child = windowsDetachedChild(4242, (pid, signal) => { + signals.push({ pid, signal }); + return true; +}); +check( + "C04 Windows detached child kill forwards the exact pid and signal", + child.kill("SIGTERM") === true && signals.length === 1 && signals[0]?.pid === 4242 && signals[0]?.signal === "SIGTERM", + JSON.stringify(signals), +); +const goneSignals: Array<{ pid: number; signal?: NodeJS.Signals | number }> = []; +const gone = Object.assign(new Error("gone"), { code: "ESRCH" }); +const goneChild = windowsDetachedChild(4343, (pid, signal) => { + goneSignals.push({ pid, signal }); + throw gone; +}); +let goneResult: boolean | undefined; +let goneError: unknown; +try { goneResult = goneChild.kill("SIGTERM"); } +catch (error) { goneError = error; } +check( + "C05 Windows detached child reports an already-gone pid as false after attempting the exact signal", + goneError === undefined && goneResult === false && goneSignals.length === 1 && goneSignals[0]?.pid === 4343 && goneSignals[0]?.signal === "SIGTERM", + JSON.stringify({ goneResult, goneError: String(goneError), goneSignals }), +); +check( + "C06 Windows detached child is active under the same null-state guard as a real child", + child.exitCode === null && child.signalCode === null && !(child.exitCode !== null || child.signalCode !== null), + JSON.stringify({ exitCode: child.exitCode, signalCode: child.signalCode }), +); +try { + assertDetachedChildExitObservable(child); + check("C07 Windows detached child exit observation fails loud instead of pretending it exited", false, "did not throw"); +} catch (error) { + check( + "C07 Windows detached child exit observation fails loud instead of pretending it exited", + error instanceof Error && error.message === "detached child process 4242 cannot have its exit observed", + error, + ); +} + +const signalFailure = Object.assign(new Error("not permitted"), { code: "EPERM" }); +const failingChild = windowsDetachedChild(4444, () => { throw signalFailure; }); +const bindFailure = new Error("bind failed"); +let removedPid: number | undefined; +let bindCaught: unknown; +try { + await rethrowUnboundListenerFailureForTest( + failingChild, + bindFailure, + undefined, + (pid) => { removedPid = pid; }, + ); +} catch (error) { + bindCaught = error; +} +check( + "C08 an unbound-listener non-ESRCH teardown failure keeps the pidfile", + removedPid === undefined, + removedPid, +); +check( + "C09 an unbound-listener teardown failure preserves the primary error and attaches the cleanup failure", + bindCaught === bindFailure && bindFailure.cause === signalFailure, + bindCaught, +); + +const unboundGoneFailure = new Error("bind failed after exit"); +let unboundGonePidRemoved: number | undefined; +let unboundGoneCaught: unknown; +try { + await rethrowUnboundListenerFailureForTest( + goneChild, + unboundGoneFailure, + undefined, + (pid) => { unboundGonePidRemoved = pid; }, + ); +} catch (error) { + unboundGoneCaught = error; +} +check( + "C10 an unbound-listener ESRCH result removes the stale pidfile and preserves the bind error", + unboundGonePidRemoved === 4343 && unboundGoneCaught === unboundGoneFailure && unboundGoneFailure.cause === undefined, + { unboundGonePidRemoved, unboundGoneCaught }, +); + +const readinessFailure = new Error("nats-server did not become reachable - see log"); +let notReadyPidRemoved = false; +let readinessCaught: unknown; +try { + await rethrowNotReadyListenerFailureForTest(failingChild, readinessFailure, () => { notReadyPidRemoved = true; }); +} catch (error) { + readinessCaught = error; +} +check( + "C11 a not-ready non-ESRCH kill failure keeps the bound-listener pidfile", + !notReadyPidRemoved, + notReadyPidRemoved, +); +check( + "C12 a not-ready kill failure preserves the readiness error and attaches the signal failure", + readinessCaught === readinessFailure && readinessFailure.cause === signalFailure, + readinessCaught, +); + +const readinessGoneFailure = new Error("nats-server exited before readiness"); +let notReadyGonePidRemoved = false; +let readinessGoneCaught: unknown; +try { + await rethrowNotReadyListenerFailureForTest(goneChild, readinessGoneFailure, () => { notReadyGonePidRemoved = true; }); +} catch (error) { + readinessGoneCaught = error; +} +check( + "C13 a not-ready ESRCH result removes the stale pidfile and preserves the readiness error", + notReadyGonePidRemoved && readinessGoneCaught === readinessGoneFailure && readinessGoneFailure.cause === undefined, + { notReadyGonePidRemoved, readinessGoneCaught }, +); + +const postStartFailure = new Error("postStart failed"); +const postStartSignalFailure = Object.assign(new Error("not permitted"), { code: "EPERM" }); +const postStartChild = { pid: 4545, kill() { throw postStartSignalFailure; } } as unknown as ChildProcess; +let postStartPidRemoved = false; +let postStartCaught: unknown; +try { + await rethrowPostStartListenerFailureForTest(postStartChild, postStartFailure, () => { postStartPidRemoved = true; }); +} catch (error) { + postStartCaught = error; +} +check( + "C14 a postStart non-ESRCH signal failure keeps the pidfile", + !postStartPidRemoved, + postStartPidRemoved, +); +check( + "C15 a postStart signal failure preserves the postStart error and attaches the signal failure", + postStartCaught === postStartFailure && postStartFailure.cause === postStartSignalFailure, + postStartCaught, +); + +const postStartGoneFailure = new Error("postStart failed after exit"); +const postStartGoneChild = { pid: 4646, kill() { return false; } } as unknown as ChildProcess; +let gonePidRemoved = false; +let postStartGoneCaught: unknown; +try { + await rethrowPostStartListenerFailureForTest(postStartGoneChild, postStartGoneFailure, () => { gonePidRemoved = true; }); +} catch (error) { + postStartGoneCaught = error; +} +check( + "C16 a postStart ESRCH result removes the stale pidfile and preserves the postStart error", + gonePidRemoved && postStartGoneCaught === postStartGoneFailure && postStartGoneFailure.cause === undefined, + { gonePidRemoved, postStartGoneCaught }, +); + +console.log(`\nWINDOWS DETACHED SPAWN SMOKE ${failures === 0 ? "OK ✅" : "FAILED ❌"}`); +process.exit(failures === 0 ? 0 : 1); diff --git a/docs/README.md b/docs/README.md index ef10d7790..2e2a93eda 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,7 +25,7 @@ Task dispatch: | Task | Page | First command / tool | |---|---|---| -| Install + start a local mesh, non-interactive | [Quickstart](getting-started.md) | `npx cotal-ai setup --yes && npx cotal-ai up --detach` | +| Install + start a local mesh, non-interactive | [Quickstart](getting-started.md) | `npx cotal-ai setup --yes && npx cotal-ai up` | | Put an agent on the mesh | [Quickstart](getting-started.md) | `cotal spawn` | | Message peers from inside a session | [MCP tool catalog](mcp-tools.md) | `cotal_send` · `cotal_dm` · `cotal_anycast` | | Spawn / define a teammate at runtime | [MCP tool catalog](mcp-tools.md) | `cotal_spawn` · `cotal_persona` | diff --git a/docs/cli.md b/docs/cli.md index b60931a72..0af69f00b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -37,7 +37,7 @@ runtimes ship this way. | Set up & lifecycle | [`down`](#down) | Stop the whole stack, selected registered components, or a manifest deploy | | Set up & lifecycle | [`backup`](#backup-and-restore) | Create an offline full-space or registry-only artifact from a preserved cut | | Set up & lifecycle | [`clean`](#clean) | Configurable cleanup: purge history (live), or wipe the local store / identity (stopped) | -| Set up & lifecycle | [`meshes`](#meshes-use-status) | List the running meshes on this machine | +| Set up & lifecycle | [`meshes`](#meshes-use-status) | List the recorded meshes on this machine | | Set up & lifecycle | [`use`](#meshes-use-status) | Set the default mesh a bare `cotal spawn` joins | | Set up & lifecycle | [`status`](#meshes-use-status) | Read-only diagnostics for setup, processes, and the selected mesh | | Agents & personas | [`spawn`](#spawn) | Launch an agent from a persona (foreground, or `--detach` via the manager) | @@ -125,7 +125,7 @@ current. ## up ```bash -cotal up [--detach] [--open] [--space ] [--server ] [--channels ] [--runtime ] +cotal up [--foreground] [--open] [--space ] [--server ] [--channels ] [--runtime ] cotal up --user-auth --idp [--exchange-public-port --exchange-public-url [--exchange-trusted-proxy]] cotal up --tls-cert --tls-key # serve broker TLS (both, or neither) cotal up --restore [--restore-only registry] [--accept-missing-source] @@ -148,7 +148,8 @@ cotal up -f [--dry-run] [--runtime ] | `--exchange-public-port ` | none | With `--user-auth`: add the public exchange face on this loopback port, for an HTTPS reverse proxy to forward to | | `--exchange-public-url ` | none | With `--exchange-public-port`: advertise the reverse proxy's HTTPS URL in discovery | | `--exchange-trusted-proxy` | off | With `--exchange-public-port`: attribute public failure buckets to the last `X-Forwarded-For` hop. Enable only when the listener is reachable solely through a trusted proxy; otherwise the socket address is used | -| `--detach` | off | Run in the background (stop with `cotal down`) | +| `--detach` | on | Deprecated compatibility spelling. It is a no-op because normal `up` is already detached | +| `--foreground` | off | Debug in the invoking terminal. Normal `up` detaches the stack so it survives shells, SSH sessions, agents, and CI runners ending. On Windows, Cotal first proves the current job permits process breakaway; otherwise it refuses before launch and points to this flag | | `--tls-cert ` | — | PEM certificate to serve TLS with. Must be given together with `--tls-key`. The pair is validated **before** the broker starts — readability, private-key mode, that the two match, the validity window, and that the certificate covers the host clients will dial — because `nats-server` starts happily on an expired certificate and only the client then fails. The decision is recorded, so a later bare `cotal up` after a `cotal down` keeps serving TLS rather than silently reverting to cleartext | | `--tls-key ` | — | PEM private key for `--tls-cert`. Refused if group- or other-readable (tighten to `600`) | | `--file `, `-f` | — | Launch a whole mesh from a manifest | @@ -156,10 +157,10 @@ cotal up -f [--dry-run] [--runtime ] | `--runtime ` | `pty` (or the manifest's, with `-f`) | Agent runtime for the mesh manager (`pty` built in; others are installed extensions, explicit-only). Resolved + probed before the broker starts; an uninstalled/unreachable runtime fails loud. With `-f`, overrides the manifest's runtime | | `--rotate-sys` | off | Rotate the space's system account and re-mint its two `$SYS` creds. Needs a stopped mesh; refused with `--open` | -`cotal up` boots a local nats-server with JetStream and, in auth mode (the default), JWT auth and -per-agent ACLs; `--detach` records the mesh so `cotal spawn` from any directory can find it. With no +`cotal up` boots a detached local nats-server with JetStream and, in auth mode (the default), JWT auth and +per-agent ACLs. A launch that starts the broker records the mesh as self-hosted so `cotal spawn` from any directory can find it and a stopped stack remains restartable from its recorded root. A refresh that merely finds a broker already answering preserves an operator-owned manual record. With no `--server`, it auto-selects a free port if the default address is taken; an explicit `--server` -stays fail-loud on collision. `--detach` also brings up the control plane (delivery daemon in auth +stays fail-loud on collision. It also brings up the control plane (delivery daemon in auth mode, then the manager). The `-f` form is a [manifest deploy](#manifest-deploys); see [Run a mesh](run-a-mesh.md). @@ -185,7 +186,7 @@ config, so it runs as part of a boot: ```bash cotal down -cotal up --rotate-sys --detach # agents reconnect; nothing is re-provisioned +cotal up --rotate-sys # agents reconnect; nothing is re-provisioned cotal doctor auth # both $SYS creds healthy again, 30 days out ``` @@ -421,8 +422,7 @@ cotal use cotal status [--space ] [--server ] [--components] ``` -`meshes` lists the meshes this machine knows; a `*` marks the `current` default a bare -`cotal spawn` joins. +`meshes` lists the meshes this machine knows, including stopped self-hosted ones tagged `offline`; a `*` marks the `current` default a bare `cotal spawn` joins. Run on a terminal with the space or `--server` missing, **`meshes add` is guided**: it asks for the one thing that cannot be derived (the broker URL), probes it, and tells you what answered - open or @@ -433,8 +433,9 @@ than an error. Anything you pass on the command line is taken as given and not a a terminal - a script, an agent, CI - nothing prompts and the flag form's errors stand (`COTAL_NO_PROMPT=1` forces that too). -`cotal up` and `cotal down` maintain their own records. `meshes add` registers a mesh they cannot -speak for: one running on another machine, a shared broker, a hosted space. `--root` is the folder +`cotal up` writes a durable self-hosted record; `cotal down` stops the stack but leaves that record +so a stopped mesh remains listed and restartable. `meshes add` registers a mesh they cannot speak +for: one running on another machine, a shared broker, a hosted space. `--root` is the folder whose `.cotal/auth` holds that mesh's credentials and whose `.cotal/agents` holds its personas (default: the project you run it in) — the registry stores that path, never a secret. `--mode` defaults to `auth` when the root holds the space's account record and to `open` otherwise. The @@ -849,7 +850,7 @@ cotal supervise [--runtime ] [--space ] [--server ] [--spawn ` | — | Comma-separated personas to pre-spawn at startup | The manager is the agent supervisor and control plane: it answers `spawn --detach`, `stop`, `ps`, -`attach`, and the `cotal_*` manager tools. `cotal up --detach` starts one for you; run `supervise` +`attach`, and the `cotal_*` manager tools. `cotal up` starts one for you; run `supervise` directly to recover a dead manager or drive a custom runtime. Default runtime is `pty`; install an optional provider first (`cotal ext add @cotal-ai/orca`, `@cotal-ai/tmux`, `@cotal-ai/cmux`, or `@cotal-ai/herdr`) and select it explicitly. A missing provider or app fails loudly; there is no fallback. See [Deploy](deploy.md). @@ -1299,7 +1300,7 @@ keyed beta intake; without one it goes to the public `cotal.ai` intake and requi ## Server daemons Two long-lived infra roles ship with the CLI. They are not part of everyday operation; the delivery -daemon comes up automatically with `cotal up --detach` in auth mode. +daemon comes up automatically with `cotal up` in auth mode. ```bash cotal deliver --space [--server ] [--creds ] diff --git a/docs/getting-started.md b/docs/getting-started.md index 79c0af951..176b49161 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -84,7 +84,7 @@ When it finishes, nothing is running yet; it prints the commands to start things whole loop is three commands: ```bash -cotal up --detach # start the mesh + delivery daemon + manager (JWT-authed by default) +cotal up # start the detached mesh + delivery daemon + manager (JWT-authed by default) cotal spawn # launch your agent here and talk to it (Ctrl-C to leave) cotal down # stop everything ``` @@ -138,7 +138,7 @@ Every later `cotal setup` prints a **read-only status card**: cotal · status ✓ NATS nats://127.0.0.1:4222 ✓ plugin installed -○ mesh down · start: cotal up --detach +○ mesh down · start: cotal up ○ web down · start: cotal web ○ manager not running · start: cotal up, or: cotal supervise ``` @@ -157,7 +157,7 @@ peers, spawn teammates, and send feedback (the full surface is the [MCP tool catalog](mcp-tools.md)). The same things are available as commands: ```bash -cotal up --detach # start the mesh + delivery daemon + manager +cotal up # start the detached mesh + delivery daemon + manager cotal status # detailed setup, process, registry, and live mesh status cotal spawn # your agent (edit .cotal/agents/default.md) cotal spawn david # a guided expert, needs `cotal setup --demo` first (also sven, me) @@ -188,14 +188,14 @@ A coding agent can set Cotal up for you with two non-interactive commands: ```bash npx cotal-ai setup --yes # configure: install the plugin + seed one agent (launches nothing) -npx cotal-ai up --detach # start the mesh + delivery daemon + manager +npx cotal-ai up # start the detached mesh + delivery daemon + manager ``` `setup --yes` accepts every default with no prompts and exits non-zero with the log path if a step fails, so an agent or a CI job can check the result (add `--demo` for the guided team). -`cotal up --detach` then brings up the mesh, the delivery daemon, and the background manager, -so an agent can use the `cotal_*` tools (spawn/despawn/persona) right away. `cotal down` -stops the background processes. +`cotal up` then brings up the mesh, the delivery daemon, and the background manager, so an +agent can use the `cotal_*` tools (spawn/despawn/persona) right away. The stack does not depend +on the invoking process staying alive. `cotal down` stops the background processes. ## Troubleshooting diff --git a/docs/run-a-mesh.md b/docs/run-a-mesh.md index f4ee72546..e3b07ec07 100644 --- a/docs/run-a-mesh.md +++ b/docs/run-a-mesh.md @@ -17,6 +17,12 @@ operator-only maintenance verbs. Every command's full flag set is in the - **Manager**: a detached supervisor answering the control plane, so `cotal spawn --detach` and the `cotal_spawn` tool work right after `up`. +The stack is detached by default and does not depend on the invoking shell, SSH connection, agent +session, or CI runner staying alive. On Windows, Cotal probes the current process job and launches +with `CREATE_BREAKAWAY_FROM_JOB` when required and permitted. If the job forbids breakaway, `up` +refuses before starting any detached process and names `--foreground`, which remains the debugging +path in the invoking terminal. + Three modes: - **Default (static auth).** JWT-authed, on by default: sender authenticity and per-agent @@ -130,16 +136,19 @@ the registry; a missing provider or app throws, never silently falls back ## From any directory: the mesh registry -`cotal up` records each running mesh in a machine-local registry +`cotal up` records each provisioned mesh as **self-hosted** in a machine-local registry (`~/.cotal/meshes/space..json`, named by a case-safe hex encoding of the space: broker URL, the project root holding its creds and -personas, and its mode). So a bare `cotal spawn ` from *any* directory joins the -running mesh with the right credentials instead of mistaking the cwd for a space: +personas, and its mode). The record survives downtime. A command targeting a stopped mesh says +where it is recorded and tells you to run `cotal up` there to restart, instead of denying the mesh +exists. A bare `cotal spawn ` from *any* directory joins a running mesh with the right +credentials instead of mistaking the cwd for a space: - `cotal use ` sets the default from every directory, including inside another mesh's project. `--space ` overrides it for one command. - With no live selected default, a project with its own `.cotal/` resolves to that project's mesh; otherwise one running mesh is used automatically and several are an error. -- `cotal meshes` lists them (a `*` marks the default); `cotal down` removes the entry. +- `cotal meshes` lists them (a `*` marks the default); stopped self-hosted meshes stay listed as + `self-hosted · offline`. `cotal meshes rm ` deliberately removes a record. The registry stores a *path*, never a secret; trust material stays in each project's `.cotal/auth`. If the mesh is down or won't take your creds, spawn fails with one @@ -273,9 +282,9 @@ down: cotal down --preserve-state cotal backup create ./space-backup # full by default # later: deliberately resume the unchanged source -cotal up --detach +cotal up # or, from another preserved cut, restore before the normal listener opens -cotal up --restore ./space-backup --detach +cotal up --restore ./space-backup ``` Use `--store-dir` on both preservation and backup for a custom JetStream store. `registry` is the diff --git a/docs/setup-internals.md b/docs/setup-internals.md index 439a0a951..6e32e82c6 100644 --- a/docs/setup-internals.md +++ b/docs/setup-internals.md @@ -21,7 +21,7 @@ It is two-tier, gated on a machine marker. - splash → intro → core **checks** (Node >= 22; **locate** `nats-server`: located, never started) → **connector picker** → write the demo personas (david/sven/me) and seed the generic `default` → **offer a global install** (`offerGlobalInstall`) → onboarded marker → a finale that - lists the commands to start things (`cotal up --detach`, `cotal web`, `cotal spawn …`, + lists the commands to start things (`cotal up`, `cotal web`, `cotal spawn …`, `cotal console`, `cotal down`). Nothing is running when it returns. - The old `--auth` / `--open` flags are **gone**: they set the mesh MODE at launch time, and setup no longer launches; mode is now `cotal up [--open]`'s concern (an unknown-option error names @@ -32,7 +32,7 @@ re-offer the **global install** (`offerGlobalInstall`, same `isNpx()` + PATH-sca run — so a repeat `npx cotal-ai setup` on a machine that still lacks a durable `cotal` finally installs it), then print the **status card** (`readyCard`). The card is **read-only probes** (`machineStatus`/`meshStatus`/`webUp`/`managerUp` for NATS, the plugin, the mesh, the web dashboard, and the manager) and for anything down it prints the exact command to start it -(`cotal up --detach`, `cotal web`, `cotal supervise`). Displaying state never depends on it; setup +(`cotal up`, `cotal web`, `cotal supervise`). Displaying state never depends on it; setup still launches nothing. Steps run in-process via `runSteps` @@ -52,7 +52,7 @@ persona `cotal spawn me` drives. **`--yes`** forces non-interactive accept-all even on a TTY: optional plus `confirm` steps run (so the demo personas are written), the global install takes its default, and a failure aborts with the log path and a non-zero exit. It still launches nothing. The control plane comes up with -`cotal up --detach`. This is the agent/CI contract; keep it working. +`cotal up`. This is the agent/CI contract; keep it working. ## Invariants @@ -83,7 +83,7 @@ fail-loud on collision. - **Mesh:** `startMeshDetached` ([`commands/up.ts`](../implementations/cli/src/commands/up.ts)) is the one place that boots a - background nats-server (foreground `up` and `up --detach` both route through it). Writes + background nats-server (default detached `up`; `--foreground` is the debug path). Writes `.cotal/nats.pid` and tails `.cotal/nats.log`. - **Delivery daemon:** `startDeliveryDetached` / `ensureDelivery` ([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)) re-execs `cotal diff --git a/extensions/connector-core/src/docs-bundle.generated.ts b/extensions/connector-core/src/docs-bundle.generated.ts index c9c9d9bd6..f680e7ed1 100644 --- a/extensions/connector-core/src/docs-bundle.generated.ts +++ b/extensions/connector-core/src/docs-bundle.generated.ts @@ -19,7 +19,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "Quickstart", "kind": "Start here (informative)", "summary": "Paste this into any coding agent (Claude Code, OpenCode, Cursor, Codex) and it will do the whole page for you:", - "body": "# Quickstart\n\n> **Start here** (informative) · **For:** everyone · **Next:** [Connect Claude](connect-claude.md) · [Define a team](define-a-team.md) · [Watch a mesh](watch-a-mesh.md)\n\n## Set up with your agent\n\nPaste this into any coding agent (Claude Code, OpenCode, Cursor, Codex) and it will do\nthe whole page for you:\n\n```text wrap\nRead https://docs.cotal.ai/prompt.md, then set up Cotal on this machine: install it, start a local mesh, and put an agent on it.\n```\n\nTo do it by hand instead, keep reading: this page takes you from install to a running\nlocal mesh with an agent on it, in a few minutes.\n\n## Install and run\n\n```bash\ncurl -fsSL https://get.cotal.ai | sh\n```\n\nThat is the whole install on a machine with nothing on it. The script finds a Node 22+ or\ninstalls a verified one of its own, puts `cotal` in `~/.local/bin`, adds that to your PATH,\nand runs guided setup. It never uses sudo and writes nothing outside your home directory.\nRead it first at [get.cotal.ai](https://get.cotal.ai); it is served as plain text for that\nreason. Useful flags: `--dry-run` to see the plan, `--no-modify-path` to leave your shell rc\nalone, `--no-setup` to install only. Pass them through the pipe as\n`| sh -s -- --dry-run`.\n\nOn Windows, or if you already run Node 22+ and would rather use npm directly:\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH\ncotal setup # one-time, configure-only; launches nothing\n```\n\nCotal runs natively on Windows, but the installer above is a POSIX shell script, so npm is the\nroute there (or run the installer under WSL).\n\nBare `cotal` prints help; `cotal setup` runs the guided setup. `npx cotal-ai setup` works\ntoo and offers to install the global `cotal` at the end. Declining is fine: the hints stay\n`npx cotal-ai …`, and the background processes `cotal up` starts invoke their own resolved\npath rather than a global `cotal`.\n\nRequirements:\n\n- Node 22 or newer. The installer handles this for you; it downloads an official Node build\n and checks it against the SHA-256 sums published beside it on nodejs.org.\n- A glibc system. Cotal's terminal layer ships prebuilt native binaries for glibc only, so\n musl distributions (Alpine) are not supported yet and the installer refuses them rather\n than leaving you with an install that cannot start.\n- A `nats-server` binary, version 2.12 or newer (the control surface uses its message\n schedules and per-message TTLs, and fails loud at connect against an older broker). The\n one that ships with the package is new enough; if you already have `nats-server` on your\n PATH, Cotal uses that instead, so make sure it is 2.12+.\n\nTo uninstall: `rm -rf ~/.local/share/cotal ~/.local/bin/cotal` removes what the installer wrote,\n`rm -rf ~/.cotal` removes your meshes, agents and credentials, and the `# cotal` block it added\nto your shell rc can be deleted.\n\n## First run\n\n`cotal setup` is configure-only: it prepares your machine and starts nothing. The first\ntime, it walks you through:\n\n1. **Checks.** Verifies Node 22+ and locates a `nats-server` (the bundled one, or your\n own on PATH). Located only; nothing starts.\n2. **Picks connectors.** Choose which agents join your mesh (Claude or OpenCode; detected\n ones are pre-selected). Claude installs a plugin, because its wake channel needs one.\n OpenCode needs no install; it auto-wires when you `cotal spawn` it.\n3. **Seeds one agent.** The generic `default` persona that a bare `cotal spawn` launches;\n edit it to taste. `cotal setup --demo` additionally seeds a guided team to talk to:\n **david** (the engineer, how Cotal works), **sven** (the guide, what to build), and\n **me** (the session you drive). Every file setup writes is announced with a\n `→ wrote …` line.\n4. **Nothing to install for the dashboard.** `@cotal-ai/web` ships inside `cotal-ai` and is\n seeded automatically on first run (like the built-in connectors), so `cotal web` works out\n of the box and tracks your CLI version on upgrade.\n5. **Offers a global install.** Run via `npx` with no global `cotal`, it offers to\n `npm i -g cotal-ai` so you can just type `cotal`.\n\nWhen it finishes, nothing is running yet; it prints the commands to start things. The\nwhole loop is three commands:\n\n```bash\ncotal up --detach # start the mesh + delivery daemon + manager (JWT-authed by default)\ncotal spawn # launch your agent here and talk to it (Ctrl-C to leave)\ncotal down # stop everything\n```\n\nOpen the browser dashboard with `cotal web` (it ships with `cotal-ai`, seeded automatically). Add the\nguided expert team with `cotal setup --demo`, then `cotal spawn\ndavid` (or `sven`, or `me`). Watch the mesh in this terminal anytime with `cotal console`:\n\n![The cotal console: a live roster of agents and their all-activity feed in a terminal TUI](../assets/quickstart.gif)\n\n`cotal up` is JWT-authed by default (sender authenticity plus per-agent ACLs), starts the\nserver-side [delivery daemon](delivery-daemon.md) as the durable backstop, and starts a\ndetached manager so `cotal spawn --detach` / `cotal_spawn` work right after.\n`cotal up --open` gives you an open, loopback-only, live-only mesh instead (no auth, no\ndaemon) for quick local experiments.\n\nFor a mesh where **people sign in** instead of handing out creds files, start it with\n`cotal up --user-auth --idp `: each human runs `cotal login --idp ` once,\nthe operator grants their agents with `cotal actor grant --sub ` (a full\ngrant by default: all channels, may spawn; narrow it with `--allow-subscribe` /\n`--allow-publish` / `--scope`), and every connect is authorized live against that grant\n(revoke and it's gone). See [identity & auth](identity-and-auth.md).\n\nIf a step fails, setup offers to hand you to an interactive Claude session that has the\nfailure context. Type `/exit` to return, and it retries.\n\n## The primitives\n\nThe vocabulary behind those three commands, which every other page builds on:\n\n| Primitive | What it is |\n|---|---|\n| **Space** | One collaboration, isolated from other spaces. Your mesh is a space. |\n| **Endpoint** | Any software on the mesh: a long-lived connection with presence. |\n| **Agent node** | An endpoint with identity, role, and tags (what `cotal spawn` launches). |\n| **Channel** | A named topic participants broadcast on and subscribe to. |\n| **Direct message** | A message addressed to one peer. |\n| **Presence** | The live roster: who is here, `idle` / `waiting` / `working` / `offline`. |\n| **History** | Recent messages a late joiner replays. |\n\nDelivery comes in three modes: **multicast** (to a channel), **unicast** (to one peer),\nand **anycast** (to any one holder of a role). More in\n[Presence & delivery](presence-and-delivery.md); the full term list is in the\n[glossary](glossary.md).\n\n## After the first run\n\nEvery later `cotal setup` prints a **read-only status card**:\n\n```\ncotal · status\n✓ NATS nats://127.0.0.1:4222\n✓ plugin installed\n○ mesh down · start: cotal up --detach\n○ web down · start: cotal web\n○ manager not running · start: cotal up, or: cotal supervise\n```\n\nIt probes the current folder (the mesh, the browser dashboard, and the manager behind\n`cotal_spawn` / `despawn` / `persona`) and shows the exact start command for anything\nthat is down. It starts nothing itself.\n\nThe dashboard ships with `cotal-ai` and is seeded automatically on first run. It runs at\n`http://cotal.localhost:7799` once you start it with `cotal web` (works in Chrome,\nFirefox, and Edge; on Safari use `http://127.0.0.1:7799`). If a seeded copy is damaged,\n`cotal ext seed --repair` restores it.\n\nYou drive Cotal through an agent: spawn one and talk to it. It has the tools to message\npeers, spawn teammates, and send feedback (the full surface is the\n[MCP tool catalog](mcp-tools.md)). The same things are available as commands:\n\n```bash\ncotal up --detach # start the mesh + delivery daemon + manager\ncotal status # detailed setup, process, registry, and live mesh status\ncotal spawn # your agent (edit .cotal/agents/default.md)\ncotal spawn david # a guided expert, needs `cotal setup --demo` first (also sven, me)\ncotal console --space main # live mesh view in the terminal (TUI)\ncotal web --space main # open the browser dashboard\ncotal down # stop the background mesh, delivery daemon, and manager\n```\n\nFeedback flows through your agent too: tell it \"send feedback: ...\" and it reports it for\nyou (built-in `cotal_feedback`), or run `cotal feedback \"\"`.\n\n`cotal setup --demo` adds the guided team (david, sven, me) to an already-configured machine.\n`cotal setup --full` redoes the whole guided flow (team included), for example to repair\nsomething. Defaults (persona, harness, model selection) and day-to-day operation are in\n[Run a mesh](run-a-mesh.md); every command and flag is in the [CLI reference](cli.md).\n\n## Launch a team from a manifest\n\nThe guided flow gives you one agent (or the expert team with `--demo`). To run a **specific\nteam** (your own channels, agents, and who may read and post where), describe it once in a\n`cotal.yaml` and launch it with `cotal up -f cotal.yaml`. The walkthrough is\n**[Define a team](define-a-team.md)**; the file format is the\n[manifest reference](manifest.md).\n\n## For agents and CI\n\nA coding agent can set Cotal up for you with two non-interactive commands:\n\n```bash\nnpx cotal-ai setup --yes # configure: install the plugin + seed one agent (launches nothing)\nnpx cotal-ai up --detach # start the mesh + delivery daemon + manager\n```\n\n`setup --yes` accepts every default with no prompts and exits non-zero with the log path if a\nstep fails, so an agent or a CI job can check the result (add `--demo` for the guided team).\n`cotal up --detach` then brings up the mesh, the delivery daemon, and the background manager,\nso an agent can use the `cotal_*` tools (spawn/despawn/persona) right away. `cotal down`\nstops the background processes.\n\n## Troubleshooting\n\n- The full log is at `.cotal/setup.log` (and `.cotal/nats.log` for the server).\n- Re-running setup is safe. It reuses a running web and keeps your files.\n- Set `COTAL_SKIP_ASSIST=1` to disable the Claude handoff offer on failures.\n\nNext: put your own agent on the mesh ([Connectors](connectors.md) compares them:\n[Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n[Hermes](connect-hermes.md) · [pi](connect-pi.md)), declare a team\n([Define a team](define-a-team.md)), or watch it live ([Watch a mesh](watch-a-mesh.md)).\n" + "body": "# Quickstart\n\n> **Start here** (informative) · **For:** everyone · **Next:** [Connect Claude](connect-claude.md) · [Define a team](define-a-team.md) · [Watch a mesh](watch-a-mesh.md)\n\n## Set up with your agent\n\nPaste this into any coding agent (Claude Code, OpenCode, Cursor, Codex) and it will do\nthe whole page for you:\n\n```text wrap\nRead https://docs.cotal.ai/prompt.md, then set up Cotal on this machine: install it, start a local mesh, and put an agent on it.\n```\n\nTo do it by hand instead, keep reading: this page takes you from install to a running\nlocal mesh with an agent on it, in a few minutes.\n\n## Install and run\n\n```bash\ncurl -fsSL https://get.cotal.ai | sh\n```\n\nThat is the whole install on a machine with nothing on it. The script finds a Node 22+ or\ninstalls a verified one of its own, puts `cotal` in `~/.local/bin`, adds that to your PATH,\nand runs guided setup. It never uses sudo and writes nothing outside your home directory.\nRead it first at [get.cotal.ai](https://get.cotal.ai); it is served as plain text for that\nreason. Useful flags: `--dry-run` to see the plan, `--no-modify-path` to leave your shell rc\nalone, `--no-setup` to install only. Pass them through the pipe as\n`| sh -s -- --dry-run`.\n\nOn Windows, or if you already run Node 22+ and would rather use npm directly:\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH\ncotal setup # one-time, configure-only; launches nothing\n```\n\nCotal runs natively on Windows, but the installer above is a POSIX shell script, so npm is the\nroute there (or run the installer under WSL).\n\nBare `cotal` prints help; `cotal setup` runs the guided setup. `npx cotal-ai setup` works\ntoo and offers to install the global `cotal` at the end. Declining is fine: the hints stay\n`npx cotal-ai …`, and the background processes `cotal up` starts invoke their own resolved\npath rather than a global `cotal`.\n\nRequirements:\n\n- Node 22 or newer. The installer handles this for you; it downloads an official Node build\n and checks it against the SHA-256 sums published beside it on nodejs.org.\n- A glibc system. Cotal's terminal layer ships prebuilt native binaries for glibc only, so\n musl distributions (Alpine) are not supported yet and the installer refuses them rather\n than leaving you with an install that cannot start.\n- A `nats-server` binary, version 2.12 or newer (the control surface uses its message\n schedules and per-message TTLs, and fails loud at connect against an older broker). The\n one that ships with the package is new enough; if you already have `nats-server` on your\n PATH, Cotal uses that instead, so make sure it is 2.12+.\n\nTo uninstall: `rm -rf ~/.local/share/cotal ~/.local/bin/cotal` removes what the installer wrote,\n`rm -rf ~/.cotal` removes your meshes, agents and credentials, and the `# cotal` block it added\nto your shell rc can be deleted.\n\n## First run\n\n`cotal setup` is configure-only: it prepares your machine and starts nothing. The first\ntime, it walks you through:\n\n1. **Checks.** Verifies Node 22+ and locates a `nats-server` (the bundled one, or your\n own on PATH). Located only; nothing starts.\n2. **Picks connectors.** Choose which agents join your mesh (Claude or OpenCode; detected\n ones are pre-selected). Claude installs a plugin, because its wake channel needs one.\n OpenCode needs no install; it auto-wires when you `cotal spawn` it.\n3. **Seeds one agent.** The generic `default` persona that a bare `cotal spawn` launches;\n edit it to taste. `cotal setup --demo` additionally seeds a guided team to talk to:\n **david** (the engineer, how Cotal works), **sven** (the guide, what to build), and\n **me** (the session you drive). Every file setup writes is announced with a\n `→ wrote …` line.\n4. **Nothing to install for the dashboard.** `@cotal-ai/web` ships inside `cotal-ai` and is\n seeded automatically on first run (like the built-in connectors), so `cotal web` works out\n of the box and tracks your CLI version on upgrade.\n5. **Offers a global install.** Run via `npx` with no global `cotal`, it offers to\n `npm i -g cotal-ai` so you can just type `cotal`.\n\nWhen it finishes, nothing is running yet; it prints the commands to start things. The\nwhole loop is three commands:\n\n```bash\ncotal up # start the detached mesh + delivery daemon + manager (JWT-authed by default)\ncotal spawn # launch your agent here and talk to it (Ctrl-C to leave)\ncotal down # stop everything\n```\n\nOpen the browser dashboard with `cotal web` (it ships with `cotal-ai`, seeded automatically). Add the\nguided expert team with `cotal setup --demo`, then `cotal spawn\ndavid` (or `sven`, or `me`). Watch the mesh in this terminal anytime with `cotal console`:\n\n![The cotal console: a live roster of agents and their all-activity feed in a terminal TUI](../assets/quickstart.gif)\n\n`cotal up` is JWT-authed by default (sender authenticity plus per-agent ACLs), starts the\nserver-side [delivery daemon](delivery-daemon.md) as the durable backstop, and starts a\ndetached manager so `cotal spawn --detach` / `cotal_spawn` work right after.\n`cotal up --open` gives you an open, loopback-only, live-only mesh instead (no auth, no\ndaemon) for quick local experiments.\n\nFor a mesh where **people sign in** instead of handing out creds files, start it with\n`cotal up --user-auth --idp `: each human runs `cotal login --idp ` once,\nthe operator grants their agents with `cotal actor grant --sub ` (a full\ngrant by default: all channels, may spawn; narrow it with `--allow-subscribe` /\n`--allow-publish` / `--scope`), and every connect is authorized live against that grant\n(revoke and it's gone). See [identity & auth](identity-and-auth.md).\n\nIf a step fails, setup offers to hand you to an interactive Claude session that has the\nfailure context. Type `/exit` to return, and it retries.\n\n## The primitives\n\nThe vocabulary behind those three commands, which every other page builds on:\n\n| Primitive | What it is |\n|---|---|\n| **Space** | One collaboration, isolated from other spaces. Your mesh is a space. |\n| **Endpoint** | Any software on the mesh: a long-lived connection with presence. |\n| **Agent node** | An endpoint with identity, role, and tags (what `cotal spawn` launches). |\n| **Channel** | A named topic participants broadcast on and subscribe to. |\n| **Direct message** | A message addressed to one peer. |\n| **Presence** | The live roster: who is here, `idle` / `waiting` / `working` / `offline`. |\n| **History** | Recent messages a late joiner replays. |\n\nDelivery comes in three modes: **multicast** (to a channel), **unicast** (to one peer),\nand **anycast** (to any one holder of a role). More in\n[Presence & delivery](presence-and-delivery.md); the full term list is in the\n[glossary](glossary.md).\n\n## After the first run\n\nEvery later `cotal setup` prints a **read-only status card**:\n\n```\ncotal · status\n✓ NATS nats://127.0.0.1:4222\n✓ plugin installed\n○ mesh down · start: cotal up\n○ web down · start: cotal web\n○ manager not running · start: cotal up, or: cotal supervise\n```\n\nIt probes the current folder (the mesh, the browser dashboard, and the manager behind\n`cotal_spawn` / `despawn` / `persona`) and shows the exact start command for anything\nthat is down. It starts nothing itself.\n\nThe dashboard ships with `cotal-ai` and is seeded automatically on first run. It runs at\n`http://cotal.localhost:7799` once you start it with `cotal web` (works in Chrome,\nFirefox, and Edge; on Safari use `http://127.0.0.1:7799`). If a seeded copy is damaged,\n`cotal ext seed --repair` restores it.\n\nYou drive Cotal through an agent: spawn one and talk to it. It has the tools to message\npeers, spawn teammates, and send feedback (the full surface is the\n[MCP tool catalog](mcp-tools.md)). The same things are available as commands:\n\n```bash\ncotal up # start the detached mesh + delivery daemon + manager\ncotal status # detailed setup, process, registry, and live mesh status\ncotal spawn # your agent (edit .cotal/agents/default.md)\ncotal spawn david # a guided expert, needs `cotal setup --demo` first (also sven, me)\ncotal console --space main # live mesh view in the terminal (TUI)\ncotal web --space main # open the browser dashboard\ncotal down # stop the background mesh, delivery daemon, and manager\n```\n\nFeedback flows through your agent too: tell it \"send feedback: ...\" and it reports it for\nyou (built-in `cotal_feedback`), or run `cotal feedback \"\"`.\n\n`cotal setup --demo` adds the guided team (david, sven, me) to an already-configured machine.\n`cotal setup --full` redoes the whole guided flow (team included), for example to repair\nsomething. Defaults (persona, harness, model selection) and day-to-day operation are in\n[Run a mesh](run-a-mesh.md); every command and flag is in the [CLI reference](cli.md).\n\n## Launch a team from a manifest\n\nThe guided flow gives you one agent (or the expert team with `--demo`). To run a **specific\nteam** (your own channels, agents, and who may read and post where), describe it once in a\n`cotal.yaml` and launch it with `cotal up -f cotal.yaml`. The walkthrough is\n**[Define a team](define-a-team.md)**; the file format is the\n[manifest reference](manifest.md).\n\n## For agents and CI\n\nA coding agent can set Cotal up for you with two non-interactive commands:\n\n```bash\nnpx cotal-ai setup --yes # configure: install the plugin + seed one agent (launches nothing)\nnpx cotal-ai up # start the detached mesh + delivery daemon + manager\n```\n\n`setup --yes` accepts every default with no prompts and exits non-zero with the log path if a\nstep fails, so an agent or a CI job can check the result (add `--demo` for the guided team).\n`cotal up` then brings up the mesh, the delivery daemon, and the background manager, so an\nagent can use the `cotal_*` tools (spawn/despawn/persona) right away. The stack does not depend\non the invoking process staying alive. `cotal down` stops the background processes.\n\n## Troubleshooting\n\n- The full log is at `.cotal/setup.log` (and `.cotal/nats.log` for the server).\n- Re-running setup is safe. It reuses a running web and keeps your files.\n- Set `COTAL_SKIP_ASSIST=1` to disable the Claude handoff offer on failures.\n\nNext: put your own agent on the mesh ([Connectors](connectors.md) compares them:\n[Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n[Hermes](connect-hermes.md) · [pi](connect-pi.md)), declare a team\n([Define a team](define-a-team.md)), or watch it live ([Watch a mesh](watch-a-mesh.md)).\n" }, { "slug": "architecture", @@ -75,7 +75,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "`cotal` CLI reference", "kind": "Reference: describes the TypeScript reference implementation (the `cotal` CLI), not the wire contract.", "summary": "cotal is the operator command line for the reference implementation: bring a mesh up, mint identities, launch agents, watch what they do, and tear it all down.", - "body": "# `cotal` CLI reference\n\n> **Reference**: describes the TypeScript reference implementation (the `cotal` CLI), not the wire contract. · **For:** operators · **Wire contract:** [SPEC](../SPEC.md)\n\n`cotal` is the operator command line for the reference implementation: bring a mesh up, mint\nidentities, launch agents, watch what they do, and tear it all down. It is a thin client over the\nwire contract: the normative subjects and schemas live in the [SPEC](../SPEC.md); this page is\nlookup material for the commands, not a walkthrough; if you are new, start with\n[Getting started](getting-started.md).\n\n## Running it\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH (needs Node 22+)\ncotal --help # every command, grouped\ncotal --version # cotal-ai version + each installed extension's (also `cotal -v`)\ncotal --help # one command's flags and usage\n```\n\n`npx cotal-ai ` runs it without a global install; in a dev clone, `pnpm cotal `\nruns it through `tsx` with no build step. Bare `cotal` prints help. Every command generates its own\n`--help`, usage, and shell completion from its declared flags.\n\nCommands come from the surfaces the binary composes: the base mesh CLI, the manager\n(`supervise`), and the delivery daemon (`deliver`), plus any operator-installed extensions.\n`cotal ext add ` installs any registry providers a package contributes: commands,\nruntimes, and local process lifecycle descriptors. The `web` dashboard and optional manager\nruntimes ship this way.\n\n## Commands\n\n| Area | Command | Purpose |\n|---|---|---|\n| Set up & lifecycle | [`setup`](#setup) | Guided, configure-only setup (installs, seeds personas; launches nothing) |\n| Set up & lifecycle | [`update`](#update) | Reconcile first-party extensions and check or opt into a coherent CLI upgrade |\n| Set up & lifecycle | [`up`](#up) | Start a local mesh (nats-server + JetStream), or boot a whole manifest with `-f` |\n| Set up & lifecycle | [`down`](#down) | Stop the whole stack, selected registered components, or a manifest deploy |\n| Set up & lifecycle | [`backup`](#backup-and-restore) | Create an offline full-space or registry-only artifact from a preserved cut |\n| Set up & lifecycle | [`clean`](#clean) | Configurable cleanup: purge history (live), or wipe the local store / identity (stopped) |\n| Set up & lifecycle | [`meshes`](#meshes-use-status) | List the running meshes on this machine |\n| Set up & lifecycle | [`use`](#meshes-use-status) | Set the default mesh a bare `cotal spawn` joins |\n| Set up & lifecycle | [`status`](#meshes-use-status) | Read-only diagnostics for setup, processes, and the selected mesh |\n| Agents & personas | [`spawn`](#spawn) | Launch an agent from a persona (foreground, or `--detach` via the manager) |\n| Agents & personas | [`models`](#models) | List connector model catalogs and variants from the manager |\n| Agents & personas | [`ps`](#ps-stop-attach) | List managed agents and their mesh status |\n| Agents & personas | [`stop`](#ps-stop-attach) | Ask the manager to stop a managed agent |\n| Agents & personas | [`attach`](#ps-stop-attach) | Stream and drive a managed agent's terminal (pty runtime) |\n| Agents & personas | [`input`](#input) | Type one line into a managed agent's terminal without attaching |\n| Agents & personas | [`personas`](#personas) | List, show, edit, create, or remove local personas |\n| Agents & personas | [`supervise`](#supervise) | Run a manager daemon (the agent supervisor / control plane) |\n| Agents & personas | [`runtimes`](#runtimes) | List the agent runtimes the manager can spawn through and whether each is reachable |\n| Agents & personas | [`reconcile-gate`](#reconcile-gate) | Unfreeze an issuance gate left frozen by a crashed restart when the successor cannot boot-heal it (holder gone, complete CONNZ sweep) |\n| Messaging & watching | [`endpoints`](#endpoints) | List every endpoint in the live presence roster, including infrastructure |\n| Messaging & watching | [`describe` / `invoke`](#describe-invoke) | Resolve a v0.4 service's command surface off the wire; invoke one command by name |\n| Messaging & watching | [`send`](#send) | Send one message, then exit: DM a peer, post a channel, or ask a role |\n| Messaging & watching | [`channels`](#channels) | Inspect or set the channel registry |\n| Messaging & watching | [`history`](#history) | Clear retained message history |\n| Messaging & watching | [`console`](#console) | Live protocol view for a space (TUI, or `--plain` line stream) |\n| Messaging & watching | [`web`](#web) | Browser dashboard (installed as the `@cotal-ai/web` extension) |\n| Auth & meshes | [`mint`](#mint) | Mint a creds file for a space (static auth mode) |\n| Auth & meshes | [`login`](#login-logout) | Sign in to a per-user-auth mesh's IdP (once per machine) |\n| Auth & meshes | [`logout`](#login-logout) | Revoke the IdP session and clear the cached login |\n| Auth & meshes | [`actor`](#actor) | Manage a user-auth space's actor ledger (grant / revoke / list) |\n| Auth & meshes | [`doctor`](#doctor) | Credential-health diagnosis and repair (`doctor auth`) |\n| Auth & meshes | [`join`](#join) | Join a space as your own presence (interactive) |\n| Manifest | [`topology`](#manifest-deploys) | Validate and view a mesh manifest's access graph (read-only) |\n| Extensions & misc | [`ext`](#ext) | Install / remove operator CLI extensions |\n| Extensions & misc | [`completion`](#completion) | Print or install shell completion |\n| Extensions & misc | [`feedback`](#feedback) | Send feedback to the Cotal developers |\n| Extensions & misc | [`deliver`](#server-daemons) | Run the server-side Plane-3 delivery daemon |\n| Extensions & misc | [`feedback-intake`](#server-daemons) | Run a self-hosted feedback intake server |\n\nThe manifest modes of `up`, `spawn`, and `down` (`-f `) plus `topology` are covered\ntogether under [Manifest deploys](#manifest-deploys).\n\n## setup\n\n```bash\ncotal setup [--full] [--demo] [--yes]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--full` | off | Redo the full guided flow (implies `--demo`) |\n| `--demo` | off | Also seed the guided expert team (`david`, `sven`, `me`) |\n| `--yes`, `-y` | off | Non-interactive accept-all (for agents / CI) |\n\nGuided setup is **configure-only**: it checks prerequisites, installs the Claude Code plugin, and\nseeds persona files, and it launches nothing (no mesh, no web, no manager). First run gets the\nnarrated flow; later runs print a status card. By default it seeds one `default` persona; the\n`david`/`sven`/`me` team is opt-in via `--demo`. See [Getting started](getting-started.md) and, for\nmaintainers, [setup internals](setup-internals.md).\n\n## update\n\n```bash\ncotal update [--self]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--self` | off | If a newer release exists, install that exact validated `cotal-ai` version globally and reconcile through the newly installed binary |\n\nWithout `--self`, `update` keeps the installed first-party surfaces coherent with the running\nbinary: it force-reconciles the four built-in connectors, then reinstalls other `@cotal-ai/*`\noperator extensions at the binary's exact version. Each extension runs in an isolated child, so one\nfailure cannot poison later replays. It then checks npm; a newer binary is an informational notice\nwith `cotal update --self` as the next command, not an automatic install.\n\nWith `--self`, the npm check happens first. When a newer release exists, Cotal installs the exact\nversion it validated, resolves and verifies that package in npm's global root, then launches that\nbinary to reconcile connectors and first-party extensions to the new generation. An npx or dev-clone\ninvocation therefore installs and continues through a separate global copy; it never claims the\nalready-running process changed. If the binary is current, `--self` performs the normal local\nreconcile without reinstalling it.\n\nThird-party extensions are listed with their installed version and recorded spec but are not\nauto-updated in v1. Floating third-party updates require `@cotal-ai/*` peer-range validation and are\na future follow-up. A failed connector/extension install, npm metadata check, or requested global\ninstall is reported and makes the command exit nonzero. Independent extension attempts continue so\nthe output includes every failure; an unavailable npm registry does not undo a completed local\nreconcile, but the command still exits nonzero because it could not establish that the install is\ncurrent.\n\n## up\n\n```bash\ncotal up [--detach] [--open] [--space ] [--server ] [--channels ] [--runtime ]\ncotal up --user-auth --idp [--exchange-public-port --exchange-public-url [--exchange-trusted-proxy]]\ncotal up --tls-cert --tls-key # serve broker TLS (both, or neither)\ncotal up --restore [--restore-only registry] [--accept-missing-source]\ncotal up -f [--dry-run] [--runtime ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--server ` | auto (free local port) | Listen URL override |\n| `--host ` | — | Bind host override. With no `--server`, the broker URL is derived from it, so `--host ` alone is enough to make a mesh reachable at that address; a `--host`/`--server` pair naming different addresses is refused. A wildcard bind (`0.0.0.0`, `::`) keeps a dialable loopback URL. Recorded on the mesh and reused by every later manager launch, so a repair or resume keeps remote [`attach`](#ps-stop-attach) working |\n| `--space ` | the folder's name | Space name |\n| `--store-dir ` | — | JetStream store directory |\n| `--channels ` | `.cotal/channels.json` if present | Channel-registry seed file (JSON). An explicit path that is missing is an error |\n| `--restore ` | — | Restore a completed offline backup before exposing the normal listener |\n| `--restore-only registry` | artifact selection | Restore only the registry component |\n| `--accept-missing-source` | off | Explicit disaster consent when the inode-bound preserved source is absent |\n| `--open` | off (auth) | Unauthenticated dev mesh: no JWT, no ACLs |\n| `--user-auth` | off | Per-user auth: people `cotal login`; connects are authorized against the actor ledger |\n| `--idp ` | — | With `--user-auth`: the IdP auth base URL to pin on first enable |\n| `--exchange-public-port ` | none | With `--user-auth`: add the public exchange face on this loopback port, for an HTTPS reverse proxy to forward to |\n| `--exchange-public-url ` | none | With `--exchange-public-port`: advertise the reverse proxy's HTTPS URL in discovery |\n| `--exchange-trusted-proxy` | off | With `--exchange-public-port`: attribute public failure buckets to the last `X-Forwarded-For` hop. Enable only when the listener is reachable solely through a trusted proxy; otherwise the socket address is used |\n| `--detach` | off | Run in the background (stop with `cotal down`) |\n| `--tls-cert ` | — | PEM certificate to serve TLS with. Must be given together with `--tls-key`. The pair is validated **before** the broker starts — readability, private-key mode, that the two match, the validity window, and that the certificate covers the host clients will dial — because `nats-server` starts happily on an expired certificate and only the client then fails. The decision is recorded, so a later bare `cotal up` after a `cotal down` keeps serving TLS rather than silently reverting to cleartext |\n| `--tls-key ` | — | PEM private key for `--tls-cert`. Refused if group- or other-readable (tighten to `600`) |\n| `--file `, `-f` | — | Launch a whole mesh from a manifest |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--runtime ` | `pty` (or the manifest's, with `-f`) | Agent runtime for the mesh manager (`pty` built in; others are installed extensions, explicit-only). Resolved + probed before the broker starts; an uninstalled/unreachable runtime fails loud. With `-f`, overrides the manifest's runtime |\n| `--rotate-sys` | off | Rotate the space's system account and re-mint its two `$SYS` creds. Needs a stopped mesh; refused with `--open` |\n\n`cotal up` boots a local nats-server with JetStream and, in auth mode (the default), JWT auth and\nper-agent ACLs; `--detach` records the mesh so `cotal spawn` from any directory can find it. With no\n`--server`, it auto-selects a free port if the default address is taken; an explicit `--server`\nstays fail-loud on collision. `--detach` also brings up the control plane (delivery daemon in auth\nmode, then the manager). The `-f` form is a [manifest deploy](#manifest-deploys); see\n[Run a mesh](run-a-mesh.md).\n\n`--user-auth --idp ` starts the space's auth service alongside the broker: the NATS\nauth callout plus its capability-gated local exchange, and optionally the closed public exchange\nface configured by the three `--exchange-*` flags above. The service is torn down with `cotal down`,\nand a re-run of `cotal up` heals a dead service on a running broker. `--user-auth` and `--open`\ncontradict each other and are refused loudly; a running broker cannot change auth mode\nwithout a `cotal down` first. See [identity & auth](identity-and-auth.md).\n\n`--rotate-sys` renews the two `$SYS` credentials (`membership-observer`, `connection-evictor`).\nThey carry a 30-day expiry and nothing re-signs them in place, because the system-account seed is\nnever persisted, so they are renewed by issuing a **new system account** under the same broker\noperator and minting fresh creds against it. A plain re-`up` does **not** do this: it reuses the\nexisting trust record, and its `$SYS` creds along with it.\n\nThe rotation is safe to run on a real space, with one operational cost. The data account, the account\nsigning key, every agent credential minted from it, and the JetStream store are all untouched; what\ndies is the retired system account, and with it any out-of-band copy of the old `$SYS` creds, on every\nbroker that loads the rotated config. The cost is that **earlier full backups stop being restorable**\n(see below), so this is not a no-consequence operation. It needs the broker to restart on the rewritten\nconfig, so it runs as part of a boot:\n\n```bash\ncotal down\ncotal up --rotate-sys --detach # agents reconnect; nothing is re-provisioned\ncotal doctor auth # both $SYS creds healthy again, 30 days out\n```\n\nA rotation is a stopped, fresh boot, and anything that is not one refuses it, all for the same reason\n(the on-disk material and the broker it runs on must never end up on different generations):\n\n- a live mesh, because the running broker would keep serving the retired account;\n- an open mesh, whether that comes from `--open` or from `broker.auth: false` in a manifest, which\n has no system account at all;\n- `--restore`, because reinstating a trust root and superseding it in one command leaves no way to\n say which authority the mesh came up on;\n- an unfinished restore or resume attempt on this root, including one `cotal up` would recover on\n its own, because those paths can adopt a live listener and return without booting a broker;\n- a root that hosts more than one space, because the system account lives in the shared broker\n record and a rotation would retire every tenant's, while the root holds one `$SYS` cred pair\n pinned to one data account.\n\nTwo things to know before you run it:\n\n- **The retirement is config-load-bound.** Old `$SYS` creds are refused by any broker that loads the\n rotated config. A stale `nats-server` still running the *previous* config in memory would keep\n honouring them, so stop every broker for this root first. `--rotate-sys` refuses if this root's\n mesh is recorded as running, if anything unidentified is answering at the address it was given, or\n if the root's pid file names a live (or unreadable) process. Those are Cotal's own ownership\n records, not a scan of the process table: a `nats-server` you started by hand against this root's\n `server.conf` on some other port writes none of them and will not be seen. Do not run one.\n- **It invalidates earlier full backups.** A full artifact binds to the trust chain it was taken\n against, and that commitment covers the operator JWT and the system account. Every full backup\n taken before a rotation refuses to restore afterwards, so take a fresh `cotal backup` once the\n rotated mesh is up. `cotal up --restore` names this case when the data account still matches.\n\nThe commit is not atomic (a trust-record write plus two credential writes), so an interrupted\nrotation leaves the record ahead of the creds. That split is detected rather than silent: every\n`cotal up` on an auth mesh, and every `cotal doctor auth`, compares each `$SYS` cred's issuer against\nthe persisted record and names the retired account. `up` warns rather than refusing, because these\ncreds power the membership graph and live eviction, both of which degrade fail-soft; the mesh is not\nworth taking down over them. Re-running the rotation heals it, at the cost of one generation.\n\nWhile those creds are expired the mesh keeps delivering messages, but the\n[membership feed](delivery-daemon.md) and live connection eviction stay down; `cotal doctor auth`\nand the manager's log both name the credential and this repair.\n\n## down\n\n```bash\ncotal down\ncotal down --preserve-state [--store-dir ]\ncotal down manager [delivery auth web nats ...]\ncotal down web [--space ]\ncotal down -f | --run [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--file `, `-f` | — | Tear down this manifest's deploy |\n| `--run ` | — | Tear down one `spawn -f` run by id |\n| `--space ` | current mesh | With components: the mesh whose target-addressed components (e.g. `web`) to stop |\n| `--dry-run` | off | Print the manifest teardown or selected components, mutate nothing |\n| `--preserve-state` | off | Bare whole stack only: fence the manager, retain principals and durable state, stop and prove the stack down, then publish `ready` |\n| `--store-dir ` | `.cotal/nats` | With `--preserve-state`: the actual store path (required for a custom store) |\n\nBare `cotal down` stops the whole local stack in dependency order. Positional component names stop\nonly those self-registered local processes; for example, `cotal down manager` leaves delivery and\nthe broker running, and `cotal down web` is available when the web extension is installed. A\ncomponent that starts target-resolved (the web dashboard) is stopped the same way: `cotal down web`\nresolves the mesh exactly like `cotal web` (registry current mesh first, `--space` to name one), so\nit works from any directory; the other components always stop under the folder you run it in. The\n`-f` / `--run` forms tear down a [manifest deploy](#manifest-deploys) without stopping the whole mesh\nand cannot be combined with component names. Stopping `nats` alone is refused while an unselected\nregistered daemon is still live; include those components or use bare `cotal down`.\n\nNormal `down` remains destructive at the logical identity/durable layer. `--preserve-state` is a\ndifferent maintenance transition: it suppresses leave/deprovision cleanup, persists the manager's\nsame-principal resume inventory, stops the entire stack without removing run/auth artifacts, and\npublishes a stable inode-bound cut only after every recorded process is proven stopped and the exact\nrecorded NATS endpoint is unreachable. A missing or stale broker pidfile never counts as stopped. The\nattempt is bound durably before the manager is fenced, the resume document and attempt-bound\n`cut-intent` are fsynced before manager commit, and the manager's commitment itself is journaled\n(`cut-committed`) before any process stops. A retry after a crash at any of those boundaries reuses\nthe exact recorded attempt and finishes the remaining stop and endpoint proofs idempotently, without\nneeding the (by then intentionally dead) manager. A partial cut never publishes `ready`. It cannot\nbe combined with component names, manifest teardown, or `--dry-run`.\n\n## clean\n\n```bash\ncotal clean --force\ncotal clean restore-attempt --attempt --force\ncotal clean restore-fallback --attempt --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | `history`: target mesh |\n| `--dms` | off | `history`: also clear DM history |\n| `--store-dir ` | `.cotal/nats` | `store`/`all`: JetStream store directory |\n| `--force` | — | Required: destructive, no prompting |\n| `--attempt ` | — | `restore-attempt`: exact stale pre-commit attempt; `restore-fallback`: matching healthy committed restore |\n\nOne configurable cleanup verb; every target requires `--force`.\n\n- `history` purges the retained message backlog on the **running** broker (channels, plus DMs\n with `--dms`). The same operation as [`history clear`](#history), which stays as an alias.\n- `store` deletes the **stopped** mesh's JetStream store (`.cotal/nats`): streams, durable\n consumers, and messages. This is the reset for stale on-disk broker state, e.g. durables\n minted by an older, incompatible Cotal generation surviving a `down`/`up` cycle.\n- `all` is `store` plus the space identity (`.cotal/auth`), the local creds and markers tied to\n it, any crash residue a normal `down` would have swept (stale pidfiles, `run/`), and the mesh's\n registry entry; the next `cotal up` mints a fresh identity.\n\n`history` needs the mesh up; `store` and `all` refuse while any recorded mesh process is still\nalive or any same-root recorded broker endpoint remains reachable (run `cotal down` first). They\nalso refuse outright on a root that holds accounts for several spaces: the store and the broker\ntrust record are shared by every space on the broker, so both targets would take out all of them\nand no `--space` can narrow that. `down`, `backup` and `up --restore` refuse there for the same\nreason. `cotal status` lists the tenants on such a root. Personas\n(`.cotal/agents`) and logs are never touched. A custom\nstore location is not recorded anywhere, so `--store-dir` must repeat whatever the mesh was\nlaunched with. Custom cleanup targets must contain either the Cotal store-generation marker or a\nreal `jetstream/` store directory; filesystem roots, project roots, and Cotal auth/maintenance trees\nare always refused.\n\n`store` and `all` also refuse every maintenance journal state. After a healthy committed restore,\n`restore-fallback` is the only supported way to remove the recorded unchanged old-store inode; it\nnever deletes the active target, requires both the exact attempt id and `--force`, and retires the\ncompleted restore journal so a later `down --preserve-state` can start a new backup cycle.\n\n## backup and restore\n\n```bash\ncotal down --preserve-state [--store-dir ]\ncotal backup create [--only full|registry] [--store-dir ]\ncotal up --restore [--restore-only registry] [--accept-missing-source]\n```\n\nBackup is offline-only. It requires the stable `ready` record from `down --preserve-state`, an exact\nstore match, no live recorded process, and an unreachable exact endpoint from the recorded cut.\nThat endpoint is probed immediately before cloning, so a live broker with a missing or stale pidfile\nis still refused. It claims the cut, reflink/copies the stopped source to a\nprivate attempt clone, and opens only that clone on a random loopback bootstrap broker with an\nindependent parent/deadline watchdog. It validates the canonical stream and pull-consumer inventory,\nwrites native snapshots with consumers excluded, and stores conservative contiguous ACK-floor\ncheckpoints separately. The original store is never opened by the backup broker, and the stack is\nnot restarted implicitly. Artifact destinations must not overlap the preserved source or maintenance\nattempt tree. Restore artifacts and targets likewise cannot nest inside or contain each other, the\npreserved source, or the maintenance attempt tree.\n\n`full` is the default and indivisible: channel registry, CHAT/DM/TASK/INBOX/DLV, ACL, MEMBERS, and\nvalidated durable checkpoints. `registry` is the sole partial artifact. Presence, derived membership\nfeed, leases, native ephemeral/history consumers, credentials, keys, tokens, owner secrets, and actor\nledger files are excluded. Artifacts are exclusively created `0700`; snapshot/checkpoint files and\nthe manifest are `0600`; `manifest.json` is written last with exact sizes and SHA-256 values. The\ndirectory is trusted operator input: hashes detect corruption, not malicious rewriting.\n\nRestore validates and stages the exact allowlisted artifact bytes before moving or creating a store.\nIt requires the same space and existing trust state. The whole pre-commit window holds a journaled\nliveness claim (coordinator, watchdogs, brokers, absolute deadline): ordinary `up` and a repeated\n`up --restore` refuse while the claim is live, and a stale attempt is recovered only after the\ndeadline has elapsed and every recorded owner is proven dead — automatically by a retried\n`up --restore`, or explicitly with `cotal clean restore-attempt --attempt --force`. Nothing\never rolls back a live attempt. A registry-only artifact restores as registry-only whether or not\n`--restore-only registry` is passed; omitted infrastructure is always created and the exact\npost-restore stream inventory is asserted before commit intent. Ordinary `up` from a preserved cut\nresumes only the exact recorded source store and runtime; a contradicting `--store-dir` or\n`--runtime` fails in preflight. Authenticated restores validate the complete\nspace trust bundle before staging, including nkeys, seed matches, JWTs, signers, and space binding;\nfull restores commit to the validated operator, system-account, data-account, and active-signer root\nchain in addition to the static/user authority fingerprint. Because the system account is part of that\ncommitment, a [`cotal up --rotate-sys`](#up) makes every full artifact taken before it unrestorable\nagainst this root: take a fresh full backup after each rotation. The composed commitment is revalidated\nimmediately before store mutation and never includes secret seeds. Restore never creates fresh auth.\nSame-path restores atomically retain the old\nsource at the journaled fallback path; alternate targets retain it in place; a missing canonical\nsource needs explicit `--accept-missing-source`. Quarantine and target restores use current canonical\nconfigs on isolated random-loopback brokers, never expose native snapshot consumers, and publish a\ncommit-intent immediately before the normal listener starts. Archive bytes never instantiate the real\ntarget: after quarantine validation, every stream is re-snapshotted from the validated quarantine\nstate into attempt-owned sanitized files, and the target is restored solely from those. Before that boundary, failure rolls back\nthe attempt-owned target; after it, ambiguity preserves both stores and records forward-repair\nrecourse. The cooperative maintenance lock excludes Cotal commands, not arbitrary raw NATS processes.\n\nBootstrap brokers in every auth mode — including open — mount the store under a local account with\nrandom operation-specific logins only, each carrying the exact per-phase subject permission matrix;\nnormal static credentials and user-auth sentinel/bearer connections are rejected, and no auth\nservice or callout starts. Open mode differs only in its account label, never in authority. Inventory, each stream snapshot,\nrestore initiation, exact upload id, validation, and each checkpoint recreation use separate exact\nauthorities. Every checkpoint carries the source stream's message/first/last sequence state and must\nmatch its snapshot record before mutation; core then derives and validates the only allowed start\npolicy. TASK is not a CLI exception: the same core checkpoint API recreates its canonical `DeliverAll`\nWorkQueue durable because acknowledged tasks are absent from retention and NATS forbids a\nstart-sequence policy there. Registry-only restore creates every omitted canonical stream and transient\nbucket on the isolated target before the normal listener is exposed. It deliberately does not resume\nretained agents or recreate their DM/DLV/TASK/ACL state; their identity material stays retained and\nstopped rather than being reprovisioned into a partial restore.\n\nAfter listener readiness, the manager starts attempt-bound, validates retained credentials/tokens\nwithout granting or reprovisioning, and resumes the exact persisted principals under cleanup\nsuppression. Registry-only restore uses the same flow with an empty agent set. `commitResume` is an\nidempotent validation barrier only: success must be `awaitingFinalize` with an attempt-bound 64-hex\ncommit token and does not release suppression. Under the workspace lock, the CLI first fsyncs that\nexact evidence as `manager-committed` (restore) or `resume-committed` (ordinary resume), then calls\ntoken-bound `finalizeResume`; only an `active` response for the exact token releases suppression. The\nCLI records the same token in finalization evidence before a restore becomes `active`, or before an\nordinary resume retires and consumes the marker. Re-entry from either committed state skips the prior\nidempotent activation/commit phases, retries finalization with the durable token, and finishes the\nworkspace transition. Failure before finalization preserves the committed state and cleanup\nsuppression; it is not rewritten through a degraded transition. Re-entry between any two earlier\nboundaries reuses the same attempt and may retry the idempotent phases without deleting retained state. A missing or\nchanged per-agent dependency is a named fail-closed result; the journal becomes degraded and remains\navailable for forward repair. A retry from `resume-intent`,\n`resume-active`, or `resume-degraded` reuses the same attempt and inventory after the prior listener is\nproven stopped. Every normal restore listener has an unguessable attempt-bound NATS server name. The\nCLI fsyncs its exact name/nonce, canonical endpoint, process owner, and generation-bound target identity\nimmediately after spawn. Re-entry accepts a surviving listener only when its INFO server name, live PID\nrecord, endpoint, and target identity all match that proof; degraded restore repair then moves through\nthe guarded workspace transition only after manager commit. If an uncommitted bound owner is provably\ndead, recovery retires that exact proof under the maintenance lock and binds a fresh listener for the\nsame attempt, endpoint, and target with a new nonce and server name. A live foreign/mismatched listener\nor ambiguous owner is preserved and refused, never adopted by reachability alone. A reconstructed\ncommit/degraded attempt without either the exact bound proof or a durable dead-listener replacement\nrecord fails closed even when the recorded port is free. A later ordinary startup may pass an `active`\nrestore only when its details prove manager commit and its exact recorded listener is dead.\n\n## meshes, use, status\n\n```bash\ncotal meshes\ncotal meshes add # guided, on a terminal\ncotal meshes add --server [--root ] [--mode auth|open|user] [--tls] [--force]\ncotal meshes add --mode user (--user-auth-file | --from )\ncotal meshes rm [ …] [--force]\ncotal use \ncotal status [--space ] [--server ] [--components]\n```\n\n`meshes` lists the meshes this machine knows; a `*` marks the `current` default a bare\n`cotal spawn` joins.\n\nRun on a terminal with the space or `--server` missing, **`meshes add` is guided**: it asks for the\none thing that cannot be derived (the broker URL), probes it, and tells you what answered - open or\nrequiring credentials. It then offers the spaces your `--root` already holds credentials for, states\nthe mode as a fact about that broker rather than asking, and shows the exact record before writing\nanything. A broker that does not answer, or a space name already registered, becomes a choice rather\nthan an error. Anything you pass on the command line is taken as given and not asked again. Without\na terminal - a script, an agent, CI - nothing prompts and the flag form's errors stand\n(`COTAL_NO_PROMPT=1` forces that too).\n\n`cotal up` and `cotal down` maintain their own records. `meshes add` registers a mesh they cannot\nspeak for: one running on another machine, a shared broker, a hosted space. `--root` is the folder\nwhose `.cotal/auth` holds that mesh's credentials and whose `.cotal/agents` holds its personas\n(default: the project you run it in) — the registry stores that path, never a secret. `--mode`\ndefaults to `auth` when the root holds the space's account record and to `open` otherwise. The\nbroker is probed before anything is recorded, so a wrong address, or credentials that mesh will\nnot accept, fails here instead of at the first `spawn`; `--force` records without verifying (and\nreplaces an existing record).\n\nA hostname or public address is registrable only when the connection will **require TLS**: pass\n`--tls`, or use a `tls://` URL — the scheme is recorded as enforced intent, so every later dial\nthrough the record demands the handshake (and `meshes add tls://…` against a plaintext broker is\nrefused at registration). Without required TLS the fence is unchanged: loopback and\nprivate-overlay literals only, and RFC1918 addresses are refused in both modes — a cafe LAN is\nprivate, not yours.\n\nA **user-auth** mesh registers from supplied pinned trust, never guessed: `--user-auth-file`\ntakes the bundle exported where the mesh runs; `--from` asks before it dials the address at all,\nthen fetches its `/.well-known/cotal-mesh` discovery document (HTTPS only), displays the pins, and\nasks again before adopting them. Neither fetch follows redirects: a 302 can move a pinned fetch\nonto plaintext or onto another host, so it is refused rather than followed, and the pinned\nexchange must itself be an `https://` URL — except for an exchange on this machine, where plain\n`http://` is accepted for a loopback *literal* (`127.0.0.1`, `::1`, any spelling of them) but not\nfor `localhost`, which is a name rather than an address. Registration verifies that exchange answers `/health`\nand `/jwks` as the pinned issuer, and that the broker itself refuses a bare connect — the\nauth-required refusal is the pass. The sentinel credentials land in a 0600 file under the entry's root; the registry\nrecords only the path.\n\n`meshes rm` drops records — it never stops a mesh. For a mesh running on this machine `cotal down`\nis the right verb, and `rm` says so unless you pass `--force`. A record you added by hand is only\nremoved by something that names it — `meshes rm`, or an `add --force` replacement — or by a\n`cotal up` that actually starts the broker for that same space, server and root, which becomes that\nmesh and so takes the record over (a `cotal up` for that space anywhere else refuses instead).\nNothing that merely *infers* a record is stale touches it: an\nunreachable broker is listed `offline` and stays, and `cotal down` / `cotal clean all` leave it\nalone even when it shares a root with the project they are tearing down, because nothing on this\nmachine could write it back.\n\n`use ` sets that default; the selection applies from every directory,\nincluding inside another mesh's project. `status` is a read-only report: machine prerequisites\n(starting with the installed `cotal-ai` version), the installed extensions and their versions, this\nfolder's `.cotal/`, the recorded meshes, and a live snapshot of the selected mesh (roster, channels,\nmembership feed). `status` takes `--space` / `--server` to pick the mesh to inspect; it starts\nnothing.\n\n`cotal status --components` adds a fail-loud per-component health pass. It reads **each\ncomponent's own control surface**, rather than treating a PID, a lease, or a successful probe of a\nsibling as proof that the component serves. It prints one of `serving`, `absent`, `not-serving`, or\n`refused` for each component and exits `0`, `1`, `2`, or `3` respectively (the highest observed\nstate wins):\n\n- **manager** — local PID record, its liveness-lease holder and PID, then the manager's own typed\n `status` service reachability from this host. Builds without a startup-phase report say\n `phase not reported by this manager build`; that is never a blank green state.\n- **delivery** — local PID record, its ready lease (`ready` is the daemon's own bound-control\n signal), and the latest `renewal.json` adoption verdict. A re-signed credential and a\n broker-accepted adoption stay distinct facts.\n- **web** — local PID record and the dashboard's own loopback `/api/meta` response, which must name\n the same PID and its requested port. A different process on the port, an unreadable PID command,\n or an unrecognizable process record is `refused`, not a green default-port guess.\n- **broker** — the registered mesh URL dialed from this host with its recorded TLS requirement.\n\n`absent` means Cotal has no live local component record (or has a stale record); `not-serving`\nmeans the component record is live but its service/readiness surface did not answer or is not ready.\nThose are intentionally separate exit cases. A failed or unreadable probe is `refused`, never an\nabsent component or a clean zero.\n\n## spawn\n\n```bash\ncotal spawn [] [--detach] [--name ] [--agent ] [--model ] [--variant ] [--prompt ] [--cwd ]\ncotal spawn -f [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | resolved mesh | Target space |\n| `--server ` | registry entry | Broker URL override |\n| `--creds ` | — | Control-caller creds for an off-registry manager (`--detach` only) |\n| `--name ` | persona's `name:` | Presence-name override (does not choose the persona) |\n| `--config ` | — | Persona catalog name or file path; wins over the positional |\n| `--agent ` | `COTAL_DEFAULT_AGENT`, else `claude` | Connector type (`claude`, `opencode`, `jcode`, `hermes`, …) |\n| `--role ` | persona's `role:` | Role override |\n| `--model ` | persona's `model:` | Model override |\n| `--variant ` | persona's `variant:` | Model variant override (connector-defined; e.g. OpenCode reasoning tiers) |\n| `--cwd ` | this cwd | Working directory to root the agent at |\n| `--prompt ` | — | Initial prompt auto-submitted at start |\n| `--resume ` | — | Fork an existing session id into the mesh (claude only) |\n| `--events` / `--no-events` | off | Publish the session's structured event plane to its own event channel |\n| `--share-tools ` | none | Share named operator MCP servers with the agent |\n| `--subscribe ` | persona's | Channel read-set override |\n| `--allow-subscribe ` | = subscribe | Read-ACL override |\n| `--allow-publish ` | deny | Post-ACL override |\n| `--detach`, `-d` | off | Launch via the manager into a detached PTY (reattach with `cotal attach`) |\n| `--on ` | class anycast | With `--detach` only: pin the launch to one manager instance id (the whole id, as `ps` prints it). Refused on a foreground spawn (no manager to pin), with `-f` (a manifest deploy launches through the manager class queue), and when empty |\n| `--file `, `-f` | — | Deploy a manifest onto the running mesh |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--allow-stale ` | — | With `-f`: waive named stale agents (apply-only) |\n| `--runtime ` | manifest's | With `-f`: override the manifest's runtime |\n\n`--events` turns on the session's **event plane**: a stream of structured events describing what\nthe agent did, rather than the prose it wrote, on a channel of its own. The channel is named after\nthe agent's principal, `events..`, never after its display name, because two live\nagents are allowed to share a display name and would then share a stream. The launch grants publish\nrights on exactly that one channel, foreground and detached alike, and a connector that does not\npublish an event plane refuses the flag rather than starting a session whose events have nowhere to\ngo.\n\nThe flag and the grant are separate on purpose. Holding publish rights on a channel is not a request\nto publish to it, so writing an event channel into an agent file's `allowPublish` does not turn the\nplane on: only the launch does.\n\nThe persona (`--config` > positional > `COTAL_DEFAULT_PERSONA` > `default`) is loaded from the\ntarget mesh's `.cotal/agents/`; the launch flags override the file. Foreground runs the agent\nattached to your terminal; `--detach` hands the launch to the running manager. Both modes get the\ndurable backstop on a mesh that runs the delivery daemon; `--live-only` skips it for a foreground\nspawn (messages posted while it is disconnected are then not replayed). A foreground exit retires\nthe agent's creds and broker footprint, like a manager despawn. A `--detach` spawn is an\n**action**: the manager accepts it and returns the allocated identity at once, then the launch\nfollows to a terminal outcome rather than blocking (see [the control surface](control-surface.md)).\nSee [Connect Claude Code](connect-claude.md) and [Agent files](agent-files.md); `-f` is a\n[manifest deploy](#manifest-deploys). (`cotal start` was merged into `cotal spawn --detach`.)\n\n## models\n\n```bash\ncotal models [--agent ] [--refresh]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--agent ` | all registered connectors | Connector whose catalog to list |\n| `--refresh` | off | Ask the connector to refresh its provider cache |\n\nAsks the running manager for each connector's model catalog (model ids plus their variants)\nfor connectors that expose one (OpenCode today; a connector without a catalog says so). Pick a\nresult with `cotal spawn --model --variant `.\n\n## endpoints\n\n```bash\ncotal endpoints [--space ] [--server ] [--creds ]\n```\n\nLists the mesh presence roster: agents, the manager, and any other protocol endpoint, with each\nendpoint's role, kind, status, and current activity. Unlike `ps`, this is a read-only presence view;\nit is not limited to child processes owned by the manager.\n\n## describe, invoke\n\n```bash\ncotal describe [--space ]\ncotal invoke [--args ''] [--space ]\ncotal invoke --name [--admin] [--space ]\n```\n\nThe generic v0.4 service surface. `describe` resolves a registered endpoint's command set off the\nwire - the reserved `describe` command answers the registered contract digests, the schemas are\nfetched from the space's content-addressed contract store, recompiled, and verified against those\ndigests - and prints each command with its capability class and targeting shape. `invoke` calls one\ncommand by name: `--args` is a JSON object validated against the fetched input schema *before*\npublish; a targeted command takes `--name ` (resolved to the agent's current principal via\n`ps`) or `--self`. `--admin` uses the admin instrument credential, whose cross-agent reach rides\nthe operator-only `any` authorization mode. Neither command has compile-time knowledge of any\nendpoint's schemas - this is the same trust chain every built-in control command now uses. Needs an\nauth mesh: the manager registers its service on both static and per-user meshes (a signed-in user\nrides their bearer; cross-agent reach needs the `admin` scope). An open mesh has no service\nregistry.\n\n## ps, stop, attach\n\n```bash\ncotal ps [--on ] [--wide | --json] [--space ]\ncotal stop --name [--on ] [--space ]\ncotal attach --name [--on ] [--no-reconnect] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | — | Managed agent to stop / attach (required) |\n| `--on ` | class anycast (`ps`: class scatter) | Pin to one manager instance id (multi-manager space); takes the whole id as `ps` prints it, not a prefix. An empty value (`--on \"\"`, an unset shell variable) is refused, never treated as absent |\n| `--wide` (`ps`) | off | After each seat's compact row, print the per-seat facts the manager already records: model pin (and variant), `cwd`, `pid`, spawner, lifecycle uid, and the owning manager's instance id and host. A fact the manager did not record (no model pinned, or a runtime that owns no real process) prints nothing, never a placeholder |\n| `--json` (`ps`) | off | Machine-readable: one JSON object per seat per line, exactly the row the manager sent. Instance headers and errors go to stderr, so stdout is pure rows. Mutually exclusive with `--wide` |\n| `--no-reconnect` (`attach`) | off | End the attach when its session ends, instead of re-establishing it. For scripts that want one run and one exit code |\n\nThese are operator clients over the running manager's control plane. `ps` prints two facts per\nmanaged agent, because they answer different questions: the process fact from the manager's own\nruntime handle (`running` with its uptime, or `exited` with how long it ran), and the mesh fact from\nthe roster (`idle` / `working` / `waiting` / `mesh offline`, or `not in roster` when the seat has no\npresence row at all: a seat that has not joined yet, or one that never did). A seat can be `running` and `mesh offline` at once: the process is alive and\nits presence has lapsed. On a user-auth mesh `ps` also renders each managed agent's last\ncredential-refresh outcome, fail-closed.\n\n**Mode split (chosen up front, never try-scatter-then-degrade):**\n\n- **Static / open mesh.** Bare `ps` is a **class scatter**: it freezes the live manager class from\n the records registry, merges every registered instance's agents grouped and attributed per\n instance, and a non-answering instance is shown as `registered, no answer within the deadline`\n (never silently omitted). That label is the whole claim: the instance is registered and did not\n answer. It does not say the host is down, because a dead host never deregisters itself and a\n live one can be slow; if it is gone, deregister it.\n `--on ` pins the read to one exact instance id instead. A wrong pin fails loud\n rather than falling through: a well-formed id that no live manager carries is reported as\n `manager instance did not answer` (nothing else is asked), and a credential without that\n instance's rail is reported as refused by the broker, not as an unresponsive manager. A manager\n that answers with a refusal is shown with its own cause; \"no manager reachable\" is said only when\n nothing answered at all. If the scatter's own registry read fails (the freeze or the reconcile),\n `ps` says the manager registry could not be read rather than pronouncing on the managers, which\n may all be up.\n\n**`stop` and `attach` route by seat locality.** A seat can only be stopped or attached by the\nmanager actually running it, and the class queue does not know which one that is. So on a\nstatic/open mesh both verbs first ask every registered instance which one hosts the named seat, then\naddress that instance directly. You do not need `--on` for this — it happens by default.\n\n`--on ` remains the override, for when you already know where the seat lives or the\nlookup itself is degraded. It is also the **only** route on a **user-auth mesh**: a ledger-scoped\nbearer does not hold the registry-read rows the lookup needs, so there the verbs stay on the class\nqueue unless you pin them yourself.\n\nIf the seat is found on no reachable instance, the error says so — how many managers answered, and\nwhich ones did not — rather than reporting a bare `no agent `. That distinction matters\nbecause a single manager cannot tell \"hosted elsewhere\" from \"does not exist\": it answers\n`not-found` for both.\n- **User-auth mesh.** `cotal ps` reports what **one** manager knows about your agents (an `ep.one`\n read against the manager's in-memory roster, owner-filtered). It does **not** report other\n manager instances, and it cannot tell you that one is down — an unreachable manager is absent\n from the list, not flagged. Completeness across a multi-manager user-auth space is not claimed.\n A manager that does not answer fails the command outright (exit non-zero), rather than printing\n an empty list that could be read as \"no agents\". Your ledger row needs the `admin` scope to\n reach `ps` at all; `spawn` alone is refused by the broker (the ep tier boundary).\n\n`attach` streams and drives an agent's terminal on the `pty` runtime; detach with the escape key\n(Ctrl-] by default; see [`COTAL_DETACH_KEY`](config.md)). It does so over a one-use, holder-bound\nmesh session ([SPEC](../SPEC.md) §13.6): the manager replies with a signed session grant (never a\n`127.0.0.1` URL), the CLI redeems it once over the broker, and the browser console (`cotal console`)\ndrives the same session. `stop` and `attach` need a running manager to talk to. On a static mesh\nthey are cross-agent admin operations. On a user-auth mesh, your own agents (any agent under your\nowner) need only the `spawn` scope; another owner's agent needs `admin` on your ledger row\n([identity & auth](identity-and-auth.md)). Launch detached agents with [`spawn --detach`](#spawn).\n\n**`attach` reconnects when the link dies.** A session lives on a network link, and a laptop that\nsleeps, a VPN that drops or a wifi handover kills it. When that happens `attach` prints\n`[cotal: connection lost, reconnecting]` on stderr and starts asking the manager for a new session:\na fresh grant, a fresh per-session credential, a fresh connection, so every attempt re-runs the same\nauthorization the first attach did. On success it prints `[cotal: reconnected]`, the manager repaints\nthe seat's current screen the way it does for any attach, and you carry on in the same terminal.\nRetries wait 1s, 2s, 5s, 10s, then 30s, for as long as the seat exists. The detach key is read the\nwhole time the loop runs, the waits and the attempts alike, so a reconnect never traps you: press it\nwhile a session is being established and the attach ends there, and a session that lands behind the\npress is handed back to the manager rather than left holding a slot. Everything else you type while\nthere is no session is dropped rather than queued, so keystrokes aimed at a terminal that turned out\nto be frozen, Ctrl-C included, are not delivered to the agent by a reconnect you did not know had\nhappened. That starts before the first session, not at the first reconnect: at a terminal, `attach`\nreads and drops what you type while it is still resolving the mesh, so a key struck at a prompt that\nhas not come up yet does not reach the agent when it does.\n\nWith stdin a **pipe** the contract is the opposite, and deliberately so. `printf 'ls\\n' | cotal\nattach --name web` is a script's input rather than an operator at a frozen screen, so it is buffered\nby the stream and delivered to the seat when the session opens, exactly as it always was. That holds\nin every window, not just before the first session: a pipe keeps buffering across a reconnect too, so\n`tail -f log | cotal attach --name web` does not lose the part of its feed written while the link was\ndown. Only a terminal gets the reader; `--no-reconnect` keeps the old behaviour on both.\n\nIt stops on its own when reconnecting cannot help, and says why: a manager that refuses the attach\nexits non-zero with the manager's own message, and a reconnect that finds the seat no longer there\n(despawned, or its agent exited while the link was down) exits cleanly with `seat is gone`.\nA refusal that could still pass, such as a manager at its session ceiling, is relayed in the\nmanager's own words while the loop keeps trying, once per refusal rather than once per attempt.\nPressing the detach key, or the agent's process exiting while you are attached, ends the attach as\nit always did. `--no-reconnect` turns all of this off and restores the single-session behaviour,\nwhich is what a script wants.\n\nEach reconnect also hands the abandoned session back to the manager, over the first link that can\ncarry the message, so an attach that flaps does not eat the manager's session slots one outage at a\ntime. If that message never gets a link, the attach says so when it ends.\n\nWhich mesh `attach` resolves also decides **whose trust it redeems with**. Redeeming a session grant\nmeans minting a short-lived, session-scoped credential from the space's seed, and that seed comes\nfrom the root the mesh resolved to, never from a `.cotal` found by walking up from whichever\ndirectory you happen to be standing in. The difference is not hypothetical: `~/.cotal` exists on\nevery install because the mesh registry lives there, so a command run anywhere under your home\ndirectory but outside a project used to mint from your home directory's trust and present it to a\nbroker that trusts a different chain, which surfaced as a bare authorization failure that named\nnothing. A directory that does hold another chain for the same space is now reported on the way\npast, and not obeyed:\n\n```text\n! this directory resolves to /Users/you, whose .cotal/auth holds a DIFFERENT trust chain for space \"team\".\n attach used /Users/you/projects/app, the root this mesh resolved to. The other one is not being used, and is worth a look.\n```\n\nWhen the resolved mesh holds no seed at all, `attach` refuses and names what it resolved, the broker\nand the root, instead of describing a directory it did not use.\n\nTerminal bytes stream over the mesh; the manager's own HTTP/WS face serves the console. That endpoint binds\n**loopback by default**, so nothing is exposed by accident; `cotal up --host ` passes its bind\naddress down, which is what lets you attach to an agent whose manager runs on another machine. A\nbare `cotal supervise` and an embedded manager stay machine-local. Set it directly with\n`supervise --console-host `.\n\nThat address is **recorded on the mesh** and carried forward, because it is a decision rather than\nsomething later commands can work out for themselves (a broker dial address is not a manager bind\naddress). Every later manager launch for the same mesh reuses it — a same-root `cotal up` repair,\nadopting a preserved or restored listener, a `spawn -f` manifest deploy — so a manager replacement\ndoes not quietly move a reachable attach face back to loopback. Passing `--host` again overrides it,\nso you can widen or narrow exposure whenever you like; a mesh that never asked stays loopback-only\nand records nothing.\n\nBecause that face carries terminal read and write for every managed agent, it is credentialed in two\ntiers. A mesh caller receives a **ticket** bound to the single agent the manager just authorized,\nsingle-use and short-lived, so one authorized attach can never be re-pointed at someone else's\nagent. The **console token** is the operator's own, reaches every agent, and is printed only to the\nmanager's output. The roster, the live feed, and the PTY stream all answer `401` without one; the\nstatic console shell is served openly, since it describes no agent.\n\n## input\n\n```bash\ncotal input --name --text [--no-enter] [--on ] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | | Managed agent to type into (required) |\n| `--text ` | | The text to type, taken verbatim (required) |\n| `--no-enter` | off | Type the text and stop there, without pressing Enter |\n| `--on ` | class anycast | Pin to one manager instance id, exactly as [`attach`](#ps-stop-attach) |\n\nTypes one line into a running agent's terminal, as if you had typed it there, and returns. This is\nthe half of [`attach`](#ps-stop-attach) that a program wants: `attach` is a live stream that holds a\nsession open and expects a terminal on your side, so a script, a cron job or a web UI cannot use it\nto send a single line. `input` is one authorized call.\n\nWhat it is for is **harness commands**. A line beginning with `/` is not chat and not a message: it\nis something the agent's own harness handles, and the only way in is the keyboard.\n\n```bash\ncotal input --name reviewer --text \"/compact\" # ask the harness to compact its context\ncotal input --name reviewer --text \"/model opus\" # switch its model\ncotal input --name reviewer --text \"hold on that PR\" # ordinary typing works too\n```\n\n**Quoting.** `--text` takes a value, so a payload starting with `/` survives as written. A payload\nstarting with a dash needs the `=` form, because the shell-style `--text --foo` is ambiguous and is\nrefused rather than guessed:\n\n```bash\ncotal input --name reviewer --text=--verbose # dash-leading text: use --text=\n```\n\nEnter is pressed by default, since a command typed but never submitted has not been delivered.\n`--no-enter` types the text and leaves it sitting at the prompt, which is how you stage a line and\nsend it later.\n\nNothing comes back but a delivery receipt (`✓ sent 9 bytes to reviewer`, counting the trailing\ncarriage return). Whatever the agent does next shows up where its output already goes: the mesh, its\ntranscript, or an `attach`.\n\n**This one is operator-only, and more narrowly than `stop` or `attach`.** Those two are granted to\nanything holding `spawn`, so an agent can stop and attach to seats under its own owner. `input` is\nnot: it is granted only to operator credentials, which on a user-auth mesh means your ledger row\nneeds the `admin` scope, the same scope [`ps`](#ps-stop-attach) already needs there. The reason is\nthat a write into a terminal is control of whatever is running in it, and on a user-auth mesh the\nown-owner rule covers every seat under you, not only the ones you launched: a `spawn`-scoped agent\ncould otherwise type into a sibling it never started. Seat locality is still resolved for you.\n\nOnly the `pty` runtime can be typed into. The external terminal runtimes (`tmux`, `cmux`, `orca`,\n`herdr`) attach to a process they do not own, so they have no input stream for it and the command\nrefuses by name rather than dropping the keystroke.\n\n## personas\n\n```bash\ncotal personas list [-v] [--running]\ncotal personas show \ncotal personas edit \ncotal personas new (--prompt | --from ) [--role ] [--model ]\ncotal personas rm --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh's persona catalog |\n| `--role ` | — | `new`: the persona's role |\n| `--model ` | — | `new`: the persona's model |\n| `--prompt ` | — | `new`: the persona's prompt text |\n| `--from ` | — | `new`: seed the prompt from a file |\n| `--verbose`, `-v` | off | `list`: include role / model / description |\n| `--running` | off | `list`: mark personas live on the mesh |\n| `--force` | — | `rm`: required, delete without prompting |\n\nPersonas are the local agent files under `.cotal/agents/` that `cotal spawn` launches. See\n[Agent files](agent-files.md) for the file format.\n\n## supervise\n\n```bash\ncotal supervise [--runtime ] [--space ] [--server ] [--spawn ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space to supervise |\n| `--server ` | hosting mesh, or matching registered mesh | Broker URL. A registered mesh supplies it when omitted; a different explicit value is refused. |\n| `--runtime ` | `pty` | Agent runtime (`pty` built in; extension runtimes are explicit-only) |\n| `--console-port ` | — | Protocol-console port |\n| `--console-host ` | loopback | Bind host for the console + attach endpoint. Loopback keeps it machine-local; `cotal up` passes the address it bound the broker to, which is what lets `cotal attach` reach this manager from another machine |\n| `--roster ` | — | Declarative roster to boot at startup |\n| `--launch ` | — | Resolved manifest launch spec (from `up -f` / `spawn -f`) |\n| `--spawn ` | — | Comma-separated personas to pre-spawn at startup |\n\nThe manager is the agent supervisor and control plane: it answers `spawn --detach`, `stop`, `ps`,\n`attach`, and the `cotal_*` manager tools. `cotal up --detach` starts one for you; run `supervise`\ndirectly to recover a dead manager or drive a custom runtime. Default runtime is `pty`; install an\noptional provider first (`cotal ext add @cotal-ai/orca`, `@cotal-ai/tmux`, `@cotal-ai/cmux`, or `@cotal-ai/herdr`) and\nselect it explicitly. A missing provider or app fails loudly; there is no fallback. See [Deploy](deploy.md).\n\nA `meshes add --mode user` entry is a **participant** registration, not hosting authority. A\nparticipant may run `supervise` only when the host advertises the remote manager authority service\nand the signed-in actor has the dedicated `supervise` ledger scope. The CLI obtains the closed,\nloopback-only `manager-service` view; `spawn` and `admin` do not substitute for that scope. The\nhost issues the manager's public-nkey JWT material through its lifecycle-bound prepare → activate\n→ renew protocol, never by handing the participant a signer or static provisioner credential.\n\nWithout that advertised host service or scope, `supervise` refuses before it starts a manager.\nRun `cotal spawn` without `--detach` to launch a foreground agent, or ask the space host to enable\nthe authority service and grant `supervise` for detached agents. If a running remote manager loses\nrenewal, it reports degraded state and refuses unsafe new starts and restarts; live agents are not\nsilently replaced. Do not run `cotal down` or `cotal up` on a participant machine to repair this\ncondition.\n\n## reconcile-gate\n\n```bash\ncotal reconcile-gate [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the frozen gate lives in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint whose gate is frozen |\n| `--instance ` | this folder's persisted manager instance | Instance id |\n\n**When you need this.** A manager restart killed partway through — after it began deregistering,\nbefore the new incarnation finished — leaves the endpoint's issuance gate *frozen*, held by a\nprocess that no longer exists. The freeze is what stops two incarnations serving at once, which is\ncorrect. The successor manager now completes that dead registration itself on boot, using the same\nguard this command uses: it acts only when the freeze-holder is affirmatively gone under a complete\nCONNZ sweep (`gone` and `sweepComplete=true`), then abort-reopens the gate at generation+1 with\nprocessEpoch unchanged and continues the normal takeover. Live, unknown, unestablishable, and\nwrong-op-kind still refuse; there is no TTL.\n\nUse this command when that boot path cannot run — the delivery daemon is down, you are repairing a\nnon-manager endpoint, or you want to lift the freeze without starting a manager. It checks that the\nholder really is gone, prints what it found, and then finishes the dead operation exactly as the\ninterrupted restart would have: revoke the old credentials, evict their holders with verification,\nand reopen the gate.\n\n**It refuses far more often than it acts, on purpose**, and always says which check stopped it:\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `holder-alive` | The freeze-holder still has a live connection — a manager *is* running | Stop that process first. Reconciling would evict a live manager's credentials |\n| `holder-unknown` | The connection sweep could not prove the holder absent | Not safe to proceed: an unprovable holder is treated as a live one. Re-run once the broker answers completely |\n| `liveness-unestablishable` | The delivery daemon could not be asked at all | Start it (`cotal up` runs it) and re-run. Silence is never read as death |\n| `not-frozen` / `no-gate` | The gate is open, or there is no gate at that coordinate | Nothing to repair — check `--endpoint` / `--instance` |\n| `wrong-op-kind` | Frozen under a takeover or retirement, not a registration | Out of scope for this command; it will not reinterpret another operation's intent |\n| `eviction-unverified` | The holder looked gone but eviction could not be verified | The gate is left frozen, unchanged. Investigate the broker before retrying |\n| `raced` | A newer manager moved the gate mid-repair | Re-run `cotal doctor` and look again |\n\nThere is no `--force`, and no path that discards gate state: the only way this reopens a gate is by\nproving the holder is gone and then completing the operation properly.\n\n## deregister-instance\n\n```bash\ncotal deregister-instance [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the instance is registered in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint the instance serves |\n| `--instance ` | this folder's persisted manager instance | Instance id, the whole id as `cotal ps` prints it |\n\n**When you need this.** The service registry records *registration*, not liveness, and nothing in\nthe model expires a row. A manager that stops cleanly removes its own registration. One whose host\ndied without writing anything cannot, so its record goes on claiming a live instance forever: every\nclass scatter in that space freezes the dead slot in, and `cotal ps`, `stop` and `attach` each pay\ntheir whole deadline waiting for a machine that is never coming back. A laptop that was reimaged, a\ncontainer that was deleted, a box that will not be back on the network: those registrations have no\nother exit.\n\nThis command is that exit. It asks the instance first, and it removes a record only when the broker\naffirms the instance's own rail is empty: nothing subscribed there. Then it deletes the\nregistration's two records keys, each pinned to the revision it read, and prints what it removed.\n\n**Silence alone never passes.** An unanswered describe is what a dead host, a wedged process and a\nslow one all look like, and a hung process still holds its subscriptions, so the broker sees\ninterest on its rail. That instance is refused and the observation is printed. A dead process holds\nno connection and therefore no subscription, so a real corpse is still removed.\n\n**It refuses rather than guesses**, and says which check stopped it:\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `instance-answered` | The instance answered a pinned describe. It is alive | Nothing to repair. If it is wedged rather than gone, stop the process first; its own clean stop removes the record |\n| `instance-not-affirmed-gone` | It did not answer, and the broker did not report its rail empty, which is what a held subscription looks like: slow or hung, not affirmed gone | Nothing was removed. Stop the process; its record goes on its own clean stop, or re-run this once it is down |\n| `liveness-unestablishable` | The probe itself failed, so nothing was learned | Fix the probe's path (credential, broker) and re-run. A probe that could not run is never read as death |\n| `not-registered` | No registration at that coordinate | Check `--instance` and `--endpoint`. This takes the whole id, never a prefix |\n| `superseded` | The record moved between the read and the delete | Something is writing to it. Nothing was removed; re-observe before retrying |\n\nThere is no `--force` and no sweep: silence is not death, and a rule that removed rows on silence\nwould eventually remove a live instance that was merely slow. An operator names one instance, the\nbroker's verdict on its rail is what authorizes the removal, and the guard's job is to show them\nthey named a dead one. Removal is not a one way door either. The same instance re-registers over\nthe tombstone on its next start, under the same identity.\n\n## runtimes\n\n```bash\ncotal runtimes\n```\n\nLists every agent runtime the manager can spawn through: the built-in `pty`, the official providers\n(`orca`, `tmux`, `cmux`, `herdr`), and any custom provider installed via `cotal ext add`. Each installed\nprovider is probed so you can see what is actually reachable on this machine before selecting it:\n\n```\npty built in\norca installed · reachable @cotal-ai/orca\ntmux available · cotal ext add @cotal-ai/tmux\ncmux available · cotal ext add @cotal-ai/cmux\nherdr available · cotal ext add @cotal-ai/herdr\n```\n\n`installed · reachable` / `unreachable` is the provider's own `available()` probe; `available` means\nit is a known runtime you can add with the shown command. Selecting an unknown or uninstalled runtime\nvia `up`/`spawn --runtime ` fails loud and, for a known one, points at the exact `cotal ext add`\npackage — there is no silent fallback to `pty`.\n\n## send\n\n```bash\ncotal send dm \"\" [--space ] [--server ] [--creds ]\ncotal send msg \"\"\ncotal send ask \"\"\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and (off-registry) which credential |\n\nOne-shot messaging: connect, send a single direct message (`dm`), channel post (`msg`), or role\nask/anycast (`ask`), then exit. For a running conversation, agents use the mesh tools instead\n([MCP tools](mcp-tools.md)).\n\n## channels\n\n```bash\ncotal channels list\ncotal channels set [--replay | --no-replay] [--window ] [--desc ] [--instructions ]\ncotal channels default --replay | --no-replay\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--replay` / `--no-replay` | — | `set`/`default`: replay history to new joiners, or not |\n| `--window ` | — | `set`: replay window size |\n| `--desc ` | — | `set`: one-line channel description |\n| `--instructions ` | — | `set`: instructions shown to joiners |\n\nInspects and edits the channel registry: replay policy, description, and joiner instructions. ACL\nsemantics (who may read or post) are set at mint / provision time, not here; see\n[Channels and permissions](channels-and-permissions.md). On a user-auth mesh, `list` rides your\nown login as is; `set` and `default` edit the registry over a short-lived\nchannel-writer view, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n\n## history\n\n```bash\ncotal history clear --force [--dms] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--dms` | off | Also clear DM history |\n| `--force` | — | Required: clear without prompting |\n\nPurges retained channel history; `--dms` extends it to direct-message history. An alias of\n[`clean history`](#clean). On a user-auth mesh the purge rides a short-lived purger view over\nyour login, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n## console\n\n```bash\ncotal console [--plain] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to watch |\n| `--plain` | off | Line stream instead of the TUI |\n\nA live protocol view for a space: a lazygit-style TUI, or a plain line stream on `--plain`. On a\nuser-auth mesh it rides the read-only admin view over your login, which needs ledger scope\n`admin`. See [Watch a mesh](watch-a-mesh.md).\n\n## web\n\n```bash\ncotal web [--detach] [--port ] [--no-open] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to serve |\n| `--port ` | `7799` | HTTP port |\n| `--detach` | off | Run in the background; stop with `cotal down web` or bare `cotal down` |\n| `--no-open` | off | Don't open the browser |\n\nThe browser observability dashboard: presence, channels, and a live feed. It is **not** part of\n`cotal up`: it ships inside `cotal-ai` as the `@cotal-ai/web` extension, seeded automatically on first\nrun (like the built-in connectors) so it always matches your CLI version. It self-registers `cotal web`\ninto this surface and serves\n`http://cotal.localhost:7799` (loopback; `*.localhost` resolves in Chrome/Firefox/Edge; Safari may\nneed `http://127.0.0.1:7799`). On a user-auth mesh the dashboard rides the read-only admin view\nover your login, and a channel purge asks for its own channel-purger view per click; both need\nledger scope `admin`. Detached mode re-execs the current Cotal installation, writes diagnostics to\nthe mesh root's `.cotal/web.log`, and reports success only after the HTTP server answers. It requires\na recorded mesh root, but can be launched from any directory once `cotal up` has recorded the mesh.\nSee [Watch a mesh](watch-a-mesh.md).\n\n## mint\n\n```bash\ncotal mint [--profile ] [--out ] [--signer]\ncotal mint --provision [--role ] [--space ] [--server ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--profile ` | `agent` | Credential profile |\n| `--out ` | `.cotal/auth/creds/.creds` | Output path |\n| `--signer` | off | Emit a stripped account-signing file instead |\n| `--force` | off | With `--signer`: overwrite an existing file |\n| `--allow-subscribe ` | the agent file's, else subscribe | Read-ACL override, **agent profile only**: `observer` and `admin` carry a fixed read set, and `mint` refuses this flag there rather than narrowing nothing |\n| `--allow-publish ` | the agent file's, else deny | Post-ACL override, **agent profile only** |\n| `--role ` | the agent file's | Agent profile: the anycast task queue the identity pulls (`svc_`) |\n| `--provision` | off | Agent profile: also pre-create the identity's bind-only DM/deliver durables (and its role's task queue) on the live mesh, so the credential can consume |\n| `--space `, `--server ` | the resolved mesh | With `--provision`: which mesh to provision on |\n\nMints a NATS creds file for a space in **static** auth mode, scoped to a profile and (optionally)\nexplicit read/post ACLs. `--signer` emits an account-signing file for delegating minting to another\nhost. A per-user-auth space refuses `mint`: agents there join under a logged-in user\n([`login`](#login-logout) + [`actor grant`](#actor)), never via a handed-out creds file. See\n[Identity and auth](identity-and-auth.md).\n\nA plain mint is creds only: the identity can publish within its post ACL at once, but on an authed\nmesh its DM inbox and task queue are provisioner-pre-created and bind-only, so a **consuming**\nconnect fails until they exist. `--provision` performs that pre-create in the same command (a\nprovisioner cred is minted from the space's trust material, used, and dropped), so a long-running\nclient you start yourself can receive DMs and role anycasts like a spawned seat. The command prints\nthe identity's principal (its wire id) and lifecycle uid; a consuming client passes that uid as its\n`lifecycleUid`. Agent profile only; an open mesh needs none of this (peers self-create there). The\nmesh it provisions on must be the one this folder's auth is for - same space and same account key -\nso `--provision` can never quietly mint under another root's trust material.\n\n## login, logout\n\n```bash\ncotal login --idp [--client-id ]\ncotal logout --idp \n```\n\nSigns you in to a per-user-auth mesh's IdP (device code flow) and caches the session; run it\nonce per machine. It prints your IdP subject, the id the operator grants against. After a\nlogin, every command on that mesh works under your identity: each connect takes a fresh IdP\nproof, exchanges it locally for a short-lived bearer, and is authorized against the actor\nledger at connect time. `logout` revokes the IdP session and clears the cache. See\n[identity & auth](identity-and-auth.md).\n\n## actor\n\n```bash\n# an upsert of the WHOLE row: a flag left off is the WIDE default below, not \"unchanged\"\ncotal actor grant --sub [--scope a,b] [--allow-subscribe a,b] [--allow-publish a,b] [--role ] [--label ]\ncotal actor revoke (--sub | --owner )\ncotal actor list\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | the folder's | Space whose ledger to manage |\n| `--sub ` | — | The IdP subject (shown by `cotal login`) the actor belongs to |\n| `--owner ` | — | The derived owner token (alternative to `--sub`) |\n| `--scope ` | `spawn,role:default` | Capability scope (`''` = none; `spawn` = may run agents; `role:` = may delegate role r; `admin` = cross-agent control; `supervise` = eligible for the closed remote manager-service view when the host enables it) |\n| `--allow-subscribe ` | `>` (all channels) | Channel read ACL; the user's envelope, their agents can never read beyond it |\n| `--allow-publish ` | `>` (all channels) | Channel post ACL; also the envelope for their agents' posting |\n| `--role ` | — | Role (scopes the task-queue consumer) |\n| `--label ` | — | Display label for `actor list` (never the IdP subject) |\n\nThe actor ledger is the single authorization source of a user-auth space: no row, no access.\nA bare `grant` is the **full** envelope (all channels, may spawn); the flags narrow it. A\nre-grant **replaces the whole row**, not the one field you name, so to add a capability spell\nevery field out: the new scope plus the row's current read set, post set, role and label\n(`cotal actor list` shows what a row holds). A field left off does not stay as it was, it\nreverts to the wide default in the table above, which is how a narrow reader becomes a reader\nof every channel. `supervise` is separate from `spawn` and `admin`: it only makes a signed-in\nperson eligible for the host-provided closed remote manager-service view; it does not grant\nmanagement of another owner or a general host profile. `revoke` denies the next exchange and\nthe next connect with no restart, and evicts the principal's live connections. Managed-agent rows\n(written by the spawn path) live in a disjoint row space this command never touches. See\n[identity & auth](identity-and-auth.md).\n\n## doctor\n\n```bash\ncotal doctor auth [--fix]\n```\n\nCredential-health diagnosis and repair for this folder's mesh: renders every managed\ncredential as healthy / near-expiry / expired and ends in `healthy` or the exact next\ncommand; `--fix` applies the repairs it can. The one surface every stale-credential error\npoints at.\n\n## join\n\n```bash\ncotal join --space --name [--role ] [--channel ]\ncotal join --link | --token \n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and which credential |\n| `--name ` | — | Your presence name |\n| `--role ` | — | Your role |\n| `--channel ` | — | Channel to join |\n| `--kind ` | `agent` | Endpoint kind |\n| `--link ` | — | Join link (`cotal://…`) |\n| `--token ` | — | Join token |\n| `--lifecycle-uid ` | — | Required with `--creds`: the lifecycle UID minted alongside the credential (`COTAL_LIFECYCLE_UID` works too). A credential's durable grants name exact lifecycle-keyed resources, so `join` refuses to invent one |\n| `--tls` | off | Connect over TLS |\n\nAn interactive presence: join a space under your own name and role, without launching an agent\nharness. A `--link` or `--token` supplies the where and the auth in one value. See\n[Spaces](spaces.md) and [Identity and auth](identity-and-auth.md).\n\n## Manifest deploys\n\nA `cotal.yaml` manifest declares a whole mesh (channels, personas, roles, and ACLs) in one file.\nThree commands consume it, plus a read-only validator:\n\n```bash\ncotal up -f cotal.yaml # boot a fresh mesh from the manifest\ncotal spawn -f cotal.yaml # deploy the manifest additively onto a running mesh\ncotal down -f cotal.yaml # tear that deploy down (or --run for one run)\ncotal topology view -f cotal.yaml # validate + view the access graph, change nothing\n```\n\n`up -f` and `spawn -f` differ in target: `up -f` brings up a new broker and applies the manifest;\n`spawn -f` requires an already-reachable mesh and applies additively (ownership-scoped). On a\nuser-auth mesh, `spawn -f` deploys over your own login (the deployer view, gated on ledger scope\n`spawn`): the manifest's agents land under your owner, a manifest claiming another owner is\nrefused, and seeding new channels additionally needs scope `admin`. Both take\n`--dry-run` to print the plan without mutating anything. `topology` validates the manifest and\nrenders its channel / role / ACL graph. See [Define a team](define-a-team.md) and the\n[manifest reference](manifest.md).\n\n## ext\n\n```bash\ncotal ext # same as `list`\ncotal ext add \ncotal ext remove \ncotal ext list\ncotal ext root # print just the install prefix (scriptable)\ncotal ext seed [--repair|--reset|--force]\n```\n\nOperator-installed extensions: `add` installs an npm package into a cotal-owned prefix and records\nevery registry provider it contributes. Commands appear in help, completion, and dispatch; runtime\nproviders are lazy-loaded by commands such as `supervise`; local process providers participate in\n`status` and selective `down`. `remove` and `list` manage them. The `@cotal-ai/web` dashboard is the\ncanonical command/process example. Installed packages and their location are described in\n[config](config.md).\n\nBare `cotal ext` lists the inventory, headed by the install prefix. That prefix is a cotal-owned npm\nroot kept **separate** from npm's own global tree, so these packages never show up in `npm list -g` —\n`cotal ext` (or the Extensions section of `cotal status`) is the canonical inventory. `cotal ext root`\nprints only the path, for scripts. The versions shown are the manifest pin recorded at add time.\n\nRemoving an extension that owns a running local process is refused with the mesh root and its\n`cotal down ` command; stop it first so uninstalling the package never strands a process\nwhose lifecycle provider is gone.\n\n### Built-in connectors are seeded extensions\n\nThe first-party agent connectors (`claude`, `opencode`, `codex`, `hermes`, `jcode`, `pi`) are not compiled into\nthe binary. They are seeded on first run through the **same** `ext add` path a third party uses, and\nappear in `cotal ext list` like any other extension. So you can remove one you do not want\n(`cotal ext remove @cotal-ai/connector-hermes`), and a deliberately-removed connector STAYS removed\nacross upgrades. `cotal ext add ` adds a third-party connector the same way. The web\ndashboard (`@cotal-ai/web`, providing `command:web`) is the seventh built-in seeded on the same path.\n\n`cotal ext seed` is the maintenance entry for that seeding (it runs automatically on the first real\ncommand of each boot, so you rarely call it):\n\n| Flag | Meaning |\n|---|---|\n| (none) | Reconcile: seed any never-seeded built-in, refresh a seeded one whose version the binary bumped, leave a removed one removed. A no-op once current. |\n| `--repair` | Recover after an interrupted seed or a lost authority (rebuilds the interrupted connector; restores the removed-vs-never-seeded record from its durable backup). |\n| `--reset` | Discard the record and re-seed all seven built-ins (the six connectors plus the web dashboard). **Resurrects any you removed.** Rebuilds cleanly over corrupt seed state. |\n| `--force` | Re-seed the built-ins even when the version stamp is current or a downgrade. |\n\nThe default connector for a bare `cotal spawn` (no `--agent`) is `claude`; set `COTAL_DEFAULT_AGENT`\n(e.g. `opencode`) to change it. An `--agent` naming a removed connector fails loud with the exact\n`cotal ext add` to restore it. Set `COTAL_SKIP_CONNECTOR_SEED=1` to turn off the automatic first-run\nseed/refresh entirely (for a controlled or offline setup that manages connectors by hand); `cotal ext\nseed` still runs on request.\n\n## completion\n\n```bash\ncotal completion # print a stub to eval / source\ncotal completion install [shell] # install it persistently\n```\n\nPrints or installs shell completion. Completion candidates come from each command's declared flags\nand, where useful, live mesh state (spaces, personas, managed agents) resolved offline.\n\n## feedback\n\n```bash\ncotal feedback \"\" [--type ] [--email ] [--details ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--type ` | — | `bug` \\| `idea` \\| `friction` \\| `praise` \\| `other` |\n| `--details ` | — | Longer free-form details |\n| `--severity ` | — | `low` \\| `medium` \\| `high` |\n| `--area ` | — | The part of Cotal this concerns |\n| `--email ` | git email | Contact email (required on the keyless public path) |\n| `--name ` | — | Your name (optional) |\n| `--url ` | keyed / public intake | Intake URL override |\n| `--key ` | `COTAL_FEEDBACK_KEY` | Feedback key |\n\nSends feedback to the Cotal developers. With a key (`--key` / `COTAL_FEEDBACK_KEY`) it routes to the\nkeyed beta intake; without one it goes to the public `cotal.ai` intake and requires a contact email\n(`--email` / `COTAL_FEEDBACK_EMAIL`, else your git email). Run a self-hosted intake with\n[`feedback-intake`](#server-daemons).\n\n## Server daemons\n\nTwo long-lived infra roles ship with the CLI. They are not part of everyday operation; the delivery\ndaemon comes up automatically with `cotal up --detach` in auth mode.\n\n```bash\ncotal deliver --space [--server ] [--creds ]\ncotal auth-service --space --server [--port ] [--exchange-public-port ] [--exchange-public-url ] [--exchange-trusted-proxy]\ncotal feedback-intake --keys [--port ] [--creds ]\n```\n\n`auth-service` runs a user-auth space's identity plane: the NATS auth callout, the\ncapability-gated local exchange and JWKS, and, when `--exchange-public-port` is set, the closed public\nexchange/discovery face forwarded by an HTTPS reverse proxy. `--exchange-public-url` is the proxy URL\nadvertised to clients; `--exchange-trusted-proxy` opts into last-hop `X-Forwarded-For` attribution.\n`cotal up --user-auth` starts and supervises the service for you, so you run it directly only to\nrecover one by hand.\n\n`deliver` runs the server-side Plane-3 delivery daemon: the durable backstop and membership/ACL\nauthority. It is auth-mode-only and single-instance (`--shard`/`--shards` accept only `N=1`);\n`--dev-mint` mints a scoped cred from the local signer for standalone dev. See the\n[delivery daemon](delivery-daemon.md). `feedback-intake` runs a self-hosted feedback server\n(requires `--keys` and a scoped `--creds`), announcing submissions into a space channel; flags\ninclude `--host`/`--port`, `--store`, `--space`/`--channel`, `--max-bytes`, and `--rate-limit`.\n\n## Plumbing\n\n`cotal __complete ` is the internal entry the shell-completion stubs call to emit candidates\nfor the current command line; you never run it directly. `cotal agent-bearer` is machine-facing\nplumbing on user-auth meshes: spawned agents exec it to print a fresh short-lived bearer from their\nspawn-time secret; you never run it directly either. Its local arm uses `--dir` to discover the\ncapability-gated loopback service. A remotely enrolled, already-granted agent instead receives\n`--exchange-url ` in its launch argv: that arm sends `{owner, actor, actorToken}` to the\npinned public exchange with no local capability, follows no redirects, and refuses every non-HTTPS\nURL because the actor token is the credential in the request body. (`cotal start` is a removed tombstone: it\nerrors and points you to `cotal spawn --detach`.)\n" + "body": "# `cotal` CLI reference\n\n> **Reference**: describes the TypeScript reference implementation (the `cotal` CLI), not the wire contract. · **For:** operators · **Wire contract:** [SPEC](../SPEC.md)\n\n`cotal` is the operator command line for the reference implementation: bring a mesh up, mint\nidentities, launch agents, watch what they do, and tear it all down. It is a thin client over the\nwire contract: the normative subjects and schemas live in the [SPEC](../SPEC.md); this page is\nlookup material for the commands, not a walkthrough; if you are new, start with\n[Getting started](getting-started.md).\n\n## Running it\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH (needs Node 22+)\ncotal --help # every command, grouped\ncotal --version # cotal-ai version + each installed extension's (also `cotal -v`)\ncotal --help # one command's flags and usage\n```\n\n`npx cotal-ai ` runs it without a global install; in a dev clone, `pnpm cotal `\nruns it through `tsx` with no build step. Bare `cotal` prints help. Every command generates its own\n`--help`, usage, and shell completion from its declared flags.\n\nCommands come from the surfaces the binary composes: the base mesh CLI, the manager\n(`supervise`), and the delivery daemon (`deliver`), plus any operator-installed extensions.\n`cotal ext add ` installs any registry providers a package contributes: commands,\nruntimes, and local process lifecycle descriptors. The `web` dashboard and optional manager\nruntimes ship this way.\n\n## Commands\n\n| Area | Command | Purpose |\n|---|---|---|\n| Set up & lifecycle | [`setup`](#setup) | Guided, configure-only setup (installs, seeds personas; launches nothing) |\n| Set up & lifecycle | [`update`](#update) | Reconcile first-party extensions and check or opt into a coherent CLI upgrade |\n| Set up & lifecycle | [`up`](#up) | Start a local mesh (nats-server + JetStream), or boot a whole manifest with `-f` |\n| Set up & lifecycle | [`down`](#down) | Stop the whole stack, selected registered components, or a manifest deploy |\n| Set up & lifecycle | [`backup`](#backup-and-restore) | Create an offline full-space or registry-only artifact from a preserved cut |\n| Set up & lifecycle | [`clean`](#clean) | Configurable cleanup: purge history (live), or wipe the local store / identity (stopped) |\n| Set up & lifecycle | [`meshes`](#meshes-use-status) | List the recorded meshes on this machine |\n| Set up & lifecycle | [`use`](#meshes-use-status) | Set the default mesh a bare `cotal spawn` joins |\n| Set up & lifecycle | [`status`](#meshes-use-status) | Read-only diagnostics for setup, processes, and the selected mesh |\n| Agents & personas | [`spawn`](#spawn) | Launch an agent from a persona (foreground, or `--detach` via the manager) |\n| Agents & personas | [`models`](#models) | List connector model catalogs and variants from the manager |\n| Agents & personas | [`ps`](#ps-stop-attach) | List managed agents and their mesh status |\n| Agents & personas | [`stop`](#ps-stop-attach) | Ask the manager to stop a managed agent |\n| Agents & personas | [`attach`](#ps-stop-attach) | Stream and drive a managed agent's terminal (pty runtime) |\n| Agents & personas | [`input`](#input) | Type one line into a managed agent's terminal without attaching |\n| Agents & personas | [`personas`](#personas) | List, show, edit, create, or remove local personas |\n| Agents & personas | [`supervise`](#supervise) | Run a manager daemon (the agent supervisor / control plane) |\n| Agents & personas | [`runtimes`](#runtimes) | List the agent runtimes the manager can spawn through and whether each is reachable |\n| Agents & personas | [`reconcile-gate`](#reconcile-gate) | Unfreeze an issuance gate left frozen by a crashed restart when the successor cannot boot-heal it (holder gone, complete CONNZ sweep) |\n| Messaging & watching | [`endpoints`](#endpoints) | List every endpoint in the live presence roster, including infrastructure |\n| Messaging & watching | [`describe` / `invoke`](#describe-invoke) | Resolve a v0.4 service's command surface off the wire; invoke one command by name |\n| Messaging & watching | [`send`](#send) | Send one message, then exit: DM a peer, post a channel, or ask a role |\n| Messaging & watching | [`channels`](#channels) | Inspect or set the channel registry |\n| Messaging & watching | [`history`](#history) | Clear retained message history |\n| Messaging & watching | [`console`](#console) | Live protocol view for a space (TUI, or `--plain` line stream) |\n| Messaging & watching | [`web`](#web) | Browser dashboard (installed as the `@cotal-ai/web` extension) |\n| Auth & meshes | [`mint`](#mint) | Mint a creds file for a space (static auth mode) |\n| Auth & meshes | [`login`](#login-logout) | Sign in to a per-user-auth mesh's IdP (once per machine) |\n| Auth & meshes | [`logout`](#login-logout) | Revoke the IdP session and clear the cached login |\n| Auth & meshes | [`actor`](#actor) | Manage a user-auth space's actor ledger (grant / revoke / list) |\n| Auth & meshes | [`doctor`](#doctor) | Credential-health diagnosis and repair (`doctor auth`) |\n| Auth & meshes | [`join`](#join) | Join a space as your own presence (interactive) |\n| Manifest | [`topology`](#manifest-deploys) | Validate and view a mesh manifest's access graph (read-only) |\n| Extensions & misc | [`ext`](#ext) | Install / remove operator CLI extensions |\n| Extensions & misc | [`completion`](#completion) | Print or install shell completion |\n| Extensions & misc | [`feedback`](#feedback) | Send feedback to the Cotal developers |\n| Extensions & misc | [`deliver`](#server-daemons) | Run the server-side Plane-3 delivery daemon |\n| Extensions & misc | [`feedback-intake`](#server-daemons) | Run a self-hosted feedback intake server |\n\nThe manifest modes of `up`, `spawn`, and `down` (`-f `) plus `topology` are covered\ntogether under [Manifest deploys](#manifest-deploys).\n\n## setup\n\n```bash\ncotal setup [--full] [--demo] [--yes]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--full` | off | Redo the full guided flow (implies `--demo`) |\n| `--demo` | off | Also seed the guided expert team (`david`, `sven`, `me`) |\n| `--yes`, `-y` | off | Non-interactive accept-all (for agents / CI) |\n\nGuided setup is **configure-only**: it checks prerequisites, installs the Claude Code plugin, and\nseeds persona files, and it launches nothing (no mesh, no web, no manager). First run gets the\nnarrated flow; later runs print a status card. By default it seeds one `default` persona; the\n`david`/`sven`/`me` team is opt-in via `--demo`. See [Getting started](getting-started.md) and, for\nmaintainers, [setup internals](setup-internals.md).\n\n## update\n\n```bash\ncotal update [--self]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--self` | off | If a newer release exists, install that exact validated `cotal-ai` version globally and reconcile through the newly installed binary |\n\nWithout `--self`, `update` keeps the installed first-party surfaces coherent with the running\nbinary: it force-reconciles the four built-in connectors, then reinstalls other `@cotal-ai/*`\noperator extensions at the binary's exact version. Each extension runs in an isolated child, so one\nfailure cannot poison later replays. It then checks npm; a newer binary is an informational notice\nwith `cotal update --self` as the next command, not an automatic install.\n\nWith `--self`, the npm check happens first. When a newer release exists, Cotal installs the exact\nversion it validated, resolves and verifies that package in npm's global root, then launches that\nbinary to reconcile connectors and first-party extensions to the new generation. An npx or dev-clone\ninvocation therefore installs and continues through a separate global copy; it never claims the\nalready-running process changed. If the binary is current, `--self` performs the normal local\nreconcile without reinstalling it.\n\nThird-party extensions are listed with their installed version and recorded spec but are not\nauto-updated in v1. Floating third-party updates require `@cotal-ai/*` peer-range validation and are\na future follow-up. A failed connector/extension install, npm metadata check, or requested global\ninstall is reported and makes the command exit nonzero. Independent extension attempts continue so\nthe output includes every failure; an unavailable npm registry does not undo a completed local\nreconcile, but the command still exits nonzero because it could not establish that the install is\ncurrent.\n\n## up\n\n```bash\ncotal up [--foreground] [--open] [--space ] [--server ] [--channels ] [--runtime ]\ncotal up --user-auth --idp [--exchange-public-port --exchange-public-url [--exchange-trusted-proxy]]\ncotal up --tls-cert --tls-key # serve broker TLS (both, or neither)\ncotal up --restore [--restore-only registry] [--accept-missing-source]\ncotal up -f [--dry-run] [--runtime ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--server ` | auto (free local port) | Listen URL override |\n| `--host ` | — | Bind host override. With no `--server`, the broker URL is derived from it, so `--host ` alone is enough to make a mesh reachable at that address; a `--host`/`--server` pair naming different addresses is refused. A wildcard bind (`0.0.0.0`, `::`) keeps a dialable loopback URL. Recorded on the mesh and reused by every later manager launch, so a repair or resume keeps remote [`attach`](#ps-stop-attach) working |\n| `--space ` | the folder's name | Space name |\n| `--store-dir ` | — | JetStream store directory |\n| `--channels ` | `.cotal/channels.json` if present | Channel-registry seed file (JSON). An explicit path that is missing is an error |\n| `--restore ` | — | Restore a completed offline backup before exposing the normal listener |\n| `--restore-only registry` | artifact selection | Restore only the registry component |\n| `--accept-missing-source` | off | Explicit disaster consent when the inode-bound preserved source is absent |\n| `--open` | off (auth) | Unauthenticated dev mesh: no JWT, no ACLs |\n| `--user-auth` | off | Per-user auth: people `cotal login`; connects are authorized against the actor ledger |\n| `--idp ` | — | With `--user-auth`: the IdP auth base URL to pin on first enable |\n| `--exchange-public-port ` | none | With `--user-auth`: add the public exchange face on this loopback port, for an HTTPS reverse proxy to forward to |\n| `--exchange-public-url ` | none | With `--exchange-public-port`: advertise the reverse proxy's HTTPS URL in discovery |\n| `--exchange-trusted-proxy` | off | With `--exchange-public-port`: attribute public failure buckets to the last `X-Forwarded-For` hop. Enable only when the listener is reachable solely through a trusted proxy; otherwise the socket address is used |\n| `--detach` | on | Deprecated compatibility spelling. It is a no-op because normal `up` is already detached |\n| `--foreground` | off | Debug in the invoking terminal. Normal `up` detaches the stack so it survives shells, SSH sessions, agents, and CI runners ending. On Windows, Cotal first proves the current job permits process breakaway; otherwise it refuses before launch and points to this flag |\n| `--tls-cert ` | — | PEM certificate to serve TLS with. Must be given together with `--tls-key`. The pair is validated **before** the broker starts — readability, private-key mode, that the two match, the validity window, and that the certificate covers the host clients will dial — because `nats-server` starts happily on an expired certificate and only the client then fails. The decision is recorded, so a later bare `cotal up` after a `cotal down` keeps serving TLS rather than silently reverting to cleartext |\n| `--tls-key ` | — | PEM private key for `--tls-cert`. Refused if group- or other-readable (tighten to `600`) |\n| `--file `, `-f` | — | Launch a whole mesh from a manifest |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--runtime ` | `pty` (or the manifest's, with `-f`) | Agent runtime for the mesh manager (`pty` built in; others are installed extensions, explicit-only). Resolved + probed before the broker starts; an uninstalled/unreachable runtime fails loud. With `-f`, overrides the manifest's runtime |\n| `--rotate-sys` | off | Rotate the space's system account and re-mint its two `$SYS` creds. Needs a stopped mesh; refused with `--open` |\n\n`cotal up` boots a detached local nats-server with JetStream and, in auth mode (the default), JWT auth and\nper-agent ACLs. A launch that starts the broker records the mesh as self-hosted so `cotal spawn` from any directory can find it and a stopped stack remains restartable from its recorded root. A refresh that merely finds a broker already answering preserves an operator-owned manual record. With no\n`--server`, it auto-selects a free port if the default address is taken; an explicit `--server`\nstays fail-loud on collision. It also brings up the control plane (delivery daemon in auth\nmode, then the manager). The `-f` form is a [manifest deploy](#manifest-deploys); see\n[Run a mesh](run-a-mesh.md).\n\n`--user-auth --idp ` starts the space's auth service alongside the broker: the NATS\nauth callout plus its capability-gated local exchange, and optionally the closed public exchange\nface configured by the three `--exchange-*` flags above. The service is torn down with `cotal down`,\nand a re-run of `cotal up` heals a dead service on a running broker. `--user-auth` and `--open`\ncontradict each other and are refused loudly; a running broker cannot change auth mode\nwithout a `cotal down` first. See [identity & auth](identity-and-auth.md).\n\n`--rotate-sys` renews the two `$SYS` credentials (`membership-observer`, `connection-evictor`).\nThey carry a 30-day expiry and nothing re-signs them in place, because the system-account seed is\nnever persisted, so they are renewed by issuing a **new system account** under the same broker\noperator and minting fresh creds against it. A plain re-`up` does **not** do this: it reuses the\nexisting trust record, and its `$SYS` creds along with it.\n\nThe rotation is safe to run on a real space, with one operational cost. The data account, the account\nsigning key, every agent credential minted from it, and the JetStream store are all untouched; what\ndies is the retired system account, and with it any out-of-band copy of the old `$SYS` creds, on every\nbroker that loads the rotated config. The cost is that **earlier full backups stop being restorable**\n(see below), so this is not a no-consequence operation. It needs the broker to restart on the rewritten\nconfig, so it runs as part of a boot:\n\n```bash\ncotal down\ncotal up --rotate-sys # agents reconnect; nothing is re-provisioned\ncotal doctor auth # both $SYS creds healthy again, 30 days out\n```\n\nA rotation is a stopped, fresh boot, and anything that is not one refuses it, all for the same reason\n(the on-disk material and the broker it runs on must never end up on different generations):\n\n- a live mesh, because the running broker would keep serving the retired account;\n- an open mesh, whether that comes from `--open` or from `broker.auth: false` in a manifest, which\n has no system account at all;\n- `--restore`, because reinstating a trust root and superseding it in one command leaves no way to\n say which authority the mesh came up on;\n- an unfinished restore or resume attempt on this root, including one `cotal up` would recover on\n its own, because those paths can adopt a live listener and return without booting a broker;\n- a root that hosts more than one space, because the system account lives in the shared broker\n record and a rotation would retire every tenant's, while the root holds one `$SYS` cred pair\n pinned to one data account.\n\nTwo things to know before you run it:\n\n- **The retirement is config-load-bound.** Old `$SYS` creds are refused by any broker that loads the\n rotated config. A stale `nats-server` still running the *previous* config in memory would keep\n honouring them, so stop every broker for this root first. `--rotate-sys` refuses if this root's\n mesh is recorded as running, if anything unidentified is answering at the address it was given, or\n if the root's pid file names a live (or unreadable) process. Those are Cotal's own ownership\n records, not a scan of the process table: a `nats-server` you started by hand against this root's\n `server.conf` on some other port writes none of them and will not be seen. Do not run one.\n- **It invalidates earlier full backups.** A full artifact binds to the trust chain it was taken\n against, and that commitment covers the operator JWT and the system account. Every full backup\n taken before a rotation refuses to restore afterwards, so take a fresh `cotal backup` once the\n rotated mesh is up. `cotal up --restore` names this case when the data account still matches.\n\nThe commit is not atomic (a trust-record write plus two credential writes), so an interrupted\nrotation leaves the record ahead of the creds. That split is detected rather than silent: every\n`cotal up` on an auth mesh, and every `cotal doctor auth`, compares each `$SYS` cred's issuer against\nthe persisted record and names the retired account. `up` warns rather than refusing, because these\ncreds power the membership graph and live eviction, both of which degrade fail-soft; the mesh is not\nworth taking down over them. Re-running the rotation heals it, at the cost of one generation.\n\nWhile those creds are expired the mesh keeps delivering messages, but the\n[membership feed](delivery-daemon.md) and live connection eviction stay down; `cotal doctor auth`\nand the manager's log both name the credential and this repair.\n\n## down\n\n```bash\ncotal down\ncotal down --preserve-state [--store-dir ]\ncotal down manager [delivery auth web nats ...]\ncotal down web [--space ]\ncotal down -f | --run [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--file `, `-f` | — | Tear down this manifest's deploy |\n| `--run ` | — | Tear down one `spawn -f` run by id |\n| `--space ` | current mesh | With components: the mesh whose target-addressed components (e.g. `web`) to stop |\n| `--dry-run` | off | Print the manifest teardown or selected components, mutate nothing |\n| `--preserve-state` | off | Bare whole stack only: fence the manager, retain principals and durable state, stop and prove the stack down, then publish `ready` |\n| `--store-dir ` | `.cotal/nats` | With `--preserve-state`: the actual store path (required for a custom store) |\n\nBare `cotal down` stops the whole local stack in dependency order. Positional component names stop\nonly those self-registered local processes; for example, `cotal down manager` leaves delivery and\nthe broker running, and `cotal down web` is available when the web extension is installed. A\ncomponent that starts target-resolved (the web dashboard) is stopped the same way: `cotal down web`\nresolves the mesh exactly like `cotal web` (registry current mesh first, `--space` to name one), so\nit works from any directory; the other components always stop under the folder you run it in. The\n`-f` / `--run` forms tear down a [manifest deploy](#manifest-deploys) without stopping the whole mesh\nand cannot be combined with component names. Stopping `nats` alone is refused while an unselected\nregistered daemon is still live; include those components or use bare `cotal down`.\n\nNormal `down` remains destructive at the logical identity/durable layer. `--preserve-state` is a\ndifferent maintenance transition: it suppresses leave/deprovision cleanup, persists the manager's\nsame-principal resume inventory, stops the entire stack without removing run/auth artifacts, and\npublishes a stable inode-bound cut only after every recorded process is proven stopped and the exact\nrecorded NATS endpoint is unreachable. A missing or stale broker pidfile never counts as stopped. The\nattempt is bound durably before the manager is fenced, the resume document and attempt-bound\n`cut-intent` are fsynced before manager commit, and the manager's commitment itself is journaled\n(`cut-committed`) before any process stops. A retry after a crash at any of those boundaries reuses\nthe exact recorded attempt and finishes the remaining stop and endpoint proofs idempotently, without\nneeding the (by then intentionally dead) manager. A partial cut never publishes `ready`. It cannot\nbe combined with component names, manifest teardown, or `--dry-run`.\n\n## clean\n\n```bash\ncotal clean --force\ncotal clean restore-attempt --attempt --force\ncotal clean restore-fallback --attempt --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | `history`: target mesh |\n| `--dms` | off | `history`: also clear DM history |\n| `--store-dir ` | `.cotal/nats` | `store`/`all`: JetStream store directory |\n| `--force` | — | Required: destructive, no prompting |\n| `--attempt ` | — | `restore-attempt`: exact stale pre-commit attempt; `restore-fallback`: matching healthy committed restore |\n\nOne configurable cleanup verb; every target requires `--force`.\n\n- `history` purges the retained message backlog on the **running** broker (channels, plus DMs\n with `--dms`). The same operation as [`history clear`](#history), which stays as an alias.\n- `store` deletes the **stopped** mesh's JetStream store (`.cotal/nats`): streams, durable\n consumers, and messages. This is the reset for stale on-disk broker state, e.g. durables\n minted by an older, incompatible Cotal generation surviving a `down`/`up` cycle.\n- `all` is `store` plus the space identity (`.cotal/auth`), the local creds and markers tied to\n it, any crash residue a normal `down` would have swept (stale pidfiles, `run/`), and the mesh's\n registry entry; the next `cotal up` mints a fresh identity.\n\n`history` needs the mesh up; `store` and `all` refuse while any recorded mesh process is still\nalive or any same-root recorded broker endpoint remains reachable (run `cotal down` first). They\nalso refuse outright on a root that holds accounts for several spaces: the store and the broker\ntrust record are shared by every space on the broker, so both targets would take out all of them\nand no `--space` can narrow that. `down`, `backup` and `up --restore` refuse there for the same\nreason. `cotal status` lists the tenants on such a root. Personas\n(`.cotal/agents`) and logs are never touched. A custom\nstore location is not recorded anywhere, so `--store-dir` must repeat whatever the mesh was\nlaunched with. Custom cleanup targets must contain either the Cotal store-generation marker or a\nreal `jetstream/` store directory; filesystem roots, project roots, and Cotal auth/maintenance trees\nare always refused.\n\n`store` and `all` also refuse every maintenance journal state. After a healthy committed restore,\n`restore-fallback` is the only supported way to remove the recorded unchanged old-store inode; it\nnever deletes the active target, requires both the exact attempt id and `--force`, and retires the\ncompleted restore journal so a later `down --preserve-state` can start a new backup cycle.\n\n## backup and restore\n\n```bash\ncotal down --preserve-state [--store-dir ]\ncotal backup create [--only full|registry] [--store-dir ]\ncotal up --restore [--restore-only registry] [--accept-missing-source]\n```\n\nBackup is offline-only. It requires the stable `ready` record from `down --preserve-state`, an exact\nstore match, no live recorded process, and an unreachable exact endpoint from the recorded cut.\nThat endpoint is probed immediately before cloning, so a live broker with a missing or stale pidfile\nis still refused. It claims the cut, reflink/copies the stopped source to a\nprivate attempt clone, and opens only that clone on a random loopback bootstrap broker with an\nindependent parent/deadline watchdog. It validates the canonical stream and pull-consumer inventory,\nwrites native snapshots with consumers excluded, and stores conservative contiguous ACK-floor\ncheckpoints separately. The original store is never opened by the backup broker, and the stack is\nnot restarted implicitly. Artifact destinations must not overlap the preserved source or maintenance\nattempt tree. Restore artifacts and targets likewise cannot nest inside or contain each other, the\npreserved source, or the maintenance attempt tree.\n\n`full` is the default and indivisible: channel registry, CHAT/DM/TASK/INBOX/DLV, ACL, MEMBERS, and\nvalidated durable checkpoints. `registry` is the sole partial artifact. Presence, derived membership\nfeed, leases, native ephemeral/history consumers, credentials, keys, tokens, owner secrets, and actor\nledger files are excluded. Artifacts are exclusively created `0700`; snapshot/checkpoint files and\nthe manifest are `0600`; `manifest.json` is written last with exact sizes and SHA-256 values. The\ndirectory is trusted operator input: hashes detect corruption, not malicious rewriting.\n\nRestore validates and stages the exact allowlisted artifact bytes before moving or creating a store.\nIt requires the same space and existing trust state. The whole pre-commit window holds a journaled\nliveness claim (coordinator, watchdogs, brokers, absolute deadline): ordinary `up` and a repeated\n`up --restore` refuse while the claim is live, and a stale attempt is recovered only after the\ndeadline has elapsed and every recorded owner is proven dead — automatically by a retried\n`up --restore`, or explicitly with `cotal clean restore-attempt --attempt --force`. Nothing\never rolls back a live attempt. A registry-only artifact restores as registry-only whether or not\n`--restore-only registry` is passed; omitted infrastructure is always created and the exact\npost-restore stream inventory is asserted before commit intent. Ordinary `up` from a preserved cut\nresumes only the exact recorded source store and runtime; a contradicting `--store-dir` or\n`--runtime` fails in preflight. Authenticated restores validate the complete\nspace trust bundle before staging, including nkeys, seed matches, JWTs, signers, and space binding;\nfull restores commit to the validated operator, system-account, data-account, and active-signer root\nchain in addition to the static/user authority fingerprint. Because the system account is part of that\ncommitment, a [`cotal up --rotate-sys`](#up) makes every full artifact taken before it unrestorable\nagainst this root: take a fresh full backup after each rotation. The composed commitment is revalidated\nimmediately before store mutation and never includes secret seeds. Restore never creates fresh auth.\nSame-path restores atomically retain the old\nsource at the journaled fallback path; alternate targets retain it in place; a missing canonical\nsource needs explicit `--accept-missing-source`. Quarantine and target restores use current canonical\nconfigs on isolated random-loopback brokers, never expose native snapshot consumers, and publish a\ncommit-intent immediately before the normal listener starts. Archive bytes never instantiate the real\ntarget: after quarantine validation, every stream is re-snapshotted from the validated quarantine\nstate into attempt-owned sanitized files, and the target is restored solely from those. Before that boundary, failure rolls back\nthe attempt-owned target; after it, ambiguity preserves both stores and records forward-repair\nrecourse. The cooperative maintenance lock excludes Cotal commands, not arbitrary raw NATS processes.\n\nBootstrap brokers in every auth mode — including open — mount the store under a local account with\nrandom operation-specific logins only, each carrying the exact per-phase subject permission matrix;\nnormal static credentials and user-auth sentinel/bearer connections are rejected, and no auth\nservice or callout starts. Open mode differs only in its account label, never in authority. Inventory, each stream snapshot,\nrestore initiation, exact upload id, validation, and each checkpoint recreation use separate exact\nauthorities. Every checkpoint carries the source stream's message/first/last sequence state and must\nmatch its snapshot record before mutation; core then derives and validates the only allowed start\npolicy. TASK is not a CLI exception: the same core checkpoint API recreates its canonical `DeliverAll`\nWorkQueue durable because acknowledged tasks are absent from retention and NATS forbids a\nstart-sequence policy there. Registry-only restore creates every omitted canonical stream and transient\nbucket on the isolated target before the normal listener is exposed. It deliberately does not resume\nretained agents or recreate their DM/DLV/TASK/ACL state; their identity material stays retained and\nstopped rather than being reprovisioned into a partial restore.\n\nAfter listener readiness, the manager starts attempt-bound, validates retained credentials/tokens\nwithout granting or reprovisioning, and resumes the exact persisted principals under cleanup\nsuppression. Registry-only restore uses the same flow with an empty agent set. `commitResume` is an\nidempotent validation barrier only: success must be `awaitingFinalize` with an attempt-bound 64-hex\ncommit token and does not release suppression. Under the workspace lock, the CLI first fsyncs that\nexact evidence as `manager-committed` (restore) or `resume-committed` (ordinary resume), then calls\ntoken-bound `finalizeResume`; only an `active` response for the exact token releases suppression. The\nCLI records the same token in finalization evidence before a restore becomes `active`, or before an\nordinary resume retires and consumes the marker. Re-entry from either committed state skips the prior\nidempotent activation/commit phases, retries finalization with the durable token, and finishes the\nworkspace transition. Failure before finalization preserves the committed state and cleanup\nsuppression; it is not rewritten through a degraded transition. Re-entry between any two earlier\nboundaries reuses the same attempt and may retry the idempotent phases without deleting retained state. A missing or\nchanged per-agent dependency is a named fail-closed result; the journal becomes degraded and remains\navailable for forward repair. A retry from `resume-intent`,\n`resume-active`, or `resume-degraded` reuses the same attempt and inventory after the prior listener is\nproven stopped. Every normal restore listener has an unguessable attempt-bound NATS server name. The\nCLI fsyncs its exact name/nonce, canonical endpoint, process owner, and generation-bound target identity\nimmediately after spawn. Re-entry accepts a surviving listener only when its INFO server name, live PID\nrecord, endpoint, and target identity all match that proof; degraded restore repair then moves through\nthe guarded workspace transition only after manager commit. If an uncommitted bound owner is provably\ndead, recovery retires that exact proof under the maintenance lock and binds a fresh listener for the\nsame attempt, endpoint, and target with a new nonce and server name. A live foreign/mismatched listener\nor ambiguous owner is preserved and refused, never adopted by reachability alone. A reconstructed\ncommit/degraded attempt without either the exact bound proof or a durable dead-listener replacement\nrecord fails closed even when the recorded port is free. A later ordinary startup may pass an `active`\nrestore only when its details prove manager commit and its exact recorded listener is dead.\n\n## meshes, use, status\n\n```bash\ncotal meshes\ncotal meshes add # guided, on a terminal\ncotal meshes add --server [--root ] [--mode auth|open|user] [--tls] [--force]\ncotal meshes add --mode user (--user-auth-file | --from )\ncotal meshes rm [ …] [--force]\ncotal use \ncotal status [--space ] [--server ] [--components]\n```\n\n`meshes` lists the meshes this machine knows, including stopped self-hosted ones tagged `offline`; a `*` marks the `current` default a bare `cotal spawn` joins.\n\nRun on a terminal with the space or `--server` missing, **`meshes add` is guided**: it asks for the\none thing that cannot be derived (the broker URL), probes it, and tells you what answered - open or\nrequiring credentials. It then offers the spaces your `--root` already holds credentials for, states\nthe mode as a fact about that broker rather than asking, and shows the exact record before writing\nanything. A broker that does not answer, or a space name already registered, becomes a choice rather\nthan an error. Anything you pass on the command line is taken as given and not asked again. Without\na terminal - a script, an agent, CI - nothing prompts and the flag form's errors stand\n(`COTAL_NO_PROMPT=1` forces that too).\n\n`cotal up` writes a durable self-hosted record; `cotal down` stops the stack but leaves that record\nso a stopped mesh remains listed and restartable. `meshes add` registers a mesh they cannot speak\nfor: one running on another machine, a shared broker, a hosted space. `--root` is the folder\nwhose `.cotal/auth` holds that mesh's credentials and whose `.cotal/agents` holds its personas\n(default: the project you run it in) — the registry stores that path, never a secret. `--mode`\ndefaults to `auth` when the root holds the space's account record and to `open` otherwise. The\nbroker is probed before anything is recorded, so a wrong address, or credentials that mesh will\nnot accept, fails here instead of at the first `spawn`; `--force` records without verifying (and\nreplaces an existing record).\n\nA hostname or public address is registrable only when the connection will **require TLS**: pass\n`--tls`, or use a `tls://` URL — the scheme is recorded as enforced intent, so every later dial\nthrough the record demands the handshake (and `meshes add tls://…` against a plaintext broker is\nrefused at registration). Without required TLS the fence is unchanged: loopback and\nprivate-overlay literals only, and RFC1918 addresses are refused in both modes — a cafe LAN is\nprivate, not yours.\n\nA **user-auth** mesh registers from supplied pinned trust, never guessed: `--user-auth-file`\ntakes the bundle exported where the mesh runs; `--from` asks before it dials the address at all,\nthen fetches its `/.well-known/cotal-mesh` discovery document (HTTPS only), displays the pins, and\nasks again before adopting them. Neither fetch follows redirects: a 302 can move a pinned fetch\nonto plaintext or onto another host, so it is refused rather than followed, and the pinned\nexchange must itself be an `https://` URL — except for an exchange on this machine, where plain\n`http://` is accepted for a loopback *literal* (`127.0.0.1`, `::1`, any spelling of them) but not\nfor `localhost`, which is a name rather than an address. Registration verifies that exchange answers `/health`\nand `/jwks` as the pinned issuer, and that the broker itself refuses a bare connect — the\nauth-required refusal is the pass. The sentinel credentials land in a 0600 file under the entry's root; the registry\nrecords only the path.\n\n`meshes rm` drops records — it never stops a mesh. For a mesh running on this machine `cotal down`\nis the right verb, and `rm` says so unless you pass `--force`. A record you added by hand is only\nremoved by something that names it — `meshes rm`, or an `add --force` replacement — or by a\n`cotal up` that actually starts the broker for that same space, server and root, which becomes that\nmesh and so takes the record over (a `cotal up` for that space anywhere else refuses instead).\nNothing that merely *infers* a record is stale touches it: an\nunreachable broker is listed `offline` and stays, and `cotal down` / `cotal clean all` leave it\nalone even when it shares a root with the project they are tearing down, because nothing on this\nmachine could write it back.\n\n`use ` sets that default; the selection applies from every directory,\nincluding inside another mesh's project. `status` is a read-only report: machine prerequisites\n(starting with the installed `cotal-ai` version), the installed extensions and their versions, this\nfolder's `.cotal/`, the recorded meshes, and a live snapshot of the selected mesh (roster, channels,\nmembership feed). `status` takes `--space` / `--server` to pick the mesh to inspect; it starts\nnothing.\n\n`cotal status --components` adds a fail-loud per-component health pass. It reads **each\ncomponent's own control surface**, rather than treating a PID, a lease, or a successful probe of a\nsibling as proof that the component serves. It prints one of `serving`, `absent`, `not-serving`, or\n`refused` for each component and exits `0`, `1`, `2`, or `3` respectively (the highest observed\nstate wins):\n\n- **manager** — local PID record, its liveness-lease holder and PID, then the manager's own typed\n `status` service reachability from this host. Builds without a startup-phase report say\n `phase not reported by this manager build`; that is never a blank green state.\n- **delivery** — local PID record, its ready lease (`ready` is the daemon's own bound-control\n signal), and the latest `renewal.json` adoption verdict. A re-signed credential and a\n broker-accepted adoption stay distinct facts.\n- **web** — local PID record and the dashboard's own loopback `/api/meta` response, which must name\n the same PID and its requested port. A different process on the port, an unreadable PID command,\n or an unrecognizable process record is `refused`, not a green default-port guess.\n- **broker** — the registered mesh URL dialed from this host with its recorded TLS requirement.\n\n`absent` means Cotal has no live local component record (or has a stale record); `not-serving`\nmeans the component record is live but its service/readiness surface did not answer or is not ready.\nThose are intentionally separate exit cases. A failed or unreadable probe is `refused`, never an\nabsent component or a clean zero.\n\n## spawn\n\n```bash\ncotal spawn [] [--detach] [--name ] [--agent ] [--model ] [--variant ] [--prompt ] [--cwd ]\ncotal spawn -f [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | resolved mesh | Target space |\n| `--server ` | registry entry | Broker URL override |\n| `--creds ` | — | Control-caller creds for an off-registry manager (`--detach` only) |\n| `--name ` | persona's `name:` | Presence-name override (does not choose the persona) |\n| `--config ` | — | Persona catalog name or file path; wins over the positional |\n| `--agent ` | `COTAL_DEFAULT_AGENT`, else `claude` | Connector type (`claude`, `opencode`, `jcode`, `hermes`, …) |\n| `--role ` | persona's `role:` | Role override |\n| `--model ` | persona's `model:` | Model override |\n| `--variant ` | persona's `variant:` | Model variant override (connector-defined; e.g. OpenCode reasoning tiers) |\n| `--cwd ` | this cwd | Working directory to root the agent at |\n| `--prompt ` | — | Initial prompt auto-submitted at start |\n| `--resume ` | — | Fork an existing session id into the mesh (claude only) |\n| `--events` / `--no-events` | off | Publish the session's structured event plane to its own event channel |\n| `--share-tools ` | none | Share named operator MCP servers with the agent |\n| `--subscribe ` | persona's | Channel read-set override |\n| `--allow-subscribe ` | = subscribe | Read-ACL override |\n| `--allow-publish ` | deny | Post-ACL override |\n| `--detach`, `-d` | off | Launch via the manager into a detached PTY (reattach with `cotal attach`) |\n| `--on ` | class anycast | With `--detach` only: pin the launch to one manager instance id (the whole id, as `ps` prints it). Refused on a foreground spawn (no manager to pin), with `-f` (a manifest deploy launches through the manager class queue), and when empty |\n| `--file `, `-f` | — | Deploy a manifest onto the running mesh |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--allow-stale ` | — | With `-f`: waive named stale agents (apply-only) |\n| `--runtime ` | manifest's | With `-f`: override the manifest's runtime |\n\n`--events` turns on the session's **event plane**: a stream of structured events describing what\nthe agent did, rather than the prose it wrote, on a channel of its own. The channel is named after\nthe agent's principal, `events..`, never after its display name, because two live\nagents are allowed to share a display name and would then share a stream. The launch grants publish\nrights on exactly that one channel, foreground and detached alike, and a connector that does not\npublish an event plane refuses the flag rather than starting a session whose events have nowhere to\ngo.\n\nThe flag and the grant are separate on purpose. Holding publish rights on a channel is not a request\nto publish to it, so writing an event channel into an agent file's `allowPublish` does not turn the\nplane on: only the launch does.\n\nThe persona (`--config` > positional > `COTAL_DEFAULT_PERSONA` > `default`) is loaded from the\ntarget mesh's `.cotal/agents/`; the launch flags override the file. Foreground runs the agent\nattached to your terminal; `--detach` hands the launch to the running manager. Both modes get the\ndurable backstop on a mesh that runs the delivery daemon; `--live-only` skips it for a foreground\nspawn (messages posted while it is disconnected are then not replayed). A foreground exit retires\nthe agent's creds and broker footprint, like a manager despawn. A `--detach` spawn is an\n**action**: the manager accepts it and returns the allocated identity at once, then the launch\nfollows to a terminal outcome rather than blocking (see [the control surface](control-surface.md)).\nSee [Connect Claude Code](connect-claude.md) and [Agent files](agent-files.md); `-f` is a\n[manifest deploy](#manifest-deploys). (`cotal start` was merged into `cotal spawn --detach`.)\n\n## models\n\n```bash\ncotal models [--agent ] [--refresh]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--agent ` | all registered connectors | Connector whose catalog to list |\n| `--refresh` | off | Ask the connector to refresh its provider cache |\n\nAsks the running manager for each connector's model catalog (model ids plus their variants)\nfor connectors that expose one (OpenCode today; a connector without a catalog says so). Pick a\nresult with `cotal spawn --model --variant `.\n\n## endpoints\n\n```bash\ncotal endpoints [--space ] [--server ] [--creds ]\n```\n\nLists the mesh presence roster: agents, the manager, and any other protocol endpoint, with each\nendpoint's role, kind, status, and current activity. Unlike `ps`, this is a read-only presence view;\nit is not limited to child processes owned by the manager.\n\n## describe, invoke\n\n```bash\ncotal describe [--space ]\ncotal invoke [--args ''] [--space ]\ncotal invoke --name [--admin] [--space ]\n```\n\nThe generic v0.4 service surface. `describe` resolves a registered endpoint's command set off the\nwire - the reserved `describe` command answers the registered contract digests, the schemas are\nfetched from the space's content-addressed contract store, recompiled, and verified against those\ndigests - and prints each command with its capability class and targeting shape. `invoke` calls one\ncommand by name: `--args` is a JSON object validated against the fetched input schema *before*\npublish; a targeted command takes `--name ` (resolved to the agent's current principal via\n`ps`) or `--self`. `--admin` uses the admin instrument credential, whose cross-agent reach rides\nthe operator-only `any` authorization mode. Neither command has compile-time knowledge of any\nendpoint's schemas - this is the same trust chain every built-in control command now uses. Needs an\nauth mesh: the manager registers its service on both static and per-user meshes (a signed-in user\nrides their bearer; cross-agent reach needs the `admin` scope). An open mesh has no service\nregistry.\n\n## ps, stop, attach\n\n```bash\ncotal ps [--on ] [--wide | --json] [--space ]\ncotal stop --name [--on ] [--space ]\ncotal attach --name [--on ] [--no-reconnect] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | — | Managed agent to stop / attach (required) |\n| `--on ` | class anycast (`ps`: class scatter) | Pin to one manager instance id (multi-manager space); takes the whole id as `ps` prints it, not a prefix. An empty value (`--on \"\"`, an unset shell variable) is refused, never treated as absent |\n| `--wide` (`ps`) | off | After each seat's compact row, print the per-seat facts the manager already records: model pin (and variant), `cwd`, `pid`, spawner, lifecycle uid, and the owning manager's instance id and host. A fact the manager did not record (no model pinned, or a runtime that owns no real process) prints nothing, never a placeholder |\n| `--json` (`ps`) | off | Machine-readable: one JSON object per seat per line, exactly the row the manager sent. Instance headers and errors go to stderr, so stdout is pure rows. Mutually exclusive with `--wide` |\n| `--no-reconnect` (`attach`) | off | End the attach when its session ends, instead of re-establishing it. For scripts that want one run and one exit code |\n\nThese are operator clients over the running manager's control plane. `ps` prints two facts per\nmanaged agent, because they answer different questions: the process fact from the manager's own\nruntime handle (`running` with its uptime, or `exited` with how long it ran), and the mesh fact from\nthe roster (`idle` / `working` / `waiting` / `mesh offline`, or `not in roster` when the seat has no\npresence row at all: a seat that has not joined yet, or one that never did). A seat can be `running` and `mesh offline` at once: the process is alive and\nits presence has lapsed. On a user-auth mesh `ps` also renders each managed agent's last\ncredential-refresh outcome, fail-closed.\n\n**Mode split (chosen up front, never try-scatter-then-degrade):**\n\n- **Static / open mesh.** Bare `ps` is a **class scatter**: it freezes the live manager class from\n the records registry, merges every registered instance's agents grouped and attributed per\n instance, and a non-answering instance is shown as `registered, no answer within the deadline`\n (never silently omitted). That label is the whole claim: the instance is registered and did not\n answer. It does not say the host is down, because a dead host never deregisters itself and a\n live one can be slow; if it is gone, deregister it.\n `--on ` pins the read to one exact instance id instead. A wrong pin fails loud\n rather than falling through: a well-formed id that no live manager carries is reported as\n `manager instance did not answer` (nothing else is asked), and a credential without that\n instance's rail is reported as refused by the broker, not as an unresponsive manager. A manager\n that answers with a refusal is shown with its own cause; \"no manager reachable\" is said only when\n nothing answered at all. If the scatter's own registry read fails (the freeze or the reconcile),\n `ps` says the manager registry could not be read rather than pronouncing on the managers, which\n may all be up.\n\n**`stop` and `attach` route by seat locality.** A seat can only be stopped or attached by the\nmanager actually running it, and the class queue does not know which one that is. So on a\nstatic/open mesh both verbs first ask every registered instance which one hosts the named seat, then\naddress that instance directly. You do not need `--on` for this — it happens by default.\n\n`--on ` remains the override, for when you already know where the seat lives or the\nlookup itself is degraded. It is also the **only** route on a **user-auth mesh**: a ledger-scoped\nbearer does not hold the registry-read rows the lookup needs, so there the verbs stay on the class\nqueue unless you pin them yourself.\n\nIf the seat is found on no reachable instance, the error says so — how many managers answered, and\nwhich ones did not — rather than reporting a bare `no agent `. That distinction matters\nbecause a single manager cannot tell \"hosted elsewhere\" from \"does not exist\": it answers\n`not-found` for both.\n- **User-auth mesh.** `cotal ps` reports what **one** manager knows about your agents (an `ep.one`\n read against the manager's in-memory roster, owner-filtered). It does **not** report other\n manager instances, and it cannot tell you that one is down — an unreachable manager is absent\n from the list, not flagged. Completeness across a multi-manager user-auth space is not claimed.\n A manager that does not answer fails the command outright (exit non-zero), rather than printing\n an empty list that could be read as \"no agents\". Your ledger row needs the `admin` scope to\n reach `ps` at all; `spawn` alone is refused by the broker (the ep tier boundary).\n\n`attach` streams and drives an agent's terminal on the `pty` runtime; detach with the escape key\n(Ctrl-] by default; see [`COTAL_DETACH_KEY`](config.md)). It does so over a one-use, holder-bound\nmesh session ([SPEC](../SPEC.md) §13.6): the manager replies with a signed session grant (never a\n`127.0.0.1` URL), the CLI redeems it once over the broker, and the browser console (`cotal console`)\ndrives the same session. `stop` and `attach` need a running manager to talk to. On a static mesh\nthey are cross-agent admin operations. On a user-auth mesh, your own agents (any agent under your\nowner) need only the `spawn` scope; another owner's agent needs `admin` on your ledger row\n([identity & auth](identity-and-auth.md)). Launch detached agents with [`spawn --detach`](#spawn).\n\n**`attach` reconnects when the link dies.** A session lives on a network link, and a laptop that\nsleeps, a VPN that drops or a wifi handover kills it. When that happens `attach` prints\n`[cotal: connection lost, reconnecting]` on stderr and starts asking the manager for a new session:\na fresh grant, a fresh per-session credential, a fresh connection, so every attempt re-runs the same\nauthorization the first attach did. On success it prints `[cotal: reconnected]`, the manager repaints\nthe seat's current screen the way it does for any attach, and you carry on in the same terminal.\nRetries wait 1s, 2s, 5s, 10s, then 30s, for as long as the seat exists. The detach key is read the\nwhole time the loop runs, the waits and the attempts alike, so a reconnect never traps you: press it\nwhile a session is being established and the attach ends there, and a session that lands behind the\npress is handed back to the manager rather than left holding a slot. Everything else you type while\nthere is no session is dropped rather than queued, so keystrokes aimed at a terminal that turned out\nto be frozen, Ctrl-C included, are not delivered to the agent by a reconnect you did not know had\nhappened. That starts before the first session, not at the first reconnect: at a terminal, `attach`\nreads and drops what you type while it is still resolving the mesh, so a key struck at a prompt that\nhas not come up yet does not reach the agent when it does.\n\nWith stdin a **pipe** the contract is the opposite, and deliberately so. `printf 'ls\\n' | cotal\nattach --name web` is a script's input rather than an operator at a frozen screen, so it is buffered\nby the stream and delivered to the seat when the session opens, exactly as it always was. That holds\nin every window, not just before the first session: a pipe keeps buffering across a reconnect too, so\n`tail -f log | cotal attach --name web` does not lose the part of its feed written while the link was\ndown. Only a terminal gets the reader; `--no-reconnect` keeps the old behaviour on both.\n\nIt stops on its own when reconnecting cannot help, and says why: a manager that refuses the attach\nexits non-zero with the manager's own message, and a reconnect that finds the seat no longer there\n(despawned, or its agent exited while the link was down) exits cleanly with `seat is gone`.\nA refusal that could still pass, such as a manager at its session ceiling, is relayed in the\nmanager's own words while the loop keeps trying, once per refusal rather than once per attempt.\nPressing the detach key, or the agent's process exiting while you are attached, ends the attach as\nit always did. `--no-reconnect` turns all of this off and restores the single-session behaviour,\nwhich is what a script wants.\n\nEach reconnect also hands the abandoned session back to the manager, over the first link that can\ncarry the message, so an attach that flaps does not eat the manager's session slots one outage at a\ntime. If that message never gets a link, the attach says so when it ends.\n\nWhich mesh `attach` resolves also decides **whose trust it redeems with**. Redeeming a session grant\nmeans minting a short-lived, session-scoped credential from the space's seed, and that seed comes\nfrom the root the mesh resolved to, never from a `.cotal` found by walking up from whichever\ndirectory you happen to be standing in. The difference is not hypothetical: `~/.cotal` exists on\nevery install because the mesh registry lives there, so a command run anywhere under your home\ndirectory but outside a project used to mint from your home directory's trust and present it to a\nbroker that trusts a different chain, which surfaced as a bare authorization failure that named\nnothing. A directory that does hold another chain for the same space is now reported on the way\npast, and not obeyed:\n\n```text\n! this directory resolves to /Users/you, whose .cotal/auth holds a DIFFERENT trust chain for space \"team\".\n attach used /Users/you/projects/app, the root this mesh resolved to. The other one is not being used, and is worth a look.\n```\n\nWhen the resolved mesh holds no seed at all, `attach` refuses and names what it resolved, the broker\nand the root, instead of describing a directory it did not use.\n\nTerminal bytes stream over the mesh; the manager's own HTTP/WS face serves the console. That endpoint binds\n**loopback by default**, so nothing is exposed by accident; `cotal up --host ` passes its bind\naddress down, which is what lets you attach to an agent whose manager runs on another machine. A\nbare `cotal supervise` and an embedded manager stay machine-local. Set it directly with\n`supervise --console-host `.\n\nThat address is **recorded on the mesh** and carried forward, because it is a decision rather than\nsomething later commands can work out for themselves (a broker dial address is not a manager bind\naddress). Every later manager launch for the same mesh reuses it — a same-root `cotal up` repair,\nadopting a preserved or restored listener, a `spawn -f` manifest deploy — so a manager replacement\ndoes not quietly move a reachable attach face back to loopback. Passing `--host` again overrides it,\nso you can widen or narrow exposure whenever you like; a mesh that never asked stays loopback-only\nand records nothing.\n\nBecause that face carries terminal read and write for every managed agent, it is credentialed in two\ntiers. A mesh caller receives a **ticket** bound to the single agent the manager just authorized,\nsingle-use and short-lived, so one authorized attach can never be re-pointed at someone else's\nagent. The **console token** is the operator's own, reaches every agent, and is printed only to the\nmanager's output. The roster, the live feed, and the PTY stream all answer `401` without one; the\nstatic console shell is served openly, since it describes no agent.\n\n## input\n\n```bash\ncotal input --name --text [--no-enter] [--on ] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | | Managed agent to type into (required) |\n| `--text ` | | The text to type, taken verbatim (required) |\n| `--no-enter` | off | Type the text and stop there, without pressing Enter |\n| `--on ` | class anycast | Pin to one manager instance id, exactly as [`attach`](#ps-stop-attach) |\n\nTypes one line into a running agent's terminal, as if you had typed it there, and returns. This is\nthe half of [`attach`](#ps-stop-attach) that a program wants: `attach` is a live stream that holds a\nsession open and expects a terminal on your side, so a script, a cron job or a web UI cannot use it\nto send a single line. `input` is one authorized call.\n\nWhat it is for is **harness commands**. A line beginning with `/` is not chat and not a message: it\nis something the agent's own harness handles, and the only way in is the keyboard.\n\n```bash\ncotal input --name reviewer --text \"/compact\" # ask the harness to compact its context\ncotal input --name reviewer --text \"/model opus\" # switch its model\ncotal input --name reviewer --text \"hold on that PR\" # ordinary typing works too\n```\n\n**Quoting.** `--text` takes a value, so a payload starting with `/` survives as written. A payload\nstarting with a dash needs the `=` form, because the shell-style `--text --foo` is ambiguous and is\nrefused rather than guessed:\n\n```bash\ncotal input --name reviewer --text=--verbose # dash-leading text: use --text=\n```\n\nEnter is pressed by default, since a command typed but never submitted has not been delivered.\n`--no-enter` types the text and leaves it sitting at the prompt, which is how you stage a line and\nsend it later.\n\nNothing comes back but a delivery receipt (`✓ sent 9 bytes to reviewer`, counting the trailing\ncarriage return). Whatever the agent does next shows up where its output already goes: the mesh, its\ntranscript, or an `attach`.\n\n**This one is operator-only, and more narrowly than `stop` or `attach`.** Those two are granted to\nanything holding `spawn`, so an agent can stop and attach to seats under its own owner. `input` is\nnot: it is granted only to operator credentials, which on a user-auth mesh means your ledger row\nneeds the `admin` scope, the same scope [`ps`](#ps-stop-attach) already needs there. The reason is\nthat a write into a terminal is control of whatever is running in it, and on a user-auth mesh the\nown-owner rule covers every seat under you, not only the ones you launched: a `spawn`-scoped agent\ncould otherwise type into a sibling it never started. Seat locality is still resolved for you.\n\nOnly the `pty` runtime can be typed into. The external terminal runtimes (`tmux`, `cmux`, `orca`,\n`herdr`) attach to a process they do not own, so they have no input stream for it and the command\nrefuses by name rather than dropping the keystroke.\n\n## personas\n\n```bash\ncotal personas list [-v] [--running]\ncotal personas show \ncotal personas edit \ncotal personas new (--prompt | --from ) [--role ] [--model ]\ncotal personas rm --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh's persona catalog |\n| `--role ` | — | `new`: the persona's role |\n| `--model ` | — | `new`: the persona's model |\n| `--prompt ` | — | `new`: the persona's prompt text |\n| `--from ` | — | `new`: seed the prompt from a file |\n| `--verbose`, `-v` | off | `list`: include role / model / description |\n| `--running` | off | `list`: mark personas live on the mesh |\n| `--force` | — | `rm`: required, delete without prompting |\n\nPersonas are the local agent files under `.cotal/agents/` that `cotal spawn` launches. See\n[Agent files](agent-files.md) for the file format.\n\n## supervise\n\n```bash\ncotal supervise [--runtime ] [--space ] [--server ] [--spawn ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space to supervise |\n| `--server ` | hosting mesh, or matching registered mesh | Broker URL. A registered mesh supplies it when omitted; a different explicit value is refused. |\n| `--runtime ` | `pty` | Agent runtime (`pty` built in; extension runtimes are explicit-only) |\n| `--console-port ` | — | Protocol-console port |\n| `--console-host ` | loopback | Bind host for the console + attach endpoint. Loopback keeps it machine-local; `cotal up` passes the address it bound the broker to, which is what lets `cotal attach` reach this manager from another machine |\n| `--roster ` | — | Declarative roster to boot at startup |\n| `--launch ` | — | Resolved manifest launch spec (from `up -f` / `spawn -f`) |\n| `--spawn ` | — | Comma-separated personas to pre-spawn at startup |\n\nThe manager is the agent supervisor and control plane: it answers `spawn --detach`, `stop`, `ps`,\n`attach`, and the `cotal_*` manager tools. `cotal up` starts one for you; run `supervise`\ndirectly to recover a dead manager or drive a custom runtime. Default runtime is `pty`; install an\noptional provider first (`cotal ext add @cotal-ai/orca`, `@cotal-ai/tmux`, `@cotal-ai/cmux`, or `@cotal-ai/herdr`) and\nselect it explicitly. A missing provider or app fails loudly; there is no fallback. See [Deploy](deploy.md).\n\nA `meshes add --mode user` entry is a **participant** registration, not hosting authority. A\nparticipant may run `supervise` only when the host advertises the remote manager authority service\nand the signed-in actor has the dedicated `supervise` ledger scope. The CLI obtains the closed,\nloopback-only `manager-service` view; `spawn` and `admin` do not substitute for that scope. The\nhost issues the manager's public-nkey JWT material through its lifecycle-bound prepare → activate\n→ renew protocol, never by handing the participant a signer or static provisioner credential.\n\nWithout that advertised host service or scope, `supervise` refuses before it starts a manager.\nRun `cotal spawn` without `--detach` to launch a foreground agent, or ask the space host to enable\nthe authority service and grant `supervise` for detached agents. If a running remote manager loses\nrenewal, it reports degraded state and refuses unsafe new starts and restarts; live agents are not\nsilently replaced. Do not run `cotal down` or `cotal up` on a participant machine to repair this\ncondition.\n\n## reconcile-gate\n\n```bash\ncotal reconcile-gate [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the frozen gate lives in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint whose gate is frozen |\n| `--instance ` | this folder's persisted manager instance | Instance id |\n\n**When you need this.** A manager restart killed partway through — after it began deregistering,\nbefore the new incarnation finished — leaves the endpoint's issuance gate *frozen*, held by a\nprocess that no longer exists. The freeze is what stops two incarnations serving at once, which is\ncorrect. The successor manager now completes that dead registration itself on boot, using the same\nguard this command uses: it acts only when the freeze-holder is affirmatively gone under a complete\nCONNZ sweep (`gone` and `sweepComplete=true`), then abort-reopens the gate at generation+1 with\nprocessEpoch unchanged and continues the normal takeover. Live, unknown, unestablishable, and\nwrong-op-kind still refuse; there is no TTL.\n\nUse this command when that boot path cannot run — the delivery daemon is down, you are repairing a\nnon-manager endpoint, or you want to lift the freeze without starting a manager. It checks that the\nholder really is gone, prints what it found, and then finishes the dead operation exactly as the\ninterrupted restart would have: revoke the old credentials, evict their holders with verification,\nand reopen the gate.\n\n**It refuses far more often than it acts, on purpose**, and always says which check stopped it:\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `holder-alive` | The freeze-holder still has a live connection — a manager *is* running | Stop that process first. Reconciling would evict a live manager's credentials |\n| `holder-unknown` | The connection sweep could not prove the holder absent | Not safe to proceed: an unprovable holder is treated as a live one. Re-run once the broker answers completely |\n| `liveness-unestablishable` | The delivery daemon could not be asked at all | Start it (`cotal up` runs it) and re-run. Silence is never read as death |\n| `not-frozen` / `no-gate` | The gate is open, or there is no gate at that coordinate | Nothing to repair — check `--endpoint` / `--instance` |\n| `wrong-op-kind` | Frozen under a takeover or retirement, not a registration | Out of scope for this command; it will not reinterpret another operation's intent |\n| `eviction-unverified` | The holder looked gone but eviction could not be verified | The gate is left frozen, unchanged. Investigate the broker before retrying |\n| `raced` | A newer manager moved the gate mid-repair | Re-run `cotal doctor` and look again |\n\nThere is no `--force`, and no path that discards gate state: the only way this reopens a gate is by\nproving the holder is gone and then completing the operation properly.\n\n## deregister-instance\n\n```bash\ncotal deregister-instance [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the instance is registered in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint the instance serves |\n| `--instance ` | this folder's persisted manager instance | Instance id, the whole id as `cotal ps` prints it |\n\n**When you need this.** The service registry records *registration*, not liveness, and nothing in\nthe model expires a row. A manager that stops cleanly removes its own registration. One whose host\ndied without writing anything cannot, so its record goes on claiming a live instance forever: every\nclass scatter in that space freezes the dead slot in, and `cotal ps`, `stop` and `attach` each pay\ntheir whole deadline waiting for a machine that is never coming back. A laptop that was reimaged, a\ncontainer that was deleted, a box that will not be back on the network: those registrations have no\nother exit.\n\nThis command is that exit. It asks the instance first, and it removes a record only when the broker\naffirms the instance's own rail is empty: nothing subscribed there. Then it deletes the\nregistration's two records keys, each pinned to the revision it read, and prints what it removed.\n\n**Silence alone never passes.** An unanswered describe is what a dead host, a wedged process and a\nslow one all look like, and a hung process still holds its subscriptions, so the broker sees\ninterest on its rail. That instance is refused and the observation is printed. A dead process holds\nno connection and therefore no subscription, so a real corpse is still removed.\n\n**It refuses rather than guesses**, and says which check stopped it:\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `instance-answered` | The instance answered a pinned describe. It is alive | Nothing to repair. If it is wedged rather than gone, stop the process first; its own clean stop removes the record |\n| `instance-not-affirmed-gone` | It did not answer, and the broker did not report its rail empty, which is what a held subscription looks like: slow or hung, not affirmed gone | Nothing was removed. Stop the process; its record goes on its own clean stop, or re-run this once it is down |\n| `liveness-unestablishable` | The probe itself failed, so nothing was learned | Fix the probe's path (credential, broker) and re-run. A probe that could not run is never read as death |\n| `not-registered` | No registration at that coordinate | Check `--instance` and `--endpoint`. This takes the whole id, never a prefix |\n| `superseded` | The record moved between the read and the delete | Something is writing to it. Nothing was removed; re-observe before retrying |\n\nThere is no `--force` and no sweep: silence is not death, and a rule that removed rows on silence\nwould eventually remove a live instance that was merely slow. An operator names one instance, the\nbroker's verdict on its rail is what authorizes the removal, and the guard's job is to show them\nthey named a dead one. Removal is not a one way door either. The same instance re-registers over\nthe tombstone on its next start, under the same identity.\n\n## runtimes\n\n```bash\ncotal runtimes\n```\n\nLists every agent runtime the manager can spawn through: the built-in `pty`, the official providers\n(`orca`, `tmux`, `cmux`, `herdr`), and any custom provider installed via `cotal ext add`. Each installed\nprovider is probed so you can see what is actually reachable on this machine before selecting it:\n\n```\npty built in\norca installed · reachable @cotal-ai/orca\ntmux available · cotal ext add @cotal-ai/tmux\ncmux available · cotal ext add @cotal-ai/cmux\nherdr available · cotal ext add @cotal-ai/herdr\n```\n\n`installed · reachable` / `unreachable` is the provider's own `available()` probe; `available` means\nit is a known runtime you can add with the shown command. Selecting an unknown or uninstalled runtime\nvia `up`/`spawn --runtime ` fails loud and, for a known one, points at the exact `cotal ext add`\npackage — there is no silent fallback to `pty`.\n\n## send\n\n```bash\ncotal send dm \"\" [--space ] [--server ] [--creds ]\ncotal send msg \"\"\ncotal send ask \"\"\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and (off-registry) which credential |\n\nOne-shot messaging: connect, send a single direct message (`dm`), channel post (`msg`), or role\nask/anycast (`ask`), then exit. For a running conversation, agents use the mesh tools instead\n([MCP tools](mcp-tools.md)).\n\n## channels\n\n```bash\ncotal channels list\ncotal channels set [--replay | --no-replay] [--window ] [--desc ] [--instructions ]\ncotal channels default --replay | --no-replay\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--replay` / `--no-replay` | — | `set`/`default`: replay history to new joiners, or not |\n| `--window ` | — | `set`: replay window size |\n| `--desc ` | — | `set`: one-line channel description |\n| `--instructions ` | — | `set`: instructions shown to joiners |\n\nInspects and edits the channel registry: replay policy, description, and joiner instructions. ACL\nsemantics (who may read or post) are set at mint / provision time, not here; see\n[Channels and permissions](channels-and-permissions.md). On a user-auth mesh, `list` rides your\nown login as is; `set` and `default` edit the registry over a short-lived\nchannel-writer view, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n\n## history\n\n```bash\ncotal history clear --force [--dms] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--dms` | off | Also clear DM history |\n| `--force` | — | Required: clear without prompting |\n\nPurges retained channel history; `--dms` extends it to direct-message history. An alias of\n[`clean history`](#clean). On a user-auth mesh the purge rides a short-lived purger view over\nyour login, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n## console\n\n```bash\ncotal console [--plain] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to watch |\n| `--plain` | off | Line stream instead of the TUI |\n\nA live protocol view for a space: a lazygit-style TUI, or a plain line stream on `--plain`. On a\nuser-auth mesh it rides the read-only admin view over your login, which needs ledger scope\n`admin`. See [Watch a mesh](watch-a-mesh.md).\n\n## web\n\n```bash\ncotal web [--detach] [--port ] [--no-open] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to serve |\n| `--port ` | `7799` | HTTP port |\n| `--detach` | off | Run in the background; stop with `cotal down web` or bare `cotal down` |\n| `--no-open` | off | Don't open the browser |\n\nThe browser observability dashboard: presence, channels, and a live feed. It is **not** part of\n`cotal up`: it ships inside `cotal-ai` as the `@cotal-ai/web` extension, seeded automatically on first\nrun (like the built-in connectors) so it always matches your CLI version. It self-registers `cotal web`\ninto this surface and serves\n`http://cotal.localhost:7799` (loopback; `*.localhost` resolves in Chrome/Firefox/Edge; Safari may\nneed `http://127.0.0.1:7799`). On a user-auth mesh the dashboard rides the read-only admin view\nover your login, and a channel purge asks for its own channel-purger view per click; both need\nledger scope `admin`. Detached mode re-execs the current Cotal installation, writes diagnostics to\nthe mesh root's `.cotal/web.log`, and reports success only after the HTTP server answers. It requires\na recorded mesh root, but can be launched from any directory once `cotal up` has recorded the mesh.\nSee [Watch a mesh](watch-a-mesh.md).\n\n## mint\n\n```bash\ncotal mint [--profile ] [--out ] [--signer]\ncotal mint --provision [--role ] [--space ] [--server ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--profile ` | `agent` | Credential profile |\n| `--out ` | `.cotal/auth/creds/.creds` | Output path |\n| `--signer` | off | Emit a stripped account-signing file instead |\n| `--force` | off | With `--signer`: overwrite an existing file |\n| `--allow-subscribe ` | the agent file's, else subscribe | Read-ACL override, **agent profile only**: `observer` and `admin` carry a fixed read set, and `mint` refuses this flag there rather than narrowing nothing |\n| `--allow-publish ` | the agent file's, else deny | Post-ACL override, **agent profile only** |\n| `--role ` | the agent file's | Agent profile: the anycast task queue the identity pulls (`svc_`) |\n| `--provision` | off | Agent profile: also pre-create the identity's bind-only DM/deliver durables (and its role's task queue) on the live mesh, so the credential can consume |\n| `--space `, `--server ` | the resolved mesh | With `--provision`: which mesh to provision on |\n\nMints a NATS creds file for a space in **static** auth mode, scoped to a profile and (optionally)\nexplicit read/post ACLs. `--signer` emits an account-signing file for delegating minting to another\nhost. A per-user-auth space refuses `mint`: agents there join under a logged-in user\n([`login`](#login-logout) + [`actor grant`](#actor)), never via a handed-out creds file. See\n[Identity and auth](identity-and-auth.md).\n\nA plain mint is creds only: the identity can publish within its post ACL at once, but on an authed\nmesh its DM inbox and task queue are provisioner-pre-created and bind-only, so a **consuming**\nconnect fails until they exist. `--provision` performs that pre-create in the same command (a\nprovisioner cred is minted from the space's trust material, used, and dropped), so a long-running\nclient you start yourself can receive DMs and role anycasts like a spawned seat. The command prints\nthe identity's principal (its wire id) and lifecycle uid; a consuming client passes that uid as its\n`lifecycleUid`. Agent profile only; an open mesh needs none of this (peers self-create there). The\nmesh it provisions on must be the one this folder's auth is for - same space and same account key -\nso `--provision` can never quietly mint under another root's trust material.\n\n## login, logout\n\n```bash\ncotal login --idp [--client-id ]\ncotal logout --idp \n```\n\nSigns you in to a per-user-auth mesh's IdP (device code flow) and caches the session; run it\nonce per machine. It prints your IdP subject, the id the operator grants against. After a\nlogin, every command on that mesh works under your identity: each connect takes a fresh IdP\nproof, exchanges it locally for a short-lived bearer, and is authorized against the actor\nledger at connect time. `logout` revokes the IdP session and clears the cache. See\n[identity & auth](identity-and-auth.md).\n\n## actor\n\n```bash\n# an upsert of the WHOLE row: a flag left off is the WIDE default below, not \"unchanged\"\ncotal actor grant --sub [--scope a,b] [--allow-subscribe a,b] [--allow-publish a,b] [--role ] [--label ]\ncotal actor revoke (--sub | --owner )\ncotal actor list\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | the folder's | Space whose ledger to manage |\n| `--sub ` | — | The IdP subject (shown by `cotal login`) the actor belongs to |\n| `--owner ` | — | The derived owner token (alternative to `--sub`) |\n| `--scope ` | `spawn,role:default` | Capability scope (`''` = none; `spawn` = may run agents; `role:` = may delegate role r; `admin` = cross-agent control; `supervise` = eligible for the closed remote manager-service view when the host enables it) |\n| `--allow-subscribe ` | `>` (all channels) | Channel read ACL; the user's envelope, their agents can never read beyond it |\n| `--allow-publish ` | `>` (all channels) | Channel post ACL; also the envelope for their agents' posting |\n| `--role ` | — | Role (scopes the task-queue consumer) |\n| `--label ` | — | Display label for `actor list` (never the IdP subject) |\n\nThe actor ledger is the single authorization source of a user-auth space: no row, no access.\nA bare `grant` is the **full** envelope (all channels, may spawn); the flags narrow it. A\nre-grant **replaces the whole row**, not the one field you name, so to add a capability spell\nevery field out: the new scope plus the row's current read set, post set, role and label\n(`cotal actor list` shows what a row holds). A field left off does not stay as it was, it\nreverts to the wide default in the table above, which is how a narrow reader becomes a reader\nof every channel. `supervise` is separate from `spawn` and `admin`: it only makes a signed-in\nperson eligible for the host-provided closed remote manager-service view; it does not grant\nmanagement of another owner or a general host profile. `revoke` denies the next exchange and\nthe next connect with no restart, and evicts the principal's live connections. Managed-agent rows\n(written by the spawn path) live in a disjoint row space this command never touches. See\n[identity & auth](identity-and-auth.md).\n\n## doctor\n\n```bash\ncotal doctor auth [--fix]\n```\n\nCredential-health diagnosis and repair for this folder's mesh: renders every managed\ncredential as healthy / near-expiry / expired and ends in `healthy` or the exact next\ncommand; `--fix` applies the repairs it can. The one surface every stale-credential error\npoints at.\n\n## join\n\n```bash\ncotal join --space --name [--role ] [--channel ]\ncotal join --link | --token \n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and which credential |\n| `--name ` | — | Your presence name |\n| `--role ` | — | Your role |\n| `--channel ` | — | Channel to join |\n| `--kind ` | `agent` | Endpoint kind |\n| `--link ` | — | Join link (`cotal://…`) |\n| `--token ` | — | Join token |\n| `--lifecycle-uid ` | — | Required with `--creds`: the lifecycle UID minted alongside the credential (`COTAL_LIFECYCLE_UID` works too). A credential's durable grants name exact lifecycle-keyed resources, so `join` refuses to invent one |\n| `--tls` | off | Connect over TLS |\n\nAn interactive presence: join a space under your own name and role, without launching an agent\nharness. A `--link` or `--token` supplies the where and the auth in one value. See\n[Spaces](spaces.md) and [Identity and auth](identity-and-auth.md).\n\n## Manifest deploys\n\nA `cotal.yaml` manifest declares a whole mesh (channels, personas, roles, and ACLs) in one file.\nThree commands consume it, plus a read-only validator:\n\n```bash\ncotal up -f cotal.yaml # boot a fresh mesh from the manifest\ncotal spawn -f cotal.yaml # deploy the manifest additively onto a running mesh\ncotal down -f cotal.yaml # tear that deploy down (or --run for one run)\ncotal topology view -f cotal.yaml # validate + view the access graph, change nothing\n```\n\n`up -f` and `spawn -f` differ in target: `up -f` brings up a new broker and applies the manifest;\n`spawn -f` requires an already-reachable mesh and applies additively (ownership-scoped). On a\nuser-auth mesh, `spawn -f` deploys over your own login (the deployer view, gated on ledger scope\n`spawn`): the manifest's agents land under your owner, a manifest claiming another owner is\nrefused, and seeding new channels additionally needs scope `admin`. Both take\n`--dry-run` to print the plan without mutating anything. `topology` validates the manifest and\nrenders its channel / role / ACL graph. See [Define a team](define-a-team.md) and the\n[manifest reference](manifest.md).\n\n## ext\n\n```bash\ncotal ext # same as `list`\ncotal ext add \ncotal ext remove \ncotal ext list\ncotal ext root # print just the install prefix (scriptable)\ncotal ext seed [--repair|--reset|--force]\n```\n\nOperator-installed extensions: `add` installs an npm package into a cotal-owned prefix and records\nevery registry provider it contributes. Commands appear in help, completion, and dispatch; runtime\nproviders are lazy-loaded by commands such as `supervise`; local process providers participate in\n`status` and selective `down`. `remove` and `list` manage them. The `@cotal-ai/web` dashboard is the\ncanonical command/process example. Installed packages and their location are described in\n[config](config.md).\n\nBare `cotal ext` lists the inventory, headed by the install prefix. That prefix is a cotal-owned npm\nroot kept **separate** from npm's own global tree, so these packages never show up in `npm list -g` —\n`cotal ext` (or the Extensions section of `cotal status`) is the canonical inventory. `cotal ext root`\nprints only the path, for scripts. The versions shown are the manifest pin recorded at add time.\n\nRemoving an extension that owns a running local process is refused with the mesh root and its\n`cotal down ` command; stop it first so uninstalling the package never strands a process\nwhose lifecycle provider is gone.\n\n### Built-in connectors are seeded extensions\n\nThe first-party agent connectors (`claude`, `opencode`, `codex`, `hermes`, `jcode`, `pi`) are not compiled into\nthe binary. They are seeded on first run through the **same** `ext add` path a third party uses, and\nappear in `cotal ext list` like any other extension. So you can remove one you do not want\n(`cotal ext remove @cotal-ai/connector-hermes`), and a deliberately-removed connector STAYS removed\nacross upgrades. `cotal ext add ` adds a third-party connector the same way. The web\ndashboard (`@cotal-ai/web`, providing `command:web`) is the seventh built-in seeded on the same path.\n\n`cotal ext seed` is the maintenance entry for that seeding (it runs automatically on the first real\ncommand of each boot, so you rarely call it):\n\n| Flag | Meaning |\n|---|---|\n| (none) | Reconcile: seed any never-seeded built-in, refresh a seeded one whose version the binary bumped, leave a removed one removed. A no-op once current. |\n| `--repair` | Recover after an interrupted seed or a lost authority (rebuilds the interrupted connector; restores the removed-vs-never-seeded record from its durable backup). |\n| `--reset` | Discard the record and re-seed all seven built-ins (the six connectors plus the web dashboard). **Resurrects any you removed.** Rebuilds cleanly over corrupt seed state. |\n| `--force` | Re-seed the built-ins even when the version stamp is current or a downgrade. |\n\nThe default connector for a bare `cotal spawn` (no `--agent`) is `claude`; set `COTAL_DEFAULT_AGENT`\n(e.g. `opencode`) to change it. An `--agent` naming a removed connector fails loud with the exact\n`cotal ext add` to restore it. Set `COTAL_SKIP_CONNECTOR_SEED=1` to turn off the automatic first-run\nseed/refresh entirely (for a controlled or offline setup that manages connectors by hand); `cotal ext\nseed` still runs on request.\n\n## completion\n\n```bash\ncotal completion # print a stub to eval / source\ncotal completion install [shell] # install it persistently\n```\n\nPrints or installs shell completion. Completion candidates come from each command's declared flags\nand, where useful, live mesh state (spaces, personas, managed agents) resolved offline.\n\n## feedback\n\n```bash\ncotal feedback \"\" [--type ] [--email ] [--details ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--type ` | — | `bug` \\| `idea` \\| `friction` \\| `praise` \\| `other` |\n| `--details ` | — | Longer free-form details |\n| `--severity ` | — | `low` \\| `medium` \\| `high` |\n| `--area ` | — | The part of Cotal this concerns |\n| `--email ` | git email | Contact email (required on the keyless public path) |\n| `--name ` | — | Your name (optional) |\n| `--url ` | keyed / public intake | Intake URL override |\n| `--key ` | `COTAL_FEEDBACK_KEY` | Feedback key |\n\nSends feedback to the Cotal developers. With a key (`--key` / `COTAL_FEEDBACK_KEY`) it routes to the\nkeyed beta intake; without one it goes to the public `cotal.ai` intake and requires a contact email\n(`--email` / `COTAL_FEEDBACK_EMAIL`, else your git email). Run a self-hosted intake with\n[`feedback-intake`](#server-daemons).\n\n## Server daemons\n\nTwo long-lived infra roles ship with the CLI. They are not part of everyday operation; the delivery\ndaemon comes up automatically with `cotal up` in auth mode.\n\n```bash\ncotal deliver --space [--server ] [--creds ]\ncotal auth-service --space --server [--port ] [--exchange-public-port ] [--exchange-public-url ] [--exchange-trusted-proxy]\ncotal feedback-intake --keys [--port ] [--creds ]\n```\n\n`auth-service` runs a user-auth space's identity plane: the NATS auth callout, the\ncapability-gated local exchange and JWKS, and, when `--exchange-public-port` is set, the closed public\nexchange/discovery face forwarded by an HTTPS reverse proxy. `--exchange-public-url` is the proxy URL\nadvertised to clients; `--exchange-trusted-proxy` opts into last-hop `X-Forwarded-For` attribution.\n`cotal up --user-auth` starts and supervises the service for you, so you run it directly only to\nrecover one by hand.\n\n`deliver` runs the server-side Plane-3 delivery daemon: the durable backstop and membership/ACL\nauthority. It is auth-mode-only and single-instance (`--shard`/`--shards` accept only `N=1`);\n`--dev-mint` mints a scoped cred from the local signer for standalone dev. See the\n[delivery daemon](delivery-daemon.md). `feedback-intake` runs a self-hosted feedback server\n(requires `--keys` and a scoped `--creds`), announcing submissions into a space channel; flags\ninclude `--host`/`--port`, `--store`, `--space`/`--channel`, `--max-bytes`, and `--rate-limit`.\n\n## Plumbing\n\n`cotal __complete ` is the internal entry the shell-completion stubs call to emit candidates\nfor the current command line; you never run it directly. `cotal agent-bearer` is machine-facing\nplumbing on user-auth meshes: spawned agents exec it to print a fresh short-lived bearer from their\nspawn-time secret; you never run it directly either. Its local arm uses `--dir` to discover the\ncapability-gated loopback service. A remotely enrolled, already-granted agent instead receives\n`--exchange-url ` in its launch argv: that arm sends `{owner, actor, actorToken}` to the\npinned public exchange with no local capability, follows no redirects, and refuses every non-HTTPS\nURL because the actor token is the credential in the request body. (`cotal start` is a removed tombstone: it\nerrors and points you to `cotal spawn --detach`.)\n" }, { "slug": "config", @@ -222,7 +222,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "Run a mesh", "kind": "Guide (informative)", "summary": "Day-to-day operation of a local mesh: what cotal up actually runs, how spawning resolves personas, harnesses, and models, how to reach a mesh from any directory, and the operator-only maintenance v…", - "body": "# Run a mesh\n\n> **Guide** (informative) · **For:** operators · **Prereqs:** [Quickstart](getting-started.md)\n\nDay-to-day operation of a local mesh: what `cotal up` actually runs, how spawning\nresolves personas, harnesses, and models, how to reach a mesh from any directory, and the\noperator-only maintenance verbs. Every command's full flag set is in the\n[CLI reference](cli.md).\n\n## The stack\n\n`cotal up` brings up the whole local stack and bare `cotal down` stops it:\n\n- **Broker**: a local `nats-server` (logs to `.cotal/nats.log`).\n- **Delivery daemon**: the durable backstop, auth mode only\n ([what it does](delivery-daemon.md)).\n- **Manager**: a detached supervisor answering the control plane, so\n `cotal spawn --detach` and the `cotal_spawn` tool work right after `up`.\n\nThree modes:\n\n- **Default (static auth).** JWT-authed, on by default: sender authenticity and per-agent\n ACLs, enforced by the broker ([how](identity-and-auth.md)).\n- **`--user-auth --idp `.** Per-user auth: people `cotal login` once, the operator\n grants their agents on the actor ledger, and every connect is authorized live against\n that grant. Starts the space's auth service alongside the broker\n ([how](identity-and-auth.md)).\n- **`--open`.** An unauthenticated, live-only dev mesh (no auth, no delivery daemon). For\n quick local experiments.\n\nThe broker and local services bind **loopback** by default. `--host 0.0.0.0` widens the broker\nbind independently of the auth mode, so \"network-reachable\" never silently means\n\"unauthenticated\". With no explicit `--server`, `cotal up` auto-selects a free local port when\nthe default address is already held by another project; an explicit `--server` fails loud on\ncollision.\n\nA user-auth mesh can expose only its credential exchange through an operator-owned HTTPS reverse\nproxy while leaving the existing local exchange untouched:\n\n```bash\ncotal up --user-auth --idp https://idp.example/api/auth \\\n --exchange-public-port 7443 \\\n --exchange-public-url https://auth.example\n```\n\nThe public listener itself still binds `127.0.0.1:7443`; configure the proxy to terminate TLS and\nforward to it. It serves only `/health`, `/jwks`, `/exchange`, and `/.well-known/cotal-mesh` with\nthe documented methods. It needs no local file capability: the signed IdP JWT or managed-agent\nactor token is the proof, while the original loopback listener remains capability-gated. Add\n`--exchange-trusted-proxy` only when that listener is reachable exclusively through your trusted\nproxy; it keys failure throttling by the last `X-Forwarded-For` hop instead of the socket address.\nThe well-known bundle includes IdP pins and a deny-all sentinel credential, so fetch it only from\nthe configured HTTPS origin. To change these listener flags, stop and restart the mesh; a refresh\nof an already-running service does not replace its bind or proxy policy. See\n[Identity & auth](identity-and-auth.md#per-user-auth-people-sign-in) for the trust boundary.\n\n`cotal status` prints the detailed setup, process, registry, and live mesh status;\n`cotal setup` (after the first run) prints the compact card.\n\nStop one part without tearing down the mesh by naming its registered component: `cotal down\nmanager`, `cotal down delivery`, or `cotal down web`. Component names from installed extensions\njoin the same surface; `cotal down` with no names retains whole-stack behavior.\n\n## Remote supervised agents\n\nOn a remote user-auth mesh, foreground `cotal spawn` remains the default participant path. A\nparticipant can run detached agents only after the host advertises and operates the remote manager\nauthority service, and the participant's actor-ledger row includes `supervise`. This is not implied\nby `spawn` or `admin`.\n\nThe participant's loopback/operator exchange obtains one closed `manager-service` view for its\nordinary derived owner, a fixed server-selected manager actor, and one opaque manager instance.\nThe host, not the participant, issues the public-nkey JWT material via the replay-safe,\nlifecycle-bound prepare → activate → renew exchange. It never exports the space signer, a static\nprovisioner credential, or generic storage authority. The manager may provision only descendants\nof that same owner, with host validation at each provision.\n\nWhen the authority service, login, or renewal is unavailable, the remote manager degrades\nfail-closed: it refuses new agents, restarts, and credential replacement rather than pretending\nlocal authority exists. Existing agents remain live only while their independent credentials are\nvalid. Restore service and renew successfully before asking it to recover an agent. See\n[Identity & auth](identity-and-auth.md#remote-manager-authority) and the [CLI\nreference](cli.md#supervise).\n\n## Spawning agents\n\n```bash\ncotal spawn # foreground: your default agent, in this terminal\ncotal spawn reviewer --detach # supervised: the manager runs it in a PTY\ncotal attach --name reviewer # watch/type into a detached agent (Ctrl-] detaches)\ncotal ps # what the manager is running\ncotal stop --name reviewer # stop one\n```\n\nHow a spawn resolves:\n\n- **Persona.** A bare `cotal spawn` uses `.cotal/agents/default.md`; a positional name\n picks `.cotal/agents/.md`; `--config` takes an explicit ref or path. Set\n `COTAL_DEFAULT_PERSONA=` to change the fallback. Fields and format:\n [agent files](agent-files.md).\n- **Harness.** Claude by default; `--agent opencode` / `--agent hermes` / `--agent pi` per\n spawn, or `COTAL_DEFAULT_AGENT` to change the default. Compared in\n [Connectors](connectors.md); per-connector guides:\n [Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n [Hermes](connect-hermes.md) · [pi](connect-pi.md).\n- **Model.** `--model` overrides the persona file's `model:` (Claude: `opus` / `sonnet` or\n a full id; OpenCode: `provider/model`). Connectors that expose a catalog report it via\n `cotal models --agent opencode`: model ids plus available variants; pick one with\n `--model provider/model --variant high`.\n- **Tools.** A spawned agent gets only the cotal tools by default; share your own MCP\n servers deliberately with `--share-tools` ([config](config.md)).\n- **Launch options.** `--opt key=value` (repeatable) passes a native harness flag straight\n through; a persona or manifest `launchOptions:` mapping does the same declaratively (a\n `--opt` wins per key). It is a **raw passthrough**, with no allow/deny list: Claude renders\n each as `--key value` (a bare `--key` for an empty value), OpenCode merges them into its\n agent config, and Hermes has no option surface so it fails loud. The trust boundary is the\n `spawn` capability itself, not the flag set, so granting `spawn` is host-launch authority\n ([security](security.md)). A key must be a plain flag name; malformed or prototype-polluting\n keys are refused.\n\nDetach from an attached PTY with **Ctrl-]** (the agent keeps running); rebind it with\n`COTAL_DETACH_KEY=ctrl-` when it clashes with a keybinding inside the agent's TUI.\n\n**Runtimes.** The manager spawns into a **pty** it owns by default. Optional runtimes are installed\nthrough the extension surface, for example `cotal ext add @cotal-ai/orca`, then selected with\n`--runtime orca` (similarly `@cotal-ai/tmux`, `@cotal-ai/cmux`, and `@cotal-ai/herdr`). They put teammates in native\nterminal surfaces rather than manager-owned PTYs. Runtime names are open-ended and resolved from\nthe registry; a missing provider or app throws, never silently falls back\n([architecture](architecture.md)).\n\n## From any directory: the mesh registry\n\n`cotal up` records each running mesh in a machine-local registry\n(`~/.cotal/meshes/space..json`, named by a case-safe hex encoding of the space: broker URL, the project root holding its creds and\npersonas, and its mode). So a bare `cotal spawn ` from *any* directory joins the\nrunning mesh with the right credentials instead of mistaking the cwd for a space:\n\n- `cotal use ` sets the default from every directory, including inside another mesh's\n project. `--space ` overrides it for one command.\n- With no live selected default, a project with its own `.cotal/` resolves to that project's\n mesh; otherwise one running mesh is used automatically and several are an error.\n- `cotal meshes` lists them (a `*` marks the default); `cotal down` removes the entry.\n\nThe registry stores a *path*, never a secret; trust material stays in each project's\n`.cotal/auth`. If the mesh is down or won't take your creds, spawn fails with one\nsentence, never a raw NATS trace.\n\n### Meshes you did not start here\n\nA mesh running on another machine has no `cotal up` on this one, so register it by hand:\n\n```bash\ncotal meshes add # guided: asks for the broker, probes it, offers what it finds\ncotal meshes add optiplex --server nats://100.90.12.34:4222 --root ~/meshes/optiplex \\\n --allow-unencrypted-overlay # see below: an overlay address needs this\ncotal meshes rm optiplex\n```\n\nOn a terminal, a bare `cotal meshes add` walks you through it: it probes the broker you name and\nreports whether it is open or requires credentials, offers the spaces the folder already holds\ncredentials for, and shows the record before writing it. Scripts and agents keep the flag form -\nwithout a terminal nothing prompts.\n\n`--root` is the local folder holding that mesh's `.cotal/auth` and `.cotal/agents` (its personas);\nthe mode is inferred from what that folder holds.\n\n**Know what you are copying.** For an authenticated mesh that folder carries the space's account\n**signing seed**, which is the authority to mint any identity in the space. A machine holding it\nis a certificate authority for the mesh rather than a client of it: anyone who reads it can\nimpersonate any agent, read every retained channel and DM, change ACLs, and keep issuing\nthemselves credentials. There is no per-machine revocation; undoing it means rotating the signing\nkey and re-minting every credential in the space. Copy it only to machines you would trust with\nthe whole mesh. `cotal mint` on its own does not substitute here: registering an `auth` mesh needs\nsigning material that composes, which a minted user credential is not. The\nbroker is probed before the record is written, so a bad address or a credential that mesh will not\naccept fails at registration rather than at your first `spawn` (`--force` records it without verifying —\nuseful when the mesh is simply down right now).\n\n#### Which addresses you may register\n\nRegistering a mesh is how this machine starts sending agent credentials to a broker it does not\nrun. NATS announces itself in plaintext before anyone authenticates, so an attacker on the path\ncan pose as the broker and read the credential out of the connect — unless the connection\n**requires TLS**, which is recorded on the entry and enforced on every dial through it.\n\nWhat the record will require decides what you may register:\n\n- **Without required TLS**, the address is the gate: **loopback** (`127.0.0.0/8`, `::1`), or\n **your private overlay** (`100.64.0.0/10`, `fd7a:115c:a1e0::/48`) with\n `--allow-unencrypted-overlay`, because the protection is real only while the tunnel is running\n and this command cannot check that for you. Hostnames are refused — whoever answers the lookup\n would be choosing which machine receives your credentials.\n- **With required TLS** (`--tls`, or a `tls://` URL — the scheme is recorded and enforced, not\n cosmetic), a **hostname or public address** is accepted too: the certificate chain and\n hostname check pick the peer, not the resolver. A `tls://` registration against a broker that\n cannot complete the handshake fails at registration — unless you pass `--force`, which records\n the entry without verifying it at all — and on every later dial regardless.\n\nOrdinary private ranges like `10.x` and `192.168.x` are refused in **both** modes: a café's wifi\nis a private network too, being private is not the same as being yours, and no public CA issues\ncertificates for those ranges. How an address is *spelled* changes nothing: `[::ffff:192.168.1.10]`,\n`3232235786`, `0300.0250.01.012` and `192.168.257` are all private addresses that your machine\nwould dial as such, so each gets the same refusal as its dotted form. `--force` does not waive any of this — it exists for a mesh that\nis *down*, not for sending credentials somewhere unsafe.\n\n#### Registering a hosted user-auth mesh\n\nA user-auth space's IdP pins are established where the mesh runs and are never guessed. Register\none from **supplied** trust: `--user-auth-file bundle.json` (exported on the mesh's machine), or\n`--from https://…/.well-known/cotal-mesh`, which asks before it contacts the address at all,\nfetches the discovery document over HTTPS, shows you the pins, and asks again before adopting\nthem. Redirects are refused rather than followed — a 302 can walk a pinned fetch down to\nplaintext or onto another host — and the pinned exchange must be an `https://` URL too. The one\nexception is an exchange on **this machine**, where nothing leaves the box: plain `http://` is\naccepted for a loopback *literal* (`127.0.0.1`, `::1`, and any spelling of them), but **not** for\n`localhost`, which is a name a hosts entry or a poisoned lookup could point elsewhere — use the\nliteral. Registration checks that the pinned exchange\nanswers `/health` and `/jwks` as the pinned issuer and that the broker refuses a bare connect —\nthat refusal is the pass. The bundle's sentinel credentials are written to a private (0600) file\nunder the entry's root; the registry itself never carries the secret.\n\n**Without required TLS**, an overlay address is **refused unless you accept the dependency\nexplicitly**, with `--allow-unencrypted-overlay`. The address is not the guarantee: it is protected\nwhile the tunnel is up, and if the tunnel is down that range is ordinary carrier-grade NAT and\nwhoever answers the dial receives your credentials. Only you can know which it is, so the command\nasks you to say so. Your acceptance is recorded on the mesh entry rather than printed and\nforgotten, and the guided form asks the same question instead of taking the flag.\n\n**With required TLS** (`--tls`, or a `tls://` URL) that consent is no longer asked for, and the\nflag is not needed: the handshake is what protects the connection, so the acceptance it stood in\nfor has been replaced by proof rather than promise. `cotal meshes add --server\nnats://100.64.0.1 --tls` registers an overlay address with no prompt, no flag and no recorded\nacceptance — this is the \"the flag disappears once the broker can be served over TLS\" case, and it\nhas now arrived.\n\nThis gate is on **registration**. `cotal join --creds --server ` deliberately takes an\nexplicit connection at face value and does not consult the registry, so it is not covered — join\nthat way only to an address you would have registered.\n\nRecords added this way are removed only by something that names them. A mesh this machine started\ncan be dropped on a hunch — a failed liveness probe, a `cotal down` in its project — because\n`cotal up` writes the record straight back. One you registered by hand cannot be reconstructed, so\nnothing removes it by inference: an unreachable broker is shown as `offline` in `cotal meshes`, and\n`cotal down` / `cotal clean all` leave it alone even when `--root` pointed at the project they are\ntearing down. A `cotal up` for that space refuses outright (naming `cotal meshes rm`) unless it is\nthat same endpoint: finding a broker already answering there is a refresh that starts nothing and\nleaves the record's provenance alone, while actually starting the broker for that space, server and\nroot makes this machine the one running it, so the record becomes an ordinary local one that\n`cotal down` clears. `cotal meshes rm` drops it and re-registering with `--force` replaces it. `rm`\nonly forgets a mesh — to stop one running here, use `cotal down`.\n\n## Watching\n\n`cotal console` is the terminal view (TUI on a real terminal, plain line stream when\npiped); `cotal web` is the browser dashboard. Both are read-only observers; the\nwalkthrough is [Watch a mesh](watch-a-mesh.md).\n\n## History\n\nRetained history is operator-owned. `cotal clean history --force` purges a space's\nretained channel history; `--dms` also purges DMs (`cotal history clear` is an alias).\nIt is deliberately **not** an agent tool: agents cannot wipe the record\n([identity & auth](identity-and-auth.md)). For a **stopped** mesh, `cotal clean store\n--force` deletes the on-disk JetStream store outright, and `cotal clean all --force`\nalso resets the space identity ([CLI reference](cli.md#clean)).\n\n## Offline backup\n\nFor a coherent durable cut, preserve the whole stack first, then create the artifact while it stays\ndown:\n\n```bash\ncotal down --preserve-state\ncotal backup create ./space-backup # full by default\n# later: deliberately resume the unchanged source\ncotal up --detach\n# or, from another preserved cut, restore before the normal listener opens\ncotal up --restore ./space-backup --detach\n```\n\nUse `--store-dir` on both preservation and backup for a custom JetStream store. `registry` is the\nonly partial selection (`backup create ... --only registry`; `up --restore ... --restore-only\nregistry`). Backup never stops or restarts a mesh implicitly, never opens the original store, and\ndoes not contain credentials or trust secrets. Backup/restore in every auth mode — open included —\nuses isolated, operation-specific maintenance logins; normal agent credentials cannot enter that\nlistener. Full\nrestore requires the same space and exact current local trust continuity, recreates conservative\nconsumer checkpoints bound to their snapshot stream sequence state, and resumes retained agents under\ntheir original principals. The trust commitment includes the cryptographically validated full\noperator/system/data-account root chain as well as static/user authority state. A registry-only\nrestore completes canonical empty infrastructure but leaves retained agents stopped because their\nDM/DLV/TASK/ACL state is outside that selection. Authenticated restore validates the complete space\ntrust bundle before staging or changing the preserved store. Interrupted ordinary resume retries the\nsame durable attempt after its prior listener is stopped. Restore re-entry can recover a surviving normal listener\nonly when its attempt nonce, NATS server name, process owner, endpoint, and target-store identity all\nmatch the fsynced proof. A provably dead uncommitted owner is retired under lock and replaced with a\nfresh attempt-bound listener; an occupied foreign listener or ambiguous owner is never adopted. The\nmanager commit validates while retained cleanup is still suppressed; the CLI durably records its\nattempt-bound 64-hex token in `manager-committed` / `resume-committed` before `finalizeResume` can\nrelease suppression. A retry from either committed state goes straight to exact-token finalization;\nfailure preserves the committed gate and retained cleanup suppression. Missing commit evidence,\ninterrupted finalization, a live recorded endpoint despite missing pidfiles, or ambiguous proof fails closed. See the [CLI\nbackup and restore contract](cli.md#backup-and-restore) for artifact, checkpoint, fallback,\ndisaster-consent, and degraded-recovery details.\n\n## Personas from the CLI\n\n`cotal personas` manages the local catalog offline: `list` (`--running` overlays live\nmarkers), `show `, `edit ` (re-validates on save), `new `, `rm \n--force`. The runtime counterpart is the `cotal_persona` tool, which goes over the wire\nwith the manager's ownership checks. Fields: [agent files](agent-files.md).\n\n## Manager restart and a frozen issuance gate\n\nA manager that dies mid-registration leaves its issuance gate *frozen* under that registration\nop. The freeze is correct: it stops two incarnations serving at once. The successor now completes\nthat dead op on boot, using the same guard as [`cotal reconcile-gate`](cli.md#reconcile-gate): it\nacts only when the freeze-holder is affirmatively gone under a complete CONNZ sweep (`gone` and\n`sweepComplete=true`), abort-reopens the gate (generation+1, processEpoch unchanged), and continues\nthe normal takeover. A live holder, an incomplete sweep, or an unreachable delivery daemon still\nrefuses — silence is never death, and there is no TTL. Use `cotal reconcile-gate` when the boot\npath cannot run (daemon down, a non-manager endpoint, or you want to lift the freeze without\nstarting a manager).\n\n## When something looks absent\n\nPermission denials are **loud, never silent**: an over-tight ACL shows up as a logged\ndenial on the endpoint, not as a peer that mysteriously looks absent. Check\n`.cotal/manager.log`, `.cotal/delivery.log`, and `.cotal/nats.log`; `cotal status` shows\nwhat is actually running. The access rules are collected in\n[Channels & permissions](channels-and-permissions.md).\n" + "body": "# Run a mesh\n\n> **Guide** (informative) · **For:** operators · **Prereqs:** [Quickstart](getting-started.md)\n\nDay-to-day operation of a local mesh: what `cotal up` actually runs, how spawning\nresolves personas, harnesses, and models, how to reach a mesh from any directory, and the\noperator-only maintenance verbs. Every command's full flag set is in the\n[CLI reference](cli.md).\n\n## The stack\n\n`cotal up` brings up the whole local stack and bare `cotal down` stops it:\n\n- **Broker**: a local `nats-server` (logs to `.cotal/nats.log`).\n- **Delivery daemon**: the durable backstop, auth mode only\n ([what it does](delivery-daemon.md)).\n- **Manager**: a detached supervisor answering the control plane, so\n `cotal spawn --detach` and the `cotal_spawn` tool work right after `up`.\n\nThe stack is detached by default and does not depend on the invoking shell, SSH connection, agent\nsession, or CI runner staying alive. On Windows, Cotal probes the current process job and launches\nwith `CREATE_BREAKAWAY_FROM_JOB` when required and permitted. If the job forbids breakaway, `up`\nrefuses before starting any detached process and names `--foreground`, which remains the debugging\npath in the invoking terminal.\n\nThree modes:\n\n- **Default (static auth).** JWT-authed, on by default: sender authenticity and per-agent\n ACLs, enforced by the broker ([how](identity-and-auth.md)).\n- **`--user-auth --idp `.** Per-user auth: people `cotal login` once, the operator\n grants their agents on the actor ledger, and every connect is authorized live against\n that grant. Starts the space's auth service alongside the broker\n ([how](identity-and-auth.md)).\n- **`--open`.** An unauthenticated, live-only dev mesh (no auth, no delivery daemon). For\n quick local experiments.\n\nThe broker and local services bind **loopback** by default. `--host 0.0.0.0` widens the broker\nbind independently of the auth mode, so \"network-reachable\" never silently means\n\"unauthenticated\". With no explicit `--server`, `cotal up` auto-selects a free local port when\nthe default address is already held by another project; an explicit `--server` fails loud on\ncollision.\n\nA user-auth mesh can expose only its credential exchange through an operator-owned HTTPS reverse\nproxy while leaving the existing local exchange untouched:\n\n```bash\ncotal up --user-auth --idp https://idp.example/api/auth \\\n --exchange-public-port 7443 \\\n --exchange-public-url https://auth.example\n```\n\nThe public listener itself still binds `127.0.0.1:7443`; configure the proxy to terminate TLS and\nforward to it. It serves only `/health`, `/jwks`, `/exchange`, and `/.well-known/cotal-mesh` with\nthe documented methods. It needs no local file capability: the signed IdP JWT or managed-agent\nactor token is the proof, while the original loopback listener remains capability-gated. Add\n`--exchange-trusted-proxy` only when that listener is reachable exclusively through your trusted\nproxy; it keys failure throttling by the last `X-Forwarded-For` hop instead of the socket address.\nThe well-known bundle includes IdP pins and a deny-all sentinel credential, so fetch it only from\nthe configured HTTPS origin. To change these listener flags, stop and restart the mesh; a refresh\nof an already-running service does not replace its bind or proxy policy. See\n[Identity & auth](identity-and-auth.md#per-user-auth-people-sign-in) for the trust boundary.\n\n`cotal status` prints the detailed setup, process, registry, and live mesh status;\n`cotal setup` (after the first run) prints the compact card.\n\nStop one part without tearing down the mesh by naming its registered component: `cotal down\nmanager`, `cotal down delivery`, or `cotal down web`. Component names from installed extensions\njoin the same surface; `cotal down` with no names retains whole-stack behavior.\n\n## Remote supervised agents\n\nOn a remote user-auth mesh, foreground `cotal spawn` remains the default participant path. A\nparticipant can run detached agents only after the host advertises and operates the remote manager\nauthority service, and the participant's actor-ledger row includes `supervise`. This is not implied\nby `spawn` or `admin`.\n\nThe participant's loopback/operator exchange obtains one closed `manager-service` view for its\nordinary derived owner, a fixed server-selected manager actor, and one opaque manager instance.\nThe host, not the participant, issues the public-nkey JWT material via the replay-safe,\nlifecycle-bound prepare → activate → renew exchange. It never exports the space signer, a static\nprovisioner credential, or generic storage authority. The manager may provision only descendants\nof that same owner, with host validation at each provision.\n\nWhen the authority service, login, or renewal is unavailable, the remote manager degrades\nfail-closed: it refuses new agents, restarts, and credential replacement rather than pretending\nlocal authority exists. Existing agents remain live only while their independent credentials are\nvalid. Restore service and renew successfully before asking it to recover an agent. See\n[Identity & auth](identity-and-auth.md#remote-manager-authority) and the [CLI\nreference](cli.md#supervise).\n\n## Spawning agents\n\n```bash\ncotal spawn # foreground: your default agent, in this terminal\ncotal spawn reviewer --detach # supervised: the manager runs it in a PTY\ncotal attach --name reviewer # watch/type into a detached agent (Ctrl-] detaches)\ncotal ps # what the manager is running\ncotal stop --name reviewer # stop one\n```\n\nHow a spawn resolves:\n\n- **Persona.** A bare `cotal spawn` uses `.cotal/agents/default.md`; a positional name\n picks `.cotal/agents/.md`; `--config` takes an explicit ref or path. Set\n `COTAL_DEFAULT_PERSONA=` to change the fallback. Fields and format:\n [agent files](agent-files.md).\n- **Harness.** Claude by default; `--agent opencode` / `--agent hermes` / `--agent pi` per\n spawn, or `COTAL_DEFAULT_AGENT` to change the default. Compared in\n [Connectors](connectors.md); per-connector guides:\n [Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n [Hermes](connect-hermes.md) · [pi](connect-pi.md).\n- **Model.** `--model` overrides the persona file's `model:` (Claude: `opus` / `sonnet` or\n a full id; OpenCode: `provider/model`). Connectors that expose a catalog report it via\n `cotal models --agent opencode`: model ids plus available variants; pick one with\n `--model provider/model --variant high`.\n- **Tools.** A spawned agent gets only the cotal tools by default; share your own MCP\n servers deliberately with `--share-tools` ([config](config.md)).\n- **Launch options.** `--opt key=value` (repeatable) passes a native harness flag straight\n through; a persona or manifest `launchOptions:` mapping does the same declaratively (a\n `--opt` wins per key). It is a **raw passthrough**, with no allow/deny list: Claude renders\n each as `--key value` (a bare `--key` for an empty value), OpenCode merges them into its\n agent config, and Hermes has no option surface so it fails loud. The trust boundary is the\n `spawn` capability itself, not the flag set, so granting `spawn` is host-launch authority\n ([security](security.md)). A key must be a plain flag name; malformed or prototype-polluting\n keys are refused.\n\nDetach from an attached PTY with **Ctrl-]** (the agent keeps running); rebind it with\n`COTAL_DETACH_KEY=ctrl-` when it clashes with a keybinding inside the agent's TUI.\n\n**Runtimes.** The manager spawns into a **pty** it owns by default. Optional runtimes are installed\nthrough the extension surface, for example `cotal ext add @cotal-ai/orca`, then selected with\n`--runtime orca` (similarly `@cotal-ai/tmux`, `@cotal-ai/cmux`, and `@cotal-ai/herdr`). They put teammates in native\nterminal surfaces rather than manager-owned PTYs. Runtime names are open-ended and resolved from\nthe registry; a missing provider or app throws, never silently falls back\n([architecture](architecture.md)).\n\n## From any directory: the mesh registry\n\n`cotal up` records each provisioned mesh as **self-hosted** in a machine-local registry\n(`~/.cotal/meshes/space..json`, named by a case-safe hex encoding of the space: broker URL, the project root holding its creds and\npersonas, and its mode). The record survives downtime. A command targeting a stopped mesh says\nwhere it is recorded and tells you to run `cotal up` there to restart, instead of denying the mesh\nexists. A bare `cotal spawn ` from *any* directory joins a running mesh with the right\ncredentials instead of mistaking the cwd for a space:\n\n- `cotal use ` sets the default from every directory, including inside another mesh's\n project. `--space ` overrides it for one command.\n- With no live selected default, a project with its own `.cotal/` resolves to that project's\n mesh; otherwise one running mesh is used automatically and several are an error.\n- `cotal meshes` lists them (a `*` marks the default); stopped self-hosted meshes stay listed as\n `self-hosted · offline`. `cotal meshes rm ` deliberately removes a record.\n\nThe registry stores a *path*, never a secret; trust material stays in each project's\n`.cotal/auth`. If the mesh is down or won't take your creds, spawn fails with one\nsentence, never a raw NATS trace.\n\n### Meshes you did not start here\n\nA mesh running on another machine has no `cotal up` on this one, so register it by hand:\n\n```bash\ncotal meshes add # guided: asks for the broker, probes it, offers what it finds\ncotal meshes add optiplex --server nats://100.90.12.34:4222 --root ~/meshes/optiplex \\\n --allow-unencrypted-overlay # see below: an overlay address needs this\ncotal meshes rm optiplex\n```\n\nOn a terminal, a bare `cotal meshes add` walks you through it: it probes the broker you name and\nreports whether it is open or requires credentials, offers the spaces the folder already holds\ncredentials for, and shows the record before writing it. Scripts and agents keep the flag form -\nwithout a terminal nothing prompts.\n\n`--root` is the local folder holding that mesh's `.cotal/auth` and `.cotal/agents` (its personas);\nthe mode is inferred from what that folder holds.\n\n**Know what you are copying.** For an authenticated mesh that folder carries the space's account\n**signing seed**, which is the authority to mint any identity in the space. A machine holding it\nis a certificate authority for the mesh rather than a client of it: anyone who reads it can\nimpersonate any agent, read every retained channel and DM, change ACLs, and keep issuing\nthemselves credentials. There is no per-machine revocation; undoing it means rotating the signing\nkey and re-minting every credential in the space. Copy it only to machines you would trust with\nthe whole mesh. `cotal mint` on its own does not substitute here: registering an `auth` mesh needs\nsigning material that composes, which a minted user credential is not. The\nbroker is probed before the record is written, so a bad address or a credential that mesh will not\naccept fails at registration rather than at your first `spawn` (`--force` records it without verifying —\nuseful when the mesh is simply down right now).\n\n#### Which addresses you may register\n\nRegistering a mesh is how this machine starts sending agent credentials to a broker it does not\nrun. NATS announces itself in plaintext before anyone authenticates, so an attacker on the path\ncan pose as the broker and read the credential out of the connect — unless the connection\n**requires TLS**, which is recorded on the entry and enforced on every dial through it.\n\nWhat the record will require decides what you may register:\n\n- **Without required TLS**, the address is the gate: **loopback** (`127.0.0.0/8`, `::1`), or\n **your private overlay** (`100.64.0.0/10`, `fd7a:115c:a1e0::/48`) with\n `--allow-unencrypted-overlay`, because the protection is real only while the tunnel is running\n and this command cannot check that for you. Hostnames are refused — whoever answers the lookup\n would be choosing which machine receives your credentials.\n- **With required TLS** (`--tls`, or a `tls://` URL — the scheme is recorded and enforced, not\n cosmetic), a **hostname or public address** is accepted too: the certificate chain and\n hostname check pick the peer, not the resolver. A `tls://` registration against a broker that\n cannot complete the handshake fails at registration — unless you pass `--force`, which records\n the entry without verifying it at all — and on every later dial regardless.\n\nOrdinary private ranges like `10.x` and `192.168.x` are refused in **both** modes: a café's wifi\nis a private network too, being private is not the same as being yours, and no public CA issues\ncertificates for those ranges. How an address is *spelled* changes nothing: `[::ffff:192.168.1.10]`,\n`3232235786`, `0300.0250.01.012` and `192.168.257` are all private addresses that your machine\nwould dial as such, so each gets the same refusal as its dotted form. `--force` does not waive any of this — it exists for a mesh that\nis *down*, not for sending credentials somewhere unsafe.\n\n#### Registering a hosted user-auth mesh\n\nA user-auth space's IdP pins are established where the mesh runs and are never guessed. Register\none from **supplied** trust: `--user-auth-file bundle.json` (exported on the mesh's machine), or\n`--from https://…/.well-known/cotal-mesh`, which asks before it contacts the address at all,\nfetches the discovery document over HTTPS, shows you the pins, and asks again before adopting\nthem. Redirects are refused rather than followed — a 302 can walk a pinned fetch down to\nplaintext or onto another host — and the pinned exchange must be an `https://` URL too. The one\nexception is an exchange on **this machine**, where nothing leaves the box: plain `http://` is\naccepted for a loopback *literal* (`127.0.0.1`, `::1`, and any spelling of them), but **not** for\n`localhost`, which is a name a hosts entry or a poisoned lookup could point elsewhere — use the\nliteral. Registration checks that the pinned exchange\nanswers `/health` and `/jwks` as the pinned issuer and that the broker refuses a bare connect —\nthat refusal is the pass. The bundle's sentinel credentials are written to a private (0600) file\nunder the entry's root; the registry itself never carries the secret.\n\n**Without required TLS**, an overlay address is **refused unless you accept the dependency\nexplicitly**, with `--allow-unencrypted-overlay`. The address is not the guarantee: it is protected\nwhile the tunnel is up, and if the tunnel is down that range is ordinary carrier-grade NAT and\nwhoever answers the dial receives your credentials. Only you can know which it is, so the command\nasks you to say so. Your acceptance is recorded on the mesh entry rather than printed and\nforgotten, and the guided form asks the same question instead of taking the flag.\n\n**With required TLS** (`--tls`, or a `tls://` URL) that consent is no longer asked for, and the\nflag is not needed: the handshake is what protects the connection, so the acceptance it stood in\nfor has been replaced by proof rather than promise. `cotal meshes add --server\nnats://100.64.0.1 --tls` registers an overlay address with no prompt, no flag and no recorded\nacceptance — this is the \"the flag disappears once the broker can be served over TLS\" case, and it\nhas now arrived.\n\nThis gate is on **registration**. `cotal join --creds --server ` deliberately takes an\nexplicit connection at face value and does not consult the registry, so it is not covered — join\nthat way only to an address you would have registered.\n\nRecords added this way are removed only by something that names them. A mesh this machine started\ncan be dropped on a hunch — a failed liveness probe, a `cotal down` in its project — because\n`cotal up` writes the record straight back. One you registered by hand cannot be reconstructed, so\nnothing removes it by inference: an unreachable broker is shown as `offline` in `cotal meshes`, and\n`cotal down` / `cotal clean all` leave it alone even when `--root` pointed at the project they are\ntearing down. A `cotal up` for that space refuses outright (naming `cotal meshes rm`) unless it is\nthat same endpoint: finding a broker already answering there is a refresh that starts nothing and\nleaves the record's provenance alone, while actually starting the broker for that space, server and\nroot makes this machine the one running it, so the record becomes an ordinary local one that\n`cotal down` clears. `cotal meshes rm` drops it and re-registering with `--force` replaces it. `rm`\nonly forgets a mesh — to stop one running here, use `cotal down`.\n\n## Watching\n\n`cotal console` is the terminal view (TUI on a real terminal, plain line stream when\npiped); `cotal web` is the browser dashboard. Both are read-only observers; the\nwalkthrough is [Watch a mesh](watch-a-mesh.md).\n\n## History\n\nRetained history is operator-owned. `cotal clean history --force` purges a space's\nretained channel history; `--dms` also purges DMs (`cotal history clear` is an alias).\nIt is deliberately **not** an agent tool: agents cannot wipe the record\n([identity & auth](identity-and-auth.md)). For a **stopped** mesh, `cotal clean store\n--force` deletes the on-disk JetStream store outright, and `cotal clean all --force`\nalso resets the space identity ([CLI reference](cli.md#clean)).\n\n## Offline backup\n\nFor a coherent durable cut, preserve the whole stack first, then create the artifact while it stays\ndown:\n\n```bash\ncotal down --preserve-state\ncotal backup create ./space-backup # full by default\n# later: deliberately resume the unchanged source\ncotal up\n# or, from another preserved cut, restore before the normal listener opens\ncotal up --restore ./space-backup\n```\n\nUse `--store-dir` on both preservation and backup for a custom JetStream store. `registry` is the\nonly partial selection (`backup create ... --only registry`; `up --restore ... --restore-only\nregistry`). Backup never stops or restarts a mesh implicitly, never opens the original store, and\ndoes not contain credentials or trust secrets. Backup/restore in every auth mode — open included —\nuses isolated, operation-specific maintenance logins; normal agent credentials cannot enter that\nlistener. Full\nrestore requires the same space and exact current local trust continuity, recreates conservative\nconsumer checkpoints bound to their snapshot stream sequence state, and resumes retained agents under\ntheir original principals. The trust commitment includes the cryptographically validated full\noperator/system/data-account root chain as well as static/user authority state. A registry-only\nrestore completes canonical empty infrastructure but leaves retained agents stopped because their\nDM/DLV/TASK/ACL state is outside that selection. Authenticated restore validates the complete space\ntrust bundle before staging or changing the preserved store. Interrupted ordinary resume retries the\nsame durable attempt after its prior listener is stopped. Restore re-entry can recover a surviving normal listener\nonly when its attempt nonce, NATS server name, process owner, endpoint, and target-store identity all\nmatch the fsynced proof. A provably dead uncommitted owner is retired under lock and replaced with a\nfresh attempt-bound listener; an occupied foreign listener or ambiguous owner is never adopted. The\nmanager commit validates while retained cleanup is still suppressed; the CLI durably records its\nattempt-bound 64-hex token in `manager-committed` / `resume-committed` before `finalizeResume` can\nrelease suppression. A retry from either committed state goes straight to exact-token finalization;\nfailure preserves the committed gate and retained cleanup suppression. Missing commit evidence,\ninterrupted finalization, a live recorded endpoint despite missing pidfiles, or ambiguous proof fails closed. See the [CLI\nbackup and restore contract](cli.md#backup-and-restore) for artifact, checkpoint, fallback,\ndisaster-consent, and degraded-recovery details.\n\n## Personas from the CLI\n\n`cotal personas` manages the local catalog offline: `list` (`--running` overlays live\nmarkers), `show `, `edit ` (re-validates on save), `new `, `rm \n--force`. The runtime counterpart is the `cotal_persona` tool, which goes over the wire\nwith the manager's ownership checks. Fields: [agent files](agent-files.md).\n\n## Manager restart and a frozen issuance gate\n\nA manager that dies mid-registration leaves its issuance gate *frozen* under that registration\nop. The freeze is correct: it stops two incarnations serving at once. The successor now completes\nthat dead op on boot, using the same guard as [`cotal reconcile-gate`](cli.md#reconcile-gate): it\nacts only when the freeze-holder is affirmatively gone under a complete CONNZ sweep (`gone` and\n`sweepComplete=true`), abort-reopens the gate (generation+1, processEpoch unchanged), and continues\nthe normal takeover. A live holder, an incomplete sweep, or an unreachable delivery daemon still\nrefuses — silence is never death, and there is no TTL. Use `cotal reconcile-gate` when the boot\npath cannot run (daemon down, a non-manager endpoint, or you want to lift the freeze without\nstarting a manager).\n\n## When something looks absent\n\nPermission denials are **loud, never silent**: an over-tight ACL shows up as a logged\ndenial on the endpoint, not as a peer that mysteriously looks absent. Check\n`.cotal/manager.log`, `.cotal/delivery.log`, and `.cotal/nats.log`; `cotal status` shows\nwhat is actually running. The access rules are collected in\n[Channels & permissions](channels-and-permissions.md).\n" }, { "slug": "security", @@ -236,7 +236,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "Setup internals (maintainer notes)", "kind": "Project (non-normative maintainer notes)", "summary": "cotal setup (implementations/cli/src/commands/setup.ts) is configure-only and state-independent: it checks prerequisites, installs the Claude Code plugin, and seeds persona files, and it launches n…", - "body": "# Setup internals (maintainer notes)\n\n> **Project** (non-normative maintainer notes) · **For:** maintainers changing how setup works\n>\n> How `cotal setup` works, and the cross-repo couplings it depends on. If you change one of\n> the things in the **Invariants** table, update the listed siblings in the same change, or\n> setup silently breaks for npx users.\n\n## The flow\n\n`cotal setup`\n([`implementations/cli/src/commands/setup.ts`](../implementations/cli/src/commands/setup.ts))\nis **configure-only and state-independent**: it checks prerequisites, installs the Claude Code\nplugin, and seeds persona files, and it **launches nothing**: no mesh, no web dashboard, no\nmanager, no delivery daemon, no cmux/tmux session, no demo. Starting the stack is `cotal up`; the\ndashboard is `cotal web`. Every file it writes is announced (`→ wrote …` via `provenance.wrote`).\nIt is two-tier, gated on a machine marker.\n\n**First run** (no `~/.cotal/onboarded.json`, or `--full`, or `--yes`) runs `runFirstRun(yes)`:\n\n- splash → intro → core **checks** (Node >= 22; **locate** `nats-server`: located, never\n started) → **connector picker** → write the demo personas (david/sven/me) and seed the generic\n `default` → **offer a global install** (`offerGlobalInstall`) → onboarded marker → a finale that\n lists the commands to start things (`cotal up --detach`, `cotal web`, `cotal spawn …`,\n `cotal console`, `cotal down`). Nothing is running when it returns.\n- The old `--auth` / `--open` flags are **gone**: they set the mesh MODE at launch time, and setup\n no longer launches; mode is now `cotal up [--open]`'s concern (an unknown-option error names\n them, no silent no-op).\n\n**Later runs** run `runEnsure`: re-seed the `default` persona if it's missing (announced),\nre-offer the **global install** (`offerGlobalInstall`, same `isNpx()` + PATH-scan gate as first\nrun — so a repeat `npx cotal-ai setup` on a machine that still lacks a durable `cotal` finally\ninstalls it), then print the **status card** (`readyCard`). The card is **read-only probes** (`machineStatus`/`meshStatus`/`webUp`/`managerUp` for NATS, the plugin, the mesh, the web\ndashboard, and the manager) and for anything down it prints the exact command to start it\n(`cotal up --detach`, `cotal web`, `cotal supervise`). Displaying state never depends on it; setup\nstill launches nothing.\n\nSteps run in-process via `runSteps`\n([`lib/steps.ts`](../implementations/cli/src/lib/steps.ts)). A step can be `optional` (asked\nY/n), carry a `confirm` consent prompt, or be `live` (it draws its own pane via\n[`lib/live-window.ts`](../implementations/cli/src/lib/live-window.ts)). On failure, an\ninteractive run offers a Claude handoff\n([`lib/assist.ts`](../implementations/cli/src/lib/assist.ts)).\n\nThe **connector picker** (`pickConnectors`) multiselects Claude / OpenCode (detected\npre-checked). Only **Claude** runs an install (its wake channel binds to an *installed* plugin);\n**OpenCode auto-wires at spawn** (it injects its plugin via `buildLaunch`, never writing the\nuser's config), so the picker just marks it ready. Two experts (david, the engineer; sven, the\nguide) plus the operator's own driving session (`me`) are written by default, and `me` is the\npersona `cotal spawn me` drives.\n\n**`--yes`** forces non-interactive accept-all even on a TTY: optional plus `confirm` steps run\n(so the demo personas are written), the global install takes its default, and a failure aborts\nwith the log path and a non-zero exit. It still launches nothing. The control plane comes up with\n`cotal up --detach`. This is the agent/CI contract; keep it working.\n\n## Invariants\n\n| Thing | Must stay in sync across | Why |\n|---|---|---|\n| Marketplace name **`cotal-mesh`** | `setup.ts` (materialized `marketplace.json`), `CHANNEL_REF` in [`extensions/connector-claude-code/src/extension.ts`](../extensions/connector-claude-code/src/extension.ts), repo [`.claude-plugin/marketplace.json`](../.claude-plugin/marketplace.json) | The wake channel ref `plugin:cotal@cotal-mesh` binds by this name |\n| Plugin assets | `setup.ts` copy list (`dist/mcp.cjs`, `dist/hook.cjs`, `.claude-plugin/plugin.json`, `.mcp.json`, `hooks/hooks.json`) and the connector `package.json` `files` field | Setup materializes the plugin from `Connector.pluginRoot`; missing or renamed assets break the install |\n| `Connector.pluginRoot` | [`packages/core/src/connector.ts`](../packages/core/src/connector.ts) (contract) plus set in the claude connector's `extension.ts` | How setup finds the plugin dir without importing the extension |\n| `BUNDLED_PKG_PREFIX` | [`lib/nats-bin.ts`](../implementations/cli/src/lib/nats-bin.ts) ↔ the `@eplightning/nats-server-*` `optionalDependencies` in [`implementations/cli/package.json`](../implementations/cli/package.json) | The bundled NATS binary is resolved by `${prefix}-${platform}-${arch}`. (Future: swap the prefix to our own `@cotal-ai/nats-server-*`.) |\n| Onboard marker plus `ONBOARD_VERSION` | `~/.cotal/onboarded.json` in [`lib/onboard.ts`](../implementations/cli/src/lib/onboard.ts); version const in `setup.ts` | Flips first-run vs ensure |\n| Demo-agent format | `DEMO_AGENTS` in `setup.ts` matches the frontmatter shape read by [`packages/core/src/agent-file.ts`](../packages/core/src/agent-file.ts) (same as `examples/01-lateral-coordination/agents/`) | `cotal spawn ` loads these |\n| Managed personas | each `DEMO_AGENTS` body carries a `# managed by cotal-setup` frontmatter marker; `writeDemoAgent` refreshes the file when the body changes, backing a marker-less (user-edited) file up to `.md.bak` first | Edit `DEMO_AGENTS` plus re-run setup to update david/sven/me; delete the marker line to take ownership |\n| `DEFAULT_SERVER` | [`packages/core/src/endpoint.ts`](../packages/core/src/endpoint.ts) | The address `cotal up` starts and the status card probes |\n\n## Background processes (`cotal up`)\n\n`cotal up` brings up the whole local stack in one place; since setup became configure-only\n(stage 2b), this is where the mesh and control plane start, so `cotal spawn --detach` /\n`cotal_spawn` find a manager right after `up`. The control plane comes up in cutover order:\nold-manager preflight → **delivery daemon** (auth mode only) → **manager**, via\n`ensureControlPlane`\n([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)). The detached\nprocesses, all stopped by `cotal down`:\n\nWith no explicit `--server`, `cotal up` auto-selects a free local port when the default broker\naddress is already held by another root or an unrecorded broker; an explicit `--server` remains\nfail-loud on collision.\n\n- **Mesh:** `startMeshDetached`\n ([`commands/up.ts`](../implementations/cli/src/commands/up.ts)) is the one place that boots a\n background nats-server (foreground `up` and `up --detach` both route through it). Writes\n `.cotal/nats.pid` and tails `.cotal/nats.log`.\n- **Delivery daemon:** `startDeliveryDetached` / `ensureDelivery`\n ([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)) re-execs `cotal\n deliver` detached with a pre-minted scoped `delivery.creds` (auth mode only, the durable\n backstop; open mode has none). Writes `.cotal/delivery.pid` and `.cotal/delivery.log`.\n- **Manager:** `startManagerDetached` / `ensureManager`\n ([`lib/manager-proc.ts`](../implementations/cli/src/lib/manager-proc.ts)) re-execs `cotal\n supervise` detached (pty runtime); it answers the control plane\n (`cotal_spawn` / `cotal_despawn` / `cotal_persona`). Writes `.cotal/manager.log`;\n `managerUp()` checks the pid record for setup's status card. The **manager itself** writes\n `.cotal/manager.pid`, so a supervisor started by a container entrypoint, by cron, or by hand is\n recorded the same way a detached `cotal up` is. Readers verify the recorded pid is alive and is a\n supervisor before trusting it ([Config](config.md#project-cotal)).\n\nThe **web dashboard** is *not* part of `cotal up`. It ships inside `cotal-ai` as the `@cotal-ai/web`\nextension and is seeded automatically by the boot reconcile — the same durable, version-locked path as\nthe built-in connectors (`SEEDED_EXTENSIONS`) — so it always matches the CLI version and needs no\nseparate install. Start it with `cotal web`; it records\n`.cotal/web.pid`, self-registers that process with `down`, and is addressed as\n`http://cotal.localhost:7799` (binds loopback; `*.localhost` resolves in Chrome/Firefox/Edge,\nSafari may need plain `127.0.0.1`). `webUp()` probes the port for setup's status card.\n\nAll recorded local processes self-register `local-process` descriptors. Bare `cotal down` resolves\nthe full set and stops it in dependency order; `cotal down manager` (or another component name)\nselects only that descriptor. Installed extensions cache their contributed registry keys, so the\nbase CLI does not hardcode optional package pidfiles.\n\nAll re-execs resolve this CLI via `selfArgv()` / `selfCotal()`\n([`lib/self-exec.ts`](../implementations/cli/src/lib/self-exec.ts)) = `[node, ...loaderFlags,\nentry]` (tsx loader in dev, compiled JS in prod), so they never need `cotal` on PATH; the stack\ncomes up identically via `npx`, `npm i -g`, and a dev clone.\n\nFor ergonomics only, an npx run with no global `cotal` offers to `npm i -g cotal-ai`\n(`offerGlobalInstall`, pinned to the running version): gated on `isNpx()` plus a PATH scan\n(`cotalOnPath()`, not `onPath(\"cotal\")`, since `cotal --version` is not a real command). The\ninteractive prompt defaults to yes, the non-interactive path (`--yes` or no TTY) takes the\ndefault, and a failed install is non-fatal (warn plus manual command). The same `self-exec.ts`\nexposes `displayCmd()`, the prefix (`cotal` / `npx cotal-ai` / `pnpm cotal`) used in the\nstatus-card hints so they match how you ran it.\n\n## Built-in connectors are seeded extensions\n\nThe first-party connectors (`claude`, `opencode`, `codex`, `hermes`, `pi`) are **not** static-imported by\nthe binary. The composition root (`bin/cotal.ts`) registers no connector; they self-register only when\nimported, and they are imported only once installed. On the first real command of each boot the CLI\n**seeds** them through the same `cotal ext add` path a third party uses, so they are ordinary\nextensions you can `cotal ext remove`. Code lives in [`implementations/cli/src/seed/`](../implementations/cli/src/seed/);\nthe entry is `reconcileSeededConnectors()`, gated in `runCli` before the manifest overlay so\n`ext seed --repair` survives a corrupt manifest.\n\n**What ships where.** The connectors are `devDependencies` of `cotal-ai` (not runtime deps), and a\n`prepack` step ([`bin/scripts/copy-seeded-connectors.mjs`](../bin/scripts/copy-seeded-connectors.mjs))\n`npm pack`s each into `bin/seeded-connectors//` (honoring each connector's own `files`), added to\nthe package `files`. `SEEDED_EXTENSIONS` (`@cotal-ai/workspace`) is the shared list — the connectors plus\n`web` — and the prepack asserts every bundled payload's `name` and `version` match the umbrella (the\n`fixed` changeset group keeps them lockstep), so a version-skewed payload can never be published; `web`\nalso emits `dist/web/vendor/vendor-manifest.json` (name/version/license/sha512) as the auditable\ninventory of its vendored browser libs (marked/DOMPurify ship as opaque `dist` bytes, not runtime deps).\n`seed/paths.ts:shippedSourceDir` resolves the live `extensions/` dir in a\nsource checkout and `/seeded-connectors/` in a published install. The reconcile copies\nthat payload into the durable store `seed/store//` and `ext add --install-links` reifies\nthe `file:` dep from THAT stable path (a volatile source would fail to re-reify); `ext add` then\njunction-links each `@cotal-ai/*` peer to the binary's own copy. Before the first lazy import in each\nprocess, materialization rechecks those links by realpath and rebinds stale links under the extension\nlock. This lets the registry-facing imports of a global install, npx, and source worktrees share the\nmachine prefix while each process still gets its host's single `@cotal-ai/core` registry instance;\nlauncher artifacts are self-contained and do not resolve those mutable links later.\n\n**Reconcile policy** (generation = the `cotal-ai` version): a never-seeded built-in is seeded; a\nstill-installed one WE seeded (`source: \"seeded\"`) is refreshed only when the version bumps (semver\ncompare) or under `--force`; an operator-managed official entry (a manual `ext add` at a chosen\nversion, no seeded marker) is left untouched on upgrade; a deliberately-removed one stays removed. The\n`ever-seeded` **authority** (`seed/authority.json`, mirrored to a monotonic `.bak`) is the sole arbiter\nof removed-vs-never-seeded and is unioned with its backup on read, so a truncated authority never\nresurrects a removal. Every (re)install is verified before the generation stamp is written — recorded in\nthe manifest, present on disk with its entry file resolvable, and at the generation version — so a\nversion-skewed payload fails loud (`ext seed --repair`) rather than being stamped as current. A cotal\n**older** than the store's stamped generation refuses before writing anything, rather than stamping the\nstore back down to its own version while refreshing nothing: run the newer cotal, or `ext seed --reset`\nto rebuild the store for the version you are running.\n\n**Crash safety.** One shared advisory lock ([`packages/workspace/src/advisory-lock.ts`](../packages/workspace/src/advisory-lock.ts):\natomic hard-link publish, PID + process-start liveness, bounded wait, dead-owner reclaim) guards the\nwhole reconcile and every `cotal ext` mutation; a live reconcile is waited on, not mistaken for a crash.\nA crash **cursor** is journaled before each connector mutation and cleared only at the final commit, so\na SIGKILL mid-run is detected on the next boot (fail loud → `ext seed --repair` re-installs the\ninterrupted connector before it clears the evidence). Seed children are authenticated (they carry the\nlive lock's nonce + parent PID, not a bare env flag) and record a liveness marker so a post-crash repair\nrefuses to race an orphaned installer. `ext seed --reset` quarantines corrupt manifest/authority state\naside and rebuilds. See [cli.md `ext`](cli.md#ext) for the operator-facing flags.\n" + "body": "# Setup internals (maintainer notes)\n\n> **Project** (non-normative maintainer notes) · **For:** maintainers changing how setup works\n>\n> How `cotal setup` works, and the cross-repo couplings it depends on. If you change one of\n> the things in the **Invariants** table, update the listed siblings in the same change, or\n> setup silently breaks for npx users.\n\n## The flow\n\n`cotal setup`\n([`implementations/cli/src/commands/setup.ts`](../implementations/cli/src/commands/setup.ts))\nis **configure-only and state-independent**: it checks prerequisites, installs the Claude Code\nplugin, and seeds persona files, and it **launches nothing**: no mesh, no web dashboard, no\nmanager, no delivery daemon, no cmux/tmux session, no demo. Starting the stack is `cotal up`; the\ndashboard is `cotal web`. Every file it writes is announced (`→ wrote …` via `provenance.wrote`).\nIt is two-tier, gated on a machine marker.\n\n**First run** (no `~/.cotal/onboarded.json`, or `--full`, or `--yes`) runs `runFirstRun(yes)`:\n\n- splash → intro → core **checks** (Node >= 22; **locate** `nats-server`: located, never\n started) → **connector picker** → write the demo personas (david/sven/me) and seed the generic\n `default` → **offer a global install** (`offerGlobalInstall`) → onboarded marker → a finale that\n lists the commands to start things (`cotal up`, `cotal web`, `cotal spawn …`,\n `cotal console`, `cotal down`). Nothing is running when it returns.\n- The old `--auth` / `--open` flags are **gone**: they set the mesh MODE at launch time, and setup\n no longer launches; mode is now `cotal up [--open]`'s concern (an unknown-option error names\n them, no silent no-op).\n\n**Later runs** run `runEnsure`: re-seed the `default` persona if it's missing (announced),\nre-offer the **global install** (`offerGlobalInstall`, same `isNpx()` + PATH-scan gate as first\nrun — so a repeat `npx cotal-ai setup` on a machine that still lacks a durable `cotal` finally\ninstalls it), then print the **status card** (`readyCard`). The card is **read-only probes** (`machineStatus`/`meshStatus`/`webUp`/`managerUp` for NATS, the plugin, the mesh, the web\ndashboard, and the manager) and for anything down it prints the exact command to start it\n(`cotal up`, `cotal web`, `cotal supervise`). Displaying state never depends on it; setup\nstill launches nothing.\n\nSteps run in-process via `runSteps`\n([`lib/steps.ts`](../implementations/cli/src/lib/steps.ts)). A step can be `optional` (asked\nY/n), carry a `confirm` consent prompt, or be `live` (it draws its own pane via\n[`lib/live-window.ts`](../implementations/cli/src/lib/live-window.ts)). On failure, an\ninteractive run offers a Claude handoff\n([`lib/assist.ts`](../implementations/cli/src/lib/assist.ts)).\n\nThe **connector picker** (`pickConnectors`) multiselects Claude / OpenCode (detected\npre-checked). Only **Claude** runs an install (its wake channel binds to an *installed* plugin);\n**OpenCode auto-wires at spawn** (it injects its plugin via `buildLaunch`, never writing the\nuser's config), so the picker just marks it ready. Two experts (david, the engineer; sven, the\nguide) plus the operator's own driving session (`me`) are written by default, and `me` is the\npersona `cotal spawn me` drives.\n\n**`--yes`** forces non-interactive accept-all even on a TTY: optional plus `confirm` steps run\n(so the demo personas are written), the global install takes its default, and a failure aborts\nwith the log path and a non-zero exit. It still launches nothing. The control plane comes up with\n`cotal up`. This is the agent/CI contract; keep it working.\n\n## Invariants\n\n| Thing | Must stay in sync across | Why |\n|---|---|---|\n| Marketplace name **`cotal-mesh`** | `setup.ts` (materialized `marketplace.json`), `CHANNEL_REF` in [`extensions/connector-claude-code/src/extension.ts`](../extensions/connector-claude-code/src/extension.ts), repo [`.claude-plugin/marketplace.json`](../.claude-plugin/marketplace.json) | The wake channel ref `plugin:cotal@cotal-mesh` binds by this name |\n| Plugin assets | `setup.ts` copy list (`dist/mcp.cjs`, `dist/hook.cjs`, `.claude-plugin/plugin.json`, `.mcp.json`, `hooks/hooks.json`) and the connector `package.json` `files` field | Setup materializes the plugin from `Connector.pluginRoot`; missing or renamed assets break the install |\n| `Connector.pluginRoot` | [`packages/core/src/connector.ts`](../packages/core/src/connector.ts) (contract) plus set in the claude connector's `extension.ts` | How setup finds the plugin dir without importing the extension |\n| `BUNDLED_PKG_PREFIX` | [`lib/nats-bin.ts`](../implementations/cli/src/lib/nats-bin.ts) ↔ the `@eplightning/nats-server-*` `optionalDependencies` in [`implementations/cli/package.json`](../implementations/cli/package.json) | The bundled NATS binary is resolved by `${prefix}-${platform}-${arch}`. (Future: swap the prefix to our own `@cotal-ai/nats-server-*`.) |\n| Onboard marker plus `ONBOARD_VERSION` | `~/.cotal/onboarded.json` in [`lib/onboard.ts`](../implementations/cli/src/lib/onboard.ts); version const in `setup.ts` | Flips first-run vs ensure |\n| Demo-agent format | `DEMO_AGENTS` in `setup.ts` matches the frontmatter shape read by [`packages/core/src/agent-file.ts`](../packages/core/src/agent-file.ts) (same as `examples/01-lateral-coordination/agents/`) | `cotal spawn ` loads these |\n| Managed personas | each `DEMO_AGENTS` body carries a `# managed by cotal-setup` frontmatter marker; `writeDemoAgent` refreshes the file when the body changes, backing a marker-less (user-edited) file up to `.md.bak` first | Edit `DEMO_AGENTS` plus re-run setup to update david/sven/me; delete the marker line to take ownership |\n| `DEFAULT_SERVER` | [`packages/core/src/endpoint.ts`](../packages/core/src/endpoint.ts) | The address `cotal up` starts and the status card probes |\n\n## Background processes (`cotal up`)\n\n`cotal up` brings up the whole local stack in one place; since setup became configure-only\n(stage 2b), this is where the mesh and control plane start, so `cotal spawn --detach` /\n`cotal_spawn` find a manager right after `up`. The control plane comes up in cutover order:\nold-manager preflight → **delivery daemon** (auth mode only) → **manager**, via\n`ensureControlPlane`\n([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)). The detached\nprocesses, all stopped by `cotal down`:\n\nWith no explicit `--server`, `cotal up` auto-selects a free local port when the default broker\naddress is already held by another root or an unrecorded broker; an explicit `--server` remains\nfail-loud on collision.\n\n- **Mesh:** `startMeshDetached`\n ([`commands/up.ts`](../implementations/cli/src/commands/up.ts)) is the one place that boots a\n background nats-server (default detached `up`; `--foreground` is the debug path). Writes\n `.cotal/nats.pid` and tails `.cotal/nats.log`.\n- **Delivery daemon:** `startDeliveryDetached` / `ensureDelivery`\n ([`lib/delivery-proc.ts`](../implementations/cli/src/lib/delivery-proc.ts)) re-execs `cotal\n deliver` detached with a pre-minted scoped `delivery.creds` (auth mode only, the durable\n backstop; open mode has none). Writes `.cotal/delivery.pid` and `.cotal/delivery.log`.\n- **Manager:** `startManagerDetached` / `ensureManager`\n ([`lib/manager-proc.ts`](../implementations/cli/src/lib/manager-proc.ts)) re-execs `cotal\n supervise` detached (pty runtime); it answers the control plane\n (`cotal_spawn` / `cotal_despawn` / `cotal_persona`). Writes `.cotal/manager.log`;\n `managerUp()` checks the pid record for setup's status card. The **manager itself** writes\n `.cotal/manager.pid`, so a supervisor started by a container entrypoint, by cron, or by hand is\n recorded the same way a detached `cotal up` is. Readers verify the recorded pid is alive and is a\n supervisor before trusting it ([Config](config.md#project-cotal)).\n\nThe **web dashboard** is *not* part of `cotal up`. It ships inside `cotal-ai` as the `@cotal-ai/web`\nextension and is seeded automatically by the boot reconcile — the same durable, version-locked path as\nthe built-in connectors (`SEEDED_EXTENSIONS`) — so it always matches the CLI version and needs no\nseparate install. Start it with `cotal web`; it records\n`.cotal/web.pid`, self-registers that process with `down`, and is addressed as\n`http://cotal.localhost:7799` (binds loopback; `*.localhost` resolves in Chrome/Firefox/Edge,\nSafari may need plain `127.0.0.1`). `webUp()` probes the port for setup's status card.\n\nAll recorded local processes self-register `local-process` descriptors. Bare `cotal down` resolves\nthe full set and stops it in dependency order; `cotal down manager` (or another component name)\nselects only that descriptor. Installed extensions cache their contributed registry keys, so the\nbase CLI does not hardcode optional package pidfiles.\n\nAll re-execs resolve this CLI via `selfArgv()` / `selfCotal()`\n([`lib/self-exec.ts`](../implementations/cli/src/lib/self-exec.ts)) = `[node, ...loaderFlags,\nentry]` (tsx loader in dev, compiled JS in prod), so they never need `cotal` on PATH; the stack\ncomes up identically via `npx`, `npm i -g`, and a dev clone.\n\nFor ergonomics only, an npx run with no global `cotal` offers to `npm i -g cotal-ai`\n(`offerGlobalInstall`, pinned to the running version): gated on `isNpx()` plus a PATH scan\n(`cotalOnPath()`, not `onPath(\"cotal\")`, since `cotal --version` is not a real command). The\ninteractive prompt defaults to yes, the non-interactive path (`--yes` or no TTY) takes the\ndefault, and a failed install is non-fatal (warn plus manual command). The same `self-exec.ts`\nexposes `displayCmd()`, the prefix (`cotal` / `npx cotal-ai` / `pnpm cotal`) used in the\nstatus-card hints so they match how you ran it.\n\n## Built-in connectors are seeded extensions\n\nThe first-party connectors (`claude`, `opencode`, `codex`, `hermes`, `pi`) are **not** static-imported by\nthe binary. The composition root (`bin/cotal.ts`) registers no connector; they self-register only when\nimported, and they are imported only once installed. On the first real command of each boot the CLI\n**seeds** them through the same `cotal ext add` path a third party uses, so they are ordinary\nextensions you can `cotal ext remove`. Code lives in [`implementations/cli/src/seed/`](../implementations/cli/src/seed/);\nthe entry is `reconcileSeededConnectors()`, gated in `runCli` before the manifest overlay so\n`ext seed --repair` survives a corrupt manifest.\n\n**What ships where.** The connectors are `devDependencies` of `cotal-ai` (not runtime deps), and a\n`prepack` step ([`bin/scripts/copy-seeded-connectors.mjs`](../bin/scripts/copy-seeded-connectors.mjs))\n`npm pack`s each into `bin/seeded-connectors//` (honoring each connector's own `files`), added to\nthe package `files`. `SEEDED_EXTENSIONS` (`@cotal-ai/workspace`) is the shared list — the connectors plus\n`web` — and the prepack asserts every bundled payload's `name` and `version` match the umbrella (the\n`fixed` changeset group keeps them lockstep), so a version-skewed payload can never be published; `web`\nalso emits `dist/web/vendor/vendor-manifest.json` (name/version/license/sha512) as the auditable\ninventory of its vendored browser libs (marked/DOMPurify ship as opaque `dist` bytes, not runtime deps).\n`seed/paths.ts:shippedSourceDir` resolves the live `extensions/` dir in a\nsource checkout and `/seeded-connectors/` in a published install. The reconcile copies\nthat payload into the durable store `seed/store//` and `ext add --install-links` reifies\nthe `file:` dep from THAT stable path (a volatile source would fail to re-reify); `ext add` then\njunction-links each `@cotal-ai/*` peer to the binary's own copy. Before the first lazy import in each\nprocess, materialization rechecks those links by realpath and rebinds stale links under the extension\nlock. This lets the registry-facing imports of a global install, npx, and source worktrees share the\nmachine prefix while each process still gets its host's single `@cotal-ai/core` registry instance;\nlauncher artifacts are self-contained and do not resolve those mutable links later.\n\n**Reconcile policy** (generation = the `cotal-ai` version): a never-seeded built-in is seeded; a\nstill-installed one WE seeded (`source: \"seeded\"`) is refreshed only when the version bumps (semver\ncompare) or under `--force`; an operator-managed official entry (a manual `ext add` at a chosen\nversion, no seeded marker) is left untouched on upgrade; a deliberately-removed one stays removed. The\n`ever-seeded` **authority** (`seed/authority.json`, mirrored to a monotonic `.bak`) is the sole arbiter\nof removed-vs-never-seeded and is unioned with its backup on read, so a truncated authority never\nresurrects a removal. Every (re)install is verified before the generation stamp is written — recorded in\nthe manifest, present on disk with its entry file resolvable, and at the generation version — so a\nversion-skewed payload fails loud (`ext seed --repair`) rather than being stamped as current. A cotal\n**older** than the store's stamped generation refuses before writing anything, rather than stamping the\nstore back down to its own version while refreshing nothing: run the newer cotal, or `ext seed --reset`\nto rebuild the store for the version you are running.\n\n**Crash safety.** One shared advisory lock ([`packages/workspace/src/advisory-lock.ts`](../packages/workspace/src/advisory-lock.ts):\natomic hard-link publish, PID + process-start liveness, bounded wait, dead-owner reclaim) guards the\nwhole reconcile and every `cotal ext` mutation; a live reconcile is waited on, not mistaken for a crash.\nA crash **cursor** is journaled before each connector mutation and cleared only at the final commit, so\na SIGKILL mid-run is detected on the next boot (fail loud → `ext seed --repair` re-installs the\ninterrupted connector before it clears the evidence). Seed children are authenticated (they carry the\nlive lock's nonce + parent PID, not a bare env flag) and record a liveness marker so a post-crash repair\nrefuses to race an orphaned installer. `ext seed --reset` quarantines corrupt manifest/authority state\naside and rebuilds. See [cli.md `ext`](cli.md#ext) for the operator-facing flags.\n" }, { "slug": "spaces", diff --git a/implementations/cli/smoke/mesh-root-identity.smoke.ts b/implementations/cli/smoke/mesh-root-identity.smoke.ts index 5296bbbce..b11210cab 100644 --- a/implementations/cli/smoke/mesh-root-identity.smoke.ts +++ b/implementations/cli/smoke/mesh-root-identity.smoke.ts @@ -3,12 +3,13 @@ * * Every `cotal up` decision about an already-running mesh keys on one question: is the broker that * is answering THIS project's mesh? The registry answers it by comparing a recorded `root` against - * the live `cotalRoot()`. Two spellings of that comparison exist in the tree: + * the live `cotalRoot()`. Two spellings of that comparison have existed in the tree: * * - `meshesForRoot` (`packages/workspace/src/mesh-registry.ts`), which canonicalizes both sides * via realpath, and whose own doc states the rule: "Anything comparing a live root against the * registry must go through here: a raw `===` silently misses"; - * - a raw `held.root === root`, which several call sites in `up.ts`/`down.ts` still use. + * - the historical raw `held.root === root`, which missed aliases before the `up.ts` guards were + * aligned on the registry's canonical-root rule. * * This suite measures whether those two spellings can DISAGREE about the same directory on a real * filesystem, and pins the direction of the disagreement. It proves the PRECONDITION only — that @@ -75,7 +76,7 @@ async function main(): Promise { // ---- the divergence itself, both comparisons run against the SAME live root ---- const liveRoot = physical; // what `cotalRoot()` returns from inside the project (cwd is physical) - const rawCompare = held !== undefined && held.root === liveRoot; // the `up.ts:621` spelling + const rawCompare = held !== undefined && held.root === liveRoot; // the unsafe historical spelling const canonicalCompare = meshesForRoot(liveRoot).some((m) => m.space === SPACE); // the documented rule check("the RAW `===` compare MISSES the record for the live root", rawCompare === false, `raw=${rawCompare}`); diff --git a/implementations/cli/smoke/meshes-registry.smoke.ts b/implementations/cli/smoke/meshes-registry.smoke.ts index d76899898..a71864708 100644 --- a/implementations/cli/smoke/meshes-registry.smoke.ts +++ b/implementations/cli/smoke/meshes-registry.smoke.ts @@ -6,8 +6,8 @@ * this machine could write back — a sleeping laptop silently unregistered a healthy remote mesh. * So the load-bearing assertions here are the ones about ORIGIN: * - * • an `up` record whose broker is dead is pruned; a `manual` one is KEPT and reported `offline`, - * on that sweep and on every later one; + * • a legacy `up` record whose broker is dead is pruned; a `manual` or `self-hosted` one is KEPT + * and reported `offline`, on that sweep and on every later one; * • `add` verifies against the real broker before recording, and records nothing when that fails; * • `--force` is the explicit record-without-verifying / replace escape; * • `rm` drops records, releases the `current` pointer, and refuses a mesh running here. @@ -627,13 +627,15 @@ try { recordMesh({ space: "local-dead", server: DEAD, root: localRoot, mode: "open", origin: "up", ts: new Date(0).toISOString() }); recordMesh({ space: "legacy-dead", server: DEAD, root: localRoot, mode: "open", ts: new Date(0).toISOString() }); recordMesh({ space: "remote-dead", server: DEAD, root, mode: "open", origin: "manual", ts: new Date(0).toISOString() }); + recordMesh({ space: "self-hosted-dead", server: DEAD, root: localRoot, mode: "open", origin: "self-hosted", ts: new Date(0).toISOString() }); const sweep = await pruneStaleMeshes(); check("sweep prunes a dead mesh this machine started", findMesh("local-dead") === undefined, loadMeshes()); check("sweep prunes a dead pre-origin record (absent origin = `up`)", findMesh("legacy-dead") === undefined, loadMeshes()); check("sweep KEEPS a dead operator-registered mesh", findMesh("remote-dead")?.space === "remote-dead", loadMeshes()); - check("sweep reports the kept one as offline", sweep.offline.includes("remote-dead") && sweep.pruned.includes("local-dead"), sweep); + check("sweep KEEPS a dead self-hosted mesh", findMesh("self-hosted-dead")?.space === "self-hosted-dead", loadMeshes()); + check("sweep reports the kept ones as offline", sweep.offline.includes("remote-dead") && sweep.offline.includes("self-hosted-dead") && sweep.pruned.includes("local-dead"), sweep); const sweep2 = await pruneStaleMeshes(); - check("a second sweep still keeps it (not a one-time reprieve)", findMesh("remote-dead") !== undefined && sweep2.offline.includes("remote-dead"), sweep2); + check("a second sweep still keeps them (not a one-time reprieve)", findMesh("remote-dead") !== undefined && findMesh("self-hosted-dead") !== undefined && sweep2.offline.includes("remote-dead") && sweep2.offline.includes("self-hosted-dead"), sweep2); // …and the same rule for the paths that delete by ROOT rather than by liveness. `add` defaults // --root to the project you run it in, so a hand-registered remote mesh routinely shares a root @@ -642,9 +644,11 @@ try { const shared = projectRoot("shared"); recordMesh({ space: "here", server: LIVE, root: shared, mode: "open", origin: "up", ts: new Date(0).toISOString() }); recordMesh({ space: "elsewhere", server: LIVE, root: shared, mode: "open", origin: "manual", ts: new Date(0).toISOString() }); + recordMesh({ space: "restartable", server: DEAD, root: shared, mode: "open", origin: "self-hosted", ts: new Date(0).toISOString() }); const byRoot = removeMeshesByRoot(shared); check("a root teardown drops this project's own record", byRoot.includes("here") && findMesh("here") === undefined, byRoot); check("a root teardown KEEPS a co-rooted registered mesh", findMesh("elsewhere") !== undefined, loadMeshes()); + check("a root teardown KEEPS a co-rooted self-hosted mesh", findMesh("restartable") !== undefined, loadMeshes()); check("…and does not claim it removed it", !byRoot.includes("elsewhere"), byRoot); // `clean`'s "is this root's mesh still live" guard asks the same question: a reachable REMOTE // broker is not the operator's to stop, so it must not block a local wipe forever. @@ -674,8 +678,30 @@ try { recordMesh({ space: "reclaimable", server: DEAD, root: localRoot, mode: "open", origin: "up", ts: new Date(0).toISOString() }); await claimSpace("reclaimable", LIVE, root); check("a dead `up` holder is still reclaimed (unchanged)", findMesh("reclaimable") === undefined, loadMeshes()); + recordMesh({ space: "self-hosted-other", server: DEAD, root: localRoot, mode: "open", origin: "self-hosted", ts: new Date(0).toISOString() }); + let selfHostedClaimError: Error | undefined; + await claimSpace("self-hosted-other", LIVE, root).catch((e: Error) => void (selfHostedClaimError = e)); + check("`up` refuses to steal a self-hosted space from another root", selfHostedClaimError !== undefined, selfHostedClaimError?.message); + check("…and the self-hosted record survives the refusal", findMesh("self-hosted-other") !== undefined, loadMeshes()); + check("…naming the recorded root and `cotal meshes rm` as the way through", + selfHostedClaimError?.message.includes(`recorded as self-hosted at ${localRoot}`) === true + && selfHostedClaimError.message.includes("cotal meshes rm self-hosted-other") === true, + selfHostedClaimError?.message); + recordMesh({ space: "self-hosted-same", server: DEAD, root: localRoot, mode: "open", origin: "self-hosted", ts: new Date(0).toISOString() }); + await claimSpace("self-hosted-same", LIVE, localRoot); + check("a same-root self-hosted record at a DEAD prior endpoint is kept (auto-port restart, not reclaim)", findMesh("self-hosted-same") !== undefined, loadMeshes()); + recordMesh({ space: "self-hosted-same-live", server: LIVE, root: localRoot, mode: "open", origin: "self-hosted", ts: new Date(0).toISOString() }); + let selfHostedLiveError: Error | undefined; + await claimSpace("self-hosted-same-live", DEAD, localRoot).catch((e: Error) => void (selfHostedLiveError = e)); + check("a same-root self-hosted record at a LIVE different endpoint blocks a competing broker", + selfHostedLiveError?.message.includes(`already in use by a mesh at ${LIVE}`) === true, + selfHostedLiveError?.message); + check("…and the live self-hosted record survives the refusal", findMesh("self-hosted-same-live")?.server === LIVE, loadMeshes()); removeMesh("claimed"); removeMesh("claimed-live"); + removeMesh("self-hosted-other"); + removeMesh("self-hosted-same"); + removeMesh("self-hosted-same-live"); // PROVENANCE IS NOT DOWNGRADED BY A REFRESH. Several `up` paths re-record a mesh they did not // start (the "a broker is already on this port" branch concludes it is up from reachability @@ -692,9 +718,9 @@ try { recordMesh({ space: "started-over", server: LIVE, root, mode: "open", origin: "manual", ts: new Date(0).toISOString() }); recordOurMeshForTest({ space: "started-over", server: LIVE, root, mode: "open", ts: new Date().toISOString() }, "started"); check("a launch that STARTED the broker claims the record, even over a manual one", - findMesh("started-over")?.origin === "up", findMesh("started-over")); + findMesh("started-over")?.origin === "self-hosted", findMesh("started-over")); recordOurMeshForTest({ space: "ours-now", server: LIVE, root, mode: "open", ts: new Date().toISOString() }, "started"); - check("…and still stamps `up` on a record it created", findMesh("ours-now")?.origin === "up", findMesh("ours-now")); + check("…and stamps `self-hosted` on a record it created", findMesh("ours-now")?.origin === "self-hosted", findMesh("ours-now")); // The overlay ACCEPTANCE is the same class as origin and was not carried, so a no-op refresh // silently erased a consent the operator had given. It is asserted beside origin because the two // are the same rule: a refresh starts nothing, so it may not overwrite what only the operator @@ -719,6 +745,7 @@ try { const listed = await run([]); check("list shows the offline registered mesh", listed.out.includes("remote-dead") && listed.out.includes("offline"), listed.out); check("list tags it as registered", listed.out.includes("registered"), listed.out); + check("list tags a stopped self-hosted mesh", listed.out.includes("self-hosted-dead") && listed.out.replaceAll("self-hosted-dead", "").includes("self-hosted"), listed.out); check("`meshes list` is the same as bare `meshes`", (await run(["list"])).out === listed.out); // ── rm ──────────────────────────────────────────────────────────────────────────────────────── @@ -731,6 +758,8 @@ try { const unknown = await run(["rm", "never-existed"]); check("rm of an unknown mesh exits non-zero", unknown.code === 1 && unknown.out.includes("no mesh named"), unknown.out); + removeMesh("self-hosted-dead"); + removeMesh("restartable"); recordMesh({ space: "a1", server: DEAD, root, mode: "open", origin: "manual", ts: new Date(0).toISOString() }); recordMesh({ space: "a2", server: DEAD, root, mode: "open", origin: "manual", ts: new Date(0).toISOString() }); const multi = await run(["rm", "a1", "a2"]); diff --git a/implementations/cli/src/commands/meshes.ts b/implementations/cli/src/commands/meshes.ts index 82662baca..7a2ea5839 100644 --- a/implementations/cli/src/commands/meshes.ts +++ b/implementations/cli/src/commands/meshes.ts @@ -42,10 +42,12 @@ import { addWizard, canPrompt } from "./meshes-wizard.js"; * cotal meshes add --server register a mesh this machine did NOT start * cotal meshes rm … drop records (never stops anything) * - * `up` and `down` still write and clear their own records; `add`/`rm` exist for the meshes they - * cannot speak for — one running on another machine, a shared broker, a hosted space. Those records - * are marked `manual` and are never auto-pruned (see `pruneMesh`), because this machine has no way - * to write them back: a dead broker under one is reported `offline`, not deleted. + * `up` writes a durable self-hosted record; `down` stops the stack but leaves that record so a + * dead mesh remains restartable. `add`/`rm` exist for the meshes they cannot speak for — one + * running on another machine, a shared broker, a hosted space. Those records are marked `manual` + * and are never auto-pruned (see `pruneMesh`), because this machine has no way to write them back: + * a dead broker under one is reported `offline`, not deleted. Self-hosted records are kept the + * same way: downtime is a state, not an existence denial. */ const SUBCOMMANDS = ["list", "add", "rm", "remove"] as const; @@ -79,8 +81,8 @@ export async function meshes(args: ParsedArgs): Promise { /** The registered meshes, one per line, with a `*` on the `current` default. This is how you see * what a bare `cotal spawn` would join and which `--space` names exist. The sweep runs first, so - * a mesh this machine started and lost is gone from the list; an operator-registered one whose - * broker is down stays, tagged `offline` — it is still the mesh you meant, just not up. */ + * a legacy `up` record whose broker is dead is gone from the list; a self-hosted or operator-registered + * one whose broker is down stays, tagged `offline` — it is still the mesh you meant, just not up. */ async function listMeshes(): Promise { const sweep = await pruneStaleMeshes(); const all = loadMeshes(); @@ -100,6 +102,7 @@ async function listMeshes(): Promise { const marker = m.space === current ? c.green("*") : " "; const tags = [ ...(m.origin === "manual" ? [c.dim("registered")] : []), + ...(m.origin === "self-hosted" ? [c.dim("self-hosted")] : []), ...(offline.has(m.space) ? [c.yellow("offline")] : []), ]; console.log( diff --git a/implementations/cli/src/commands/setup.ts b/implementations/cli/src/commands/setup.ts index 13acb8cfa..165300b7a 100644 --- a/implementations/cli/src/commands/setup.ts +++ b/implementations/cli/src/commands/setup.ts @@ -134,14 +134,14 @@ async function runFirstRun(yes: boolean, demo: boolean): Promise { // With --demo it names the team; without, it points at --demo (and the optional dashboard). const driveLines = demo ? [ - `${ok("✓")} start the mesh ${dim(`${cmd} up --detach`)}`, + `${ok("✓")} start the mesh ${dim(`${cmd} up`)}`, `${ok("✓")} drive a session ${dim(`${cmd} spawn me`)}`, `${ok("✓")} ask the experts ${dim(`${cmd} spawn david · ${cmd} spawn sven`)}`, `${ok("✓")} watch the mesh ${dim(`${cmd} console`)}`, `${ok("✓")} stop everything ${dim(`${cmd} down`)}`, ] : [ - `${ok("✓")} start the mesh ${dim(`${cmd} up --detach`)}`, + `${ok("✓")} start the mesh ${dim(`${cmd} up`)}`, `${ok("✓")} talk to your agent ${dim(`${cmd} spawn`)}`, `${ok("✓")} watch the mesh ${dim(`${cmd} console`)}`, `${ok("✓")} stop everything ${dim(`${cmd} down`)}`, @@ -345,11 +345,11 @@ async function readyCard(cwd: string): Promise { [ line(m.nats !== "missing", `NATS ${dim(m.nats === "missing" ? "missing" : m.nats)}`), line(m.claudePlugin, `plugin ${dim(m.claudePlugin ? "installed" : "not installed")}`), - line(mesh.reachable, `mesh ${dim(mesh.reachable ? `${mesh.server} · space ${mesh.space}` : `down · start: ${cmd} up --detach`)}`), + line(mesh.reachable, `mesh ${dim(mesh.reachable ? `${mesh.server} · space ${mesh.space}` : `down · start: ${cmd} up`)}`), line(web, `web ${dim(web ? WEB_URL : webInstalled() ? `down · start: ${cmd} web` : `not installed · retry: ${cmd} setup`)}`), line(mgr, `manager ${dim(mgr ? "running" : `not running · start: ${cmd} up, or: ${cmd} supervise`)}`), "", - `start the mesh: ${dim(`${cmd} up --detach`)}`, + `start the mesh: ${dim(`${cmd} up`)}`, // Match the hint to what's actually on disk: the guided team (with --demo) vs the one default agent. hasDemo ? `drive it: ${dim(`${cmd} spawn me`)} ${dim("(or david / sven)")}` diff --git a/implementations/cli/src/commands/status.ts b/implementations/cli/src/commands/status.ts index 25b9829c3..25343711a 100644 --- a/implementations/cli/src/commands/status.ts +++ b/implementations/cli/src/commands/status.ts @@ -197,7 +197,7 @@ async function printRegistry(): Promise { const current = getCurrent(); section("Recorded Meshes"); if (!meshes.length) { - console.log(c.dim(" none - start one with `cotal up --detach`, or register one running elsewhere with `cotal meshes add`")); + console.log(c.dim(" none - start one with `cotal up`, or register one running elsewhere with `cotal meshes add`")); return; } const pad = Math.max(...meshes.map((m) => m.space.length)); @@ -211,7 +211,7 @@ async function printRegistry(): Promise { // A `down` record means two different things, and the repair differs: a mesh this machine // started can be re-`up`ed here, one registered by hand runs somewhere this machine doesn't // control (and, unlike the others, its record is never swept away for it). - const origin = m.origin === "manual" ? c.dim(" registered") : ""; + const origin = m.origin === "manual" ? c.dim(" registered") : m.origin === "self-hosted" ? c.dim(" self-hosted") : ""; const transport = m.tlsRequired ? " tls-required" : ""; console.log( ` ${mark} ${m.space.padEnd(pad)} ${live ? c.green("reachable") : c.red("down")} ${c.dim(`${m.mode}${transport} ${m.server} ${m.root}`)}${origin}`, diff --git a/implementations/cli/src/commands/up.ts b/implementations/cli/src/commands/up.ts index bba2dda52..ef813d252 100644 --- a/implementations/cli/src/commands/up.ts +++ b/implementations/cli/src/commands/up.ts @@ -13,7 +13,6 @@ import { closeSync, lstatSync, rmSync, - realpathSync, } from "node:fs"; import { join, resolve } from "node:path"; import { @@ -59,6 +58,7 @@ import { getCurrent, loadMeshes, MEMBERSHIP_RW_CREDS_KEY, + canonicalRoot, recordMesh, meshesForRoot, removeMesh, @@ -95,6 +95,7 @@ import { readBrokerPolicy, writeBrokerPolicy, } from "@cotal-ai/workspace"; +import { assertDetachedChildExitObservable, rethrowAfterDetachedCleanup, spawnDetached } from "../lib/detached-spawn.js"; import { ensureAuthService, resolveAuthProvider, stopAuthService } from "../lib/auth-proc.js"; import { resolveSpace } from "../lib/status.js"; import { c } from "../ui.js"; @@ -166,7 +167,8 @@ export const upFlags: FlagSpec[] = [ { name: "advertised-server", type: "string", value: "", description: "with --exchange-public-port: the broker address the public bundle advertises - what participants dial (default: --server)" }, { name: "agent-provisioning-url", type: "string", value: "", description: "with --exchange-public-port: the deployment's remote agent-provisioning endpoint the public bundle advertises" }, { name: "rotate-sys", type: "boolean", description: "renew the expired/expiring $SYS creds by rotating the system account (agents, data and creds survive; needs a stopped mesh)" }, - { name: "detach", type: "boolean", description: "run in the background (stop with `cotal down`)" }, + { name: "detach", type: "boolean", description: "deprecated no-op; `up` is already detached by default" }, + { name: "foreground", type: "boolean", description: "debug in the invoking terminal; the default stack is detached" }, { name: "runtime", type: "string", value: "", description: "agent runtime for the mesh manager (default pty; extension runtimes are explicit-only, see `cotal runtimes`); with -f overrides the manifest's runtime" }, { name: "file", type: "string", short: "f", value: "", description: "launch a whole mesh from a manifest" }, { name: "dry-run", type: "boolean", description: "with -f: print the plan, mutate nothing" }, @@ -191,12 +193,14 @@ export async function up(args: ParsedArgs): Promise { const values = args.values as { server?: string; "store-dir"?: string; space?: string; open?: boolean; "user-auth"?: boolean; idp?: string; "exchange-public-port"?: string; "exchange-public-url"?: string; "exchange-trusted-proxy"?: boolean; "advertised-server"?: string; "agent-provisioning-url"?: string; - channels?: string; detach?: boolean; host?: string; runtime?: string; file?: string; "dry-run"?: boolean; + channels?: string; detach?: boolean; foreground?: boolean; host?: string; runtime?: string; file?: string; "dry-run"?: boolean; restore?: string; "restore-only"?: string; "accept-missing-source"?: boolean; "rotate-sys"?: boolean; "tls-cert"?: string; "tls-key"?: string; __restoreAttempt?: string; __ordinaryResumeAttempt?: string; }; + if (values.detach && values.foreground) + throw new Error("--detach and --foreground contradict - `cotal up` is detached by default; use --foreground only for debugging"); if (values.restore) { if (values.file || values.channels) throw new Error("--restore cannot be combined with --file/-f or --channels"); @@ -310,7 +314,7 @@ export async function up(args: ParsedArgs): Promise { server: resumeServer, storeDir: resumeStore, runtime: resumeRuntime, - detached: Boolean(values.detach), + detached: !values.foreground, serverName, serverNonce, }, @@ -323,7 +327,7 @@ export async function up(args: ParsedArgs): Promise { server: resumeServer, storeDir: resumeStore, runtime: resumeRuntime, - detached: Boolean(values.detach), + detached: !values.foreground, inventory: resume.inventory, journalState: "resume-intent", serverName, @@ -626,7 +630,7 @@ export async function up(args: ParsedArgs): Promise { // auth, and logs are root-scoped). Different root / unrecorded broker on the implicit default port // gets a fresh free port instead of making the user hunt for one. const held = loadMeshes().find((m) => m.server === server); - if (held && held.root === root && (held.space === space || values.space === undefined)) { + if (held && canonicalRoot(held.root) === canonicalRoot(root) && (held.space === space || values.space === undefined)) { // A refresh of the SAME already-running mesh — its mode is fixed by how the live broker was // started. A flag asking for a DIFFERENT mode must fail loud (silently preserving the old // mode would hand the operator a mesh on the wrong identity plane); a bare refresh keeps the @@ -815,7 +819,7 @@ export async function up(args: ParsedArgs): Promise { ); process.exit(1); } - if (values.server === undefined && (!held || held.root !== root)) { + if (values.server === undefined && (!held || canonicalRoot(held.root) !== canonicalRoot(root))) { const next = await serverWithFreePort(server, host); console.log(c.dim(`${server} is already in use by ${who}; starting "${space}" at ${next} instead`)); server = next; @@ -829,7 +833,7 @@ export async function up(args: ParsedArgs): Promise { } } - if (values.detach) { + if (!values.foreground) { const restored = resumeAttempt ? pendingRestores.get(resumeAttempt) : undefined; const { pid, source, authService, controlPlane, delivery, manager } = await startMeshDetached({ transport, @@ -933,16 +937,12 @@ export async function up(args: ParsedArgs): Promise { if (restored) try { bindSpawnedRestoreListener(restored, child.pid ?? 0, listenerStartedAt); } catch (error) { - await stopUnboundRestoreListener(child); - removeMatchingNatsPid(child.pid ?? 0); - throw error; + await rethrowUnboundListenerFailure(child, error); } if (ordinaryAttempt) try { bindSpawnedOrdinaryResumeListener(ordinaryAttempt, child.pid ?? 0, listenerStartedAt); } catch (error) { - await stopUnboundRestoreListener(child); - removeMatchingNatsPid(child.pid ?? 0); - throw error; + await rethrowUnboundListenerFailure(child, error); } releaseStartupLock(); child.on("error", (err) => { @@ -985,7 +985,7 @@ export async function up(args: ParsedArgs): Promise { // this broker by design (it is the operator's, and only they remove it), so unrecording it on // our exit would delete a registration this process never owned. const mine = findMesh(space); - if (mine && mine.origin !== "manual" && mine.server === server && mine.root === cotalRoot()) { + if (mine && mine.origin !== "manual" && mine.origin !== "self-hosted" && mine.server === server && mine.root === cotalRoot()) { removeMesh(space); if (getCurrent() === space) clearCurrent(); } @@ -1140,6 +1140,11 @@ function removeMatchingNatsPid(pid: number): void { function waitForChildExit(child: ChildProcess, timeoutMs: number): Promise { if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(true); + // The native Windows detached launcher closes the process handle after CreateProcess, so its + // ChildProcess-shaped handle can signal by pid but cannot emit an observed exit. Never turn that + // limitation into a fake successful wait: teardown must fail loud rather than erase ownership + // state while the listener's death is unknown. + assertDetachedChildExitObservable(child); return new Promise((resolveExit) => { const onExit = () => finish(true); const timer = setTimeout(() => finish(false), timeoutMs); @@ -1152,11 +1157,49 @@ function waitForChildExit(child: ChildProcess, timeoutMs: number): Promise Promise = stopUnboundRestoreListener, + removePid: (pid: number) => void = removeMatchingNatsPid, +): Promise { + return rethrowAfterDetachedCleanup(primary, async () => { + await stop(child); + removePid(child.pid ?? 0); + }); +} + +async function rethrowNotReadyListenerFailure( + child: ChildProcess, + primary: unknown, + removePid?: () => void, +): Promise { + return rethrowAfterDetachedCleanup(primary, () => { + child.kill("SIGTERM"); + removePid?.(); + }); +} + +async function rethrowPostStartListenerFailure( + child: ChildProcess, + primary: unknown, + removePid: () => void, +): Promise { + return rethrowAfterDetachedCleanup(primary, () => { + child.kill("SIGTERM"); + removePid(); + }); +} + +export const rethrowUnboundListenerFailureForTest = rethrowUnboundListenerFailure; +export const rethrowNotReadyListenerFailureForTest = rethrowNotReadyListenerFailure; +export const rethrowPostStartListenerFailureForTest = rethrowPostStartListenerFailure; + async function stopUnboundRestoreListener(child: ChildProcess): Promise { if (child.exitCode !== null || child.signalCode !== null) return; - child.kill("SIGTERM"); + if (!child.kill("SIGTERM")) return; if (await waitForChildExit(child, 5_000)) return; - child.kill("SIGKILL"); + if (!child.kill("SIGKILL")) return; if (!await waitForChildExit(child, 5_000)) throw new Error(`unbound restore listener process ${child.pid ?? "unknown"} did not exit`); } @@ -1980,7 +2023,7 @@ export interface DetachOpts { /** * Start a background nats-server (JetStream), wait until it's reachable, pre-create the * space's streams, and leave it running detached (pid in `.cotal/nats.pid`). Used by - * `up --detach`. When `onLine` is given, boot output is tailed from the + * the default `up` path. When `onLine` is given, boot output is tailed from the * log file and forwarded — the child writes to the file (not a pipe), so it survives the * parent exiting. */ @@ -2023,7 +2066,7 @@ export async function startMeshDetached( const startOffset = existsSync(logPath) ? statSync(logPath).size : 0; const fd = openSync(logPath, "a"); const listenerStartedAt = new Date().toISOString(); - const child = spawn(bin, args, { detached: true, stdio: ["ignore", fd, fd] }); + const child = spawnDetached(bin, args, { stdio: ["ignore", fd, fd], windowsLogPath: logPath }); closeSync(fd); if (opts.boundListener) { writeFileSync(cotalPath("nats.pid"), String(child.pid)); @@ -2031,12 +2074,9 @@ export async function startMeshDetached( try { opts.boundListener.onSpawn(child.pid ?? 0, listenerStartedAt); } catch (error) { - await stopUnboundRestoreListener(child); - removeMatchingNatsPid(child.pid ?? 0); - throw error; + await rethrowUnboundListenerFailure(child, error); } } - child.unref(); let tailing = Boolean(opts.onLine); if (opts.onLine) tailLines(logPath, startOffset, opts.onLine, () => !tailing); @@ -2044,9 +2084,11 @@ export async function startMeshDetached( const ready = await waitReady(server, setup?.creds); tailing = false; if (!ready) { - child.kill("SIGTERM"); - if (opts.boundListener) rmSync(cotalPath("nats.pid"), { force: true }); - throw new Error(`nats-server did not become reachable at ${server} - see ${logPath}`); + await rethrowNotReadyListenerFailure( + child, + new Error(`nats-server did not become reachable at ${server} - see ${logPath}`), + opts.boundListener ? () => rmSync(cotalPath("nats.pid"), { force: true }) : undefined, + ); } if (!opts.boundListener) writeFileSync(cotalPath("nats.pid"), String(child.pid)); if (opts.boundListener) await opts.boundListener.verify(); @@ -2069,17 +2111,16 @@ export async function startMeshDetached( try { await postStart(server, space, setup, seedFile); } catch (e) { - try { child.kill("SIGTERM"); } catch { /* already gone */ } - try { rmSync(cotalPath("nats.pid"), { force: true }); } catch { /* best effort */ } - throw e; + await rethrowPostStartListenerFailure(child, e, () => rmSync(cotalPath("nats.pid"), { force: true })); } } // USER MODE: the auth service comes up FIRST among the daemons (see the foreground path). const svc = await startUserAuthService(space, server, setup, opts.publicExchange); // Record BEFORE the control plane: the manager's fail-closed mode detection needs the - // authoritative registry entry at boot (marker-without-registry refuses). Detached: the entry - // outlives this process — `cotal down` removes it. - // Same capture-before-record as the foreground path: this also runs for a bare `up --detach` that + // authoritative registry entry at boot (marker-without-registry refuses). The entry outlives + // both this process and `cotal down`, so a stopped stack remains a known self-hosted mesh with + // this root as its restart authority. + // Same capture-before-record as the foreground path: this also runs for a bare `up` that // RESUMES an already-recorded mesh, whose exposure decision exists only in the registry entry the // call below rewrites. const effectiveAttachHost = attachHostFor(space, opts.host); @@ -2149,7 +2190,7 @@ function ensureRootForSpace(_useAuth: boolean, space: string): void { const existingAuth = loadSoleSpaceAuth(authDir(root)); const existingSpace = existingAuth?.space ?? - loadMeshes().find((m) => m.root === root || realpathSafe(m.root) === realpathSafe(root))?.space; + loadMeshes().find((m) => canonicalRoot(m.root) === canonicalRoot(root))?.space; if (!existingSpace || existingSpace === space) return; if (root !== cwd) { mkdirSync(join(cwd, ".cotal"), { recursive: true }); @@ -2159,23 +2200,29 @@ function ensureRootForSpace(_useAuth: boolean, space: string): void { throw new Error(`this folder is the root of space "${existingSpace}" (${root}/.cotal), so it can't also run "${space}" - drop \`--space\` to run "${existingSpace}", or start "${space}" from a different folder (it becomes that mesh's own root)`); } -function realpathSafe(p: string): string { - try { - return realpathSync(p); - } catch { - return p; - } -} - /** A space name maps to one mesh in the registry (the key `--space`/`use`/`down` act on). Before * starting a broker, refuse to reuse a space already claimed by a DIFFERENT live mesh — a stale/dead - * holder is reclaimed. Re-`up`ping the same mesh (same server + root) is a refresh (port-reachable - * path). NOTE: this is a best-effort sequential guard — two `cotal up --space X` racing from + * legacy holder is reclaimed. For a self-hosted record, the same server + canonical root is a + * refresh; a same-root auto-port restart may move to a different server only after the recorded + * endpoint is dead. + * NOTE: this is a best-effort sequential guard — two `cotal up --space X` racing from * different roots within the same instant can both pass the check before either records; that * concurrent case is out of scope (a single-operator CLI action), not synchronized with a lock. */ export async function claimSpace(space: string, server: string, root: string): Promise { const existing = findMesh(space); - if (!existing || (existing.server === server && existing.root === root)) return; + if (!existing || (existing.server === server && canonicalRoot(existing.root) === canonicalRoot(root))) return; + // A durable self-hosted record identifies its recorded root as the restart authority. Same-root + // is the identity check (the last server may change on an auto-port restart); a different root + // may not steal the name, live or dead — downtime is not proof the mesh is gone, and erasing the + // record would leave the original folder with nothing to restart from. + if (existing.origin === "self-hosted") { + if (canonicalRoot(existing.root) === canonicalRoot(root)) { + if (await isReachable(existing.server)) + throw new Error(`space "${space}" is already in use by a mesh at ${existing.server} (${existing.root}) - pick a different \`--space\`, or \`cotal down\` it first`); + return; + } + throw new Error(`space "${space}" is recorded as self-hosted at ${existing.root} - run \`cotal up\` there to restart it, or \`cotal meshes rm ${space}\` to drop that record first`); + } // An OPERATOR-REGISTERED holder is decided FIRST, before liveness, because liveness changes // nothing about it and the two outcomes would otherwise print the wrong remedy. It is never // reclaimed: unreachable is not proof the mesh is gone (the record describes a broker on another @@ -2214,11 +2261,11 @@ export function attachHostFor(space: string, explicit?: string): string | undefi * * The distinction is provenance, and it has to come from the call site because it cannot be * observed here: `started` covers the paths that spawned the broker or proved a listener this - * attempt owns, and stamps `origin: "up"` — a record this machine can always write back, so the - * liveness sweep and `cotal down` may drop it. `refresh` is the "a broker is already on this port" - * branch, which concludes the mesh is up from reachability alone and starts nothing; stamping `up` - * there would silently convert a hand-registered record into one the next sweep may delete, so it - * keeps whatever origin the record already had. + * attempt owns, and stamps `origin: "self-hosted"` — this root is the restart authority, so the + * liveness sweep and `cotal down` must not drop it. `refresh` is the "a broker is already on this + * port" branch, which concludes the mesh is up from reachability alone and starts nothing; + * stamping `self-hosted` there would silently convert a hand-registered record into one this + * machine now claims, so it keeps whatever origin the record already had. */ type Provenance = "started" | "refresh"; @@ -2231,7 +2278,7 @@ function recordOurMesh(m: MeshEntry, provenance: Provenance): void { const cur = getCurrent(); const usableCurrent = cur && findMesh(cur) ? cur : undefined; // compute before recording m const prior = findMesh(m.space); - const origin = provenance === "refresh" && prior?.origin === "manual" ? "manual" : "up"; + const origin = provenance === "refresh" && prior?.origin === "manual" ? "manual" : "self-hosted"; // A REFRESH starts nothing: it concluded the mesh is up from reachability alone, and rebuilds `m` // from what THIS launch knows, which is never the operator's past decisions. `origin` was already // carried across for that reason; the overlay acceptance is the same class and was not, so a diff --git a/implementations/cli/src/lib/auth-proc.ts b/implementations/cli/src/lib/auth-proc.ts index fef4afaa1..88073993b 100644 --- a/implementations/cli/src/lib/auth-proc.ts +++ b/implementations/cli/src/lib/auth-proc.ts @@ -6,7 +6,7 @@ * command via {@link selfArgv} (argv array — never shell interpolation), and delegates readiness * to the provider's `ready()` contract. No `@cotal-ai/auth` import anywhere in this package. */ -import { spawn } from "node:child_process"; +import { spawnDetached } from "./detached-spawn.js"; import { randomBytes } from "node:crypto"; import { closeSync, existsSync, ftruncateSync, linkSync, openSync, readdirSync, readFileSync, rmSync, writeSync } from "node:fs"; import { basename, dirname } from "node:path"; @@ -160,13 +160,12 @@ function startAuthServiceDetached(space: string, server: string, command: string const [node, ...self] = selfArgv(); // Internal child re-exec (the `up` that reached here already seeded); the auth service does not // launch agents, so it skips the connector seed on boot (a direct `cotal auth-service` still seeds). - const child = spawn(node, [...self, command, "--space", space, "--server", server, ...extraArgs], { - detached: true, + const child = spawnDetached(node, [...self, command, "--space", space, "--server", server, ...extraArgs], { stdio: ["ignore", fd, fd], + windowsLogPath: LOG_PATH(space), env: { ...process.env, COTAL_SKIP_CONNECTOR_SEED: "1" }, }); closeSync(fd); - child.unref(); // Replace the launcher pid with the daemon child's pid through the exclusively-created fd, // never a re-open (truncate first: the fd position sits past the launcher pid). ftruncateSync(slot.fd, 0); diff --git a/implementations/cli/src/lib/delivery-proc.ts b/implementations/cli/src/lib/delivery-proc.ts index cdf4dbc13..6b85a02e8 100644 --- a/implementations/cli/src/lib/delivery-proc.ts +++ b/implementations/cli/src/lib/delivery-proc.ts @@ -1,4 +1,4 @@ -import { spawn } from "node:child_process"; +import { spawnDetached } from "./detached-spawn.js"; import { existsSync, openSync, closeSync, writeFileSync, readFileSync, rmSync } from "node:fs"; import { DEFAULT_SERVER, @@ -118,9 +118,8 @@ export function startDeliveryDetached(o: Opts = {}): number { ]; // Internal child re-exec (the `up` that reached here already seeded); the delivery daemon does not // launch agents, so it skips the connector seed on boot (a direct `cotal deliver` still seeds). - const child = spawn(node, args, { detached: true, stdio: ["ignore", fd, fd], env: { ...process.env, COTAL_SKIP_CONNECTOR_SEED: "1" } }); + const child = spawnDetached(node, args, { stdio: ["ignore", fd, fd], windowsLogPath: cotalPath("delivery.log"), env: { ...process.env, COTAL_SKIP_CONNECTOR_SEED: "1" } }); closeSync(fd); - child.unref(); writeFileSync(PID_PATH(), String(child.pid)); return child.pid ?? 0; } diff --git a/implementations/cli/src/lib/detached-spawn.ts b/implementations/cli/src/lib/detached-spawn.ts new file mode 100644 index 000000000..a58c38dab --- /dev/null +++ b/implementations/cli/src/lib/detached-spawn.ts @@ -0,0 +1,134 @@ +import { spawn, spawnSync, type ChildProcess, type SpawnOptions } from "node:child_process"; + +export const WINDOWS_JOB_REFUSAL = + "this Windows process is in a job that does not allow process breakaway, so it cannot host a detached stack. Run `cotal up --foreground` in this terminal, or start the mesh from a session that is not job-bound"; + +export interface WindowsJobState { + inJob: boolean; + breakawayAllowed: boolean; +} + +export interface WindowsDetachedLauncher { + jobState(): WindowsJobState; + spawn(command: string, args: readonly string[], opts: SpawnOptions): ChildProcess; +} + +export function assertWindowsDetachAllowed(state: WindowsJobState): boolean { + if (state.inJob && !state.breakawayAllowed) throw new Error(WINDOWS_JOB_REFUSAL); + return state.inJob; +} + +export interface DetachedSpawnOptions extends SpawnOptions { + windowsLogPath?: string; +} + +export type SignalProcess = (pid: number, signal?: NodeJS.Signals | number) => boolean; + +function isNoSuchProcess(error: unknown): boolean { + return typeof error === "object" && error !== null && "code" in error && error.code === "ESRCH"; +} + +/** The ChildProcess-shaped handle returned after the native Windows launcher closes its process + * handle. Kept as a seam because the contract must be testable without spawning or signalling a + * real process on a Windows host. */ +export function windowsDetachedChild(pid: number, signalProcess: SignalProcess = process.kill): ChildProcess { + return { + pid, + exitCode: null, + signalCode: null, + unref() {}, + kill(signal: NodeJS.Signals | number = "SIGTERM") { + try { return signalProcess(pid, signal); } + catch (error) { + // ChildProcess.kill reports an already-gone child as `false`; keep that caller contract even + // though the pid-only Windows implementation reaches process.kill, which throws ESRCH. + if (isNoSuchProcess(error)) return false; + throw error; + } + }, + } as unknown as ChildProcess; +} + +/** A detached child may be signalable without being observable. Waiting code must reject that + * shape instead of treating absent event methods as evidence the process exited. */ +export function assertDetachedChildExitObservable(child: ChildProcess): void { + if (typeof child.once !== "function" || typeof child.off !== "function") + throw new Error(`detached child process ${child.pid ?? "unknown"} cannot have its exit observed`); +} + +/** Run detached-process cleanup without letting its failure replace the operation that made cleanup + * necessary. The primary Error remains the thrown object; a cleanup failure is attached as `cause`. */ +export async function rethrowAfterDetachedCleanup(primary: unknown, cleanup: () => void | Promise): Promise { + try { + await cleanup(); + } catch (cleanupError) { + if (primary instanceof Error) { + Object.defineProperty(primary, "cause", { value: cleanupError, configurable: true }); + throw primary; + } + throw new Error(String(primary), { cause: cleanupError }); + } + throw primary; +} + +const windowsScript = String.raw` +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; +public static class CotalDetachedProcess { + [StructLayout(LayoutKind.Sequential, CharSet=CharSet.Unicode)] public struct STARTUPINFO { public int cb; public string lpReserved; public string lpDesktop; public string lpTitle; public int dwX; public int dwY; public int dwXSize; public int dwYSize; public int dwXCountChars; public int dwYCountChars; public int dwFillAttribute; public int dwFlags; public short wShowWindow; public short cbReserved2; public IntPtr lpReserved2; public IntPtr hStdInput; public IntPtr hStdOutput; public IntPtr hStdError; } + [StructLayout(LayoutKind.Sequential)] public struct PROCESS_INFORMATION { public IntPtr hProcess; public IntPtr hThread; public int dwProcessId; public int dwThreadId; } + [StructLayout(LayoutKind.Sequential)] public struct JOBOBJECT_BASIC_LIMIT_INFORMATION { public long PerProcessUserTimeLimit; public long PerJobUserTimeLimit; public uint LimitFlags; public UIntPtr MinimumWorkingSetSize; public UIntPtr MaximumWorkingSetSize; public uint ActiveProcessLimit; public UIntPtr Affinity; public uint PriorityClass; public uint SchedulingClass; } + [StructLayout(LayoutKind.Sequential)] public struct IO_COUNTERS { public ulong ReadOperationCount; public ulong WriteOperationCount; public ulong OtherOperationCount; public ulong ReadTransferCount; public ulong WriteTransferCount; public ulong OtherTransferCount; } + [StructLayout(LayoutKind.Sequential)] public struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION { public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation; public IO_COUNTERS IoInfo; public UIntPtr ProcessMemoryLimit; public UIntPtr JobMemoryLimit; public UIntPtr PeakProcessMemoryUsed; public UIntPtr PeakJobMemoryUsed; } + [DllImport("kernel32.dll", SetLastError=true)] static extern bool IsProcessInJob(IntPtr process, IntPtr job, out bool result); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool QueryInformationJobObject(IntPtr job, int infoClass, out JOBOBJECT_EXTENDED_LIMIT_INFORMATION info, uint length, IntPtr returnedLength); + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern bool CreateProcess(string app, string commandLine, IntPtr pa, IntPtr ta, bool inherit, uint flags, IntPtr env, string cwd, ref STARTUPINFO si, out PROCESS_INFORMATION pi); + [DllImport("kernel32.dll")] static extern IntPtr GetCurrentProcess(); + [DllImport("kernel32.dll")] static extern bool CloseHandle(IntPtr handle); + [DllImport("kernel32.dll")] static extern IntPtr GetStdHandle(int id); + [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern IntPtr CreateFile(string name, uint access, uint share, IntPtr security, uint creation, uint flags, IntPtr template); + public static int JobFlags() { bool inside; if (!IsProcessInJob(GetCurrentProcess(), IntPtr.Zero, out inside)) throw new Win32Exception(); if (!inside) return 0; JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; if (!QueryInformationJobObject(IntPtr.Zero, 9, out info, (uint)Marshal.SizeOf(typeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION)), IntPtr.Zero)) throw new Win32Exception(); return (int)(info.BasicLimitInformation.LimitFlags | 0x40000000u); } + public static int Spawn(string app, string line, string cwd, string log, string[] env, bool breakaway) { var si=new STARTUPINFO(); si.cb=Marshal.SizeOf(si); si.dwFlags=0x100; si.hStdInput=GetStdHandle(-10); IntPtr output=String.IsNullOrEmpty(log)?GetStdHandle(-11):CreateFile(log,0x40000000u,3u,IntPtr.Zero,4u,0x80u,IntPtr.Zero); if(output==new IntPtr(-1)) throw new Win32Exception(); si.hStdOutput=output; si.hStdError=output; string block=String.Join("\0",env)+"\0\0"; IntPtr environment=Marshal.StringToHGlobalUni(block); PROCESS_INFORMATION pi; uint flags=0x8u|0x200u|0x400u|(breakaway?0x01000000u:0u); try { if (!CreateProcess(app,line,IntPtr.Zero,IntPtr.Zero,true,flags,environment,cwd,ref si,out pi)) throw new Win32Exception(); } finally { Marshal.FreeHGlobal(environment); } CloseHandle(pi.hThread); CloseHandle(pi.hProcess); if(!String.IsNullOrEmpty(log)) CloseHandle(output); return pi.dwProcessId; } +}`; + +function ps(command: string, stdio?: SpawnOptions["stdio"]): string { + const result = spawnSync("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", command], { encoding: "utf8", windowsHide: true, ...(stdio ? { stdio } : {}) }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`Windows detached-process probe failed: ${result.stderr.trim()}`); + return result.stdout.trim(); +} + +function quoteWindowsArg(value: string): string { + if (value.length > 0 && !/[\s"]/u.test(value)) return value; + return `"${value.replace(/(\\*)"/gu, "$1$1\\\"").replace(/(\\+)$/u, "$1$1")}"`; +} + +const nativeWindowsLauncher: WindowsDetachedLauncher = { + jobState() { + const flags = Number(ps(`Add-Type -TypeDefinition @'\n${windowsScript}\n'@; [CotalDetachedProcess]::JobFlags()`)); + if (!Number.isInteger(flags)) throw new Error("Windows detached-process probe returned an invalid job limit value"); + return { inJob: (flags & 0x40000000) !== 0, breakawayAllowed: (flags & (0x800 | 0x1000)) !== 0 }; + }, + spawn(command, args, opts: DetachedSpawnOptions) { + if (opts.stdio !== "ignore" && !opts.windowsLogPath) throw new Error("Windows detached spawn requires a durable log path"); + const state = this.jobState(); + const breakaway = assertWindowsDetachAllowed(state); + const line = [command, ...args].map(quoteWindowsArg).join(" "); + const environment = Object.entries(opts.env ?? process.env) + .filter((entry): entry is [string, string] => entry[1] !== undefined) + .map(([key, value]) => `${key}=${value}`) + .sort((a, b) => a.localeCompare(b, "en", { sensitivity: "base" })); + const payload = Buffer.from(JSON.stringify({ command, line, cwd: opts.cwd ?? process.cwd(), breakaway, log: opts.windowsLogPath, environment }), "utf8").toString("base64"); + const source = `$x=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${payload}'))|ConvertFrom-Json; Add-Type -TypeDefinition @'\n${windowsScript}\n'@; [CotalDetachedProcess]::Spawn($x.command,$x.line,$x.cwd,$x.log,[string[]]$x.environment,$x.breakaway)`; + const pid = Number(ps(source)); + return windowsDetachedChild(pid); + }, +}; + +export function spawnDetached(command: string, args: readonly string[], opts: DetachedSpawnOptions, windows: WindowsDetachedLauncher = nativeWindowsLauncher): ChildProcess { + if (process.platform === "win32") return windows.spawn(command, args, opts); + const child = spawn(command, [...args], { ...opts, detached: true, windowsHide: true }); + child.unref(); + return child; +} diff --git a/implementations/cli/src/lib/manager-proc.ts b/implementations/cli/src/lib/manager-proc.ts index 289d65b90..0909d44fe 100644 --- a/implementations/cli/src/lib/manager-proc.ts +++ b/implementations/cli/src/lib/manager-proc.ts @@ -1,9 +1,10 @@ -import { spawn, spawnSync } from "node:child_process"; +import { spawnSync } from "node:child_process"; import { existsSync, openSync, closeSync, chmodSync, writeFileSync, readFileSync, rmSync } from "node:fs"; import { DEFAULT_SERVER } from "@cotal-ai/core"; import { selfArgv } from "./self-exec.js"; import { resolveSpace } from "./status.js"; import { cotalPath } from "./paths.js"; +import { spawnDetached } from "./detached-spawn.js"; import { commandIsCotalSupervisor, parsePid, probeLiveness, readProcessCommand, MANAGER_DELIVERY_AWARE_MARKER, MANAGER_PIDFILE, type CommandReader, type LivenessProbe, @@ -150,9 +151,8 @@ export function startManagerDetached( ]; // This is an INTERNAL child re-exec: the `up`/`spawn` that reached here already ran the first-run // connector seed, so the manager skips it on boot (a direct `cotal supervise` still seeds). - const child = spawn(node, args, { detached: true, stdio: ["ignore", fd, fd], env: { ...process.env, COTAL_SKIP_CONNECTOR_SEED: "1" } }); + const child = spawnDetached(node, args, { stdio: ["ignore", fd, fd], windowsLogPath: cotalPath("manager.log"), env: { ...process.env, COTAL_SKIP_CONNECTOR_SEED: "1" } }); closeSync(fd); - child.unref(); writeFileSync(PID_PATH(), String(child.pid)); // Mark this manager as delivery-aware (non-hosting) so the delivery preflight can tell it apart from // an old Plane-3-hosting manager. Written next to the pid, removed together in stopManager / down. diff --git a/package.json b/package.json index df5d252c4..898f0092a 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,6 @@ "smoke:jcode-lifecycle": "tsx extensions/connector-jcode/smoke/jcode-lifecycle.smoke.ts", "smoke:jcode-private-lifecycle": "tsx extensions/connector-jcode/smoke/private-lifecycle.smoke.ts", "smoke:jcode-provider-disconnect": "tsx extensions/connector-jcode/smoke/jcode-provider-disconnect.smoke.ts", - "smoke:jcode-retry-policy": "tsx extensions/connector-jcode/smoke/retry-policy.smoke.ts", "smoke:jcode-live": "tsx extensions/connector-jcode/smoke/jcode-live.smoke.ts", "smoke:view": "tsx implementations/cli/smoke/view.smoke.ts", "smoke:members": "tsx packages/core/smoke/members.smoke.ts", @@ -494,7 +493,8 @@ "smoke:web-channel-alias": "tsx implementations/web/smoke/channel-alias.smoke.ts", "smoke:web-body-cap": "tsx implementations/web/smoke/body-cap.smoke.ts", "smoke:core-history-limit": "tsx packages/core/smoke/history-limit.smoke.ts", - "smoke:web-graph-boot": "tsx implementations/web/smoke/graph-boot.smoke.ts" + "smoke:web-graph-boot": "tsx implementations/web/smoke/graph-boot.smoke.ts", + "smoke:windows-detached-spawn": "tsx bin/smoke/windows-detached-spawn.smoke.ts" }, "dependencies": { "@cotal-ai/auth": "workspace:*", diff --git a/packages/workspace/src/mesh-registry.ts b/packages/workspace/src/mesh-registry.ts index 4dae4e777..d1f90354d 100644 --- a/packages/workspace/src/mesh-registry.ts +++ b/packages/workspace/src/mesh-registry.ts @@ -59,9 +59,9 @@ export interface MeshEntry { * will then send its credentials in the clear. The flag is what turns "encrypted if the server * says so" into "encrypted or refuse". */ tlsRequired?: boolean; - /** Who put this record here — and therefore what may take it out. `up` (the default, and what any - * record written without the field is) means THIS machine started the mesh: it is safe to drop - * on a liveness verdict or a local teardown, because `cotal up` writes it straight back. + /** Who put this record here — and therefore what may take it out. `self-hosted` means this machine + * provisioned the mesh and this root is its restart authority, so downtime must not erase it. + * Legacy `up` records (including records written without the field) remain pruneable. * `manual` means an operator registered it by hand (`cotal meshes add`) — typically a mesh * running on another machine, whose record nothing here can reconstruct. * @@ -69,12 +69,12 @@ export interface MeshEntry { * demonstrably BECOMES that mesh: * - `cotal meshes rm` (drops it) and `cotal meshes add --force` (replaces it); * - a `cotal up` that actually starts the broker for that same space, server and root — it now - * runs the mesh, so it claims the record and `cotal down` clears it again. + * runs the mesh, so it claims the record as `self-hosted`. * Nothing else infers its way past: not the liveness sweep, not the classified preflight prune, * not `cotal down` / `cotal clean all` sweeping a shared root, and not a `cotal up` REFRESH that * merely found a broker already answering (it starts nothing, so it keeps the origin). A `cotal * up` for that space anywhere else refuses outright rather than reclaim the name. */ - origin?: "up" | "manual"; + origin?: "up" | "manual" | "self-hosted"; /** Present and true when the operator EXPLICITLY accepted registering an overlay address that * this build cannot encrypt (`--allow-unencrypted-overlay`). Recorded rather than inferred: the * address class is re-derivable from `server`, but CONSENT is not, and a dial that happens long @@ -244,16 +244,17 @@ export function removeMesh(space: string): void { * * Every automatic deletion goes through here rather than {@link removeMesh}, because the rule is one * rule and forgetting it at a single site is the whole failure: a `manual` record (`cotal meshes - * add`) is NEVER auto-pruned. An `up` record is safe to drop — `cotal up` writes it back — but a - * manual one usually describes a mesh on ANOTHER machine, and nothing on this machine can - * reconstruct the server URL, root and mode the operator typed. A sleeping laptop or a VPN blip - * would otherwise unregister a perfectly healthy remote mesh for good (observed exactly once, and - * once was enough). An unreachable manual record is a STATE the surfaces report ("offline"), not a - * deletion; `cotal meshes rm` is how it leaves. + * add`) and a `self-hosted` record (`cotal up`) are NEVER auto-pruned. A legacy `up` record is safe + * to drop — `cotal up` writes it back as `self-hosted` — but a manual one usually describes a mesh + * on ANOTHER machine, and a self-hosted one is the restart authority for THIS root. A sleeping + * laptop or a VPN blip would otherwise unregister a perfectly healthy remote mesh for good + * (observed exactly once, and once was enough); the same erasure of a stopped local stack is the + * existence denial this helper exists to prevent. An unreachable durable record is a STATE the + * surfaces report ("offline"), not a deletion; `cotal meshes rm` is how it leaves. */ export function pruneMesh(space: string): boolean { const m = findMesh(space); - if (!m || m.origin === "manual") return false; + if (!m || m.origin === "manual" || m.origin === "self-hosted") return false; removeMesh(space); return true; } @@ -283,17 +284,16 @@ export function meshesForRoot(root: string): MeshEntry[] { * or wiped (`cotal down` / `cotal clean all`), releasing the `current` pointer per removed entry. * Returns the removed space names. * - * OPERATOR-REGISTERED entries are skipped. The root is shared, not owned: `cotal meshes add` - * defaults `--root` to the project you run it in, so registering a mesh that runs elsewhere from - * inside your own project files both records under one root. Tearing down THIS project's mesh says - * nothing about the remote one, and deleting its record here is the unrecoverable case (`down` can - * rewrite what `up` wrote; nothing rewrites a hand-registered record). `cotal meshes rm` is how one - * of those leaves. + * OPERATOR-REGISTERED and SELF-HOSTED entries are skipped. The root is shared, not owned: + * `cotal meshes add` defaults `--root` to the project you run it in, so registering a mesh that + * runs elsewhere from inside your own project files both records under one root. Tearing down THIS + * project's processes says nothing about the remote one, and deleting a self-hosted record would + * erase the restart authority `cotal up` just wrote. `cotal meshes rm` is how one of those leaves. */ export function removeMeshesByRoot(root: string): string[] { const removed: string[] = []; for (const m of meshesForRoot(root)) { - if (m.origin === "manual") continue; + if (m.origin === "manual" || m.origin === "self-hosted") continue; removeMesh(m.space); if (getCurrent() === m.space) clearCurrent(); removed.push(m.space); @@ -301,10 +301,10 @@ export function removeMeshesByRoot(root: string): string[] { return removed; } -/** The entries this root actually RUNS — what a local teardown or wipe may act on. The complement - * of the skip in {@link removeMeshesByRoot}, exported so a caller that guards on "is this root's - * mesh still live" asks about its OWN mesh: a hand-registered record co-rooted here points at a - * broker on another machine, which the operator cannot stop and must not be blocked by. */ +/** The entries this root actually RUNS — what a local liveness guard may act on. Hand-registered + * records are excluded (they point at a broker on another machine). Self-hosted records stay: + * this root is their restart authority, so a live one must still block a wipe even though + * {@link removeMeshesByRoot} will not delete the record itself. */ export function localMeshesForRoot(root: string): MeshEntry[] { return meshesForRoot(root).filter((m) => m.origin !== "manual"); } diff --git a/packages/workspace/src/preflight.ts b/packages/workspace/src/preflight.ts index 746e21294..3e15d20e4 100644 --- a/packages/workspace/src/preflight.ts +++ b/packages/workspace/src/preflight.ts @@ -190,8 +190,9 @@ async function readNatsInfoGreeting( * registry mutation stays opt-in: callers that act on the registry (`spawn`/`use`/`meshes`, the * manager control commands) invoke it; `` completion must not. * - * Only `up`-written records are candidates at all: {@link pruneMesh} keeps an operator-registered - * (`cotal meshes add`) one whatever the probe says, and this reports it as `offline` instead. + * Only legacy `up`-written records are candidates at all: {@link pruneMesh} keeps an + * operator-registered (`cotal meshes add`) or self-hosted (`cotal up`) one whatever the probe says, + * and this reports it as `offline` instead. * * **Deletion needs CONFIRMATION, not one timeout.** Pruning is destructive: a wrongly pruned mesh * costs the operator a re-`up` (or, for a remote one, the exact registration line again) and every @@ -209,7 +210,7 @@ const PRUNE_CONFIRM_TIMEOUT_MS = 5_000; const PREFLIGHT_CONFIRM_TIMEOUT_MS = 8_000; /** What one sweep did. `offline` is the entries whose broker is gone but whose record STAYS — - * operator-registered (`cotal meshes add`) meshes, which {@link pruneMesh} never deletes. A surface + * operator-registered (`cotal meshes add`) or self-hosted (`cotal up`) meshes, which {@link pruneMesh} never deletes. A surface * that lists meshes renders that as a state instead of probing every broker a second time. */ export interface MeshSweep { /** Spaces whose dead record was dropped. */ diff --git a/packages/workspace/src/render.ts b/packages/workspace/src/render.ts index d388afe31..1bcda3777 100644 --- a/packages/workspace/src/render.ts +++ b/packages/workspace/src/render.ts @@ -65,6 +65,8 @@ function renderPreflightFailure(kind: PreflightFailure, t: MeshTarget, pruned: b // remedy here — this machine can only wait for it or stop pointing at it. if (t.origin === "manual") return `✗ no broker answered at ${t.server} - "${t.space}" is registered here but its mesh is not up; start it where it runs, or \`cotal meshes rm ${t.space}\` to unregister it`; + if (t.origin === "self-hosted") + return `✗ mesh "${t.space}" is recorded at ${t.root} but not running - run \`cotal up\` there to restart`; return `✗ no mesh running at ${t.server}${pruned ? " (stale registry entry - removed)" : ""} - run \`cotal up\``; // The registry-mismatch pair, like `unreachable`, must not prescribe `cotal up` for a mesh this // machine only registered: the repair there is the credentials under `--root`, or re-registering diff --git a/website/astro.config.mjs b/website/astro.config.mjs index 883550f41..9cd18fed1 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -41,7 +41,7 @@ export default defineConfig({ 'Route by task:', '', '- [Agent setup prompt](https://docs.cotal.ai/prompt.md): self-contained setup runbook for coding agents; fetch it and run its commands directly', - '- [Quickstart](https://docs.cotal.ai/getting-started.md): install and start a local mesh with `npx cotal-ai setup --yes && npx cotal-ai up --detach`, then `npx cotal-ai spawn --detach`; verify with `npx cotal-ai status`', + '- [Quickstart](https://docs.cotal.ai/getting-started.md): install and start a local mesh with `npx cotal-ai setup --yes && npx cotal-ai up`, then `npx cotal-ai spawn --detach`; verify with `npx cotal-ai status`', '- [MCP tool catalog](https://docs.cotal.ai/mcp-tools.md): message peers from inside a session (`cotal_send` / `cotal_dm` / `cotal_anycast`) and spawn teammates (`cotal_spawn` / `cotal_persona`)', '- [Define a team](https://docs.cotal.ai/define-a-team.md): declare a team and its channels in one `cotal.yaml`, launch with `cotal up -f`', '- [Channels & permissions](https://docs.cotal.ai/channels-and-permissions.md): grant or audit channel access (`subscribe` / `allowSubscribe` / `allowPublish`)', diff --git a/website/public/.well-known/agent-skills/cotal-setup/SKILL.md b/website/public/.well-known/agent-skills/cotal-setup/SKILL.md index ad6248b8c..415534959 100644 --- a/website/public/.well-known/agent-skills/cotal-setup/SKILL.md +++ b/website/public/.well-known/agent-skills/cotal-setup/SKILL.md @@ -14,7 +14,7 @@ them. Requires Node 20+. ```sh npx cotal-ai setup --yes # configure only: seeds one agent, installs detected connectors, launches nothing -npx cotal-ai up --detach # start the mesh + delivery daemon + manager +npx cotal-ai up # start the detached mesh + delivery daemon + manager ``` `setup --yes` accepts every default with no prompts and exits non-zero with the log path diff --git a/website/public/prompt.md b/website/public/prompt.md index 45815022e..5728423ef 100644 --- a/website/public/prompt.md +++ b/website/public/prompt.md @@ -8,7 +8,7 @@ user to run them. Requires Node 20+. ```sh npx cotal-ai setup --yes # configure only: seeds one agent, installs detected connectors, launches nothing -npx cotal-ai up --detach # start the mesh + delivery daemon + manager +npx cotal-ai up # start the detached mesh + delivery daemon + manager ``` `setup --yes` accepts every default with no prompts and exits non-zero with the log path diff --git a/website/scripts/check-dist.mjs b/website/scripts/check-dist.mjs index dae4d5a9c..2c07370a6 100644 --- a/website/scripts/check-dist.mjs +++ b/website/scripts/check-dist.mjs @@ -23,9 +23,10 @@ for (const needle of ['/prompt.md', '/getting-started.md', '/mcp-tools.md', '/sp // The agent setup runbook must ship, keep its commands, and stay in step with // the Quickstart (same commands, so neither can drift alone). const runbook = readFileSync(join(dist, 'prompt.md'), 'utf8'); -for (const cmd of ['npx cotal-ai setup --yes', 'npx cotal-ai up --detach']) { +for (const cmd of ['npx cotal-ai setup --yes', 'npx cotal-ai up']) { if (!runbook.includes(cmd)) fail(`prompt.md lost its command: ${cmd}`); } +if (runbook.includes('npx cotal-ai up --detach')) fail('prompt.md still documents up --detach as the start command'); // The Quickstart ships the interactive prompt card on the page, and its // Markdown twin (plus the llms dumps) must stay clean Markdown: the plain @@ -35,9 +36,10 @@ if (!quickstartHtml.includes('agent-prompt')) fail('quickstart lost its prompt c const quickstartTwin = readFileSync(join(dist, 'getting-started.md'), 'utf8'); if (!quickstartTwin.includes('prompt.md, then set up Cotal')) fail('quickstart twin lost the setup prompt'); -for (const cmd of ['npx cotal-ai setup --yes', 'npx cotal-ai up --detach']) { +for (const cmd of ['npx cotal-ai setup --yes', 'npx cotal-ai up']) { if (!quickstartTwin.includes(cmd)) fail(`quickstart lost the runbook command: ${cmd}`); } +if (quickstartTwin.includes('npx cotal-ai up --detach')) fail('quickstart still documents up --detach as the start command'); for (const f of ['getting-started.md', 'llms-full.txt', 'llms-small.txt']) { if (readFileSync(join(dist, f), 'utf8').includes('AgentPrompt')) fail(`AgentPrompt component leaked into ${f}`); @@ -72,9 +74,10 @@ const setupSkill = readFileSync( join(dist, '.well-known', 'agent-skills', 'cotal-setup', 'SKILL.md'), 'utf8', ); -for (const cmd of ['npx cotal-ai setup --yes', 'npx cotal-ai up --detach']) { +for (const cmd of ['npx cotal-ai setup --yes', 'npx cotal-ai up']) { if (!setupSkill.includes(cmd)) fail(`cotal-setup skill lost its command: ${cmd}`); } +if (setupSkill.includes('npx cotal-ai up --detach')) fail('cotal-setup skill still documents up --detach as the start command'); const BAD = /(?:href="|\]\()(?:\.\.\/|docs\/|spec\/|packages\/|extensions\/|implementations\/|examples\/)/; function scan(dir) {