diff --git a/.changeset/calm-ravens-reconcile.md b/.changeset/calm-ravens-reconcile.md new file mode 100644 index 000000000..2cd0d039a --- /dev/null +++ b/.changeset/calm-ravens-reconcile.md @@ -0,0 +1,7 @@ +--- +"@cotal-ai/manager": patch +"@cotal-ai/cli": patch +"@cotal-ai/connector-core": patch +--- + +Report failed static lifecycle reconciliation, retry the same durable terminal with a bounded schedule, and drain accepted reconciliation work during shutdown. diff --git a/bin/smoke/ci-suites.d/4fac15840cc0d39928185484921a58e0778f546be0344c88b4c05f91beba993a.txt b/bin/smoke/ci-suites.d/4fac15840cc0d39928185484921a58e0778f546be0344c88b4c05f91beba993a.txt new file mode 100644 index 000000000..421dd4f5a --- /dev/null +++ b/bin/smoke/ci-suites.d/4fac15840cc0d39928185484921a58e0778f546be0344c88b4c05f91beba993a.txt @@ -0,0 +1 @@ +smoke:manager-reconcile-redrive diff --git a/bin/smoke/required-arg-seam.smoke.ts b/bin/smoke/required-arg-seam.smoke.ts index 0ebe56380..ee3836dd8 100644 --- a/bin/smoke/required-arg-seam.smoke.ts +++ b/bin/smoke/required-arg-seam.smoke.ts @@ -301,7 +301,10 @@ const SEAMS: Seam[] = [ // credential of its own (one smoke-side call, tls: false). // 126/93 -> 127/94: run-host-live connects under a token-pinned run-operator ANSWERING // credential to prove the broker refuses an answer on any other pause. - { fn: "standaloneConnectOpts", key: "tls", sites: 127, untypecheckedSites: 94 }, + // 127/94 -> 132/98: #774 adds one typechecked provisioner re-read before each static reconcile + // retry, plus four smoke-side connections for the isolated broker acceptance fixture (orphan write, + // observer, caller, and exact terminal gate inspection). Every site states tls: false. + { fn: "standaloneConnectOpts", key: "tls", sites: 132, untypecheckedSites: 98 }, ]; /** diff --git a/docs/cli.md b/docs/cli.md index a2d4d575e..c408fb9b8 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -530,8 +530,9 @@ sibling as proof that the component serves. It prints one of `serving`, `absent` state wins): - **manager**: local PID record, its liveness-lease holder and PID, then the manager's own typed - `status` service reachability from this host. Builds without a startup-phase report say - `phase not reported by this manager build`; that is never a blank green state. + `status` service reachability from this host. Manager builds that do not report static + reconciliation say `static reconciliation not reported by this manager build`; the line stays + visible even when the manager is otherwise `serving`. - **delivery**: local PID record, its ready lease (`ready` is the daemon's own bound-control signal), and the latest `renewal.json` adoption verdict. A re-signed credential and a broker-accepted adoption stay distinct facts. diff --git a/docs/run-a-mesh.md b/docs/run-a-mesh.md index 5e28f2878..91debe807 100644 --- a/docs/run-a-mesh.md +++ b/docs/run-a-mesh.md @@ -65,6 +65,25 @@ continues, but prints a named `connector unavailable` line and records th manager's `status` response. Available connector rows record the absolute paths boot resolved. Spawn keeps the same pre-mint check as a backstop for connectors registered after boot. +On an authenticated manager start, unfinished static lifecycle rows reconcile while the control +endpoint is already serving. The manager `status` response reports +the `staticReconciliation` state, the last sweep counts, and each failed alias with its durable +phase and literal disposition. `cotal status --components` reports the state and per-alias failure +details. A failed exact terminal is retried in the same process after 1, 5, +and 30 seconds. Each attempt re-reads the durable slot and re-enters the same deterministic terminal +operation; the delays only schedule work and never release the lifecycle fence. + +On shutdown, the manager fences new reconciliation work and waits for an exact terminal that already +started. The current serial sweep stops before its next alias, and startup cannot publish the manager +service after `stop()` completes. + +The four-attempt budget is per manager process. An exhausted row stays held and reports +`retry-exhausted` with the remedy to restart the manager. The next process derives a fresh budget +from the still-authoritative durable row. A `recovered` row remains visible until the next static +reconciliation sweep, then clears. This component reports reconciliation outcomes. It does not say +whether footprint cleanup completed independently of the terminal result; that separate durable +projection remains tracked by #1274. + There is no supported `cotal service install` command yet. Running the manager as a launchd agent or systemd user service remains operator-managed; service installation is separate from this boot-time detection behavior. diff --git a/extensions/connector-core/src/docs-bundle.generated.ts b/extensions/connector-core/src/docs-bundle.generated.ts index 95e56eaa1..52dcf986b 100644 --- a/extensions/connector-core/src/docs-bundle.generated.ts +++ b/extensions/connector-core/src/docs-bundle.generated.ts @@ -75,7 +75,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "`cotal` CLI reference", "kind": "Reference: describes the TypeScript reference implementation (the `cotal` CLI), not the wire contract.", "summary": "cotal is the operator command line for the reference implementation: bring a mesh up, mint identities, launch agents, watch what they do, and tear it all down.", - "body": "# `cotal` CLI reference\n\n> **Reference**: describes the TypeScript reference implementation (the `cotal` CLI), not the wire contract. · **For:** operators · **Wire contract:** [SPEC](../SPEC.md)\n\n`cotal` is the operator command line for the reference implementation: bring a mesh up, mint\nidentities, launch agents, watch what they do, and tear it all down. It is a thin client over the\nwire contract: the normative subjects and schemas live in the [SPEC](../SPEC.md); this page is\nlookup material for the commands, not a walkthrough; if you are new, start with\n[Getting started](getting-started.md).\n\n## Running it\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH (needs Node 22+)\ncotal --help # every command, grouped\ncotal --version # cotal-ai version + each installed extension's (also `cotal -v`)\ncotal --help # one command's flags and usage\n```\n\n`npx cotal-ai ` runs it without a global install; in a dev clone, `pnpm cotal `\nruns it through `tsx` with no build step. Bare `cotal` prints help. Every command generates its own\n`--help`, usage, and shell completion from its declared flags.\n\nCommands come from the surfaces the binary composes: the base mesh CLI, the manager\n(`supervise`), and the delivery daemon (`deliver`), plus any operator-installed extensions.\n`cotal ext add ` installs any registry providers a package contributes: commands,\nruntimes, and local process lifecycle descriptors. The `web` dashboard and optional manager\nruntimes ship this way.\n\n## Commands\n\n| Area | Command | Purpose |\n|---|---|---|\n| Set up & lifecycle | [`setup`](#setup) | Guided, configure-only setup (installs, seeds personas; launches nothing) |\n| Set up & lifecycle | [`update`](#update) | Reconcile first-party extensions and check or opt into a coherent CLI upgrade |\n| Set up & lifecycle | [`up`](#up) | Start a local mesh (nats-server + JetStream), or boot a whole manifest with `-f` |\n| Set up & lifecycle | [`down`](#down) | Stop the whole stack, selected registered components, or a manifest deploy |\n| Set up & lifecycle | [`backup`](#backups) | Create an offline full-space or registry-only artifact from a preserved cut |\n| Set up & lifecycle | [`clean`](#clean) | Configurable cleanup: purge history (live), or wipe the local store / identity (stopped) |\n| Set up & lifecycle | [`meshes`](#mesh-registry) | List the running meshes on this machine |\n| Set up & lifecycle | [`use`](#mesh-registry) | Set the default mesh a bare `cotal spawn` joins |\n| Set up & lifecycle | [`status`](#mesh-registry) | Read-only diagnostics for setup, processes, and the selected mesh |\n| Agents & personas | [`spawn`](#spawn) | Launch an agent from a persona (foreground, or `--detach` via the manager) |\n| Agents & personas | [`models`](#models) | List connector model catalogs and variants from the manager |\n| Agents & personas | [`ps`](#managed-seats) | List managed agents and their mesh status |\n| Agents & personas | [`stop`](#managed-seats) | Ask the manager to stop a managed agent |\n| Agents & personas | [`attach`](#managed-seats) | Stream and drive a managed agent's terminal (pty runtime) |\n| Agents & personas | [`input`](#input) | Type one line into a managed agent's terminal without attaching |\n| Agents & personas | [`personas`](#personas) | List, show, edit, create, or remove local personas |\n| Agents & personas | [`supervise`](#supervise) | Run a manager daemon (the agent supervisor / control plane) |\n| Agents & personas | [`runtimes`](#runtimes) | List the agent runtimes the manager can spawn through and whether each is reachable |\n| Agents & personas | [`reconcile-gate`](#reconcile-gate) | Unfreeze an issuance gate left frozen by a crashed restart when the successor cannot boot-heal it (holder gone, complete CONNZ sweep) |\n| Messaging & watching | [`endpoints`](#endpoints) | List every endpoint in the live presence roster, including infrastructure |\n| Messaging & watching | [`describe` / `invoke`](#endpoint-control) | Resolve a v0.4 service's command surface off the wire; invoke one command by name |\n| Messaging & watching | [`send`](#send) | Send one message, then exit: DM a peer, post a channel, or ask a role |\n| Messaging & watching | [`channels`](#channels) | Inspect or set the channel registry |\n| Messaging & watching | [`history`](#history) | Clear retained message history |\n| Messaging & watching | [`console`](#console) | Live protocol view for a space (TUI, or `--plain` line stream) |\n| Messaging & watching | [`web`](#web) | Browser dashboard (installed as the `@cotal-ai/web` extension) |\n| Auth & meshes | [`mint`](#mint) | Mint a creds file for a space (static auth mode) |\n| Auth & meshes | [`login`](#login) | Sign in to a per-user-auth mesh's IdP (once per machine) |\n| Auth & meshes | [`logout`](#login) | Revoke the IdP session and clear the cached login |\n| Auth & meshes | [`actor`](#actor) | Manage a user-auth space's actor ledger (grant / revoke / list) |\n| Auth & meshes | [`doctor`](#doctor) | Credential-health diagnosis and repair (`doctor auth`) |\n| Auth & meshes | [`join`](#join) | Join a space as your own presence (interactive) |\n| Manifest | [`topology`](#manifest-deploys) | Validate and view a mesh manifest's access graph (read-only) |\n| Extensions & misc | [`ext`](#ext) | Install / remove operator CLI extensions |\n| Extensions & misc | [`completion`](#completion) | Print or install shell completion |\n| Extensions & misc | [`feedback`](#feedback) | Send feedback to the Cotal developers |\n| Extensions & misc | [`deliver`](#server-daemons) | Run the server-side Plane-3 delivery daemon |\n| Workflow runs | [`run`](#run) | Operate durable workflow runs: start, resume, list, inspect, answer a checkpoint |\n| Extensions & misc | [`feedback-intake`](#server-daemons) | Run a self-hosted feedback intake server |\n\nThe manifest modes of `up`, `spawn`, and `down` (`-f `) plus `topology` are covered\ntogether under [Manifest deploys](#manifest-deploys).\n\n## setup\n\n```bash\ncotal setup [--full] [--demo] [--yes] [--skills]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--full` | off | Redo the full guided flow (implies `--demo`) |\n| `--demo` | off | Also seed the guided expert team (`david`, `sven`, `me`) |\n| `--yes`, `-y` | off | Non-interactive accept-all (for agents / CI) |\n| `--skills` | off | Reconcile Cotal skills only through installed connector providers, plus `~/.agents/skills`. Refused with `--full` or `--demo`. |\n\nGuided setup is **configure-only**: it checks prerequisites, invokes installed connectors' declared setup providers, and\nseeds persona files, and it launches nothing (no mesh, no web, no manager). First run gets the\nnarrated flow; later runs print a status card. By default it seeds one `default` persona; the\n`david`/`sven`/`me` team is opt-in via `--demo`. `cotal status` points stale Claude skills and\nout-of-date `.agents` skills at `cotal setup --skills`, not unscoped `setup`. See [Getting started](getting-started.md) and, for\nmaintainers, [setup internals](setup-internals.md).\n\nWhen a mesh resolves, setup seeds that mesh's recorded `.cotal/agents` catalog, the same catalog a\nfollowing `cotal spawn` reads. It prints the absolute destination. On a fresh machine with no mesh it\nuses this folder and says why; when several meshes are available and none is selected, it refuses\nrather than choosing a catalog.\n\n## update\n\n```bash\ncotal update [--self] [--space ] [--server ] [--creds ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--self` | off | If a newer release exists, install that exact validated `cotal-ai` version globally and reconcile through the newly installed binary |\n| `--space`, `--server`, `--creds` | resolved mesh | Select the running manager whose continuity state is reported |\n\nWithout `--self`, `update` keeps the installed first-party surfaces coherent with the running\nbinary: it force-reconciles the four built-in connectors, then reinstalls other `@cotal-ai/*`\noperator extensions at the binary's exact version. Each extension runs in an isolated child, so one\nfailure cannot poison later replays. It then checks npm; a newer binary is an informational notice\nwith `cotal update --self` as the next command, not an automatic install.\n\nAfter disk reconciliation, `update` reads the selected running manager. A manager without a\ncustody generation is reported as `legacy`: it cannot preserve its manager-owned PTYs, so the\ncommand says that this is not a hot update and prints `exact`, `fork`, `fresh`, or `drain-only`\nfor every seat. This report sends no stop, preservation-commit, or replacement command.\nIt does not preserve a running PTY. Custody transfer is not available until the custody runtime is\nimplemented. Even after compatible custody generations exist, an incompatible native\n`@lydell/node-pty` or ConPTY ABI break remains an explicit per-seat maintenance cut.\n\nWith `--self`, the selected running manager is reported before any global install. When a newer\nrelease exists, Cotal then installs the exact version it validated, resolves and verifies that\npackage in npm's global root, then launches that binary with the same `--space` / `--server` /\n`--creds` selection to reconcile connectors and first-party extensions to the new generation. An npx\nor dev-clone invocation therefore installs and continues through a separate global copy; it never\nclaims the already-running process changed. If the binary is current, `--self` performs the normal\nlocal reconcile without reinstalling it.\n\nThird-party extensions are listed with their installed version and recorded spec but are not\nauto-updated in v1. Floating third-party updates require `@cotal-ai/*` peer-range validation and are\na future follow-up. A failed connector/extension install, npm metadata check, or requested global\ninstall is reported and makes the command exit nonzero. Independent extension attempts continue so\nthe output includes every failure; an unavailable npm registry does not undo a completed local\nreconcile, but the command still exits nonzero because it could not establish that the install is\ncurrent.\n\n## up\n\n```bash\ncotal up [--detach] [--open] [--space ] [--server ] [--channels ] [--runtime ]\ncotal up --user-auth --idp [--exchange-public-port --exchange-public-url [--exchange-trusted-proxy]]\ncotal up --tls-cert --tls-key # serve broker TLS (both, or neither)\ncotal up --restore [--restore-only registry] [--accept-missing-source]\ncotal up -f [--dry-run] [--runtime ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--server ` | auto (free local port) | Listen URL override |\n| `--host ` | none | Bind host override. With no `--server`, the broker URL is derived from it, so `--host ` alone is enough to make a mesh reachable at that address; a `--host`/`--server` pair naming different addresses is refused. A wildcard bind (`0.0.0.0`, `::`) keeps a dialable loopback URL. Recorded on the mesh and reused by every later manager launch, so a repair or resume keeps remote [`attach`](#managed-seats) working |\n| `--space ` | the folder's name | Space name |\n| `--store-dir ` | none | JetStream store directory |\n| `--channels ` | `.cotal/channels.json` if present | Channel-registry seed file (JSON). An explicit path that is missing is an error |\n| `--restore ` | none | Restore a completed offline backup before exposing the normal listener |\n| `--restore-only registry` | artifact selection | Restore only the registry component |\n| `--accept-missing-source` | off | Explicit disaster consent when the inode-bound preserved source is absent |\n| `--open` | off (auth) | Unauthenticated dev mesh: no JWT, no ACLs |\n| `--user-auth` | off | Per-user auth: people `cotal login`; connects are authorized against the actor ledger |\n| `--idp ` | none | With `--user-auth`: the IdP auth base URL to pin on first enable |\n| `--exchange-public-port ` | none | With `--user-auth`: add the public exchange face on this loopback port, for an HTTPS reverse proxy to forward to |\n| `--exchange-public-url ` | none | With `--exchange-public-port`: advertise the reverse proxy's HTTPS URL in discovery |\n| `--exchange-trusted-proxy` | off | With `--exchange-public-port`: attribute public failure buckets to the last `X-Forwarded-For` hop. Enable only when the listener is reachable solely through a trusted proxy; otherwise the socket address is used |\n| `--detach` | off | Run in the background (stop with `cotal down`) |\n| `--tls-cert ` | none | PEM certificate to serve TLS with. Must be given together with `--tls-key`. Before starting the broker, Cotal checks readability, private-key mode, key/certificate match, the validity window, and host coverage. `nats-server` accepts an expired certificate and leaves the failure to clients, so Cotal performs these checks first. The decision is recorded; a later bare `cotal up` keeps serving TLS |\n| `--tls-key ` | none | PEM private key for `--tls-cert`. Refused if group- or other-readable (tighten to `600`) |\n| `--file `, `-f` | none | Launch a whole mesh from a manifest |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--runtime ` | `pty` (or the manifest's, with `-f`) | Agent runtime for the mesh manager (`pty` built in; others are installed extensions, explicit-only). Resolved + probed before the broker starts; an uninstalled/unreachable runtime fails loud. With `-f`, overrides the manifest's runtime |\n| `--rotate-sys` | off | Rotate the space's system account and re-mint its two `$SYS` creds. Needs a stopped mesh; refused with `--open` |\n\n`cotal up` boots a local nats-server with JetStream and, in auth mode (the default), JWT auth and\nper-agent ACLs; `--detach` records the mesh so `cotal spawn` from any directory can find it. With no\n`--server`, it auto-selects a free port if the default address is taken; an explicit `--server`\nstays fail-loud on collision. `--detach` also brings up the control plane (delivery daemon in auth\nmode, then the manager). The `-f` form is a [manifest deploy](#manifest-deploys); see\n[Run a mesh](run-a-mesh.md).\n\n`--user-auth --idp ` starts the space's auth service alongside the broker: the NATS\nauth callout plus its capability-gated local exchange, and optionally the closed public exchange\nface configured by the three `--exchange-*` flags above. The service is torn down with `cotal down`,\nand a re-run of `cotal up` heals a dead service on a running broker. `--user-auth` and `--open`\ncontradict each other and are refused loudly; a running broker cannot change auth mode\nwithout a `cotal down` first. See [identity & auth](identity-and-auth.md).\n\n`--rotate-sys` renews the two `$SYS` credentials (`membership-observer`, `connection-evictor`).\nThey carry a 30-day expiry and nothing re-signs them in place, because the system-account seed is\nnever persisted, so they are renewed by issuing a **new system account** under the same broker\noperator and minting fresh creds against it. A plain re-`up` does **not** do this: it reuses the\nexisting trust record, and its `$SYS` creds along with it.\n\nThe rotation is safe to run on a real space, with one operational cost. The data account, the account\nsigning key, every agent credential minted from it, and the JetStream store are all untouched; what\ndies is the retired system account, and with it any out-of-band copy of the old `$SYS` creds, on every\nbroker that loads the rotated config. The cost is that **earlier full backups stop being restorable**\n(see below), so this is not a no-consequence operation. It needs the broker to restart on the rewritten\nconfig, so it runs as part of a boot:\n\n```bash\ncotal down\ncotal up --rotate-sys --detach # agents reconnect; nothing is re-provisioned\ncotal doctor auth # both $SYS creds healthy again, 30 days out\n```\n\nA rotation is a stopped, fresh boot, and anything that is not one refuses it, all for the same reason\n(the on-disk material and the broker it runs on must never end up on different generations):\n\n- a live mesh, because the running broker would keep serving the retired account;\n- an open mesh, whether that comes from `--open` or from `broker.auth: false` in a manifest, which\n has no system account at all;\n- `--restore`, because reinstating a trust root and superseding it in one command leaves no way to\n say which authority the mesh came up on;\n- an unfinished restore or resume attempt on this root, including one `cotal up` would recover on\n its own, because those paths can adopt a live listener and return without booting a broker;\n- a root that hosts more than one space, because the system account lives in the shared broker\n record and a rotation would retire every tenant's, while the root holds one `$SYS` cred pair\n pinned to one data account.\n\nTwo things to know before you run it:\n\n- **The retirement is config-load-bound.** Old `$SYS` creds are refused by any broker that loads the\n rotated config. A stale `nats-server` still running the *previous* config in memory would keep\n honouring them, so stop every broker for this root first. `--rotate-sys` refuses if this root's\n mesh is recorded as running, if anything unidentified is answering at the address it was given, or\n if the root's pid file names a live (or unreadable) process. Those are Cotal's own ownership\n records, not a scan of the process table: a `nats-server` you started by hand against this root's\n `server.conf` on some other port writes none of them and will not be seen. Do not run one.\n- **It invalidates earlier full backups.** A full artifact binds to the trust chain it was taken\n against, and that commitment covers the operator JWT and the system account. Every full backup\n taken before a rotation refuses to restore afterwards, so take a fresh `cotal backup` once the\n rotated mesh is up. `cotal up --restore` names this case when the data account still matches.\n\nThe commit is not atomic (a trust-record write plus two credential writes), so an interrupted\nrotation leaves the record ahead of the creds. That split is detected rather than silent: every\n`cotal up` on an auth mesh, and every `cotal doctor auth`, compares each `$SYS` cred's issuer against\nthe persisted record and names the retired account. `up` warns rather than refusing, because these\ncreds power the membership graph and live eviction, both of which degrade fail-soft; the mesh is not\nworth taking down over them. Re-running the rotation heals it, at the cost of one generation.\n\nWhile those creds are expired the mesh keeps delivering messages, but the\n[membership feed](delivery-daemon.md) and live connection eviction stay down; `cotal doctor auth`\nand the manager's log both name the credential and this repair.\n\n## down\n\n```bash\ncotal down\ncotal down --preserve-state [--store-dir ]\ncotal down manager [delivery auth web nats ...]\ncotal down web [--space ]\ncotal down -f | --run [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--file `, `-f` | none | Tear down this manifest's deploy |\n| `--run ` | none | Tear down one `spawn -f` run by id |\n| `--space ` | current mesh | With components: the mesh whose target-addressed components (e.g. `web`) to stop |\n| `--dry-run` | off | Print the manifest teardown or selected components, mutate nothing |\n| `--preserve-state` | off | Bare whole stack only: fence the manager, retain principals and durable state, stop and prove the stack down, then publish `ready` |\n| `--store-dir ` | `.cotal/nats` | With `--preserve-state`: the actual store path (required for a custom store) |\n\nBare `cotal down` stops the whole local stack in dependency order. Positional component names stop\nonly those self-registered local processes; for example, `cotal down manager` leaves delivery and\nthe broker running, and `cotal down web` is available when the web extension is installed. A\ncomponent that starts target-resolved (the web dashboard) is stopped the same way: `cotal down web`\nresolves the mesh the same way as `cotal web` (registry current mesh first, `--space` to name one), so\nit works from any directory; the other components always stop under the folder you run it in. The\n`-f` / `--run` forms tear down a [manifest deploy](#manifest-deploys) without stopping the whole mesh\nand cannot be combined with component names. Stopping `nats` alone is refused while an unselected\nregistered daemon is still live; include those components or use bare `cotal down`.\n\n**Teardown verifies pinned process identity before signalling.** PIDs are recycled by every OS,\nso a recorded pid alone is not a durable target identity. `up` records each stack process's\ncreation identity in a sibling `.identity` pin, which holds the pid and the process start\nreported by the OS. Every stop path, including `down` for the broker, web and extension components,\nand the manager, delivery and auth-service stops, applies the same rule. A pin that names a different\nstart means the pid was reused, so teardown refuses and preserves it. A torn or unreadable pin also\nrefuses. Once the recorded process is stopped, rerunning teardown clears the stale record\nautomatically.\n\nThe first teardown after upgrading a running pre-pin stack has a narrower guarantee. A live record\nwith no identity pin is signalled after a loud warning that it predates identity pinning. Restarting\nthe component writes the pin, so later teardowns receive full match and mismatch protection. The\nsame warning applies on platforms where no stable start token is available.\n\nNormal `down` remains destructive at the logical identity/durable layer. `--preserve-state` is a\ndifferent maintenance transition: it suppresses leave/deprovision cleanup, persists the manager's\nsame-principal resume inventory, stops the entire stack without removing run/auth artifacts, and\npublishes a stable inode-bound cut only after every recorded process is proven stopped and the exact\nrecorded NATS endpoint is unreachable. A missing or stale broker pidfile never counts as stopped. The\nattempt is bound durably before the manager is fenced, the resume document and attempt-bound\n`cut-intent` are fsynced before manager commit, and the manager's commitment itself is journaled\n(`cut-committed`) before any process stops. A retry after a crash at any of those boundaries reuses\nthe exact recorded attempt and finishes the remaining stop and endpoint proofs idempotently, without\nneeding the (by then intentionally dead) manager. A partial cut never publishes `ready`. It cannot\nbe combined with component names, manifest teardown, or `--dry-run`.\n\n## clean\n\n```bash\ncotal clean --force\ncotal clean restore-attempt --attempt --force\ncotal clean restore-fallback --attempt --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | `history`: target mesh |\n| `--dms` | off | `history`: also clear DM history |\n| `--store-dir ` | `.cotal/nats` | `store`/`all`: JetStream store directory |\n| `--force` | none | Required: destructive, no prompting |\n| `--attempt ` | none | `restore-attempt`: exact stale pre-commit attempt; `restore-fallback`: matching healthy committed restore |\n\nOne configurable cleanup verb; every target requires `--force`.\n\n- `history` purges the retained message backlog on the **running** broker (channels, plus DMs\n with `--dms`). The same operation as [`history clear`](#history), which stays as an alias.\n- `store` deletes the **stopped** mesh's JetStream store (`.cotal/nats`): streams, durable\n consumers, and messages. This is the reset for stale on-disk broker state, e.g. durables\n minted by an older, incompatible Cotal generation surviving a `down`/`up` cycle.\n- `all` is `store` plus the space identity (`.cotal/auth`), the local creds and markers tied to\n it, any crash residue a normal `down` would have swept (stale pidfiles, `run/`), and the mesh's\n registry entry; the next `cotal up` mints a fresh identity.\n\n`history` needs the mesh up; `store` and `all` refuse while any recorded mesh process is still\nalive or any same-root recorded broker endpoint remains reachable (run `cotal down` first). They\nalso refuse outright on a root that holds accounts for several spaces: the store and the broker\ntrust record are shared by every space on the broker, so both targets would take out all of them\nand no `--space` can narrow that. `down`, `backup` and `up --restore` refuse there for the same\nreason. `cotal status` lists the tenants on such a root. Personas\n(`.cotal/agents`) and logs are never touched. A custom\nstore location is not recorded anywhere, so `--store-dir` must repeat whatever the mesh was\nlaunched with. Custom cleanup targets must contain either the Cotal store-generation marker or a\nreal `jetstream/` store directory; filesystem roots, project roots, and Cotal auth/maintenance trees\nare always refused.\n\n`store` and `all` also refuse every maintenance journal state. After a healthy committed restore,\n`restore-fallback` is the only supported way to remove the recorded unchanged old-store inode; it\nnever deletes the active target, requires both the exact attempt id and `--force`, and retires the\ncompleted restore journal so a later `down --preserve-state` can start a new backup cycle.\n\n## Backups\n\n```bash\ncotal down --preserve-state [--store-dir ]\ncotal backup create [--only full|registry] [--store-dir ]\ncotal up --restore [--restore-only registry] [--accept-missing-source]\n```\n\nBackup is offline-only. It requires the stable `ready` record from `down --preserve-state`, an exact\nstore match, no live recorded process, and an unreachable exact endpoint from the recorded cut.\nThat endpoint is probed immediately before cloning, so a live broker with a missing or stale pidfile\nis still refused. It claims the cut, reflink/copies the stopped source to a\nprivate attempt clone, and opens only that clone on a random loopback bootstrap broker with an\nindependent parent/deadline watchdog. It validates the canonical stream and pull-consumer inventory,\nwrites native snapshots with consumers excluded, and stores conservative contiguous ACK-floor\ncheckpoints separately. The original store is never opened by the backup broker, and the stack is\nnot restarted implicitly. Artifact destinations must not overlap the preserved source or maintenance\nattempt tree. Restore artifacts and targets likewise cannot nest inside or contain each other, the\npreserved source, or the maintenance attempt tree.\n\nStopped client-managed KV ordered consumers are ephemeral read residue, not backup state. Backup\nignores only the pinned client's exact stopped shapes: ordinary last-value watchers and the\nwhole-bucket scanner that uses all-history delivery to collapse concurrent tombstones. A bound\nconsumer or any lookalike with a different filter, inbox, lifetime, or other config is still refused.\n\n`full` is the default and indivisible: channel registry, CHAT/DM/TASK/INBOX/DLV, ACL, MEMBERS, and\nvalidated durable checkpoints. `registry` is the sole partial artifact. Presence, derived membership\nfeed, leases, native ephemeral/history consumers, credentials, keys, tokens, owner secrets, and actor\nledger files are excluded. `full` means every transferable message and registry stream, not every\nJetStream resource: endpoint submissions/facts/events/timers/workflow state, contract artifacts, and\nthe records/auth/session stores are nonportable control state. Restore recreates those streams empty\nwith their canonical configs before exposing the normal listener, so active endpoint runs,\nlifecycles, and sessions do not cross a backup. Artifacts are exclusively created `0700`;\nsnapshot/checkpoint files and\nthe manifest are `0600`; `manifest.json` is written last with exact sizes and SHA-256 values. The\ndirectory is trusted operator input: hashes detect corruption, not malicious rewriting.\n\nRestore validates and stages the exact allowlisted artifact bytes before moving or creating a store.\nIt requires the same space and existing trust state. The whole pre-commit window holds a journaled\nliveness claim (coordinator, watchdogs, brokers, absolute deadline): ordinary `up` and a repeated\n`up --restore` refuse while the claim is live, and a stale attempt is recovered only after the\ndeadline has elapsed and every recorded owner is proven dead. A retried `up --restore` handles this\nautomatically; an operator can also recover it explicitly with `cotal clean restore-attempt --attempt --force`. Nothing\never rolls back a live attempt. A registry-only artifact restores as registry-only whether or not\n`--restore-only registry` is passed; omitted infrastructure is always created and the exact\npost-restore stream inventory is asserted before commit intent. Ordinary `up` from a preserved cut\nresumes only the exact recorded source store and runtime; a contradicting `--store-dir` or\n`--runtime` fails in preflight. Authenticated restores validate the complete\nspace trust bundle before staging, including nkeys, seed matches, JWTs, signers, and space binding;\nfull restores commit to the validated operator, system-account, data-account, and active-signer root\nchain in addition to the static/user authority fingerprint. Because the system account is part of that\ncommitment, a [`cotal up --rotate-sys`](#up) makes every full artifact taken before it unrestorable\nagainst this root: take a fresh full backup after each rotation. The composed commitment is revalidated\nimmediately before store mutation and never includes secret seeds. Restore never creates fresh auth.\nSame-path restores atomically retain the old\nsource at the journaled fallback path; alternate targets retain it in place; a missing canonical\nsource needs explicit `--accept-missing-source`. Quarantine and target restores use current canonical\nconfigs on isolated random-loopback brokers, never expose native snapshot consumers, and publish a\ncommit-intent immediately before the normal listener starts. Archive bytes never instantiate the real\ntarget: after quarantine validation, every stream is re-snapshotted from the validated quarantine\nstate into attempt-owned sanitized files, and the target is restored solely from those. Before that boundary, failure rolls back\nthe attempt-owned target; after it, ambiguity preserves both stores and records forward-repair\nrecourse. The cooperative maintenance lock excludes Cotal commands, not arbitrary raw NATS processes.\n\nBootstrap brokers in every auth mode, including open, mount the store under a local account with\nrandom operation-specific logins only, each carrying the exact per-phase subject permission matrix;\nnormal static credentials and user-auth sentinel/bearer connections are rejected, and no auth\nservice or callout starts. Open mode differs only in its account label, never in authority. Inventory, each stream snapshot,\nrestore initiation, exact upload id, validation, and each checkpoint recreation use separate exact\nauthorities. Every checkpoint carries the source stream's message/first/last sequence state and must\nmatch its snapshot record before mutation; core then derives and validates the only allowed start\npolicy. TASK is not a CLI exception: the same core checkpoint API recreates its canonical `DeliverAll`\nWorkQueue durable because acknowledged tasks are absent from retention and NATS forbids a\nstart-sequence policy there. Registry-only restore creates every omitted canonical stream and transient\nbucket on the isolated target before the normal listener is exposed. It deliberately does not resume\nretained agents or recreate their DM/DLV/TASK/ACL state; their identity material stays retained and\nstopped rather than being reprovisioned into a partial restore.\n\nAfter listener readiness, the manager starts attempt-bound, validates retained credentials/tokens\nwithout granting or reprovisioning, and resumes the exact persisted principals under cleanup\nsuppression. Registry-only restore uses the same flow with an empty agent set. `commitResume` is an\nidempotent validation barrier only: success must be `awaitingFinalize` with an attempt-bound 64-hex\ncommit token and does not release suppression. Under the workspace lock, the CLI first fsyncs that\nexact evidence as `manager-committed` (restore) or `resume-committed` (ordinary resume), then calls\ntoken-bound `finalizeResume`; only an `active` response for the exact token releases suppression. The\nCLI records the same token in finalization evidence before a restore becomes `active`, or before an\nordinary resume retires and consumes the marker. Re-entry from either committed state skips the prior\nidempotent activation/commit phases, retries finalization with the durable token, and finishes the\nworkspace transition. Failure before finalization preserves the committed state and cleanup\nsuppression; it is not rewritten through a degraded transition. Re-entry between any two earlier\nboundaries reuses the same attempt and may retry the idempotent phases without deleting retained state. A missing or\nchanged per-agent dependency is a named fail-closed result; the journal becomes degraded and remains\navailable for forward repair. A retry from `resume-intent`,\n`resume-active`, or `resume-degraded` reuses the same attempt and inventory after the prior listener is\nproven stopped. Every normal restore listener has an unguessable attempt-bound NATS server name. The\nCLI fsyncs its exact name/nonce, canonical endpoint, process owner, and generation-bound target identity\nimmediately after spawn. Re-entry accepts a surviving listener only when its INFO server name, live PID\nrecord, endpoint, and target identity all match that proof; degraded restore repair then moves through\nthe guarded workspace transition only after manager commit. If an uncommitted bound owner is provably\ndead, recovery retires that exact proof under the maintenance lock and binds a fresh listener for the\nsame attempt, endpoint, and target with a new nonce and server name. A live foreign/mismatched listener\nor ambiguous owner is preserved and refused, never adopted by reachability alone. A reconstructed\ncommit/degraded attempt without either the exact bound proof or a durable dead-listener replacement\nrecord fails closed even when the recorded port is free. A later ordinary startup may pass an `active`\nrestore only when its details prove manager commit and its exact recorded listener is dead.\n\n## Mesh registry\n\n```bash\ncotal meshes\ncotal meshes add # guided, on a terminal\ncotal meshes add --server [--root ] [--mode auth|open|user] [--tls] [--force]\ncotal meshes add --mode user (--user-auth-file | --from )\ncotal meshes rm [ …] [--force]\ncotal use \ncotal status [--space ] [--server ] [--components]\n```\n\n`meshes` lists the meshes this machine knows; a `*` marks the `current` default a bare\n`cotal spawn` joins.\n\nRun on a terminal with the space or `--server` missing, **`meshes add` is guided**: it asks for the\none thing that cannot be derived (the broker URL), probes it, and tells you what answered - open or\nrequiring credentials. It then offers the spaces your `--root` already holds credentials for, states\nthe mode as a fact about that broker rather than asking, and shows the exact record before writing\nanything. A broker that does not answer, or a space name already registered, becomes a choice rather\nthan an error. Anything you pass on the command line is taken as given and not asked again. Without\na terminal - a script, an agent, CI - nothing prompts and the flag form's errors stand\n(`COTAL_NO_PROMPT=1` forces that too).\n\n`cotal up` and `cotal down` maintain their own records. `meshes add` registers a mesh they cannot\nspeak for: one running on another machine, a shared broker, a hosted space. `--root` is the folder\nwhose `.cotal/auth` holds that mesh's credentials and whose `.cotal/agents` holds its personas.\nThe default is the project you run it in. The registry stores that path, never a secret. `--mode`\ndefaults to `auth` when the root holds the space's account record and to `open` otherwise. The\nbroker is probed before anything is recorded, so a wrong address, or credentials that mesh will\nnot accept, fails here instead of at the first `spawn`; `--force` records without verifying (and\nreplaces an existing record).\n\nA hostname or public address is registrable only when the connection will **require TLS**. Pass\n`--tls`, or use a `tls://` URL. The scheme is recorded as enforced intent, so every later dial\nthrough the record demands the handshake (and `meshes add tls://…` against a plaintext broker is\nrefused at registration). Without required TLS the fence admits loopback and private-overlay\nliterals only. RFC1918 addresses are refused in both modes because a cafe LAN is private but does not belong to you.\n\nA **user-auth** mesh registers from supplied pinned trust, never guessed: `--user-auth-file`\ntakes the bundle exported where the mesh runs; `--from` asks before it dials the address at all,\nthen fetches its `/.well-known/cotal-mesh` discovery document (HTTPS only), displays the pins, and\nasks again before adopting them. Neither fetch follows redirects: a 302 can move a pinned fetch\nonto plaintext or onto another host, so it is refused rather than followed, and the pinned\nexchange must itself be an `https://` URL, except for an exchange on this machine, where plain\n`http://` is accepted for a loopback *literal* (`127.0.0.1`, `::1`, any spelling of them) but not\nfor `localhost`, which is a name rather than an address. Registration verifies that the exchange\nanswers `/health` and `/jwks` as the pinned issuer. It also verifies that the broker refuses a bare\nconnect; that auth-required refusal is the pass. The sentinel credentials land in a 0600 file under\nthe entry's root; the registry records only the path.\n\n`meshes rm` drops records. It never stops a mesh. For a mesh running on this machine `cotal down`\nis the right verb, and `rm` says so unless you pass `--force`. A hand-added record is removed by\n`meshes rm`, by an `add --force` replacement, or by a `cotal up` that actually starts the broker for that same space, server and root, which becomes that\nmesh and so takes the record over (a `cotal up` for that space anywhere else refuses instead).\nNothing that merely *infers* a record is stale touches it: an\nunreachable broker is listed `offline` and stays, and `cotal down` / `cotal clean all` leave it\nalone even when it shares a root with the project they are tearing down, because nothing on this\nmachine could write it back.\n\n`use ` sets that default; the selection applies from every directory,\nincluding inside another mesh's project. `status` is a read-only report: machine prerequisites\n(starting with the installed `cotal-ai` version), the installed extensions and their versions, this\nfolder's `.cotal/`, the recorded meshes, and a live snapshot of the selected mesh (roster, channels,\nmembership feed). Stale Claude skills and out-of-date `.agents` skills recommend `cotal setup --skills`,\nnot unscoped `cotal setup`. `status` takes `--space` / `--server` to pick the mesh to inspect; it starts\nnothing.\n\nPersona rows name the catalog they describe. If this folder and the selected mesh use different\ncatalogs, status names both and marks which one spawn launches from. A green `default` means the file\npasses the same agent-file loader spawn uses; a present but invalid file is reported as invalid.\n\n`cotal status --components` adds a fail-loud per-component health pass. It reads **each\ncomponent's own control surface**, rather than treating a PID, a lease, or a successful probe of a\nsibling as proof that the component serves. It prints one of `serving`, `absent`, `not-serving`, or\n`refused` for each component and exits `0`, `1`, `2`, or `3` respectively (the highest observed\nstate wins):\n\n- **manager**: local PID record, its liveness-lease holder and PID, then the manager's own typed\n `status` service reachability from this host. Builds without a startup-phase report say\n `phase not reported by this manager build`; that is never a blank green state.\n- **delivery**: local PID record, its ready lease (`ready` is the daemon's own bound-control\n signal), and the latest `renewal.json` adoption verdict. A re-signed credential and a\n broker-accepted adoption stay distinct facts.\n- **web**: local PID record and the dashboard's own loopback `/api/meta` response, which must name\n the same PID and its requested port. A different process on the port, an unreadable PID command,\n or an unrecognizable process record is `refused`, not a green default-port guess.\n- **broker**: the registered mesh URL dialed from this host with its recorded TLS requirement.\n\n`absent` means Cotal has no live local component record (or has a stale record); `not-serving`\nmeans the component record is live but its service/readiness surface did not answer or is not ready.\nThose are intentionally separate exit cases. A failed or unreadable probe is `refused`, never an\nabsent component or a clean zero.\n\n## spawn\n\n```bash\ncotal spawn [] [--detach] [--name ] [--agent ] [--model ] [--variant ] [--prompt ] [--cwd ]\ncotal spawn -f [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | resolved mesh | Target space |\n| `--server ` | registry entry | Broker URL override |\n| `--creds ` | none | Control-caller creds for an off-registry manager (`--detach` only) |\n| `--name ` | persona's `name:` | Presence-name override (does not choose the persona) |\n| `--config ` | none | Persona catalog name or file path; wins over the positional |\n| `--agent ` | persona's `agent:`, else `COTAL_DEFAULT_AGENT`, else `claude` | Connector type (`claude`, `opencode`, `jcode`, `hermes`, and so on) |\n| `--role ` | persona's `role:` | Role override |\n| `--model ` | persona's `model:` | Model override |\n| `--variant ` | persona's `variant:` | Model variant override (connector-defined; e.g. OpenCode reasoning tiers) |\n| `--cwd ` | this cwd | Working directory to root the agent at |\n| `--prompt ` | none | Initial prompt auto-submitted at start |\n| `--resume ` | none | Fork an existing session id into the mesh; only connectors that declare resume support accept it (see [the matrix](connectors.md)) |\n| `--events` / `--no-events` | off | Publish the session's structured event plane to its own event channel |\n| `--share-tools ` | none | Share named operator MCP servers with the agent |\n| `--subscribe ` | persona's | Channel read-set override |\n| `--allow-subscribe ` | = subscribe | Read-ACL override |\n| `--allow-publish ` | deny | Post-ACL override |\n| `--detach`, `-d` | off | Launch via the manager into a detached PTY (reattach with `cotal attach`) |\n| `--on ` | class anycast | With `--detach` only: pin the launch to one manager instance id (the whole id, as `ps` prints it). Refused on a foreground spawn (no manager to pin), with `-f` (a manifest deploy launches through the manager class queue), and when empty |\n| `--file `, `-f` | none | Deploy a manifest onto the running mesh |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--allow-stale ` | none | With `-f`: waive named stale agents (apply-only) |\n| `--runtime ` | manifest's | With `-f`: override the manifest's runtime |\n\n`--events` turns on the session's **event plane**: a stream of structured events describing what\nthe agent did, rather than the prose it wrote, on a channel of its own. The channel is named after\nthe agent's principal, `events..`, never after its display name, because two live\nagents are allowed to share a display name and would then share a stream. The launch grants publish\nrights on that channel alone, foreground and detached alike, and a connector that does not\npublish an event plane refuses the flag rather than starting a session whose events have nowhere to\ngo.\n\nThe flag and the grant are separate on purpose. Holding publish rights on a channel is not a request\nto publish to it, so writing an event channel into an agent file's `allowPublish` does not turn the\nplane on: only the launch does.\n\nThe persona (`--config` > positional > `COTAL_DEFAULT_PERSONA` > `default`) is loaded from the\ntarget mesh's `.cotal/agents/`; the launch flags override the file. Foreground runs the agent\nattached to your terminal; `--detach` hands the launch to the running manager. Both modes get the\ndurable backstop on a mesh that runs the delivery daemon; `--live-only` skips it for a foreground\nspawn (messages posted while it is disconnected are then not replayed). A foreground exit retires\nthe agent's creds and broker footprint, like a manager despawn. A `--detach` spawn is an\n**action**: the manager accepts it and returns the allocated identity at once, then the launch\nfollows to a terminal outcome rather than blocking (see [the control surface](control-surface.md)).\nSee [Connect Claude Code](connect-claude.md) and [Agent files](agent-files.md); `-f` is a\n[manifest deploy](#manifest-deploys). (`cotal start` was merged into `cotal spawn --detach`.)\n\n## models\n\n```bash\ncotal models [--agent ] [--refresh]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--agent ` | all registered connectors | Connector whose catalog to list |\n| `--refresh` | off | Ask the connector to refresh its provider cache |\n\nAsks the running manager for each connector's model catalog (model ids plus their variants)\nfor connectors that expose one. OpenCode and Codex query harness/provider surfaces; Jcode reads\nproviders that enable `model_catalog = true` in the operator Jcode `config.toml`. Jcode's listed\neffort tiers render as `variants (declared, not provider-verified)`, and launch can still refuse one.\nA connector without a catalog says so. Pick a result with `cotal spawn --model --variant `,\nwhere `` is the model id as the catalog printed it. OpenCode and Codex ids are the full\n`provider/model`; Jcode ids are bare (`opus-5`, not `cliproxy/opus-5`), because the provider is\nselected by the operator's Jcode config and a prefixed id is refused at launch with the bare form\nnamed.\n\n## endpoints\n\n```bash\ncotal endpoints [--space ] [--server ] [--creds ]\n```\n\nLists the mesh presence roster: agents, the manager, and any other protocol endpoint, with each\nendpoint's role, kind, status, and current activity. Unlike `ps`, this is a read-only presence view;\nit is not limited to child processes owned by the manager.\n\n## Endpoint control\n\n```bash\ncotal describe [--space ]\ncotal invoke [--args ''] [--space ]\ncotal invoke --name [--admin] [--space ]\n```\n\nThe generic v0.4 service surface. `describe` resolves a registered endpoint's command set off the\nwire - the reserved `describe` command answers the registered contract digests, the schemas are\nfetched from the space's content-addressed contract store, recompiled, and verified against those\ndigests - and prints each command with its capability class and targeting shape. `invoke` calls one\ncommand by name: `--args` is a JSON object validated against the fetched input schema *before*\npublish; a targeted command takes `--name ` (resolved to the agent's current principal through\n`inspect`) or `--self`. `--admin` uses the admin instrument credential, whose cross-agent reach rides\nthe operator-only `any` authorization mode. Neither command has compile-time knowledge of any\nendpoint's schemas - this is the same trust chain every built-in control command now uses. Needs an\nauth mesh: the manager registers its service on both static and per-user meshes (a signed-in user\nrides their bearer; each visible or invoked command still requires its existing grant, and cross-agent\nreach needs the `admin` scope). An open mesh has no service registry.\n\n## Managed seats\n\n```bash\ncotal ps [--on ] [--wide | --json] [--space ]\ncotal stop --name [--on ] [--space ]\ncotal attach --name [--on ] [--no-reconnect] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | none | Managed agent to stop / attach (required) |\n| `--on ` | class anycast (`ps`: class scatter) | Pin to one manager instance id (multi-manager space); takes the whole id as `ps` prints it, not a prefix. An empty value (`--on \"\"`, an unset shell variable) is refused, never treated as absent |\n| `--wide` (`ps`) | off | After each seat's compact row, print extra operational facts the manager records: `cwd`, `pid`, spawner, lifecycle uid, and the owning manager's instance id and host. Model and requested variant stay in the identity row rather than printing twice. A fact the manager did not record (for example a runtime with no real process) prints nothing, never a placeholder |\n| `--json` (`ps`) | off | Machine-readable: one JSON object per seat per line, copied unchanged from the manager row. Instance headers and errors go to stderr, so stdout contains only rows. Mutually exclusive with `--wide` |\n| `--no-reconnect` (`attach`) | off | End the attach when its session ends, instead of re-establishing it. For scripts that want one run and one exit code |\n\nThe human `ps` row is presentation text and is not a stable parsing target. Scripts use `--json`,\nwhich is the machine-readable row contract.\n\nThese are operator clients over the running manager's control plane. The default row includes the\nconnector, model pin, optional requested variant, and runtime as operational descriptors for the\nmanaged row. They do not make a shared display name a unique protocol identity; use `--json` when\nunambiguous owner+actor attribution is required. An omitted variant means no override was requested;\nCotal does not invent an effective provider default it cannot observe. `ps` also prints two state\nfacts per managed agent, because they answer different questions: the process fact from the manager's\nown runtime handle (`running` with its uptime, or `exited` with how long it ran), and the mesh fact\nfrom the roster (`idle` / `working` / `waiting` / `mesh offline`, or `not in roster` when the seat has\nno presence row at all: a seat that has not joined yet, or one that never did). A seat can be\n`running` and `mesh offline` at once: the process is alive and its presence has lapsed. On a user-auth\nmesh `ps` also renders each managed agent's last credential-refresh outcome, fail-closed.\n\n**Mode split (chosen up front, never try-scatter-then-degrade):**\n\n- **Static / open mesh.** Bare `ps` is a **class scatter**: it freezes the live manager class from\n the records registry, merges every registered instance's agents grouped and attributed per\n instance, and a non-answering instance is shown as `registered, no answer within the deadline`\n (never silently omitted). That label is the whole claim: the instance is registered and did not\n answer. It does not say the host is down, because a dead host never deregisters itself and a\n live one can be slow; if it is gone, deregister it.\n `--on ` pins the read to one exact instance id instead. A wrong pin fails loud\n rather than falling through: a well-formed id that no live manager carries is reported as\n `manager instance did not answer` (nothing else is asked), and a credential without that\n instance's rail is reported as refused by the broker, not as an unresponsive manager. A manager\n that answers with a refusal is shown with its own cause; \"no manager reachable\" is said only when\n nothing answered at all. If the scatter's own registry read fails (the freeze or the reconcile),\n `ps` says the manager registry could not be read rather than pronouncing on the managers, which\n may all be up.\n\n**`stop` and `attach` route by seat locality.** A seat can only be stopped or attached by the\nmanager actually running it, and the class queue does not know which one that is. So on a\nstatic/open mesh both verbs first ask every registered instance which one hosts the named seat, then\naddress that instance directly. This happens by default; you do not need `--on`.\n\n`--on ` remains the override, for when you already know where the seat lives or the\nlookup itself is degraded. It is also the **only** route on a **user-auth mesh**: a ledger-scoped\nbearer does not hold the registry-read rows the lookup needs, so there the verbs stay on the class\nqueue unless you pin them yourself.\n\nIf no reachable instance has the seat, the error reports how many managers answered and names\nthose that did not. It does not collapse that state into a bare `no agent `. That distinction matters\nbecause a single manager cannot tell \"hosted elsewhere\" from \"does not exist\": it answers\n`not-found` for both.\n- **User-auth mesh.** `cotal ps` reports what **one** manager knows about your agents (an `ep.one`\n read against the manager's in-memory roster, owner-filtered). It does **not** report other\n manager instances. It cannot tell you that one is down: an unreachable manager is absent\n from the list. Completeness across a multi-manager user-auth space is not claimed.\n A manager that does not answer fails the command outright (exit non-zero), rather than printing\n an empty list that could be read as \"no agents\". Your ledger row needs the `admin` scope to\n reach `ps` at all; `spawn` alone is refused by the broker (the ep tier boundary).\n\n`attach` streams and drives an agent's terminal on the `pty` runtime; detach with the escape key\n(Ctrl-] by default; see [`COTAL_DETACH_KEY`](config.md)). It does so over a one-use, holder-bound\nmesh session ([SPEC](../SPEC.md) §13.6): the manager replies with a signed session grant (never a\n`127.0.0.1` URL), the CLI redeems it once over the broker, and the browser console (`cotal console`)\ndrives the same session. `stop` and `attach` need a running manager to talk to. On a static mesh\nthey are cross-agent admin operations. On a user-auth mesh, your own agents (any agent under your\nowner) need only the `spawn` scope; another owner's agent needs `admin` on your ledger row\n([identity & auth](identity-and-auth.md)). Launch detached agents with [`spawn --detach`](#spawn).\n\n**`attach` reconnects when the link dies.** A session lives on a network link, and a laptop that\nsleeps, a VPN that drops or a wifi handover kills it. When that happens `attach` prints\n`[cotal: connection lost, reconnecting]` on stderr and starts asking the manager for a new session:\na fresh grant, a fresh per-session credential, a fresh connection, so every attempt re-runs the same\nauthorization the first attach did. On success it prints `[cotal: reconnected]`, the manager repaints\nthe seat's current screen the way it does for any attach, and you carry on in the same terminal.\nRetries wait 1s, 2s, 5s, 10s, then 30s, for as long as the seat exists. The detach key is read the\nwhole time the loop runs, the waits and the attempts alike, so a reconnect never traps you: press it\nwhile a session is being established and the attach ends there, and a session that lands behind the\npress is handed back to the manager rather than left holding a slot. Everything else you type while\nthere is no session is dropped rather than queued, so keystrokes aimed at a terminal that turned out\nto be frozen, Ctrl-C included, are not delivered to the agent by a reconnect you did not know had\nhappened. That starts before the first session, not at the first reconnect: at a terminal, `attach`\nreads and drops what you type while it is still resolving the mesh, so a key struck at a prompt that\nhas not come up yet does not reach the agent when it does.\n\nA **pipe** carries script input. For example, `printf 'ls\\n' | cotal attach --name web` is\nbuffered until the session opens. Buffering continues across reconnects, so\n`tail -f log | cotal attach --name web` does not lose the part of its feed written while the link was\ndown. Only a terminal gets the reader; `--no-reconnect` keeps the old behaviour on both.\n\nIt stops on its own when reconnecting cannot help, and says why: a manager that refuses the attach\nexits non-zero with the manager's own message, and a reconnect that finds the seat no longer there\n(despawned, or its agent exited while the link was down) exits cleanly with `seat is gone`.\nA refusal that could still pass, such as a manager at its session ceiling, is relayed in the\nmanager's own words while the loop keeps trying, once per refusal rather than once per attempt.\nPressing the detach key, or the agent's process exiting while you are attached, ends the attach as\nit always did. `--no-reconnect` turns all of this off and restores the single-session behaviour,\nwhich is what a script wants.\n\nEach reconnect also hands the abandoned session back to the manager, over the first link that can\ncarry the message, so an attach that flaps does not eat the manager's session slots one outage at a\ntime. If that message never gets a link, the attach says so when it ends.\n\nWhich mesh `attach` resolves also decides **whose trust it redeems with**. Redeeming a session grant\nmeans minting a short-lived, session-scoped credential from the space's seed, and that seed comes\nfrom the root the mesh resolved to, never from a `.cotal` found by walking up from whichever\ndirectory you happen to be standing in. The difference is not hypothetical: `~/.cotal` exists on\nevery install because the mesh registry lives there, so a command run anywhere under your home\ndirectory but outside a project used to mint from your home directory's trust and present it to a\nbroker that trusts a different chain, which surfaced as a bare authorization failure that named\nnothing. A directory that does hold another chain for the same space is now reported on the way\npast, and not obeyed:\n\n```text\n! this directory resolves to /Users/you, whose .cotal/auth holds a DIFFERENT trust chain for space \"team\".\n attach used /Users/you/projects/app, the root this mesh resolved to. The other one is not being used, and is worth a look.\n```\n\nWhen the resolved mesh holds no seed at all, `attach` refuses and names what it resolved, the broker\nand the root, instead of describing a directory it did not use.\n\nTerminal bytes stream over the mesh; the manager's own HTTP/WS face serves the console. That endpoint binds\n**loopback by default**, so nothing is exposed by accident; `cotal up --host ` passes its bind\naddress down, which is what lets you attach to an agent whose manager runs on another machine. A\nbare `cotal supervise` and an embedded manager stay machine-local. Set it directly with\n`supervise --console-host `.\n\nThat address is **recorded on the mesh** and carried forward, because it is a decision rather than\nsomething later commands can work out for themselves (a broker dial address is not a manager bind\naddress). Every later manager launch for the same mesh reuses it, including a same-root `cotal up` repair,\nan adopted preserved or restored listener, and a `spawn -f` manifest deploy. A manager replacement\ndoes not quietly move a reachable attach face back to loopback. Passing `--host` again overrides it,\nso you can widen or narrow exposure whenever you like; a mesh that never asked stays loopback-only\nand records nothing.\n\nBecause that face carries terminal read and write for every managed agent, it is credentialed in two\ntiers. A mesh caller receives a **ticket** bound to the single agent the manager just authorized,\nsingle-use and short-lived, so one authorized attach can never be re-pointed at someone else's\nagent. The **console token** is the operator's own, reaches every agent, and is printed only to the\nmanager's output. The roster, the live feed, and the PTY stream all answer `401` without one; the\nstatic console shell is served openly, since it describes no agent.\n\n## input\n\n```bash\ncotal input --name --text [--no-enter] [--on ] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | | Managed agent to type into (required) |\n| `--text ` | | The text to type, taken verbatim (required) |\n| `--no-enter` | off | Type the text and stop there, without pressing Enter |\n| `--on ` | class anycast | Pin to one manager instance id using the same rules as [`attach`](#managed-seats) |\n\nTypes one line into a running agent's terminal, as if you had typed it there, and returns. This is\nthe half of [`attach`](#managed-seats) that a program wants: `attach` is a live stream that holds a\nsession open and expects a terminal on your side, so a script, a cron job or a web UI cannot use it\nto send a single line. `input` is one authorized call.\n\nWhat it is for is **harness commands**. A line beginning with `/` is not chat and not a message: it\nis something the agent's own harness handles, and the only way in is the keyboard.\n\n```bash\ncotal input --name reviewer --text \"/compact\" # ask the harness to compact its context\ncotal input --name reviewer --text \"/model opus\" # switch its model\ncotal input --name reviewer --text \"hold on that PR\" # ordinary typing works too\n```\n\n**Quoting.** `--text` takes a value, so a payload starting with `/` survives as written. A payload\nstarting with a dash needs the `=` form, because the shell-style `--text --foo` is ambiguous and is\nrefused rather than guessed:\n\n```bash\ncotal input --name reviewer --text=--verbose # dash-leading text: use --text=\n```\n\nEnter is pressed by default, since a command typed but never submitted has not been delivered.\n`--no-enter` types the text and leaves it sitting at the prompt, which is how you stage a line and\nsend it later.\n\nNothing comes back but a delivery receipt (`✓ sent 9 bytes to reviewer`, counting the trailing\ncarriage return). Whatever the agent does next shows up where its output already goes: the mesh, its\ntranscript, or an `attach`.\n\n**This one is operator-only, and more narrowly than `stop` or `attach`.** Those two are granted to\nanything holding `spawn`, so an agent can stop and attach to seats under its own owner. `input` is\nnot: it is granted only to operator credentials, which on a user-auth mesh means your ledger row\nneeds the `admin` scope, the same scope [`ps`](#managed-seats) already needs there. The reason is\nthat a write into a terminal is control of whatever is running in it, and on a user-auth mesh the\nown-owner rule covers every seat under you, not only the ones you launched: a `spawn`-scoped agent\ncould otherwise type into a sibling it never started. Seat locality is still resolved for you.\n\nOnly the `pty` runtime can be typed into. The external terminal runtimes (`tmux`, `cmux`, `orca`,\n`herdr`) attach to a process they do not own, so they have no input stream for it and the command\nrefuses by name rather than dropping the keystroke.\n\n## personas\n\n```bash\ncotal personas list [-v] [--running]\ncotal personas show \ncotal personas edit \ncotal personas new (--prompt | --from ) [--role ] [--model ]\ncotal personas rm --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh's persona catalog |\n| `--role ` | none | `new`: the persona's role |\n| `--model ` | none | `new`: the persona's model |\n| `--prompt ` | none | `new`: the persona's prompt text |\n| `--from ` | none | `new`: seed the prompt from a file |\n| `--verbose`, `-v` | off | `list`: include role / model / description |\n| `--running` | off | `list`: mark personas live on the mesh |\n| `--force` | none | `rm`: required, delete without prompting |\n\nPersonas are the local agent files under the resolved mesh root's `.cotal/agents/`, the same catalog\n`cotal spawn` launches from. `--space` and `--server` therefore move every list, read, write, delete\nand completion operation to the selected mesh. An unresolved target refuses rather than falling back\nto the current directory. See [Agent files](agent-files.md) for the file format.\n\n## supervise\n\n```bash\ncotal supervise [--runtime ] [--space ] [--server ] [--spawn ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space to supervise |\n| `--server ` | hosting mesh, or matching registered mesh | Broker URL. A registered mesh supplies it when omitted; a different explicit value is refused. |\n| `--runtime ` | `pty` | Agent runtime (`pty` built in; extension runtimes are explicit-only) |\n| `--console-port ` | none | Protocol-console port |\n| `--console-host ` | loopback | Bind host for the console + attach endpoint. Loopback keeps it machine-local; `cotal up` passes the address it bound the broker to, which is what lets `cotal attach` reach this manager from another machine |\n| `--roster ` | none | Declarative roster to boot at startup |\n| `--launch ` | none | Resolved manifest launch spec (from `up -f` / `spawn -f`) |\n| `--spawn ` | none | Comma-separated personas to pre-spawn at startup |\n\nThe manager is the agent supervisor and control plane: it answers `spawn --detach`, `stop`, `ps`,\n`attach`, and the `cotal_*` manager tools. `cotal up --detach` starts one for you; run `supervise`\ndirectly to recover a dead manager or drive a custom runtime. Default runtime is `pty`; install an\noptional provider first (`cotal ext add @cotal-ai/orca`, `@cotal-ai/tmux`, `@cotal-ai/cmux`, or `@cotal-ai/herdr`) and\nselect it explicitly. A missing provider or app fails loudly; there is no fallback. See [Deploy](deploy.md).\n\nOn a normal `SIGINT`/`SIGTERM`, the manager stops every seat and requires the selected runtime to\nprove the seat is gone before it releases the manager lease or service registration. A stop that\ncannot prove exit fails loud and keeps manager authority instead of reporting a clean shutdown while\nan orphan still holds broker rails. After an abrupt manager death, the same logical successor\nterminalizes only its own durable static slots, verify-evicts the predecessor's broker principal,\nrecords that result in the lifecycle's caller-readable audit detail, and only then retires the\nlifecycle and frees the alias. Missing or unverified broker evidence keeps the slot terminalizing.\nDelivery-admin does not terminate the orphan OS process; safe successor process reaping requires\ndurable process start-identity pinning and is tracked separately.\n\nA `meshes add --mode user` entry is a **participant** registration, not hosting authority. A\nparticipant may run `supervise` only when the host advertises the remote manager authority service\nand the signed-in actor has the dedicated `supervise` ledger scope. The CLI obtains the closed,\nloopback-only `manager-service` view; `spawn` and `admin` do not substitute for that scope. The\nhost issues the manager's public-nkey JWT material through its lifecycle-bound prepare → activate\n→ renew protocol, never by handing the participant a signer or static provisioner credential.\n\nWithout that advertised host service or scope, `supervise` refuses before it starts a manager.\nRun `cotal spawn` without `--detach` to launch a foreground agent, or ask the space host to enable\nthe authority service and grant `supervise` for detached agents. If a running remote manager loses\nrenewal, it reports degraded state and refuses unsafe new starts and restarts; live agents are not\nsilently replaced. Do not run `cotal down` or `cotal up` on a participant machine to repair this\ncondition.\n\n## reconcile-gate\n\n```bash\ncotal reconcile-gate [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the frozen gate lives in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint whose gate is frozen |\n| `--instance ` | this folder's persisted manager instance | Instance id |\n\n**When you need this.** A manager restart killed after deregistration begins but before the new\nincarnation finishes leaves the endpoint's issuance gate *frozen*, held by a\nprocess that no longer exists. The freeze is what stops two incarnations serving at once, which is\ncorrect. The successor manager now completes that dead registration itself on boot, using the same\nguard this command uses: it acts only when the freeze-holder is affirmatively gone under a complete\nCONNZ sweep (`gone` and `sweepComplete=true`). If that registration's spec write already committed,\nit finishes the same freeze at the committed registration revision. If the spec did not advance, it\nabort-reopens the gate at generation+1 with processEpoch unchanged and continues the normal takeover.\nLive, unknown, unestablishable, and\nwrong-op-kind still refuse; there is no TTL.\n\nUse this command when the boot path cannot run: the delivery daemon is down, the repair targets a\nnon-manager endpoint, or you want to lift the freeze without starting a manager. It checks that the\nholder really is gone, prints what it found, and then finishes the dead operation the same way as the\ninterrupted restart would have: revoke the old credentials, evict their holders with verification,\nand reopen the gate.\n\nIf verification is interrupted, the command leaves the gate frozen and durably records each holder\nwhose eviction was already verified. A retry still repeats the freeze-holder liveness check, then\nskips only progress bound to the same registration operation, frozen-gate revision, and holder set.\nThe output reports holders completed before this attempt, completed now, and still remaining. A new\nfreeze or changed holder set starts from zero. Cursor cleanup happens only after reopen; a retained\ncursor is harmless because its old gate revision cannot authorize a later freeze.\n\n**It refuses far more often than it acts, on purpose**, and always says which check stopped it:\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `holder-alive` | The freeze-holder still has a live connection: a manager *is* running | Stop that process first. Reconciling would evict a live manager's credentials |\n| `holder-unknown` | The connection sweep could not prove the holder absent | Not safe to proceed: an unprovable holder is treated as a live one. Re-run once the broker answers completely |\n| `liveness-unestablishable` | The delivery daemon could not be asked at all | Start it (`cotal up` runs it) and re-run. Silence is never read as death |\n| `not-frozen` / `no-gate` | The gate is open, or there is no gate at that coordinate | Nothing to repair: check `--endpoint` / `--instance` |\n| `wrong-op-kind` | Frozen under a takeover or retirement, not a registration | Out of scope for this command; it will not reinterpret another operation's intent |\n| `eviction-unverified` | The holder looked gone but eviction could not be verified | The gate is left frozen, unchanged. Investigate the broker before retrying |\n| `raced` | A newer manager moved the gate mid-repair | Re-run `cotal doctor` and look again |\n\nThere is no `--force`, and no path that discards gate state: the only way this reopens a gate is by\nproving the holder is gone and then completing the operation properly.\n\n## deregister-instance\n\n```bash\ncotal deregister-instance [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the instance is registered in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint the instance serves |\n| `--instance ` | this folder's persisted manager instance | Instance id, the whole id as `cotal ps` prints it |\n\n**When you need this.** The service registry records *registration*, not liveness, and nothing in\nthe model expires a row. A manager that stops cleanly removes its own registration. One whose host\ndied without writing anything cannot, so its record goes on claiming a live instance forever: every\nclass scatter in that space freezes the dead slot in, and `cotal ps`, `stop` and `attach` each pay\ntheir whole deadline waiting for a machine that is never coming back. A laptop that was reimaged, a\ncontainer that was deleted, a box that will not be back on the network: those registrations have no\nother exit.\n\nThis command is that exit. It asks the instance first, and it removes a record only when the broker\naffirms the instance's own rail is empty: nothing subscribed there. Then it deletes the\nregistration's two records keys, each pinned to the revision it read, and prints what it removed.\n\n**Silence alone never passes.** An unanswered describe is what a dead host, a wedged process and a\nslow one all look like, and a hung process still holds its subscriptions, so the broker sees\ninterest on its rail. That instance is refused and the observation is printed. A dead process holds\nno connection and therefore no subscription, so a real corpse is still removed.\n\n**Every refusal names the failed check:**\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `instance-answered` | The instance answered a pinned describe. It is alive | Nothing to repair. If it is wedged rather than gone, stop the process first; its own clean stop removes the record |\n| `instance-not-affirmed-gone` | It did not answer, and the broker did not report its rail empty, which is what a held subscription looks like: slow or hung, not affirmed gone | Nothing was removed. Stop the process; its record goes on its own clean stop, or re-run this once it is down |\n| `liveness-unestablishable` | The probe itself failed, so nothing was learned | Fix the probe's path (credential, broker) and re-run. A probe that could not run is never read as death |\n| `not-registered` | No registration at that coordinate | Check `--instance` and `--endpoint`. This takes the whole id, never a prefix |\n| `registration-in-flight` | The instance holds the endpoint governance slot at the live issuance-gate generation, so a registration is still completing | Nothing was removed. Wait for that registration to finish, then re-run |\n| `superseded` | The record moved between the read and the delete | Something is writing to it. Nothing was removed; re-observe before retrying |\n\nThere is no `--force` and no sweep: silence is not death, and a rule that removed rows on silence\nwould eventually remove a live instance that was merely slow. An operator names one instance, the\nbroker's verdict on its rail is what authorizes the removal, and the guard's job is to show them\nthey named a dead one. Removal is not a one way door either. The same instance re-registers over\nthe tombstone on its next start, under the same identity.\n\n## runtimes\n\n```bash\ncotal runtimes\n```\n\nLists every agent runtime the manager can spawn through: the built-in `pty`, the official providers\n(`orca`, `tmux`, `cmux`, `herdr`), and any custom provider installed via `cotal ext add`. Each installed\nprovider is probed so you can see what is actually reachable on this machine before selecting it:\n\n```\npty built in\norca installed · reachable @cotal-ai/orca\ntmux available · cotal ext add @cotal-ai/tmux\ncmux available · cotal ext add @cotal-ai/cmux\nherdr available · cotal ext add @cotal-ai/herdr\n```\n\n`installed · reachable` / `unreachable` is the provider's own `available()` probe; `available` means\nit is a known runtime you can add with the shown command. Selecting an unknown or uninstalled runtime\nvia `up`/`spawn --runtime ` fails loud and, for a known one, points at the exact `cotal ext add`\npackage. There is no silent fallback to `pty`.\n\n## send\n\n```bash\ncotal send dm \"\" [--space ] [--server ] [--creds ]\ncotal send msg \"\"\ncotal send ask \"\"\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and (off-registry) which credential |\n\nOne-shot messaging: connect, send a single direct message (`dm`), channel post (`msg`), or role\nask/anycast (`ask`), then exit. For a running conversation, agents use the mesh tools instead\n([MCP tools](mcp-tools.md)).\n\n`cotal send` requires `COTAL_NAME` plus either `COTAL_ID` or both `COTAL_OWNER` and `COTAL_ACTOR`.\nIf that tuple is missing, `send` refuses before connecting so the recipient never sees a message\nattributed to a nameless command principal. A child that inherited a\nseat's environment is attributed as that seat; this command does not distinguish the two. An operator\nwho is not a live seat can set both variables for the one shot:\n\n```bash\nCOTAL_NAME= COTAL_ID= cotal send ...\n```\n\n## channels\n\n```bash\ncotal channels list\ncotal channels set [--replay | --no-replay] [--window ] [--desc ] [--instructions ]\ncotal channels default --replay | --no-replay\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--replay` / `--no-replay` | none | `set`/`default`: replay history to new joiners, or not |\n| `--window ` | none | `set`: replay window size |\n| `--desc ` | none | `set`: one-line channel description |\n| `--instructions ` | none | `set`: instructions shown to joiners |\n\nInspects and edits the channel registry: replay policy, description, and joiner instructions. ACL\nsemantics (who may read or post) are set at mint / provision time, not here; see\n[Channels and permissions](channels-and-permissions.md). On a user-auth mesh, `list` rides your\nown login as is; `set` and `default` edit the registry over a short-lived\nchannel-writer view, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n\n## history\n\n```bash\ncotal history clear --force [--dms] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--dms` | off | Also clear DM history |\n| `--force` | none | Required: clear without prompting |\n\nPurges retained channel history; `--dms` extends it to direct-message history. An alias of\n[`clean history`](#clean). On a user-auth mesh the purge rides a short-lived purger view over\nyour login, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n## console\n\n```bash\ncotal console [--plain] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to watch |\n| `--plain` | off | Line stream instead of the TUI |\n\nA live protocol view for a space: a lazygit-style TUI, or a plain line stream on `--plain`. On a\nuser-auth mesh it rides the read-only admin view over your login, which needs ledger scope\n`admin`. See [Watch a mesh](watch-a-mesh.md).\n\n## web\n\n```bash\ncotal web [--detach] [--host ] [--port ] [--no-open] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to serve |\n| `--host ` | `127.0.0.1` | Concrete HTTP bind and browser host; wildcard addresses are refused |\n| `--port ` | `7799` | HTTP port |\n| `--detach` | off | Run in the background; stop with `cotal down web` or bare `cotal down` |\n| `--no-open` | off | Don't open the browser |\n\nThe browser observability dashboard: presence, channels, and a live feed. It is **not** part of\n`cotal up`: it ships inside `cotal-ai` as the `@cotal-ai/web` extension, seeded automatically on first\nrun (like the built-in connectors) so it always matches your CLI version. It self-registers `cotal web`\ninto this surface and serves\n`http://cotal.localhost:7799` by default (loopback; `*.localhost` resolves in Chrome/Firefox/Edge; Safari may\nneed `http://127.0.0.1:7799`). On a user-auth mesh the dashboard rides the read-only admin view\nover your login, and a channel purge asks for its own channel-purger view per click; both need\nledger scope `admin`. Detached mode re-execs the current Cotal installation, writes diagnostics to\nthe mesh root's `.cotal/web.log`, and reports success only after the HTTP server answers. It requires\na recorded mesh root, but can be launched from any directory once `cotal up` has recorded the mesh.\nSee [Watch a mesh](watch-a-mesh.md).\n\n## mint\n\n```bash\ncotal mint [--profile ] [--out ] [--signer]\ncotal mint --provision [--role ] [--space ] [--server ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--profile ` | `agent` | Credential profile |\n| `--out ` | `.cotal/auth/creds/space./.creds` | Output path - the default sits under the resolved space's segment (`` is that space's hex encoding, as in [Project files](config.md#project-files)) |\n| `--signer` | off | Emit a stripped account-signing file instead |\n| `--force` | off | With `--signer`: overwrite an existing file |\n| `--allow-subscribe ` | the agent file's, else subscribe | Read-ACL override, **agent profile only**: `observer` and `admin` carry a fixed read set, and `mint` refuses this flag there rather than narrowing nothing |\n| `--allow-publish ` | the agent file's, else deny | Post-ACL override, **agent profile only** |\n| `--role ` | the agent file's | Agent profile: the anycast task queue the identity pulls (`svc_`) |\n| `--provision` | off | Agent profile: also pre-create the identity's bind-only DM/deliver durables (and its role's task queue) on the live mesh, so the credential can consume |\n| `--space `, `--server ` | the resolved mesh | Which root supplies the agent file, static trust and default credential storage; with `--provision`, also which live mesh receives the durables |\n\nMints a NATS creds file for a space in **static** auth mode, scoped to a profile and (optionally)\nexplicit read/post ACLs. `--signer` emits an account-signing file for delegating minting to another\nhost. A per-user-auth space refuses `mint`: agents there join under a logged-in user\n([`login`](#login) + [`actor grant`](#actor)), never via a handed-out creds file. See\n[Identity and auth](identity-and-auth.md).\n\nFor an agent profile, the resolved mesh root supplies the persona ACL, the signing material and the\ndefault credential destination as one authority. If the current folder also holds trust for a\ndifferent space or account, mint refuses before writing and names both roots. It never combines a\npersona from one root with credentials signed or stored under another.\n\nA plain mint is creds only: the identity can publish within its post ACL at once, but on an authed\nmesh its DM inbox and task queue are provisioner-pre-created and bind-only, so a **consuming**\nconnect fails until they exist. `--provision` performs that pre-create in the same command (a\nprovisioner cred is minted from the space's trust material, used, and dropped), so a long-running\nclient you start yourself can receive DMs and role anycasts like a spawned seat. The command prints\nthe identity's principal (its wire id) and lifecycle uid; a consuming client passes that uid as its\n`lifecycleUid`. Agent profile only; an open mesh needs none of this (peers self-create there). The\nThe same resolved authority is used for both the credential and `--provision`, so the broker\nfootprint cannot be created under a different root's trust material.\n\n## Login\n\n```bash\ncotal login --idp [--client-id ]\ncotal logout --idp \n```\n\nSigns you in to a per-user-auth mesh's IdP (device code flow) and caches the session; run it\nonce per machine. It prints your IdP subject, the id the operator grants against. After a\nlogin, every command on that mesh works under your identity: each connect takes a fresh IdP\nproof, exchanges it locally for a short-lived bearer, and is authorized against the actor\nledger at connect time. `logout` revokes the IdP session and clears the cache. See\n[identity & auth](identity-and-auth.md).\n\n## actor\n\n```bash\n# an upsert of the WHOLE row: a flag left off is the WIDE default below, not \"unchanged\"\ncotal actor grant --sub [--scope a,b] [--allow-subscribe a,b] [--allow-publish a,b] [--role ] [--label ]\ncotal actor revoke (--sub | --owner )\ncotal actor list\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | the folder's | Space whose ledger to manage |\n| `--sub ` | none | The IdP subject (shown by `cotal login`) the actor belongs to |\n| `--owner ` | none | The derived owner token (alternative to `--sub`) |\n| `--scope ` | `spawn,role:default` | Capability scope (`''` = none; `spawn` = may run agents; `role:` = may delegate role r; `admin` = cross-agent control; `supervise` = eligible for the closed remote manager-service view when the host enables it) |\n| `--allow-subscribe ` | `>` (all channels) | Channel read ACL; the user's envelope, their agents can never read beyond it |\n| `--allow-publish ` | `>` (all channels) | Channel post ACL; also the envelope for their agents' posting |\n| `--role ` | none | Role (scopes the task-queue consumer) |\n| `--label ` | none | Display label for `actor list` (never the IdP subject) |\n\nThe actor ledger is the single authorization source of a user-auth space: no row, no access.\nA bare `grant` is the **full** envelope (all channels, may spawn); the flags narrow it. A\nre-grant **replaces the whole row**, not the one field you name, so to add a capability spell\nevery field out: the new scope plus the row's current read set, post set, role and label\n(`cotal actor list` shows what a row holds). A field left off does not stay as it was, it\nreverts to the wide default in the table above, which is how a narrow reader becomes a reader\nof every channel. A re-grant retires the current interactive lifecycle through the running auth\nservice before it rotates the row, so copied bearers cannot cross an authorization update. If that\nretirement cannot be confirmed, the row is left unchanged and the command fails with the recovery\naction. `revoke` uses the same retirement before deleting the row, which lets a later grant create a\nreal successor instead of colliding with a live predecessor. `supervise` is separate from `spawn` and `admin`: it only makes a signed-in\nperson eligible for the host-provided closed remote manager-service view; it does not grant\nmanagement of another owner or a general host profile. `revoke` denies the next exchange and\nthe next connect with no restart, and evicts the principal's live connections. Managed-agent rows\n(written by the spawn path) live in a disjoint row space this command never touches. See\n[identity & auth](identity-and-auth.md).\n\n## doctor\n\n```bash\ncotal doctor auth [--fix]\n```\n\nCredential-health diagnosis and repair for this folder's mesh: renders every managed\ncredential as healthy / near-expiry / expired and ends in `healthy` or the exact next\ncommand; `--fix` applies the repairs it can. The one surface every stale-credential error\npoints at.\n\n## join\n\n```bash\ncotal join --space --name [--role ] [--channel ]\ncotal join --link | --token \n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and which credential |\n| `--name ` | none | Your presence name |\n| `--role ` | none | Your role |\n| `--channel ` | none | Channel to join |\n| `--kind ` | `agent` | Endpoint kind |\n| `--link ` | none | Join link (`cotal://…`) |\n| `--token ` | none | Join token |\n| `--lifecycle-uid ` | none | Required with `--creds`: the lifecycle UID minted alongside the credential (`COTAL_LIFECYCLE_UID` works too). A credential's durable grants name exact lifecycle-keyed resources, so `join` refuses to invent one |\n| `--tls` | off | Connect over TLS |\n\nAn interactive presence: join a space under your own name and role, without launching an agent\nharness. A `--link` or `--token` supplies the where and the auth in one value. See\n[Spaces](spaces.md) and [Identity and auth](identity-and-auth.md).\n\n## Manifest deploys\n\nA `cotal.yaml` manifest declares a whole mesh (channels, personas, roles, and ACLs) in one file.\nThree commands consume it, plus a read-only validator:\n\n```bash\ncotal up -f cotal.yaml # boot a fresh mesh from the manifest\ncotal spawn -f cotal.yaml # deploy the manifest additively onto a running mesh\ncotal down -f cotal.yaml # tear that deploy down (or --run for one run)\ncotal topology view -f cotal.yaml # validate + view the access graph, change nothing\n```\n\n`up -f` and `spawn -f` differ in target: `up -f` brings up a new broker and applies the manifest;\n`spawn -f` requires an already-reachable mesh and applies additively (ownership-scoped). On a\nuser-auth mesh, `spawn -f` deploys over your own login (the deployer view, gated on ledger scope\n`spawn`): the manifest's agents land under your owner, a manifest claiming another owner is\nrefused, and seeding new channels additionally needs scope `admin`. Both take\n`--dry-run` to print the plan without mutating anything. `topology` validates the manifest and\nrenders its channel / role / ACL graph. See [Define a team](define-a-team.md) and the\n[manifest reference](manifest.md).\n\n## ext\n\n```bash\ncotal ext # same as `list`\ncotal ext add \ncotal ext remove \ncotal ext list\ncotal ext root # print just the install prefix (scriptable)\ncotal ext seed [--repair|--reset|--force]\n```\n\nOperator-installed extensions: `add` installs an npm package into a cotal-owned prefix and records\nevery registry provider it contributes. Commands appear in help, completion, and dispatch; runtime\nproviders are lazy-loaded by commands such as `supervise`; local process providers participate in\n`status` and selective `down`. `remove` and `list` manage them. The `@cotal-ai/web` dashboard is the\ncanonical command/process example. Installed packages and their location are described in\n[config](config.md).\n\nBare `cotal ext` lists the inventory, headed by the install prefix. That prefix is a cotal-owned npm\nroot kept **separate** from npm's own global tree. These packages never show up in `npm list -g`,\n`cotal ext` (or the Extensions section of `cotal status`) is the canonical inventory. `cotal ext root`\nprints only the path, for scripts. The versions shown are the manifest pin recorded at add time.\n\nRemoving an extension that owns a running local process is refused with the mesh root and its\n`cotal down ` command; stop it first so uninstalling the package never strands a process\nwhose lifecycle provider is gone.\n\n### Built-in connectors are seeded extensions\n\nThe first-party agent connectors (`claude`, `opencode`, `codex`, `hermes`, `jcode`, `pi`) are not compiled into\nthe binary. They are seeded on first run through the **same** `ext add` path a third party uses, and\nappear in `cotal ext list` like any other extension. So you can remove one you do not want\n(`cotal ext remove @cotal-ai/connector-hermes`), and a deliberately-removed connector STAYS removed\nacross upgrades. `cotal ext add ` adds a third-party connector the same way. The web\ndashboard (`@cotal-ai/web`, providing `command:web`) is the seventh built-in seeded on the same path.\n\n`cotal ext seed` is the maintenance entry for that seeding (it runs automatically on the first real\ncommand of each boot, so you rarely call it):\n\n| Flag | Meaning |\n|---|---|\n| (none) | Reconcile: seed any never-seeded built-in, refresh a seeded one whose version the binary bumped, leave a removed one removed. A no-op once current. |\n| `--repair` | Recover after an interrupted seed or a lost authority (rebuilds the interrupted connector; restores the removed-vs-never-seeded record from its durable backup). |\n| `--reset` | Discard the record and re-seed all seven built-ins (the six connectors plus the web dashboard). **Resurrects any you removed.** Rebuilds cleanly over corrupt seed state. |\n| `--force` | Re-seed the built-ins even when the version stamp is current or a downgrade. |\n\nWhen a newer `cotal` advances the operator-global seed store to its generation, it prints one\nmigration line naming the old and new generations, the exact CLI entry that wrote the store, the\ncommit timestamp, and `seed/stamp.json`. That writer and timestamp are kept in the stamp, so a later\nolder CLI refusal can say which executable wrote the generation it will not overwrite and when.\nLegacy generation-only stamps remain readable; their refusal simply has no writer provenance to add.\n\nAn older `cotal` refuses a seed store written by a newer version. When it can verify a sufficient\n`cotal` executable on PATH or at the installer's `~/.local/bin/cotal` location, the refusal names\nthat absolute path so a reduced service PATH does not select the older binary again. Otherwise it\nkeeps the generic newer-version instruction. `--force` rebuilds the store for the running older\nversion without discarding the ever-seeded authority. `--reset` still exists for corrupt state and\nresurrects deliberately-removed connectors.\n\nA source-checkout CLI (`pnpm cotal`, `tsx bin/cotal.ts`, `node bin/cotal.ts`, or a suite child of\nthose) refuses to write or garbage-collect that store. The refusal names the path, the generation\nit declined, and `$XDG_CONFIG_HOME` as the isolation remedy. `COTAL_HOME` does not relocate this\nstore. An entry that cannot be proven as a released install is refused the same way. Isolated\nrelease tests that must seed from a checkout-shaped `bin/` set `COTAL_ALLOW_CHECKOUT_SEED=1` after\npointing `$XDG_CONFIG_HOME` at a scratch dir; that override is documented here, not on the refusal\nline. An opt-in write still records the checkout path in `seed/stamp.json` as `writtenBy`.\n\nThe default connector for a bare `cotal spawn` (no `--agent`) is the persona's `agent:` pin if it\nhas one, else `claude`; set `COTAL_DEFAULT_AGENT` (e.g. `opencode`) to change the fallback. It is\na default, so a persona that pins its harness still wins over it. An `--agent` naming a removed\nconnector fails loud with the exact\n`cotal ext add` to restore it. Set `COTAL_SKIP_CONNECTOR_SEED=1` to turn off the automatic first-run\nseed/refresh entirely (for a controlled or offline setup that manages connectors by hand); `cotal ext\nseed` still runs on request.\n\n## completion\n\n```bash\ncotal completion # print a stub to eval / source\ncotal completion install [shell] # install it persistently\n```\n\nPrints or installs shell completion. Completion candidates come from each command's declared flags\nand, where useful, live mesh state (spaces, personas, managed agents) resolved offline.\n\n## feedback\n\n```bash\ncotal feedback \"\" [--type ] [--email ] [--details ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--type ` | none | `bug` \\| `idea` \\| `friction` \\| `praise` \\| `other` |\n| `--details ` | none | Longer free-form details |\n| `--severity ` | none | `low` \\| `medium` \\| `high` |\n| `--area ` | none | The part of Cotal this concerns |\n| `--email ` | git email | Contact email (required on the keyless public path) |\n| `--name ` | none | Your name (optional) |\n| `--url ` | keyed / public intake | Intake URL override |\n| `--key ` | `COTAL_FEEDBACK_KEY` | Feedback key |\n\nSends feedback to the Cotal developers. With a key (`--key` / `COTAL_FEEDBACK_KEY`) it routes to the\nkeyed beta intake; without one it goes to the public `cotal.ai` intake and requires a contact email\n(`--email` / `COTAL_FEEDBACK_EMAIL`, else your git email). Run a self-hosted intake with\n[`feedback-intake`](#server-daemons).\n\n## run\n\nOperate durable workflow runs (cotal-lang programs) from the terminal.\n\n```bash\ncotal run start --file [--timeout ] [--local]\ncotal run resume [--local --file ]\ncotal run ps [--endpoint ]\ncotal run journal [--endpoint ]\ncotal run answer [--value ] [--artifact ] [--endpoint ] [--local --by ]\n```\n\n`start` hands the program to the mesh's manager, which validates it, mints the run id (the record\nnever takes a caller-supplied one), drives it in its own process, and answers with the id once the\nrun is recorded; a program that does not validate is refused with every problem listed. `resume`\nasks the manager to take an existing run back and continue it from its step journal; the source is\nthe recorded program, so no `--file` is taken. Neither takes `--endpoint`: the manager records\nits runs under its own endpoint, and naming another is refused. `ps` lists the run records and\n`journal` renders one run's durable records; both only inspect. `answer` resolves an open\ncheckpoint through the manager, presenting as the holder that armed it; the manager records the\nanswerer from your credential, so no `--by` is taken there. `--timeout` sets the default\ncheckpoint timeout for a drive (default 1h). `--local` drives in this process instead, over one\nconnection per invocation under the run's own credential minted from the project folder's trust\nmaterial, and is the path on a bare broker with no manager or for a run with no recorded program\n(`cotal run resume --local --file `); `answer --local` takes `--by `. A\nuser-auth mesh runs no programs yet: the manager refuses the family by name, and `--local` has no\ncredential there. The guide is [workflows](workflows.md).\n\n## Server daemons\n\nTwo long-lived infra roles ship with the CLI. They are not part of everyday operation; the delivery\ndaemon comes up automatically with `cotal up --detach` in auth mode.\n\n```bash\ncotal deliver --space [--server ] [--creds ]\ncotal auth-service --space --server [--port ] [--exchange-public-port ] [--exchange-public-url ] [--exchange-trusted-proxy]\ncotal feedback-intake --keys [--port ] [--creds ]\n```\n\n`auth-service` runs a user-auth space's identity plane: the NATS auth callout, the\ncapability-gated local exchange and JWKS, and, when `--exchange-public-port` is set, the closed public\nexchange/discovery face forwarded by an HTTPS reverse proxy. `--exchange-public-url` is the proxy URL\nadvertised to clients; `--exchange-trusted-proxy` opts into last-hop `X-Forwarded-For` attribution.\n`cotal up --user-auth` starts and supervises the service for you, so you run it directly only to\nrecover one by hand.\n\n`deliver` runs the server-side Plane-3 delivery daemon: the durable backstop and membership/ACL\nauthority. It is auth-mode-only and single-instance (`--shard`/`--shards` accept only `N=1`);\n`--dev-mint` mints a scoped cred from the local signer for standalone dev. See the\n[delivery daemon](delivery-daemon.md). `feedback-intake` runs a self-hosted feedback server\n(requires `--keys` and a scoped `--creds`), announcing submissions into a space channel; flags\ninclude `--host`/`--port`, `--store`, `--space`/`--channel`, `--max-bytes`, and `--rate-limit`.\n\n## Plumbing\n\n`cotal __complete ` is the internal entry the shell-completion stubs call to emit candidates\nfor the current command line; you never run it directly. `cotal agent-bearer` is machine-facing\nplumbing on user-auth meshes: spawned agents exec it to print a fresh short-lived bearer from their\nspawn-time secret; you never run it directly either. Its local arm uses `--dir` to discover the\ncapability-gated loopback service. A remotely enrolled, already-granted agent instead receives\n`--exchange-url ` in its launch argv: that arm sends `{owner, actor, actorToken}` to the\npinned public exchange with no local capability, follows no redirects, and refuses every non-HTTPS\nURL because the actor token is the credential in the request body. (`cotal start` is a removed tombstone: it\nerrors and points you to `cotal spawn --detach`.)\n" + "body": "# `cotal` CLI reference\n\n> **Reference**: describes the TypeScript reference implementation (the `cotal` CLI), not the wire contract. · **For:** operators · **Wire contract:** [SPEC](../SPEC.md)\n\n`cotal` is the operator command line for the reference implementation: bring a mesh up, mint\nidentities, launch agents, watch what they do, and tear it all down. It is a thin client over the\nwire contract: the normative subjects and schemas live in the [SPEC](../SPEC.md); this page is\nlookup material for the commands, not a walkthrough; if you are new, start with\n[Getting started](getting-started.md).\n\n## Running it\n\n```bash\nnpm install -g cotal-ai # puts `cotal` on your PATH (needs Node 22+)\ncotal --help # every command, grouped\ncotal --version # cotal-ai version + each installed extension's (also `cotal -v`)\ncotal --help # one command's flags and usage\n```\n\n`npx cotal-ai ` runs it without a global install; in a dev clone, `pnpm cotal `\nruns it through `tsx` with no build step. Bare `cotal` prints help. Every command generates its own\n`--help`, usage, and shell completion from its declared flags.\n\nCommands come from the surfaces the binary composes: the base mesh CLI, the manager\n(`supervise`), and the delivery daemon (`deliver`), plus any operator-installed extensions.\n`cotal ext add ` installs any registry providers a package contributes: commands,\nruntimes, and local process lifecycle descriptors. The `web` dashboard and optional manager\nruntimes ship this way.\n\n## Commands\n\n| Area | Command | Purpose |\n|---|---|---|\n| Set up & lifecycle | [`setup`](#setup) | Guided, configure-only setup (installs, seeds personas; launches nothing) |\n| Set up & lifecycle | [`update`](#update) | Reconcile first-party extensions and check or opt into a coherent CLI upgrade |\n| Set up & lifecycle | [`up`](#up) | Start a local mesh (nats-server + JetStream), or boot a whole manifest with `-f` |\n| Set up & lifecycle | [`down`](#down) | Stop the whole stack, selected registered components, or a manifest deploy |\n| Set up & lifecycle | [`backup`](#backups) | Create an offline full-space or registry-only artifact from a preserved cut |\n| Set up & lifecycle | [`clean`](#clean) | Configurable cleanup: purge history (live), or wipe the local store / identity (stopped) |\n| Set up & lifecycle | [`meshes`](#mesh-registry) | List the running meshes on this machine |\n| Set up & lifecycle | [`use`](#mesh-registry) | Set the default mesh a bare `cotal spawn` joins |\n| Set up & lifecycle | [`status`](#mesh-registry) | Read-only diagnostics for setup, processes, and the selected mesh |\n| Agents & personas | [`spawn`](#spawn) | Launch an agent from a persona (foreground, or `--detach` via the manager) |\n| Agents & personas | [`models`](#models) | List connector model catalogs and variants from the manager |\n| Agents & personas | [`ps`](#managed-seats) | List managed agents and their mesh status |\n| Agents & personas | [`stop`](#managed-seats) | Ask the manager to stop a managed agent |\n| Agents & personas | [`attach`](#managed-seats) | Stream and drive a managed agent's terminal (pty runtime) |\n| Agents & personas | [`input`](#input) | Type one line into a managed agent's terminal without attaching |\n| Agents & personas | [`personas`](#personas) | List, show, edit, create, or remove local personas |\n| Agents & personas | [`supervise`](#supervise) | Run a manager daemon (the agent supervisor / control plane) |\n| Agents & personas | [`runtimes`](#runtimes) | List the agent runtimes the manager can spawn through and whether each is reachable |\n| Agents & personas | [`reconcile-gate`](#reconcile-gate) | Unfreeze an issuance gate left frozen by a crashed restart when the successor cannot boot-heal it (holder gone, complete CONNZ sweep) |\n| Messaging & watching | [`endpoints`](#endpoints) | List every endpoint in the live presence roster, including infrastructure |\n| Messaging & watching | [`describe` / `invoke`](#endpoint-control) | Resolve a v0.4 service's command surface off the wire; invoke one command by name |\n| Messaging & watching | [`send`](#send) | Send one message, then exit: DM a peer, post a channel, or ask a role |\n| Messaging & watching | [`channels`](#channels) | Inspect or set the channel registry |\n| Messaging & watching | [`history`](#history) | Clear retained message history |\n| Messaging & watching | [`console`](#console) | Live protocol view for a space (TUI, or `--plain` line stream) |\n| Messaging & watching | [`web`](#web) | Browser dashboard (installed as the `@cotal-ai/web` extension) |\n| Auth & meshes | [`mint`](#mint) | Mint a creds file for a space (static auth mode) |\n| Auth & meshes | [`login`](#login) | Sign in to a per-user-auth mesh's IdP (once per machine) |\n| Auth & meshes | [`logout`](#login) | Revoke the IdP session and clear the cached login |\n| Auth & meshes | [`actor`](#actor) | Manage a user-auth space's actor ledger (grant / revoke / list) |\n| Auth & meshes | [`doctor`](#doctor) | Credential-health diagnosis and repair (`doctor auth`) |\n| Auth & meshes | [`join`](#join) | Join a space as your own presence (interactive) |\n| Manifest | [`topology`](#manifest-deploys) | Validate and view a mesh manifest's access graph (read-only) |\n| Extensions & misc | [`ext`](#ext) | Install / remove operator CLI extensions |\n| Extensions & misc | [`completion`](#completion) | Print or install shell completion |\n| Extensions & misc | [`feedback`](#feedback) | Send feedback to the Cotal developers |\n| Extensions & misc | [`deliver`](#server-daemons) | Run the server-side Plane-3 delivery daemon |\n| Workflow runs | [`run`](#run) | Operate durable workflow runs: start, resume, list, inspect, answer a checkpoint |\n| Extensions & misc | [`feedback-intake`](#server-daemons) | Run a self-hosted feedback intake server |\n\nThe manifest modes of `up`, `spawn`, and `down` (`-f `) plus `topology` are covered\ntogether under [Manifest deploys](#manifest-deploys).\n\n## setup\n\n```bash\ncotal setup [--full] [--demo] [--yes] [--skills]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--full` | off | Redo the full guided flow (implies `--demo`) |\n| `--demo` | off | Also seed the guided expert team (`david`, `sven`, `me`) |\n| `--yes`, `-y` | off | Non-interactive accept-all (for agents / CI) |\n| `--skills` | off | Reconcile Cotal skills only through installed connector providers, plus `~/.agents/skills`. Refused with `--full` or `--demo`. |\n\nGuided setup is **configure-only**: it checks prerequisites, invokes installed connectors' declared setup providers, and\nseeds persona files, and it launches nothing (no mesh, no web, no manager). First run gets the\nnarrated flow; later runs print a status card. By default it seeds one `default` persona; the\n`david`/`sven`/`me` team is opt-in via `--demo`. `cotal status` points stale Claude skills and\nout-of-date `.agents` skills at `cotal setup --skills`, not unscoped `setup`. See [Getting started](getting-started.md) and, for\nmaintainers, [setup internals](setup-internals.md).\n\nWhen a mesh resolves, setup seeds that mesh's recorded `.cotal/agents` catalog, the same catalog a\nfollowing `cotal spawn` reads. It prints the absolute destination. On a fresh machine with no mesh it\nuses this folder and says why; when several meshes are available and none is selected, it refuses\nrather than choosing a catalog.\n\n## update\n\n```bash\ncotal update [--self] [--space ] [--server ] [--creds ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--self` | off | If a newer release exists, install that exact validated `cotal-ai` version globally and reconcile through the newly installed binary |\n| `--space`, `--server`, `--creds` | resolved mesh | Select the running manager whose continuity state is reported |\n\nWithout `--self`, `update` keeps the installed first-party surfaces coherent with the running\nbinary: it force-reconciles the four built-in connectors, then reinstalls other `@cotal-ai/*`\noperator extensions at the binary's exact version. Each extension runs in an isolated child, so one\nfailure cannot poison later replays. It then checks npm; a newer binary is an informational notice\nwith `cotal update --self` as the next command, not an automatic install.\n\nAfter disk reconciliation, `update` reads the selected running manager. A manager without a\ncustody generation is reported as `legacy`: it cannot preserve its manager-owned PTYs, so the\ncommand says that this is not a hot update and prints `exact`, `fork`, `fresh`, or `drain-only`\nfor every seat. This report sends no stop, preservation-commit, or replacement command.\nIt does not preserve a running PTY. Custody transfer is not available until the custody runtime is\nimplemented. Even after compatible custody generations exist, an incompatible native\n`@lydell/node-pty` or ConPTY ABI break remains an explicit per-seat maintenance cut.\n\nWith `--self`, the selected running manager is reported before any global install. When a newer\nrelease exists, Cotal then installs the exact version it validated, resolves and verifies that\npackage in npm's global root, then launches that binary with the same `--space` / `--server` /\n`--creds` selection to reconcile connectors and first-party extensions to the new generation. An npx\nor dev-clone invocation therefore installs and continues through a separate global copy; it never\nclaims the already-running process changed. If the binary is current, `--self` performs the normal\nlocal reconcile without reinstalling it.\n\nThird-party extensions are listed with their installed version and recorded spec but are not\nauto-updated in v1. Floating third-party updates require `@cotal-ai/*` peer-range validation and are\na future follow-up. A failed connector/extension install, npm metadata check, or requested global\ninstall is reported and makes the command exit nonzero. Independent extension attempts continue so\nthe output includes every failure; an unavailable npm registry does not undo a completed local\nreconcile, but the command still exits nonzero because it could not establish that the install is\ncurrent.\n\n## up\n\n```bash\ncotal up [--detach] [--open] [--space ] [--server ] [--channels ] [--runtime ]\ncotal up --user-auth --idp [--exchange-public-port --exchange-public-url [--exchange-trusted-proxy]]\ncotal up --tls-cert --tls-key # serve broker TLS (both, or neither)\ncotal up --restore [--restore-only registry] [--accept-missing-source]\ncotal up -f [--dry-run] [--runtime ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--server ` | auto (free local port) | Listen URL override |\n| `--host ` | none | Bind host override. With no `--server`, the broker URL is derived from it, so `--host ` alone is enough to make a mesh reachable at that address; a `--host`/`--server` pair naming different addresses is refused. A wildcard bind (`0.0.0.0`, `::`) keeps a dialable loopback URL. Recorded on the mesh and reused by every later manager launch, so a repair or resume keeps remote [`attach`](#managed-seats) working |\n| `--space ` | the folder's name | Space name |\n| `--store-dir ` | none | JetStream store directory |\n| `--channels ` | `.cotal/channels.json` if present | Channel-registry seed file (JSON). An explicit path that is missing is an error |\n| `--restore ` | none | Restore a completed offline backup before exposing the normal listener |\n| `--restore-only registry` | artifact selection | Restore only the registry component |\n| `--accept-missing-source` | off | Explicit disaster consent when the inode-bound preserved source is absent |\n| `--open` | off (auth) | Unauthenticated dev mesh: no JWT, no ACLs |\n| `--user-auth` | off | Per-user auth: people `cotal login`; connects are authorized against the actor ledger |\n| `--idp ` | none | With `--user-auth`: the IdP auth base URL to pin on first enable |\n| `--exchange-public-port ` | none | With `--user-auth`: add the public exchange face on this loopback port, for an HTTPS reverse proxy to forward to |\n| `--exchange-public-url ` | none | With `--exchange-public-port`: advertise the reverse proxy's HTTPS URL in discovery |\n| `--exchange-trusted-proxy` | off | With `--exchange-public-port`: attribute public failure buckets to the last `X-Forwarded-For` hop. Enable only when the listener is reachable solely through a trusted proxy; otherwise the socket address is used |\n| `--detach` | off | Run in the background (stop with `cotal down`) |\n| `--tls-cert ` | none | PEM certificate to serve TLS with. Must be given together with `--tls-key`. Before starting the broker, Cotal checks readability, private-key mode, key/certificate match, the validity window, and host coverage. `nats-server` accepts an expired certificate and leaves the failure to clients, so Cotal performs these checks first. The decision is recorded; a later bare `cotal up` keeps serving TLS |\n| `--tls-key ` | none | PEM private key for `--tls-cert`. Refused if group- or other-readable (tighten to `600`) |\n| `--file `, `-f` | none | Launch a whole mesh from a manifest |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--runtime ` | `pty` (or the manifest's, with `-f`) | Agent runtime for the mesh manager (`pty` built in; others are installed extensions, explicit-only). Resolved + probed before the broker starts; an uninstalled/unreachable runtime fails loud. With `-f`, overrides the manifest's runtime |\n| `--rotate-sys` | off | Rotate the space's system account and re-mint its two `$SYS` creds. Needs a stopped mesh; refused with `--open` |\n\n`cotal up` boots a local nats-server with JetStream and, in auth mode (the default), JWT auth and\nper-agent ACLs; `--detach` records the mesh so `cotal spawn` from any directory can find it. With no\n`--server`, it auto-selects a free port if the default address is taken; an explicit `--server`\nstays fail-loud on collision. `--detach` also brings up the control plane (delivery daemon in auth\nmode, then the manager). The `-f` form is a [manifest deploy](#manifest-deploys); see\n[Run a mesh](run-a-mesh.md).\n\n`--user-auth --idp ` starts the space's auth service alongside the broker: the NATS\nauth callout plus its capability-gated local exchange, and optionally the closed public exchange\nface configured by the three `--exchange-*` flags above. The service is torn down with `cotal down`,\nand a re-run of `cotal up` heals a dead service on a running broker. `--user-auth` and `--open`\ncontradict each other and are refused loudly; a running broker cannot change auth mode\nwithout a `cotal down` first. See [identity & auth](identity-and-auth.md).\n\n`--rotate-sys` renews the two `$SYS` credentials (`membership-observer`, `connection-evictor`).\nThey carry a 30-day expiry and nothing re-signs them in place, because the system-account seed is\nnever persisted, so they are renewed by issuing a **new system account** under the same broker\noperator and minting fresh creds against it. A plain re-`up` does **not** do this: it reuses the\nexisting trust record, and its `$SYS` creds along with it.\n\nThe rotation is safe to run on a real space, with one operational cost. The data account, the account\nsigning key, every agent credential minted from it, and the JetStream store are all untouched; what\ndies is the retired system account, and with it any out-of-band copy of the old `$SYS` creds, on every\nbroker that loads the rotated config. The cost is that **earlier full backups stop being restorable**\n(see below), so this is not a no-consequence operation. It needs the broker to restart on the rewritten\nconfig, so it runs as part of a boot:\n\n```bash\ncotal down\ncotal up --rotate-sys --detach # agents reconnect; nothing is re-provisioned\ncotal doctor auth # both $SYS creds healthy again, 30 days out\n```\n\nA rotation is a stopped, fresh boot, and anything that is not one refuses it, all for the same reason\n(the on-disk material and the broker it runs on must never end up on different generations):\n\n- a live mesh, because the running broker would keep serving the retired account;\n- an open mesh, whether that comes from `--open` or from `broker.auth: false` in a manifest, which\n has no system account at all;\n- `--restore`, because reinstating a trust root and superseding it in one command leaves no way to\n say which authority the mesh came up on;\n- an unfinished restore or resume attempt on this root, including one `cotal up` would recover on\n its own, because those paths can adopt a live listener and return without booting a broker;\n- a root that hosts more than one space, because the system account lives in the shared broker\n record and a rotation would retire every tenant's, while the root holds one `$SYS` cred pair\n pinned to one data account.\n\nTwo things to know before you run it:\n\n- **The retirement is config-load-bound.** Old `$SYS` creds are refused by any broker that loads the\n rotated config. A stale `nats-server` still running the *previous* config in memory would keep\n honouring them, so stop every broker for this root first. `--rotate-sys` refuses if this root's\n mesh is recorded as running, if anything unidentified is answering at the address it was given, or\n if the root's pid file names a live (or unreadable) process. Those are Cotal's own ownership\n records, not a scan of the process table: a `nats-server` you started by hand against this root's\n `server.conf` on some other port writes none of them and will not be seen. Do not run one.\n- **It invalidates earlier full backups.** A full artifact binds to the trust chain it was taken\n against, and that commitment covers the operator JWT and the system account. Every full backup\n taken before a rotation refuses to restore afterwards, so take a fresh `cotal backup` once the\n rotated mesh is up. `cotal up --restore` names this case when the data account still matches.\n\nThe commit is not atomic (a trust-record write plus two credential writes), so an interrupted\nrotation leaves the record ahead of the creds. That split is detected rather than silent: every\n`cotal up` on an auth mesh, and every `cotal doctor auth`, compares each `$SYS` cred's issuer against\nthe persisted record and names the retired account. `up` warns rather than refusing, because these\ncreds power the membership graph and live eviction, both of which degrade fail-soft; the mesh is not\nworth taking down over them. Re-running the rotation heals it, at the cost of one generation.\n\nWhile those creds are expired the mesh keeps delivering messages, but the\n[membership feed](delivery-daemon.md) and live connection eviction stay down; `cotal doctor auth`\nand the manager's log both name the credential and this repair.\n\n## down\n\n```bash\ncotal down\ncotal down --preserve-state [--store-dir ]\ncotal down manager [delivery auth web nats ...]\ncotal down web [--space ]\ncotal down -f | --run [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--file `, `-f` | none | Tear down this manifest's deploy |\n| `--run ` | none | Tear down one `spawn -f` run by id |\n| `--space ` | current mesh | With components: the mesh whose target-addressed components (e.g. `web`) to stop |\n| `--dry-run` | off | Print the manifest teardown or selected components, mutate nothing |\n| `--preserve-state` | off | Bare whole stack only: fence the manager, retain principals and durable state, stop and prove the stack down, then publish `ready` |\n| `--store-dir ` | `.cotal/nats` | With `--preserve-state`: the actual store path (required for a custom store) |\n\nBare `cotal down` stops the whole local stack in dependency order. Positional component names stop\nonly those self-registered local processes; for example, `cotal down manager` leaves delivery and\nthe broker running, and `cotal down web` is available when the web extension is installed. A\ncomponent that starts target-resolved (the web dashboard) is stopped the same way: `cotal down web`\nresolves the mesh the same way as `cotal web` (registry current mesh first, `--space` to name one), so\nit works from any directory; the other components always stop under the folder you run it in. The\n`-f` / `--run` forms tear down a [manifest deploy](#manifest-deploys) without stopping the whole mesh\nand cannot be combined with component names. Stopping `nats` alone is refused while an unselected\nregistered daemon is still live; include those components or use bare `cotal down`.\n\n**Teardown verifies pinned process identity before signalling.** PIDs are recycled by every OS,\nso a recorded pid alone is not a durable target identity. `up` records each stack process's\ncreation identity in a sibling `.identity` pin, which holds the pid and the process start\nreported by the OS. Every stop path, including `down` for the broker, web and extension components,\nand the manager, delivery and auth-service stops, applies the same rule. A pin that names a different\nstart means the pid was reused, so teardown refuses and preserves it. A torn or unreadable pin also\nrefuses. Once the recorded process is stopped, rerunning teardown clears the stale record\nautomatically.\n\nThe first teardown after upgrading a running pre-pin stack has a narrower guarantee. A live record\nwith no identity pin is signalled after a loud warning that it predates identity pinning. Restarting\nthe component writes the pin, so later teardowns receive full match and mismatch protection. The\nsame warning applies on platforms where no stable start token is available.\n\nNormal `down` remains destructive at the logical identity/durable layer. `--preserve-state` is a\ndifferent maintenance transition: it suppresses leave/deprovision cleanup, persists the manager's\nsame-principal resume inventory, stops the entire stack without removing run/auth artifacts, and\npublishes a stable inode-bound cut only after every recorded process is proven stopped and the exact\nrecorded NATS endpoint is unreachable. A missing or stale broker pidfile never counts as stopped. The\nattempt is bound durably before the manager is fenced, the resume document and attempt-bound\n`cut-intent` are fsynced before manager commit, and the manager's commitment itself is journaled\n(`cut-committed`) before any process stops. A retry after a crash at any of those boundaries reuses\nthe exact recorded attempt and finishes the remaining stop and endpoint proofs idempotently, without\nneeding the (by then intentionally dead) manager. A partial cut never publishes `ready`. It cannot\nbe combined with component names, manifest teardown, or `--dry-run`.\n\n## clean\n\n```bash\ncotal clean --force\ncotal clean restore-attempt --attempt --force\ncotal clean restore-fallback --attempt --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | `history`: target mesh |\n| `--dms` | off | `history`: also clear DM history |\n| `--store-dir ` | `.cotal/nats` | `store`/`all`: JetStream store directory |\n| `--force` | none | Required: destructive, no prompting |\n| `--attempt ` | none | `restore-attempt`: exact stale pre-commit attempt; `restore-fallback`: matching healthy committed restore |\n\nOne configurable cleanup verb; every target requires `--force`.\n\n- `history` purges the retained message backlog on the **running** broker (channels, plus DMs\n with `--dms`). The same operation as [`history clear`](#history), which stays as an alias.\n- `store` deletes the **stopped** mesh's JetStream store (`.cotal/nats`): streams, durable\n consumers, and messages. This is the reset for stale on-disk broker state, e.g. durables\n minted by an older, incompatible Cotal generation surviving a `down`/`up` cycle.\n- `all` is `store` plus the space identity (`.cotal/auth`), the local creds and markers tied to\n it, any crash residue a normal `down` would have swept (stale pidfiles, `run/`), and the mesh's\n registry entry; the next `cotal up` mints a fresh identity.\n\n`history` needs the mesh up; `store` and `all` refuse while any recorded mesh process is still\nalive or any same-root recorded broker endpoint remains reachable (run `cotal down` first). They\nalso refuse outright on a root that holds accounts for several spaces: the store and the broker\ntrust record are shared by every space on the broker, so both targets would take out all of them\nand no `--space` can narrow that. `down`, `backup` and `up --restore` refuse there for the same\nreason. `cotal status` lists the tenants on such a root. Personas\n(`.cotal/agents`) and logs are never touched. A custom\nstore location is not recorded anywhere, so `--store-dir` must repeat whatever the mesh was\nlaunched with. Custom cleanup targets must contain either the Cotal store-generation marker or a\nreal `jetstream/` store directory; filesystem roots, project roots, and Cotal auth/maintenance trees\nare always refused.\n\n`store` and `all` also refuse every maintenance journal state. After a healthy committed restore,\n`restore-fallback` is the only supported way to remove the recorded unchanged old-store inode; it\nnever deletes the active target, requires both the exact attempt id and `--force`, and retires the\ncompleted restore journal so a later `down --preserve-state` can start a new backup cycle.\n\n## Backups\n\n```bash\ncotal down --preserve-state [--store-dir ]\ncotal backup create [--only full|registry] [--store-dir ]\ncotal up --restore [--restore-only registry] [--accept-missing-source]\n```\n\nBackup is offline-only. It requires the stable `ready` record from `down --preserve-state`, an exact\nstore match, no live recorded process, and an unreachable exact endpoint from the recorded cut.\nThat endpoint is probed immediately before cloning, so a live broker with a missing or stale pidfile\nis still refused. It claims the cut, reflink/copies the stopped source to a\nprivate attempt clone, and opens only that clone on a random loopback bootstrap broker with an\nindependent parent/deadline watchdog. It validates the canonical stream and pull-consumer inventory,\nwrites native snapshots with consumers excluded, and stores conservative contiguous ACK-floor\ncheckpoints separately. The original store is never opened by the backup broker, and the stack is\nnot restarted implicitly. Artifact destinations must not overlap the preserved source or maintenance\nattempt tree. Restore artifacts and targets likewise cannot nest inside or contain each other, the\npreserved source, or the maintenance attempt tree.\n\nStopped client-managed KV ordered consumers are ephemeral read residue, not backup state. Backup\nignores only the pinned client's exact stopped shapes: ordinary last-value watchers and the\nwhole-bucket scanner that uses all-history delivery to collapse concurrent tombstones. A bound\nconsumer or any lookalike with a different filter, inbox, lifetime, or other config is still refused.\n\n`full` is the default and indivisible: channel registry, CHAT/DM/TASK/INBOX/DLV, ACL, MEMBERS, and\nvalidated durable checkpoints. `registry` is the sole partial artifact. Presence, derived membership\nfeed, leases, native ephemeral/history consumers, credentials, keys, tokens, owner secrets, and actor\nledger files are excluded. `full` means every transferable message and registry stream, not every\nJetStream resource: endpoint submissions/facts/events/timers/workflow state, contract artifacts, and\nthe records/auth/session stores are nonportable control state. Restore recreates those streams empty\nwith their canonical configs before exposing the normal listener, so active endpoint runs,\nlifecycles, and sessions do not cross a backup. Artifacts are exclusively created `0700`;\nsnapshot/checkpoint files and\nthe manifest are `0600`; `manifest.json` is written last with exact sizes and SHA-256 values. The\ndirectory is trusted operator input: hashes detect corruption, not malicious rewriting.\n\nRestore validates and stages the exact allowlisted artifact bytes before moving or creating a store.\nIt requires the same space and existing trust state. The whole pre-commit window holds a journaled\nliveness claim (coordinator, watchdogs, brokers, absolute deadline): ordinary `up` and a repeated\n`up --restore` refuse while the claim is live, and a stale attempt is recovered only after the\ndeadline has elapsed and every recorded owner is proven dead. A retried `up --restore` handles this\nautomatically; an operator can also recover it explicitly with `cotal clean restore-attempt --attempt --force`. Nothing\never rolls back a live attempt. A registry-only artifact restores as registry-only whether or not\n`--restore-only registry` is passed; omitted infrastructure is always created and the exact\npost-restore stream inventory is asserted before commit intent. Ordinary `up` from a preserved cut\nresumes only the exact recorded source store and runtime; a contradicting `--store-dir` or\n`--runtime` fails in preflight. Authenticated restores validate the complete\nspace trust bundle before staging, including nkeys, seed matches, JWTs, signers, and space binding;\nfull restores commit to the validated operator, system-account, data-account, and active-signer root\nchain in addition to the static/user authority fingerprint. Because the system account is part of that\ncommitment, a [`cotal up --rotate-sys`](#up) makes every full artifact taken before it unrestorable\nagainst this root: take a fresh full backup after each rotation. The composed commitment is revalidated\nimmediately before store mutation and never includes secret seeds. Restore never creates fresh auth.\nSame-path restores atomically retain the old\nsource at the journaled fallback path; alternate targets retain it in place; a missing canonical\nsource needs explicit `--accept-missing-source`. Quarantine and target restores use current canonical\nconfigs on isolated random-loopback brokers, never expose native snapshot consumers, and publish a\ncommit-intent immediately before the normal listener starts. Archive bytes never instantiate the real\ntarget: after quarantine validation, every stream is re-snapshotted from the validated quarantine\nstate into attempt-owned sanitized files, and the target is restored solely from those. Before that boundary, failure rolls back\nthe attempt-owned target; after it, ambiguity preserves both stores and records forward-repair\nrecourse. The cooperative maintenance lock excludes Cotal commands, not arbitrary raw NATS processes.\n\nBootstrap brokers in every auth mode, including open, mount the store under a local account with\nrandom operation-specific logins only, each carrying the exact per-phase subject permission matrix;\nnormal static credentials and user-auth sentinel/bearer connections are rejected, and no auth\nservice or callout starts. Open mode differs only in its account label, never in authority. Inventory, each stream snapshot,\nrestore initiation, exact upload id, validation, and each checkpoint recreation use separate exact\nauthorities. Every checkpoint carries the source stream's message/first/last sequence state and must\nmatch its snapshot record before mutation; core then derives and validates the only allowed start\npolicy. TASK is not a CLI exception: the same core checkpoint API recreates its canonical `DeliverAll`\nWorkQueue durable because acknowledged tasks are absent from retention and NATS forbids a\nstart-sequence policy there. Registry-only restore creates every omitted canonical stream and transient\nbucket on the isolated target before the normal listener is exposed. It deliberately does not resume\nretained agents or recreate their DM/DLV/TASK/ACL state; their identity material stays retained and\nstopped rather than being reprovisioned into a partial restore.\n\nAfter listener readiness, the manager starts attempt-bound, validates retained credentials/tokens\nwithout granting or reprovisioning, and resumes the exact persisted principals under cleanup\nsuppression. Registry-only restore uses the same flow with an empty agent set. `commitResume` is an\nidempotent validation barrier only: success must be `awaitingFinalize` with an attempt-bound 64-hex\ncommit token and does not release suppression. Under the workspace lock, the CLI first fsyncs that\nexact evidence as `manager-committed` (restore) or `resume-committed` (ordinary resume), then calls\ntoken-bound `finalizeResume`; only an `active` response for the exact token releases suppression. The\nCLI records the same token in finalization evidence before a restore becomes `active`, or before an\nordinary resume retires and consumes the marker. Re-entry from either committed state skips the prior\nidempotent activation/commit phases, retries finalization with the durable token, and finishes the\nworkspace transition. Failure before finalization preserves the committed state and cleanup\nsuppression; it is not rewritten through a degraded transition. Re-entry between any two earlier\nboundaries reuses the same attempt and may retry the idempotent phases without deleting retained state. A missing or\nchanged per-agent dependency is a named fail-closed result; the journal becomes degraded and remains\navailable for forward repair. A retry from `resume-intent`,\n`resume-active`, or `resume-degraded` reuses the same attempt and inventory after the prior listener is\nproven stopped. Every normal restore listener has an unguessable attempt-bound NATS server name. The\nCLI fsyncs its exact name/nonce, canonical endpoint, process owner, and generation-bound target identity\nimmediately after spawn. Re-entry accepts a surviving listener only when its INFO server name, live PID\nrecord, endpoint, and target identity all match that proof; degraded restore repair then moves through\nthe guarded workspace transition only after manager commit. If an uncommitted bound owner is provably\ndead, recovery retires that exact proof under the maintenance lock and binds a fresh listener for the\nsame attempt, endpoint, and target with a new nonce and server name. A live foreign/mismatched listener\nor ambiguous owner is preserved and refused, never adopted by reachability alone. A reconstructed\ncommit/degraded attempt without either the exact bound proof or a durable dead-listener replacement\nrecord fails closed even when the recorded port is free. A later ordinary startup may pass an `active`\nrestore only when its details prove manager commit and its exact recorded listener is dead.\n\n## Mesh registry\n\n```bash\ncotal meshes\ncotal meshes add # guided, on a terminal\ncotal meshes add --server [--root ] [--mode auth|open|user] [--tls] [--force]\ncotal meshes add --mode user (--user-auth-file | --from )\ncotal meshes rm [ …] [--force]\ncotal use \ncotal status [--space ] [--server ] [--components]\n```\n\n`meshes` lists the meshes this machine knows; a `*` marks the `current` default a bare\n`cotal spawn` joins.\n\nRun on a terminal with the space or `--server` missing, **`meshes add` is guided**: it asks for the\none thing that cannot be derived (the broker URL), probes it, and tells you what answered - open or\nrequiring credentials. It then offers the spaces your `--root` already holds credentials for, states\nthe mode as a fact about that broker rather than asking, and shows the exact record before writing\nanything. A broker that does not answer, or a space name already registered, becomes a choice rather\nthan an error. Anything you pass on the command line is taken as given and not asked again. Without\na terminal - a script, an agent, CI - nothing prompts and the flag form's errors stand\n(`COTAL_NO_PROMPT=1` forces that too).\n\n`cotal up` and `cotal down` maintain their own records. `meshes add` registers a mesh they cannot\nspeak for: one running on another machine, a shared broker, a hosted space. `--root` is the folder\nwhose `.cotal/auth` holds that mesh's credentials and whose `.cotal/agents` holds its personas.\nThe default is the project you run it in. The registry stores that path, never a secret. `--mode`\ndefaults to `auth` when the root holds the space's account record and to `open` otherwise. The\nbroker is probed before anything is recorded, so a wrong address, or credentials that mesh will\nnot accept, fails here instead of at the first `spawn`; `--force` records without verifying (and\nreplaces an existing record).\n\nA hostname or public address is registrable only when the connection will **require TLS**. Pass\n`--tls`, or use a `tls://` URL. The scheme is recorded as enforced intent, so every later dial\nthrough the record demands the handshake (and `meshes add tls://…` against a plaintext broker is\nrefused at registration). Without required TLS the fence admits loopback and private-overlay\nliterals only. RFC1918 addresses are refused in both modes because a cafe LAN is private but does not belong to you.\n\nA **user-auth** mesh registers from supplied pinned trust, never guessed: `--user-auth-file`\ntakes the bundle exported where the mesh runs; `--from` asks before it dials the address at all,\nthen fetches its `/.well-known/cotal-mesh` discovery document (HTTPS only), displays the pins, and\nasks again before adopting them. Neither fetch follows redirects: a 302 can move a pinned fetch\nonto plaintext or onto another host, so it is refused rather than followed, and the pinned\nexchange must itself be an `https://` URL, except for an exchange on this machine, where plain\n`http://` is accepted for a loopback *literal* (`127.0.0.1`, `::1`, any spelling of them) but not\nfor `localhost`, which is a name rather than an address. Registration verifies that the exchange\nanswers `/health` and `/jwks` as the pinned issuer. It also verifies that the broker refuses a bare\nconnect; that auth-required refusal is the pass. The sentinel credentials land in a 0600 file under\nthe entry's root; the registry records only the path.\n\n`meshes rm` drops records. It never stops a mesh. For a mesh running on this machine `cotal down`\nis the right verb, and `rm` says so unless you pass `--force`. A hand-added record is removed by\n`meshes rm`, by an `add --force` replacement, or by a `cotal up` that actually starts the broker for that same space, server and root, which becomes that\nmesh and so takes the record over (a `cotal up` for that space anywhere else refuses instead).\nNothing that merely *infers* a record is stale touches it: an\nunreachable broker is listed `offline` and stays, and `cotal down` / `cotal clean all` leave it\nalone even when it shares a root with the project they are tearing down, because nothing on this\nmachine could write it back.\n\n`use ` sets that default; the selection applies from every directory,\nincluding inside another mesh's project. `status` is a read-only report: machine prerequisites\n(starting with the installed `cotal-ai` version), the installed extensions and their versions, this\nfolder's `.cotal/`, the recorded meshes, and a live snapshot of the selected mesh (roster, channels,\nmembership feed). Stale Claude skills and out-of-date `.agents` skills recommend `cotal setup --skills`,\nnot unscoped `cotal setup`. `status` takes `--space` / `--server` to pick the mesh to inspect; it starts\nnothing.\n\nPersona rows name the catalog they describe. If this folder and the selected mesh use different\ncatalogs, status names both and marks which one spawn launches from. A green `default` means the file\npasses the same agent-file loader spawn uses; a present but invalid file is reported as invalid.\n\n`cotal status --components` adds a fail-loud per-component health pass. It reads **each\ncomponent's own control surface**, rather than treating a PID, a lease, or a successful probe of a\nsibling as proof that the component serves. It prints one of `serving`, `absent`, `not-serving`, or\n`refused` for each component and exits `0`, `1`, `2`, or `3` respectively (the highest observed\nstate wins):\n\n- **manager**: local PID record, its liveness-lease holder and PID, then the manager's own typed\n `status` service reachability from this host. Manager builds that do not report static\n reconciliation say `static reconciliation not reported by this manager build`; the line stays\n visible even when the manager is otherwise `serving`.\n- **delivery**: local PID record, its ready lease (`ready` is the daemon's own bound-control\n signal), and the latest `renewal.json` adoption verdict. A re-signed credential and a\n broker-accepted adoption stay distinct facts.\n- **web**: local PID record and the dashboard's own loopback `/api/meta` response, which must name\n the same PID and its requested port. A different process on the port, an unreadable PID command,\n or an unrecognizable process record is `refused`, not a green default-port guess.\n- **broker**: the registered mesh URL dialed from this host with its recorded TLS requirement.\n\n`absent` means Cotal has no live local component record (or has a stale record); `not-serving`\nmeans the component record is live but its service/readiness surface did not answer or is not ready.\nThose are intentionally separate exit cases. A failed or unreadable probe is `refused`, never an\nabsent component or a clean zero.\n\n## spawn\n\n```bash\ncotal spawn [] [--detach] [--name ] [--agent ] [--model ] [--variant ] [--prompt ] [--cwd ]\ncotal spawn -f [--dry-run]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | resolved mesh | Target space |\n| `--server ` | registry entry | Broker URL override |\n| `--creds ` | none | Control-caller creds for an off-registry manager (`--detach` only) |\n| `--name ` | persona's `name:` | Presence-name override (does not choose the persona) |\n| `--config ` | none | Persona catalog name or file path; wins over the positional |\n| `--agent ` | persona's `agent:`, else `COTAL_DEFAULT_AGENT`, else `claude` | Connector type (`claude`, `opencode`, `jcode`, `hermes`, and so on) |\n| `--role ` | persona's `role:` | Role override |\n| `--model ` | persona's `model:` | Model override |\n| `--variant ` | persona's `variant:` | Model variant override (connector-defined; e.g. OpenCode reasoning tiers) |\n| `--cwd ` | this cwd | Working directory to root the agent at |\n| `--prompt ` | none | Initial prompt auto-submitted at start |\n| `--resume ` | none | Fork an existing session id into the mesh; only connectors that declare resume support accept it (see [the matrix](connectors.md)) |\n| `--events` / `--no-events` | off | Publish the session's structured event plane to its own event channel |\n| `--share-tools ` | none | Share named operator MCP servers with the agent |\n| `--subscribe ` | persona's | Channel read-set override |\n| `--allow-subscribe ` | = subscribe | Read-ACL override |\n| `--allow-publish ` | deny | Post-ACL override |\n| `--detach`, `-d` | off | Launch via the manager into a detached PTY (reattach with `cotal attach`) |\n| `--on ` | class anycast | With `--detach` only: pin the launch to one manager instance id (the whole id, as `ps` prints it). Refused on a foreground spawn (no manager to pin), with `-f` (a manifest deploy launches through the manager class queue), and when empty |\n| `--file `, `-f` | none | Deploy a manifest onto the running mesh |\n| `--dry-run` | off | With `-f`: print the plan, mutate nothing |\n| `--allow-stale ` | none | With `-f`: waive named stale agents (apply-only) |\n| `--runtime ` | manifest's | With `-f`: override the manifest's runtime |\n\n`--events` turns on the session's **event plane**: a stream of structured events describing what\nthe agent did, rather than the prose it wrote, on a channel of its own. The channel is named after\nthe agent's principal, `events..`, never after its display name, because two live\nagents are allowed to share a display name and would then share a stream. The launch grants publish\nrights on that channel alone, foreground and detached alike, and a connector that does not\npublish an event plane refuses the flag rather than starting a session whose events have nowhere to\ngo.\n\nThe flag and the grant are separate on purpose. Holding publish rights on a channel is not a request\nto publish to it, so writing an event channel into an agent file's `allowPublish` does not turn the\nplane on: only the launch does.\n\nThe persona (`--config` > positional > `COTAL_DEFAULT_PERSONA` > `default`) is loaded from the\ntarget mesh's `.cotal/agents/`; the launch flags override the file. Foreground runs the agent\nattached to your terminal; `--detach` hands the launch to the running manager. Both modes get the\ndurable backstop on a mesh that runs the delivery daemon; `--live-only` skips it for a foreground\nspawn (messages posted while it is disconnected are then not replayed). A foreground exit retires\nthe agent's creds and broker footprint, like a manager despawn. A `--detach` spawn is an\n**action**: the manager accepts it and returns the allocated identity at once, then the launch\nfollows to a terminal outcome rather than blocking (see [the control surface](control-surface.md)).\nSee [Connect Claude Code](connect-claude.md) and [Agent files](agent-files.md); `-f` is a\n[manifest deploy](#manifest-deploys). (`cotal start` was merged into `cotal spawn --detach`.)\n\n## models\n\n```bash\ncotal models [--agent ] [--refresh]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--agent ` | all registered connectors | Connector whose catalog to list |\n| `--refresh` | off | Ask the connector to refresh its provider cache |\n\nAsks the running manager for each connector's model catalog (model ids plus their variants)\nfor connectors that expose one. OpenCode and Codex query harness/provider surfaces; Jcode reads\nproviders that enable `model_catalog = true` in the operator Jcode `config.toml`. Jcode's listed\neffort tiers render as `variants (declared, not provider-verified)`, and launch can still refuse one.\nA connector without a catalog says so. Pick a result with `cotal spawn --model --variant `,\nwhere `` is the model id as the catalog printed it. OpenCode and Codex ids are the full\n`provider/model`; Jcode ids are bare (`opus-5`, not `cliproxy/opus-5`), because the provider is\nselected by the operator's Jcode config and a prefixed id is refused at launch with the bare form\nnamed.\n\n## endpoints\n\n```bash\ncotal endpoints [--space ] [--server ] [--creds ]\n```\n\nLists the mesh presence roster: agents, the manager, and any other protocol endpoint, with each\nendpoint's role, kind, status, and current activity. Unlike `ps`, this is a read-only presence view;\nit is not limited to child processes owned by the manager.\n\n## Endpoint control\n\n```bash\ncotal describe [--space ]\ncotal invoke [--args ''] [--space ]\ncotal invoke --name [--admin] [--space ]\n```\n\nThe generic v0.4 service surface. `describe` resolves a registered endpoint's command set off the\nwire - the reserved `describe` command answers the registered contract digests, the schemas are\nfetched from the space's content-addressed contract store, recompiled, and verified against those\ndigests - and prints each command with its capability class and targeting shape. `invoke` calls one\ncommand by name: `--args` is a JSON object validated against the fetched input schema *before*\npublish; a targeted command takes `--name ` (resolved to the agent's current principal through\n`inspect`) or `--self`. `--admin` uses the admin instrument credential, whose cross-agent reach rides\nthe operator-only `any` authorization mode. Neither command has compile-time knowledge of any\nendpoint's schemas - this is the same trust chain every built-in control command now uses. Needs an\nauth mesh: the manager registers its service on both static and per-user meshes (a signed-in user\nrides their bearer; each visible or invoked command still requires its existing grant, and cross-agent\nreach needs the `admin` scope). An open mesh has no service registry.\n\n## Managed seats\n\n```bash\ncotal ps [--on ] [--wide | --json] [--space ]\ncotal stop --name [--on ] [--space ]\ncotal attach --name [--on ] [--no-reconnect] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | none | Managed agent to stop / attach (required) |\n| `--on ` | class anycast (`ps`: class scatter) | Pin to one manager instance id (multi-manager space); takes the whole id as `ps` prints it, not a prefix. An empty value (`--on \"\"`, an unset shell variable) is refused, never treated as absent |\n| `--wide` (`ps`) | off | After each seat's compact row, print extra operational facts the manager records: `cwd`, `pid`, spawner, lifecycle uid, and the owning manager's instance id and host. Model and requested variant stay in the identity row rather than printing twice. A fact the manager did not record (for example a runtime with no real process) prints nothing, never a placeholder |\n| `--json` (`ps`) | off | Machine-readable: one JSON object per seat per line, copied unchanged from the manager row. Instance headers and errors go to stderr, so stdout contains only rows. Mutually exclusive with `--wide` |\n| `--no-reconnect` (`attach`) | off | End the attach when its session ends, instead of re-establishing it. For scripts that want one run and one exit code |\n\nThe human `ps` row is presentation text and is not a stable parsing target. Scripts use `--json`,\nwhich is the machine-readable row contract.\n\nThese are operator clients over the running manager's control plane. The default row includes the\nconnector, model pin, optional requested variant, and runtime as operational descriptors for the\nmanaged row. They do not make a shared display name a unique protocol identity; use `--json` when\nunambiguous owner+actor attribution is required. An omitted variant means no override was requested;\nCotal does not invent an effective provider default it cannot observe. `ps` also prints two state\nfacts per managed agent, because they answer different questions: the process fact from the manager's\nown runtime handle (`running` with its uptime, or `exited` with how long it ran), and the mesh fact\nfrom the roster (`idle` / `working` / `waiting` / `mesh offline`, or `not in roster` when the seat has\nno presence row at all: a seat that has not joined yet, or one that never did). A seat can be\n`running` and `mesh offline` at once: the process is alive and its presence has lapsed. On a user-auth\nmesh `ps` also renders each managed agent's last credential-refresh outcome, fail-closed.\n\n**Mode split (chosen up front, never try-scatter-then-degrade):**\n\n- **Static / open mesh.** Bare `ps` is a **class scatter**: it freezes the live manager class from\n the records registry, merges every registered instance's agents grouped and attributed per\n instance, and a non-answering instance is shown as `registered, no answer within the deadline`\n (never silently omitted). That label is the whole claim: the instance is registered and did not\n answer. It does not say the host is down, because a dead host never deregisters itself and a\n live one can be slow; if it is gone, deregister it.\n `--on ` pins the read to one exact instance id instead. A wrong pin fails loud\n rather than falling through: a well-formed id that no live manager carries is reported as\n `manager instance did not answer` (nothing else is asked), and a credential without that\n instance's rail is reported as refused by the broker, not as an unresponsive manager. A manager\n that answers with a refusal is shown with its own cause; \"no manager reachable\" is said only when\n nothing answered at all. If the scatter's own registry read fails (the freeze or the reconcile),\n `ps` says the manager registry could not be read rather than pronouncing on the managers, which\n may all be up.\n\n**`stop` and `attach` route by seat locality.** A seat can only be stopped or attached by the\nmanager actually running it, and the class queue does not know which one that is. So on a\nstatic/open mesh both verbs first ask every registered instance which one hosts the named seat, then\naddress that instance directly. This happens by default; you do not need `--on`.\n\n`--on ` remains the override, for when you already know where the seat lives or the\nlookup itself is degraded. It is also the **only** route on a **user-auth mesh**: a ledger-scoped\nbearer does not hold the registry-read rows the lookup needs, so there the verbs stay on the class\nqueue unless you pin them yourself.\n\nIf no reachable instance has the seat, the error reports how many managers answered and names\nthose that did not. It does not collapse that state into a bare `no agent `. That distinction matters\nbecause a single manager cannot tell \"hosted elsewhere\" from \"does not exist\": it answers\n`not-found` for both.\n- **User-auth mesh.** `cotal ps` reports what **one** manager knows about your agents (an `ep.one`\n read against the manager's in-memory roster, owner-filtered). It does **not** report other\n manager instances. It cannot tell you that one is down: an unreachable manager is absent\n from the list. Completeness across a multi-manager user-auth space is not claimed.\n A manager that does not answer fails the command outright (exit non-zero), rather than printing\n an empty list that could be read as \"no agents\". Your ledger row needs the `admin` scope to\n reach `ps` at all; `spawn` alone is refused by the broker (the ep tier boundary).\n\n`attach` streams and drives an agent's terminal on the `pty` runtime; detach with the escape key\n(Ctrl-] by default; see [`COTAL_DETACH_KEY`](config.md)). It does so over a one-use, holder-bound\nmesh session ([SPEC](../SPEC.md) §13.6): the manager replies with a signed session grant (never a\n`127.0.0.1` URL), the CLI redeems it once over the broker, and the browser console (`cotal console`)\ndrives the same session. `stop` and `attach` need a running manager to talk to. On a static mesh\nthey are cross-agent admin operations. On a user-auth mesh, your own agents (any agent under your\nowner) need only the `spawn` scope; another owner's agent needs `admin` on your ledger row\n([identity & auth](identity-and-auth.md)). Launch detached agents with [`spawn --detach`](#spawn).\n\n**`attach` reconnects when the link dies.** A session lives on a network link, and a laptop that\nsleeps, a VPN that drops or a wifi handover kills it. When that happens `attach` prints\n`[cotal: connection lost, reconnecting]` on stderr and starts asking the manager for a new session:\na fresh grant, a fresh per-session credential, a fresh connection, so every attempt re-runs the same\nauthorization the first attach did. On success it prints `[cotal: reconnected]`, the manager repaints\nthe seat's current screen the way it does for any attach, and you carry on in the same terminal.\nRetries wait 1s, 2s, 5s, 10s, then 30s, for as long as the seat exists. The detach key is read the\nwhole time the loop runs, the waits and the attempts alike, so a reconnect never traps you: press it\nwhile a session is being established and the attach ends there, and a session that lands behind the\npress is handed back to the manager rather than left holding a slot. Everything else you type while\nthere is no session is dropped rather than queued, so keystrokes aimed at a terminal that turned out\nto be frozen, Ctrl-C included, are not delivered to the agent by a reconnect you did not know had\nhappened. That starts before the first session, not at the first reconnect: at a terminal, `attach`\nreads and drops what you type while it is still resolving the mesh, so a key struck at a prompt that\nhas not come up yet does not reach the agent when it does.\n\nA **pipe** carries script input. For example, `printf 'ls\\n' | cotal attach --name web` is\nbuffered until the session opens. Buffering continues across reconnects, so\n`tail -f log | cotal attach --name web` does not lose the part of its feed written while the link was\ndown. Only a terminal gets the reader; `--no-reconnect` keeps the old behaviour on both.\n\nIt stops on its own when reconnecting cannot help, and says why: a manager that refuses the attach\nexits non-zero with the manager's own message, and a reconnect that finds the seat no longer there\n(despawned, or its agent exited while the link was down) exits cleanly with `seat is gone`.\nA refusal that could still pass, such as a manager at its session ceiling, is relayed in the\nmanager's own words while the loop keeps trying, once per refusal rather than once per attempt.\nPressing the detach key, or the agent's process exiting while you are attached, ends the attach as\nit always did. `--no-reconnect` turns all of this off and restores the single-session behaviour,\nwhich is what a script wants.\n\nEach reconnect also hands the abandoned session back to the manager, over the first link that can\ncarry the message, so an attach that flaps does not eat the manager's session slots one outage at a\ntime. If that message never gets a link, the attach says so when it ends.\n\nWhich mesh `attach` resolves also decides **whose trust it redeems with**. Redeeming a session grant\nmeans minting a short-lived, session-scoped credential from the space's seed, and that seed comes\nfrom the root the mesh resolved to, never from a `.cotal` found by walking up from whichever\ndirectory you happen to be standing in. The difference is not hypothetical: `~/.cotal` exists on\nevery install because the mesh registry lives there, so a command run anywhere under your home\ndirectory but outside a project used to mint from your home directory's trust and present it to a\nbroker that trusts a different chain, which surfaced as a bare authorization failure that named\nnothing. A directory that does hold another chain for the same space is now reported on the way\npast, and not obeyed:\n\n```text\n! this directory resolves to /Users/you, whose .cotal/auth holds a DIFFERENT trust chain for space \"team\".\n attach used /Users/you/projects/app, the root this mesh resolved to. The other one is not being used, and is worth a look.\n```\n\nWhen the resolved mesh holds no seed at all, `attach` refuses and names what it resolved, the broker\nand the root, instead of describing a directory it did not use.\n\nTerminal bytes stream over the mesh; the manager's own HTTP/WS face serves the console. That endpoint binds\n**loopback by default**, so nothing is exposed by accident; `cotal up --host ` passes its bind\naddress down, which is what lets you attach to an agent whose manager runs on another machine. A\nbare `cotal supervise` and an embedded manager stay machine-local. Set it directly with\n`supervise --console-host `.\n\nThat address is **recorded on the mesh** and carried forward, because it is a decision rather than\nsomething later commands can work out for themselves (a broker dial address is not a manager bind\naddress). Every later manager launch for the same mesh reuses it, including a same-root `cotal up` repair,\nan adopted preserved or restored listener, and a `spawn -f` manifest deploy. A manager replacement\ndoes not quietly move a reachable attach face back to loopback. Passing `--host` again overrides it,\nso you can widen or narrow exposure whenever you like; a mesh that never asked stays loopback-only\nand records nothing.\n\nBecause that face carries terminal read and write for every managed agent, it is credentialed in two\ntiers. A mesh caller receives a **ticket** bound to the single agent the manager just authorized,\nsingle-use and short-lived, so one authorized attach can never be re-pointed at someone else's\nagent. The **console token** is the operator's own, reaches every agent, and is printed only to the\nmanager's output. The roster, the live feed, and the PTY stream all answer `401` without one; the\nstatic console shell is served openly, since it describes no agent.\n\n## input\n\n```bash\ncotal input --name --text [--no-enter] [--on ] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which manager to reach |\n| `--name ` | | Managed agent to type into (required) |\n| `--text ` | | The text to type, taken verbatim (required) |\n| `--no-enter` | off | Type the text and stop there, without pressing Enter |\n| `--on ` | class anycast | Pin to one manager instance id using the same rules as [`attach`](#managed-seats) |\n\nTypes one line into a running agent's terminal, as if you had typed it there, and returns. This is\nthe half of [`attach`](#managed-seats) that a program wants: `attach` is a live stream that holds a\nsession open and expects a terminal on your side, so a script, a cron job or a web UI cannot use it\nto send a single line. `input` is one authorized call.\n\nWhat it is for is **harness commands**. A line beginning with `/` is not chat and not a message: it\nis something the agent's own harness handles, and the only way in is the keyboard.\n\n```bash\ncotal input --name reviewer --text \"/compact\" # ask the harness to compact its context\ncotal input --name reviewer --text \"/model opus\" # switch its model\ncotal input --name reviewer --text \"hold on that PR\" # ordinary typing works too\n```\n\n**Quoting.** `--text` takes a value, so a payload starting with `/` survives as written. A payload\nstarting with a dash needs the `=` form, because the shell-style `--text --foo` is ambiguous and is\nrefused rather than guessed:\n\n```bash\ncotal input --name reviewer --text=--verbose # dash-leading text: use --text=\n```\n\nEnter is pressed by default, since a command typed but never submitted has not been delivered.\n`--no-enter` types the text and leaves it sitting at the prompt, which is how you stage a line and\nsend it later.\n\nNothing comes back but a delivery receipt (`✓ sent 9 bytes to reviewer`, counting the trailing\ncarriage return). Whatever the agent does next shows up where its output already goes: the mesh, its\ntranscript, or an `attach`.\n\n**This one is operator-only, and more narrowly than `stop` or `attach`.** Those two are granted to\nanything holding `spawn`, so an agent can stop and attach to seats under its own owner. `input` is\nnot: it is granted only to operator credentials, which on a user-auth mesh means your ledger row\nneeds the `admin` scope, the same scope [`ps`](#managed-seats) already needs there. The reason is\nthat a write into a terminal is control of whatever is running in it, and on a user-auth mesh the\nown-owner rule covers every seat under you, not only the ones you launched: a `spawn`-scoped agent\ncould otherwise type into a sibling it never started. Seat locality is still resolved for you.\n\nOnly the `pty` runtime can be typed into. The external terminal runtimes (`tmux`, `cmux`, `orca`,\n`herdr`) attach to a process they do not own, so they have no input stream for it and the command\nrefuses by name rather than dropping the keystroke.\n\n## personas\n\n```bash\ncotal personas list [-v] [--running]\ncotal personas show \ncotal personas edit \ncotal personas new (--prompt | --from ) [--role ] [--model ]\ncotal personas rm --force\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh's persona catalog |\n| `--role ` | none | `new`: the persona's role |\n| `--model ` | none | `new`: the persona's model |\n| `--prompt ` | none | `new`: the persona's prompt text |\n| `--from ` | none | `new`: seed the prompt from a file |\n| `--verbose`, `-v` | off | `list`: include role / model / description |\n| `--running` | off | `list`: mark personas live on the mesh |\n| `--force` | none | `rm`: required, delete without prompting |\n\nPersonas are the local agent files under the resolved mesh root's `.cotal/agents/`, the same catalog\n`cotal spawn` launches from. `--space` and `--server` therefore move every list, read, write, delete\nand completion operation to the selected mesh. An unresolved target refuses rather than falling back\nto the current directory. See [Agent files](agent-files.md) for the file format.\n\n## supervise\n\n```bash\ncotal supervise [--runtime ] [--space ] [--server ] [--spawn ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space to supervise |\n| `--server ` | hosting mesh, or matching registered mesh | Broker URL. A registered mesh supplies it when omitted; a different explicit value is refused. |\n| `--runtime ` | `pty` | Agent runtime (`pty` built in; extension runtimes are explicit-only) |\n| `--console-port ` | none | Protocol-console port |\n| `--console-host ` | loopback | Bind host for the console + attach endpoint. Loopback keeps it machine-local; `cotal up` passes the address it bound the broker to, which is what lets `cotal attach` reach this manager from another machine |\n| `--roster ` | none | Declarative roster to boot at startup |\n| `--launch ` | none | Resolved manifest launch spec (from `up -f` / `spawn -f`) |\n| `--spawn ` | none | Comma-separated personas to pre-spawn at startup |\n\nThe manager is the agent supervisor and control plane: it answers `spawn --detach`, `stop`, `ps`,\n`attach`, and the `cotal_*` manager tools. `cotal up --detach` starts one for you; run `supervise`\ndirectly to recover a dead manager or drive a custom runtime. Default runtime is `pty`; install an\noptional provider first (`cotal ext add @cotal-ai/orca`, `@cotal-ai/tmux`, `@cotal-ai/cmux`, or `@cotal-ai/herdr`) and\nselect it explicitly. A missing provider or app fails loudly; there is no fallback. See [Deploy](deploy.md).\n\nOn a normal `SIGINT`/`SIGTERM`, the manager stops every seat and requires the selected runtime to\nprove the seat is gone before it releases the manager lease or service registration. A stop that\ncannot prove exit fails loud and keeps manager authority instead of reporting a clean shutdown while\nan orphan still holds broker rails. After an abrupt manager death, the same logical successor\nterminalizes only its own durable static slots, verify-evicts the predecessor's broker principal,\nrecords that result in the lifecycle's caller-readable audit detail, and only then retires the\nlifecycle and frees the alias. Missing or unverified broker evidence keeps the slot terminalizing.\nDelivery-admin does not terminate the orphan OS process; safe successor process reaping requires\ndurable process start-identity pinning and is tracked separately.\n\nA `meshes add --mode user` entry is a **participant** registration, not hosting authority. A\nparticipant may run `supervise` only when the host advertises the remote manager authority service\nand the signed-in actor has the dedicated `supervise` ledger scope. The CLI obtains the closed,\nloopback-only `manager-service` view; `spawn` and `admin` do not substitute for that scope. The\nhost issues the manager's public-nkey JWT material through its lifecycle-bound prepare → activate\n→ renew protocol, never by handing the participant a signer or static provisioner credential.\n\nWithout that advertised host service or scope, `supervise` refuses before it starts a manager.\nRun `cotal spawn` without `--detach` to launch a foreground agent, or ask the space host to enable\nthe authority service and grant `supervise` for detached agents. If a running remote manager loses\nrenewal, it reports degraded state and refuses unsafe new starts and restarts; live agents are not\nsilently replaced. Do not run `cotal down` or `cotal up` on a participant machine to repair this\ncondition.\n\n## reconcile-gate\n\n```bash\ncotal reconcile-gate [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the frozen gate lives in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint whose gate is frozen |\n| `--instance ` | this folder's persisted manager instance | Instance id |\n\n**When you need this.** A manager restart killed after deregistration begins but before the new\nincarnation finishes leaves the endpoint's issuance gate *frozen*, held by a\nprocess that no longer exists. The freeze is what stops two incarnations serving at once, which is\ncorrect. The successor manager now completes that dead registration itself on boot, using the same\nguard this command uses: it acts only when the freeze-holder is affirmatively gone under a complete\nCONNZ sweep (`gone` and `sweepComplete=true`). If that registration's spec write already committed,\nit finishes the same freeze at the committed registration revision. If the spec did not advance, it\nabort-reopens the gate at generation+1 with processEpoch unchanged and continues the normal takeover.\nLive, unknown, unestablishable, and\nwrong-op-kind still refuse; there is no TTL.\n\nUse this command when the boot path cannot run: the delivery daemon is down, the repair targets a\nnon-manager endpoint, or you want to lift the freeze without starting a manager. It checks that the\nholder really is gone, prints what it found, and then finishes the dead operation the same way as the\ninterrupted restart would have: revoke the old credentials, evict their holders with verification,\nand reopen the gate.\n\nIf verification is interrupted, the command leaves the gate frozen and durably records each holder\nwhose eviction was already verified. A retry still repeats the freeze-holder liveness check, then\nskips only progress bound to the same registration operation, frozen-gate revision, and holder set.\nThe output reports holders completed before this attempt, completed now, and still remaining. A new\nfreeze or changed holder set starts from zero. Cursor cleanup happens only after reopen; a retained\ncursor is harmless because its old gate revision cannot authorize a later freeze.\n\n**It refuses far more often than it acts, on purpose**, and always says which check stopped it:\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `holder-alive` | The freeze-holder still has a live connection: a manager *is* running | Stop that process first. Reconciling would evict a live manager's credentials |\n| `holder-unknown` | The connection sweep could not prove the holder absent | Not safe to proceed: an unprovable holder is treated as a live one. Re-run once the broker answers completely |\n| `liveness-unestablishable` | The delivery daemon could not be asked at all | Start it (`cotal up` runs it) and re-run. Silence is never read as death |\n| `not-frozen` / `no-gate` | The gate is open, or there is no gate at that coordinate | Nothing to repair: check `--endpoint` / `--instance` |\n| `wrong-op-kind` | Frozen under a takeover or retirement, not a registration | Out of scope for this command; it will not reinterpret another operation's intent |\n| `eviction-unverified` | The holder looked gone but eviction could not be verified | The gate is left frozen, unchanged. Investigate the broker before retrying |\n| `raced` | A newer manager moved the gate mid-repair | Re-run `cotal doctor` and look again |\n\nThere is no `--force`, and no path that discards gate state: the only way this reopens a gate is by\nproving the holder is gone and then completing the operation properly.\n\n## deregister-instance\n\n```bash\ncotal deregister-instance [--space ] [--server ] [--endpoint ] [--instance ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | this folder's auth space | Space the instance is registered in |\n| `--server ` | the local mesh | Broker URL |\n| `--endpoint ` | `manager` | Endpoint the instance serves |\n| `--instance ` | this folder's persisted manager instance | Instance id, the whole id as `cotal ps` prints it |\n\n**When you need this.** The service registry records *registration*, not liveness, and nothing in\nthe model expires a row. A manager that stops cleanly removes its own registration. One whose host\ndied without writing anything cannot, so its record goes on claiming a live instance forever: every\nclass scatter in that space freezes the dead slot in, and `cotal ps`, `stop` and `attach` each pay\ntheir whole deadline waiting for a machine that is never coming back. A laptop that was reimaged, a\ncontainer that was deleted, a box that will not be back on the network: those registrations have no\nother exit.\n\nThis command is that exit. It asks the instance first, and it removes a record only when the broker\naffirms the instance's own rail is empty: nothing subscribed there. Then it deletes the\nregistration's two records keys, each pinned to the revision it read, and prints what it removed.\n\n**Silence alone never passes.** An unanswered describe is what a dead host, a wedged process and a\nslow one all look like, and a hung process still holds its subscriptions, so the broker sees\ninterest on its rail. That instance is refused and the observation is printed. A dead process holds\nno connection and therefore no subscription, so a real corpse is still removed.\n\n**Every refusal names the failed check:**\n\n| Refusal | What it means | What to do |\n|---|---|---|\n| `instance-answered` | The instance answered a pinned describe. It is alive | Nothing to repair. If it is wedged rather than gone, stop the process first; its own clean stop removes the record |\n| `instance-not-affirmed-gone` | It did not answer, and the broker did not report its rail empty, which is what a held subscription looks like: slow or hung, not affirmed gone | Nothing was removed. Stop the process; its record goes on its own clean stop, or re-run this once it is down |\n| `liveness-unestablishable` | The probe itself failed, so nothing was learned | Fix the probe's path (credential, broker) and re-run. A probe that could not run is never read as death |\n| `not-registered` | No registration at that coordinate | Check `--instance` and `--endpoint`. This takes the whole id, never a prefix |\n| `registration-in-flight` | The instance holds the endpoint governance slot at the live issuance-gate generation, so a registration is still completing | Nothing was removed. Wait for that registration to finish, then re-run |\n| `superseded` | The record moved between the read and the delete | Something is writing to it. Nothing was removed; re-observe before retrying |\n\nThere is no `--force` and no sweep: silence is not death, and a rule that removed rows on silence\nwould eventually remove a live instance that was merely slow. An operator names one instance, the\nbroker's verdict on its rail is what authorizes the removal, and the guard's job is to show them\nthey named a dead one. Removal is not a one way door either. The same instance re-registers over\nthe tombstone on its next start, under the same identity.\n\n## runtimes\n\n```bash\ncotal runtimes\n```\n\nLists every agent runtime the manager can spawn through: the built-in `pty`, the official providers\n(`orca`, `tmux`, `cmux`, `herdr`), and any custom provider installed via `cotal ext add`. Each installed\nprovider is probed so you can see what is actually reachable on this machine before selecting it:\n\n```\npty built in\norca installed · reachable @cotal-ai/orca\ntmux available · cotal ext add @cotal-ai/tmux\ncmux available · cotal ext add @cotal-ai/cmux\nherdr available · cotal ext add @cotal-ai/herdr\n```\n\n`installed · reachable` / `unreachable` is the provider's own `available()` probe; `available` means\nit is a known runtime you can add with the shown command. Selecting an unknown or uninstalled runtime\nvia `up`/`spawn --runtime ` fails loud and, for a known one, points at the exact `cotal ext add`\npackage. There is no silent fallback to `pty`.\n\n## send\n\n```bash\ncotal send dm \"\" [--space ] [--server ] [--creds ]\ncotal send msg \"\"\ncotal send ask \"\"\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and (off-registry) which credential |\n\nOne-shot messaging: connect, send a single direct message (`dm`), channel post (`msg`), or role\nask/anycast (`ask`), then exit. For a running conversation, agents use the mesh tools instead\n([MCP tools](mcp-tools.md)).\n\n`cotal send` requires `COTAL_NAME` plus either `COTAL_ID` or both `COTAL_OWNER` and `COTAL_ACTOR`.\nIf that tuple is missing, `send` refuses before connecting so the recipient never sees a message\nattributed to a nameless command principal. A child that inherited a\nseat's environment is attributed as that seat; this command does not distinguish the two. An operator\nwho is not a live seat can set both variables for the one shot:\n\n```bash\nCOTAL_NAME= COTAL_ID= cotal send ...\n```\n\n## channels\n\n```bash\ncotal channels list\ncotal channels set [--replay | --no-replay] [--window ] [--desc ] [--instructions ]\ncotal channels default --replay | --no-replay\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--replay` / `--no-replay` | none | `set`/`default`: replay history to new joiners, or not |\n| `--window ` | none | `set`: replay window size |\n| `--desc ` | none | `set`: one-line channel description |\n| `--instructions ` | none | `set`: instructions shown to joiners |\n\nInspects and edits the channel registry: replay policy, description, and joiner instructions. ACL\nsemantics (who may read or post) are set at mint / provision time, not here; see\n[Channels and permissions](channels-and-permissions.md). On a user-auth mesh, `list` rides your\nown login as is; `set` and `default` edit the registry over a short-lived\nchannel-writer view, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n\n## history\n\n```bash\ncotal history clear --force [--dms] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Target mesh |\n| `--dms` | off | Also clear DM history |\n| `--force` | none | Required: clear without prompting |\n\nPurges retained channel history; `--dms` extends it to direct-message history. An alias of\n[`clean history`](#clean). On a user-auth mesh the purge rides a short-lived purger view over\nyour login, which needs ledger scope `admin` ([Identity & auth](identity-and-auth.md)).\n\n## console\n\n```bash\ncotal console [--plain] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to watch |\n| `--plain` | off | Line stream instead of the TUI |\n\nA live protocol view for a space: a lazygit-style TUI, or a plain line stream on `--plain`. On a\nuser-auth mesh it rides the read-only admin view over your login, which needs ledger scope\n`admin`. See [Watch a mesh](watch-a-mesh.md).\n\n## web\n\n```bash\ncotal web [--detach] [--host ] [--port ] [--no-open] [--space ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Space to serve |\n| `--host ` | `127.0.0.1` | Concrete HTTP bind and browser host; wildcard addresses are refused |\n| `--port ` | `7799` | HTTP port |\n| `--detach` | off | Run in the background; stop with `cotal down web` or bare `cotal down` |\n| `--no-open` | off | Don't open the browser |\n\nThe browser observability dashboard: presence, channels, and a live feed. It is **not** part of\n`cotal up`: it ships inside `cotal-ai` as the `@cotal-ai/web` extension, seeded automatically on first\nrun (like the built-in connectors) so it always matches your CLI version. It self-registers `cotal web`\ninto this surface and serves\n`http://cotal.localhost:7799` by default (loopback; `*.localhost` resolves in Chrome/Firefox/Edge; Safari may\nneed `http://127.0.0.1:7799`). On a user-auth mesh the dashboard rides the read-only admin view\nover your login, and a channel purge asks for its own channel-purger view per click; both need\nledger scope `admin`. Detached mode re-execs the current Cotal installation, writes diagnostics to\nthe mesh root's `.cotal/web.log`, and reports success only after the HTTP server answers. It requires\na recorded mesh root, but can be launched from any directory once `cotal up` has recorded the mesh.\nSee [Watch a mesh](watch-a-mesh.md).\n\n## mint\n\n```bash\ncotal mint [--profile ] [--out ] [--signer]\ncotal mint --provision [--role ] [--space ] [--server ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--profile ` | `agent` | Credential profile |\n| `--out ` | `.cotal/auth/creds/space./.creds` | Output path - the default sits under the resolved space's segment (`` is that space's hex encoding, as in [Project files](config.md#project-files)) |\n| `--signer` | off | Emit a stripped account-signing file instead |\n| `--force` | off | With `--signer`: overwrite an existing file |\n| `--allow-subscribe ` | the agent file's, else subscribe | Read-ACL override, **agent profile only**: `observer` and `admin` carry a fixed read set, and `mint` refuses this flag there rather than narrowing nothing |\n| `--allow-publish ` | the agent file's, else deny | Post-ACL override, **agent profile only** |\n| `--role ` | the agent file's | Agent profile: the anycast task queue the identity pulls (`svc_`) |\n| `--provision` | off | Agent profile: also pre-create the identity's bind-only DM/deliver durables (and its role's task queue) on the live mesh, so the credential can consume |\n| `--space `, `--server ` | the resolved mesh | Which root supplies the agent file, static trust and default credential storage; with `--provision`, also which live mesh receives the durables |\n\nMints a NATS creds file for a space in **static** auth mode, scoped to a profile and (optionally)\nexplicit read/post ACLs. `--signer` emits an account-signing file for delegating minting to another\nhost. A per-user-auth space refuses `mint`: agents there join under a logged-in user\n([`login`](#login) + [`actor grant`](#actor)), never via a handed-out creds file. See\n[Identity and auth](identity-and-auth.md).\n\nFor an agent profile, the resolved mesh root supplies the persona ACL, the signing material and the\ndefault credential destination as one authority. If the current folder also holds trust for a\ndifferent space or account, mint refuses before writing and names both roots. It never combines a\npersona from one root with credentials signed or stored under another.\n\nA plain mint is creds only: the identity can publish within its post ACL at once, but on an authed\nmesh its DM inbox and task queue are provisioner-pre-created and bind-only, so a **consuming**\nconnect fails until they exist. `--provision` performs that pre-create in the same command (a\nprovisioner cred is minted from the space's trust material, used, and dropped), so a long-running\nclient you start yourself can receive DMs and role anycasts like a spawned seat. The command prints\nthe identity's principal (its wire id) and lifecycle uid; a consuming client passes that uid as its\n`lifecycleUid`. Agent profile only; an open mesh needs none of this (peers self-create there). The\nThe same resolved authority is used for both the credential and `--provision`, so the broker\nfootprint cannot be created under a different root's trust material.\n\n## Login\n\n```bash\ncotal login --idp [--client-id ]\ncotal logout --idp \n```\n\nSigns you in to a per-user-auth mesh's IdP (device code flow) and caches the session; run it\nonce per machine. It prints your IdP subject, the id the operator grants against. After a\nlogin, every command on that mesh works under your identity: each connect takes a fresh IdP\nproof, exchanges it locally for a short-lived bearer, and is authorized against the actor\nledger at connect time. `logout` revokes the IdP session and clears the cache. See\n[identity & auth](identity-and-auth.md).\n\n## actor\n\n```bash\n# an upsert of the WHOLE row: a flag left off is the WIDE default below, not \"unchanged\"\ncotal actor grant --sub [--scope a,b] [--allow-subscribe a,b] [--allow-publish a,b] [--role ] [--label ]\ncotal actor revoke (--sub | --owner )\ncotal actor list\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` | the folder's | Space whose ledger to manage |\n| `--sub ` | none | The IdP subject (shown by `cotal login`) the actor belongs to |\n| `--owner ` | none | The derived owner token (alternative to `--sub`) |\n| `--scope ` | `spawn,role:default` | Capability scope (`''` = none; `spawn` = may run agents; `role:` = may delegate role r; `admin` = cross-agent control; `supervise` = eligible for the closed remote manager-service view when the host enables it) |\n| `--allow-subscribe ` | `>` (all channels) | Channel read ACL; the user's envelope, their agents can never read beyond it |\n| `--allow-publish ` | `>` (all channels) | Channel post ACL; also the envelope for their agents' posting |\n| `--role ` | none | Role (scopes the task-queue consumer) |\n| `--label ` | none | Display label for `actor list` (never the IdP subject) |\n\nThe actor ledger is the single authorization source of a user-auth space: no row, no access.\nA bare `grant` is the **full** envelope (all channels, may spawn); the flags narrow it. A\nre-grant **replaces the whole row**, not the one field you name, so to add a capability spell\nevery field out: the new scope plus the row's current read set, post set, role and label\n(`cotal actor list` shows what a row holds). A field left off does not stay as it was, it\nreverts to the wide default in the table above, which is how a narrow reader becomes a reader\nof every channel. A re-grant retires the current interactive lifecycle through the running auth\nservice before it rotates the row, so copied bearers cannot cross an authorization update. If that\nretirement cannot be confirmed, the row is left unchanged and the command fails with the recovery\naction. `revoke` uses the same retirement before deleting the row, which lets a later grant create a\nreal successor instead of colliding with a live predecessor. `supervise` is separate from `spawn` and `admin`: it only makes a signed-in\nperson eligible for the host-provided closed remote manager-service view; it does not grant\nmanagement of another owner or a general host profile. `revoke` denies the next exchange and\nthe next connect with no restart, and evicts the principal's live connections. Managed-agent rows\n(written by the spawn path) live in a disjoint row space this command never touches. See\n[identity & auth](identity-and-auth.md).\n\n## doctor\n\n```bash\ncotal doctor auth [--fix]\n```\n\nCredential-health diagnosis and repair for this folder's mesh: renders every managed\ncredential as healthy / near-expiry / expired and ends in `healthy` or the exact next\ncommand; `--fix` applies the repairs it can. The one surface every stale-credential error\npoints at.\n\n## join\n\n```bash\ncotal join --space --name [--role ] [--channel ]\ncotal join --link | --token \n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--space ` / `--server ` / `--creds ` | resolved mesh | Which mesh, and which credential |\n| `--name ` | none | Your presence name |\n| `--role ` | none | Your role |\n| `--channel ` | none | Channel to join |\n| `--kind ` | `agent` | Endpoint kind |\n| `--link ` | none | Join link (`cotal://…`) |\n| `--token ` | none | Join token |\n| `--lifecycle-uid ` | none | Required with `--creds`: the lifecycle UID minted alongside the credential (`COTAL_LIFECYCLE_UID` works too). A credential's durable grants name exact lifecycle-keyed resources, so `join` refuses to invent one |\n| `--tls` | off | Connect over TLS |\n\nAn interactive presence: join a space under your own name and role, without launching an agent\nharness. A `--link` or `--token` supplies the where and the auth in one value. See\n[Spaces](spaces.md) and [Identity and auth](identity-and-auth.md).\n\n## Manifest deploys\n\nA `cotal.yaml` manifest declares a whole mesh (channels, personas, roles, and ACLs) in one file.\nThree commands consume it, plus a read-only validator:\n\n```bash\ncotal up -f cotal.yaml # boot a fresh mesh from the manifest\ncotal spawn -f cotal.yaml # deploy the manifest additively onto a running mesh\ncotal down -f cotal.yaml # tear that deploy down (or --run for one run)\ncotal topology view -f cotal.yaml # validate + view the access graph, change nothing\n```\n\n`up -f` and `spawn -f` differ in target: `up -f` brings up a new broker and applies the manifest;\n`spawn -f` requires an already-reachable mesh and applies additively (ownership-scoped). On a\nuser-auth mesh, `spawn -f` deploys over your own login (the deployer view, gated on ledger scope\n`spawn`): the manifest's agents land under your owner, a manifest claiming another owner is\nrefused, and seeding new channels additionally needs scope `admin`. Both take\n`--dry-run` to print the plan without mutating anything. `topology` validates the manifest and\nrenders its channel / role / ACL graph. See [Define a team](define-a-team.md) and the\n[manifest reference](manifest.md).\n\n## ext\n\n```bash\ncotal ext # same as `list`\ncotal ext add \ncotal ext remove \ncotal ext list\ncotal ext root # print just the install prefix (scriptable)\ncotal ext seed [--repair|--reset|--force]\n```\n\nOperator-installed extensions: `add` installs an npm package into a cotal-owned prefix and records\nevery registry provider it contributes. Commands appear in help, completion, and dispatch; runtime\nproviders are lazy-loaded by commands such as `supervise`; local process providers participate in\n`status` and selective `down`. `remove` and `list` manage them. The `@cotal-ai/web` dashboard is the\ncanonical command/process example. Installed packages and their location are described in\n[config](config.md).\n\nBare `cotal ext` lists the inventory, headed by the install prefix. That prefix is a cotal-owned npm\nroot kept **separate** from npm's own global tree. These packages never show up in `npm list -g`,\n`cotal ext` (or the Extensions section of `cotal status`) is the canonical inventory. `cotal ext root`\nprints only the path, for scripts. The versions shown are the manifest pin recorded at add time.\n\nRemoving an extension that owns a running local process is refused with the mesh root and its\n`cotal down ` command; stop it first so uninstalling the package never strands a process\nwhose lifecycle provider is gone.\n\n### Built-in connectors are seeded extensions\n\nThe first-party agent connectors (`claude`, `opencode`, `codex`, `hermes`, `jcode`, `pi`) are not compiled into\nthe binary. They are seeded on first run through the **same** `ext add` path a third party uses, and\nappear in `cotal ext list` like any other extension. So you can remove one you do not want\n(`cotal ext remove @cotal-ai/connector-hermes`), and a deliberately-removed connector STAYS removed\nacross upgrades. `cotal ext add ` adds a third-party connector the same way. The web\ndashboard (`@cotal-ai/web`, providing `command:web`) is the seventh built-in seeded on the same path.\n\n`cotal ext seed` is the maintenance entry for that seeding (it runs automatically on the first real\ncommand of each boot, so you rarely call it):\n\n| Flag | Meaning |\n|---|---|\n| (none) | Reconcile: seed any never-seeded built-in, refresh a seeded one whose version the binary bumped, leave a removed one removed. A no-op once current. |\n| `--repair` | Recover after an interrupted seed or a lost authority (rebuilds the interrupted connector; restores the removed-vs-never-seeded record from its durable backup). |\n| `--reset` | Discard the record and re-seed all seven built-ins (the six connectors plus the web dashboard). **Resurrects any you removed.** Rebuilds cleanly over corrupt seed state. |\n| `--force` | Re-seed the built-ins even when the version stamp is current or a downgrade. |\n\nWhen a newer `cotal` advances the operator-global seed store to its generation, it prints one\nmigration line naming the old and new generations, the exact CLI entry that wrote the store, the\ncommit timestamp, and `seed/stamp.json`. That writer and timestamp are kept in the stamp, so a later\nolder CLI refusal can say which executable wrote the generation it will not overwrite and when.\nLegacy generation-only stamps remain readable; their refusal simply has no writer provenance to add.\n\nAn older `cotal` refuses a seed store written by a newer version. When it can verify a sufficient\n`cotal` executable on PATH or at the installer's `~/.local/bin/cotal` location, the refusal names\nthat absolute path so a reduced service PATH does not select the older binary again. Otherwise it\nkeeps the generic newer-version instruction. `--force` rebuilds the store for the running older\nversion without discarding the ever-seeded authority. `--reset` still exists for corrupt state and\nresurrects deliberately-removed connectors.\n\nA source-checkout CLI (`pnpm cotal`, `tsx bin/cotal.ts`, `node bin/cotal.ts`, or a suite child of\nthose) refuses to write or garbage-collect that store. The refusal names the path, the generation\nit declined, and `$XDG_CONFIG_HOME` as the isolation remedy. `COTAL_HOME` does not relocate this\nstore. An entry that cannot be proven as a released install is refused the same way. Isolated\nrelease tests that must seed from a checkout-shaped `bin/` set `COTAL_ALLOW_CHECKOUT_SEED=1` after\npointing `$XDG_CONFIG_HOME` at a scratch dir; that override is documented here, not on the refusal\nline. An opt-in write still records the checkout path in `seed/stamp.json` as `writtenBy`.\n\nThe default connector for a bare `cotal spawn` (no `--agent`) is the persona's `agent:` pin if it\nhas one, else `claude`; set `COTAL_DEFAULT_AGENT` (e.g. `opencode`) to change the fallback. It is\na default, so a persona that pins its harness still wins over it. An `--agent` naming a removed\nconnector fails loud with the exact\n`cotal ext add` to restore it. Set `COTAL_SKIP_CONNECTOR_SEED=1` to turn off the automatic first-run\nseed/refresh entirely (for a controlled or offline setup that manages connectors by hand); `cotal ext\nseed` still runs on request.\n\n## completion\n\n```bash\ncotal completion # print a stub to eval / source\ncotal completion install [shell] # install it persistently\n```\n\nPrints or installs shell completion. Completion candidates come from each command's declared flags\nand, where useful, live mesh state (spaces, personas, managed agents) resolved offline.\n\n## feedback\n\n```bash\ncotal feedback \"\" [--type ] [--email ] [--details ]\n```\n\n| Flag | Default | Meaning |\n|---|---|---|\n| `--type ` | none | `bug` \\| `idea` \\| `friction` \\| `praise` \\| `other` |\n| `--details ` | none | Longer free-form details |\n| `--severity ` | none | `low` \\| `medium` \\| `high` |\n| `--area ` | none | The part of Cotal this concerns |\n| `--email ` | git email | Contact email (required on the keyless public path) |\n| `--name ` | none | Your name (optional) |\n| `--url ` | keyed / public intake | Intake URL override |\n| `--key ` | `COTAL_FEEDBACK_KEY` | Feedback key |\n\nSends feedback to the Cotal developers. With a key (`--key` / `COTAL_FEEDBACK_KEY`) it routes to the\nkeyed beta intake; without one it goes to the public `cotal.ai` intake and requires a contact email\n(`--email` / `COTAL_FEEDBACK_EMAIL`, else your git email). Run a self-hosted intake with\n[`feedback-intake`](#server-daemons).\n\n## run\n\nOperate durable workflow runs (cotal-lang programs) from the terminal.\n\n```bash\ncotal run start --file [--timeout ] [--local]\ncotal run resume [--local --file ]\ncotal run ps [--endpoint ]\ncotal run journal [--endpoint ]\ncotal run answer [--value ] [--artifact ] [--endpoint ] [--local --by ]\n```\n\n`start` hands the program to the mesh's manager, which validates it, mints the run id (the record\nnever takes a caller-supplied one), drives it in its own process, and answers with the id once the\nrun is recorded; a program that does not validate is refused with every problem listed. `resume`\nasks the manager to take an existing run back and continue it from its step journal; the source is\nthe recorded program, so no `--file` is taken. Neither takes `--endpoint`: the manager records\nits runs under its own endpoint, and naming another is refused. `ps` lists the run records and\n`journal` renders one run's durable records; both only inspect. `answer` resolves an open\ncheckpoint through the manager, presenting as the holder that armed it; the manager records the\nanswerer from your credential, so no `--by` is taken there. `--timeout` sets the default\ncheckpoint timeout for a drive (default 1h). `--local` drives in this process instead, over one\nconnection per invocation under the run's own credential minted from the project folder's trust\nmaterial, and is the path on a bare broker with no manager or for a run with no recorded program\n(`cotal run resume --local --file `); `answer --local` takes `--by `. A\nuser-auth mesh runs no programs yet: the manager refuses the family by name, and `--local` has no\ncredential there. The guide is [workflows](workflows.md).\n\n## Server daemons\n\nTwo long-lived infra roles ship with the CLI. They are not part of everyday operation; the delivery\ndaemon comes up automatically with `cotal up --detach` in auth mode.\n\n```bash\ncotal deliver --space [--server ] [--creds ]\ncotal auth-service --space --server [--port ] [--exchange-public-port ] [--exchange-public-url ] [--exchange-trusted-proxy]\ncotal feedback-intake --keys [--port ] [--creds ]\n```\n\n`auth-service` runs a user-auth space's identity plane: the NATS auth callout, the\ncapability-gated local exchange and JWKS, and, when `--exchange-public-port` is set, the closed public\nexchange/discovery face forwarded by an HTTPS reverse proxy. `--exchange-public-url` is the proxy URL\nadvertised to clients; `--exchange-trusted-proxy` opts into last-hop `X-Forwarded-For` attribution.\n`cotal up --user-auth` starts and supervises the service for you, so you run it directly only to\nrecover one by hand.\n\n`deliver` runs the server-side Plane-3 delivery daemon: the durable backstop and membership/ACL\nauthority. It is auth-mode-only and single-instance (`--shard`/`--shards` accept only `N=1`);\n`--dev-mint` mints a scoped cred from the local signer for standalone dev. See the\n[delivery daemon](delivery-daemon.md). `feedback-intake` runs a self-hosted feedback server\n(requires `--keys` and a scoped `--creds`), announcing submissions into a space channel; flags\ninclude `--host`/`--port`, `--store`, `--space`/`--channel`, `--max-bytes`, and `--rate-limit`.\n\n## Plumbing\n\n`cotal __complete ` is the internal entry the shell-completion stubs call to emit candidates\nfor the current command line; you never run it directly. `cotal agent-bearer` is machine-facing\nplumbing on user-auth meshes: spawned agents exec it to print a fresh short-lived bearer from their\nspawn-time secret; you never run it directly either. Its local arm uses `--dir` to discover the\ncapability-gated loopback service. A remotely enrolled, already-granted agent instead receives\n`--exchange-url ` in its launch argv: that arm sends `{owner, actor, actorToken}` to the\npinned public exchange with no local capability, follows no redirects, and refuses every non-HTTPS\nURL because the actor token is the credential in the request body. (`cotal start` is a removed tombstone: it\nerrors and points you to `cotal spawn --detach`.)\n" }, { "slug": "config", @@ -229,7 +229,7 @@ export const DOCS_BUNDLE: DocsBundle = { "title": "Run a mesh", "kind": "Guide (informative)", "summary": "Day-to-day operation of a local mesh: what cotal up actually runs, how spawning resolves personas, harnesses, and models, how to reach a mesh from any directory, and the operator-only maintenance v…", - "body": "# Run a mesh\n\n> **Guide** (informative) · **For:** operators · **Prereqs:** [Quickstart](getting-started.md)\n\nDay-to-day operation of a local mesh: what `cotal up` actually runs, how spawning\nresolves personas, harnesses, and models, how to reach a mesh from any directory, and the\noperator-only maintenance verbs. Every command's full flag set is in the\n[CLI reference](cli.md).\n\n## The stack\n\n`cotal up` brings up the whole local stack and bare `cotal down` stops it:\n\n- **Broker**: a local `nats-server` (logs to `.cotal/nats.log`).\n- **Delivery daemon**: the durable backstop, auth mode only\n ([what it does](delivery-daemon.md)).\n- **Manager**: a detached supervisor answering the control plane, so\n `cotal spawn --detach` and the `cotal_spawn` tool work right after `up`.\n\nThree modes:\n\n- **Default (static auth).** JWT-authed, on by default: sender authenticity and per-agent\n ACLs, enforced by the broker ([how](identity-and-auth.md)).\n- **`--user-auth --idp `.** Per-user auth: people `cotal login` once, the operator\n grants their agents on the actor ledger, and every connect is authorized live against\n that grant. Starts the space's auth service alongside the broker\n ([how](identity-and-auth.md)).\n- **`--open`.** An unauthenticated, live-only dev mesh (no auth, no delivery daemon). For\n quick local experiments.\n\nThe broker and local services bind **loopback** by default. `--host 0.0.0.0` widens the broker\nbind independently of the auth mode, so \"network-reachable\" never silently means\n\"unauthenticated\". With no explicit `--server`, `cotal up` auto-selects a free local port when\nthe default address is already held by another project; an explicit `--server` fails loud on\ncollision.\n\nA user-auth mesh can expose only its credential exchange through an operator-owned HTTPS reverse\nproxy while leaving the existing local exchange untouched:\n\n```bash\ncotal up --user-auth --idp https://idp.example/api/auth \\\n --exchange-public-port 7443 \\\n --exchange-public-url https://auth.example\n```\n\nThe public listener itself still binds `127.0.0.1:7443`; configure the proxy to terminate TLS and\nforward to it. It serves only `/health`, `/jwks`, `/exchange`, and `/.well-known/cotal-mesh` with\nthe documented methods. It needs no local file capability: the signed IdP JWT or managed-agent\nactor token is the proof, while the original loopback listener remains capability-gated. Add\n`--exchange-trusted-proxy` only when that listener is reachable exclusively through your trusted\nproxy; it keys failure throttling by the last `X-Forwarded-For` hop instead of the socket address.\nThe well-known bundle includes IdP pins and a deny-all sentinel credential, so fetch it only from\nthe configured HTTPS origin. To change these listener flags, stop and restart the mesh; a refresh\nof an already-running service does not replace its bind or proxy policy. See\n[Identity & auth](identity-and-auth.md#per-user-authentication) for the trust boundary.\n\n`cotal status` prints the detailed setup, process, registry, and live mesh status. Its Machine\nsection names the running CLI's source checkout, installed package root, or npx package root beside\nthe version. A stale Claude skills row names the installed and CLI versions it compared. `cotal\nsetup` (after the first run) prints the compact card.\n\nBefore reporting ready, the manager resolves every installed connector's declared harness\nbinaries against its own environment. A missing binary does not stop unrelated manager work: boot\ncontinues, but prints a named `connector unavailable` line and records that reason in the\nmanager's `status` response. Available connector rows record the absolute paths boot resolved.\nSpawn keeps the same pre-mint check as a backstop for connectors registered after boot.\n\nThere is no supported `cotal service install` command yet. Running the manager as a launchd agent or\nsystemd user service remains operator-managed; service installation is separate from this boot-time\ndetection behavior.\n\nStop one part without tearing down the mesh by naming its registered component: `cotal down\nmanager`, `cotal down delivery`, or `cotal down web`. Component names from installed extensions\njoin the same surface; `cotal down` with no names retains whole-stack behavior.\n\n## Remote supervised agents\n\nOn a remote user-auth mesh, foreground `cotal spawn` remains the default participant path. A\nparticipant can run detached agents only after the host advertises and operates the remote manager\nauthority service, and the participant's actor-ledger row includes `supervise`. This is not implied\nby `spawn` or `admin`.\n\nThe participant's loopback/operator exchange obtains one closed `manager-service` view for its\nordinary derived owner, a fixed server-selected manager actor, and one opaque manager instance.\nThe host, not the participant, issues the public-nkey JWT material via the replay-safe,\nlifecycle-bound prepare → activate → renew exchange. It never exports the space signer, a static\nprovisioner credential, or generic storage authority. The manager may provision only descendants\nof that same owner, with host validation at each provision.\n\nWhen the authority service, login, or renewal is unavailable, the remote manager degrades\nfail-closed: it refuses new agents, restarts, and credential replacement rather than pretending\nlocal authority exists. Existing agents remain live only while their independent credentials are\nvalid. Restore service and renew successfully before asking it to recover an agent. See\n[Identity & auth](identity-and-auth.md#remote-manager-authority) and the [CLI\nreference](cli.md#supervise).\n\n## Spawning agents\n\n```bash\ncotal spawn # foreground: your default agent, in this terminal\ncotal spawn reviewer --detach # supervised: the manager runs it in a PTY\ncotal attach --name reviewer # watch/type into a detached agent (Ctrl-] detaches)\ncotal ps # what the manager is running\ncotal stop --name reviewer # stop one\n```\n\nHow a spawn resolves:\n\n- **Persona.** A bare `cotal spawn` uses `.cotal/agents/default.md`; a positional name\n picks `.cotal/agents/.md`; `--config` takes an explicit ref or path. Set\n `COTAL_DEFAULT_PERSONA=` to change the fallback. Fields and format:\n [agent files](agent-files.md).\n- **Harness.** Resolution order is an explicit `--agent` or `cotal_spawn` `agent` argument,\n then the persona file's `agent:` pin, then the invoking caller's `COTAL_DEFAULT_AGENT`,\n then the manager's `COTAL_DEFAULT_AGENT`, then the product default (Claude). Compared in\n [Connectors](connectors.md); per-connector guides:\n [Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n [Hermes](connect-hermes.md) · [pi](connect-pi.md).\n- **Model.** `--model` overrides the persona file's `model:` (Claude: `opus` / `sonnet` or\n a full id; OpenCode: `provider/model`). Connectors that expose a catalog report it via\n `cotal models --agent opencode`: model ids plus available variants; pick one with\n `--model provider/model --variant high`.\n- **Tools.** A spawned agent gets only the cotal tools by default; share your own MCP\n servers deliberately with `--share-tools` ([config](config.md)).\n- **Launch options.** `--opt key=value` (repeatable) passes a native harness flag straight\n through; a persona or manifest `launchOptions:` mapping does the same declaratively (a\n `--opt` wins per key). It is a **raw passthrough**, with no allow/deny list: Claude renders\n each as `--key value` (a bare `--key` for an empty value), OpenCode merges them into its\n agent config, and Hermes has no option surface so it fails loud. The trust boundary is the\n `spawn` capability itself, not the flag set, so granting `spawn` is host-launch authority\n ([security](security.md)). A key must be a plain flag name; malformed or prototype-polluting\n keys are refused.\n\nDetach from an attached PTY with **Ctrl-]** (the agent keeps running); rebind it with\n`COTAL_DETACH_KEY=ctrl-` when it clashes with a keybinding inside the agent's TUI.\n\n**Runtimes.** The manager spawns into a **pty** it owns by default. Optional runtimes are installed\nthrough the extension surface, for example `cotal ext add @cotal-ai/orca`, then selected with\n`--runtime orca` (similarly `@cotal-ai/tmux`, `@cotal-ai/cmux`, and `@cotal-ai/herdr`). They put teammates in native\nterminal surfaces rather than manager-owned PTYs. Runtime names are open-ended and resolved from\nthe registry; a missing provider or app throws, never silently falls back\n([architecture](architecture.md)).\n\n## Mesh registry\n\n`cotal up` records each running mesh in a machine-local registry\n(`~/.cotal/meshes/space..json`, named by a case-safe hex encoding of the space: broker URL, the project root holding its creds and\npersonas, and its mode). So a bare `cotal spawn ` from *any* directory joins the\nrunning mesh with the right credentials instead of mistaking the cwd for a space:\n\n- `cotal use ` sets the default from every directory, including inside another mesh's\n project. `--space ` overrides it for one command.\n- When one broker has records for several spaces, `cotal up --space ` refreshes that named\n space.\n- With no live selected default, a project with its own `.cotal/` resolves to that project's\n mesh; otherwise one running mesh is used automatically and several are an error.\n- `cotal meshes` lists them (a `*` marks the default); `cotal down` removes the entry.\n\nThe registry stores a *path*, never a secret; trust material stays in each project's\n`.cotal/auth`. If the mesh is down or won't take your creds, spawn fails with one\nsentence, never a raw NATS trace.\n\n### Meshes you did not start here\n\nA mesh running on another machine has no `cotal up` on this one, so register it by hand:\n\n```bash\ncotal meshes add # guided: asks for the broker, probes it, offers what it finds\ncotal meshes add optiplex --server nats://100.90.12.34:4222 --root ~/meshes/optiplex \\\n --allow-unencrypted-overlay # see below: an overlay address needs this\ncotal meshes rm optiplex\n```\n\nOn a terminal, a bare `cotal meshes add` walks you through it: it probes the broker you name and\nreports whether it is open or requires credentials, offers the spaces the folder already holds\ncredentials for, and shows the record before writing it. Scripts and agents keep the flag form -\nwithout a terminal nothing prompts.\n\n`--root` is the local folder holding that mesh's `.cotal/auth` and `.cotal/agents` (its personas);\nthe mode is inferred from what that folder holds.\n\n**Know what you are copying.** For an authenticated mesh that folder carries the space's account\n**signing seed**, which is the authority to mint any identity in the space. A machine holding it\nis a certificate authority for the mesh rather than a client of it: anyone who reads it can\nimpersonate any agent, read every retained channel and DM, change ACLs, and keep issuing\nthemselves credentials. There is no per-machine revocation; undoing it means rotating the signing\nkey and re-minting every credential in the space. Copy it only to machines you would trust with\nthe whole mesh. `cotal mint` on its own does not substitute here: registering an `auth` mesh needs\nsigning material that composes, which a minted user credential is not. The\nbroker is probed before the record is written, so a bad address or a credential that mesh will not\naccept fails at registration rather than at your first `spawn` (`--force` records it without verifying,\nuseful when the mesh is simply down right now).\n\n#### Which addresses you may register\n\nRegistering a mesh is how this machine starts sending agent credentials to a broker it does not\nrun. NATS announces itself in plaintext before anyone authenticates, so an attacker on the path\ncan pose as the broker and read the credential out of the connect unless the connection\n**requires TLS**, which is recorded on the entry and enforced on every dial through it.\n\nWhat the record will require decides what you may register:\n\n- **Without required TLS**, the address is the gate: **loopback** (`127.0.0.0/8`, `::1`), or\n **your private overlay** (`100.64.0.0/10`, `fd7a:115c:a1e0::/48`) with\n `--allow-unencrypted-overlay`. The tunnel provides the protection, and this command cannot check\n its state. Hostnames are refused because the lookup would choose which machine receives your\n credentials.\n- **With required TLS**, set `--tls` or use a `tls://` URL. The recorded scheme enforces the TLS\n requirement. A **hostname or public address** is accepted because the certificate chain and\n hostname check identify the peer. A registration whose broker cannot complete the handshake\n fails unless you pass `--force`, which records the entry without verification.\n\nOrdinary private ranges like `10.x` and `192.168.x` are refused in **both** modes. A café's wifi\nis private but does not belong to you, and no public CA issues certificates for those ranges. An\naddress spelling changes nothing: `[::ffff:192.168.1.10]`, `3232235786`, `0300.0250.01.012`, and\n`192.168.257` all resolve to private addresses and receive the same refusal as the dotted form.\n`--force` exists for a mesh that is down. It never permits an unsafe credential destination.\n\n#### Registering a hosted user-auth mesh\n\nA user-auth space's IdP pins are established where the mesh runs and are never guessed. Register\none from **supplied** trust: `--user-auth-file bundle.json` (exported on the mesh's machine), or\n`--from https://…/.well-known/cotal-mesh`, which asks before it contacts the address at all,\nfetches the discovery document over HTTPS, shows you the pins, and asks again before adopting\nthem. Redirects are refused because a 302 can walk a pinned fetch down to\nplaintext or onto another host, and the pinned exchange must be an `https://` URL too. The one\nexception is an exchange on **this machine**, where nothing leaves the box: plain `http://` is\naccepted for a loopback *literal* (`127.0.0.1`, `::1`, and any spelling of them), but **not** for\n`localhost`, which a hosts entry or poisoned lookup could point elsewhere. Use the\nliteral. Registration checks that the pinned exchange\nanswers `/health` and `/jwks` as the pinned issuer. It also checks that the broker refuses a\nbare connect; that refusal is the pass. The bundle's sentinel credentials are written to a private (0600) file\nunder the entry's root; the registry itself never carries the secret.\n\n**Without required TLS**, an overlay address is **refused unless you accept the dependency\nexplicitly**, with `--allow-unencrypted-overlay`. The address is not the guarantee: it is protected\nwhile the tunnel is up, and if the tunnel is down that range is ordinary carrier-grade NAT and\nwhoever answers the dial receives your credentials. Only you can know which it is, so the command\nasks you to say so. Your acceptance is recorded on the mesh entry rather than printed and\nforgotten, and the guided form asks the same question instead of taking the flag.\n\n**With required TLS** (`--tls`, or a `tls://` URL) that consent is no longer asked for, and the\nflag is not needed: the handshake is what protects the connection, so the acceptance it stood in\nfor has been replaced by proof rather than promise. `cotal meshes add --server\nnats://100.64.0.1 --tls` registers an overlay address with no prompt, no flag and no recorded\nacceptance. This is the \"the flag disappears once the broker can be served over TLS\" case, and it\nhas now arrived.\n\nThis gate is on **registration**. `cotal join --creds --server ` deliberately takes an\nexplicit connection at face value and does not consult the registry, so it is not covered. Join\nthat way only to an address you would have registered.\n\nRecords added this way are removed only by something that names them. A mesh this machine started\ncan be dropped on a hunch, such as a failed liveness probe or a `cotal down` in its project, because\n`cotal up` writes the record straight back. One you registered by hand cannot be reconstructed, so\nnothing removes it by inference: an unreachable broker is shown as `offline` in `cotal meshes`, and\n`cotal down` / `cotal clean all` leave it alone even when `--root` pointed at the project they are\ntearing down. A `cotal up` for that space refuses outright (naming `cotal meshes rm`) unless it is\nthat same endpoint: finding a broker already answering there is a refresh that starts nothing and\nleaves the record's provenance alone, while actually starting the broker for that space, server and\nroot makes this machine the one running it, so the record becomes an ordinary local one that\n`cotal down` clears. `cotal meshes rm` drops it and re-registering with `--force` replaces it. `rm`\nonly forgets a mesh. To stop one running here, use `cotal down`.\n\n## Watching\n\n`cotal console` is the terminal view (TUI on a real terminal, plain line stream when\npiped); `cotal web` is the browser dashboard. Both are read-only observers; the\nwalkthrough is [Watch a mesh](watch-a-mesh.md).\n\n## History\n\nRetained history is operator-owned. `cotal clean history --force` purges a space's\nretained channel history; `--dms` also purges DMs (`cotal history clear` is an alias).\nIt is deliberately **not** an agent tool: agents cannot wipe the record\n([identity & auth](identity-and-auth.md)). For a **stopped** mesh, `cotal clean store\n--force` deletes the on-disk JetStream store outright, and `cotal clean all --force`\nalso resets the space identity ([CLI reference](cli.md#clean)).\n\n## Offline backup\n\nFor a coherent durable cut, preserve the whole stack first, then create the artifact while it stays\ndown:\n\n```bash\ncotal down --preserve-state\ncotal backup create ./space-backup # full by default\n# later: deliberately resume the unchanged source\ncotal up --detach\n# or, from another preserved cut, restore before the normal listener opens\ncotal up --restore ./space-backup --detach\n```\n\nUse `--store-dir` on both preservation and backup for a custom JetStream store. `registry` is the\nonly partial selection (`backup create ... --only registry`; `up --restore ... --restore-only\nregistry`). Backup never stops or restarts a mesh implicitly, never opens the original store, and\ndoes not contain credentials or trust secrets. Backup/restore in every auth mode, open included,\nuses isolated, operation-specific maintenance logins; normal agent credentials cannot enter that\nlistener. Full\nrestore requires the same space and exact current local trust continuity, recreates conservative\nconsumer checkpoints bound to their snapshot stream sequence state, and resumes retained agents under\ntheir original principals. The trust commitment includes the cryptographically validated full\noperator/system/data-account root chain as well as static/user authority state. A registry-only\nrestore completes canonical empty infrastructure but leaves retained agents stopped because their\nDM/DLV/TASK/ACL state is outside that selection. Authenticated restore validates the complete space\ntrust bundle before staging or changing the preserved store. Interrupted ordinary resume retries the\nsame durable attempt after its prior listener is stopped. Restore re-entry can recover a surviving normal listener\nonly when its attempt nonce, NATS server name, process owner, endpoint, and target-store identity all\nmatch the fsynced proof. A provably dead uncommitted owner is retired under lock and replaced with a\nfresh attempt-bound listener; an occupied foreign listener or ambiguous owner is never adopted. The\nmanager commit validates while retained cleanup is still suppressed; the CLI durably records its\nattempt-bound 64-hex token in `manager-committed` / `resume-committed` before `finalizeResume` can\nrelease suppression. A retry from either committed state goes straight to exact-token finalization;\nfailure preserves the committed gate and retained cleanup suppression. Missing commit evidence,\ninterrupted finalization, a live recorded endpoint despite missing pidfiles, or ambiguous proof fails closed. See the [CLI\nbackup and restore contract](cli.md#backups) for artifact, checkpoint, fallback,\ndisaster-consent, and degraded-recovery details.\n\n## Personas from the CLI\n\n`cotal personas` manages the local catalog offline: `list` (`--running` overlays live\nmarkers), `show `, `edit ` (re-validates on save), `new `, `rm \n--force`. The runtime write is `cotal_persona`; the runtime read is `cotal_personas`\n(list / show), both over the wire with the manager's ownership checks. Fields: [agent files](agent-files.md).\n\n## Gate recovery\n\nA manager that dies mid-registration leaves its issuance gate *frozen* under that registration\nop. The freeze is correct: it stops two incarnations serving at once. The successor now completes\nthat dead op on boot, using the same guard as [`cotal reconcile-gate`](cli.md#reconcile-gate): it\nacts only when the freeze-holder is affirmatively gone under a complete CONNZ sweep (`gone` and\n`sweepComplete=true`). If the dead op's spec write committed, it finishes that same freeze\n(promote and reopen at the committed registration revision). If the spec did not advance, it\nabort-reopens the gate (generation+1, processEpoch unchanged) and continues the normal takeover.\nA live holder, an incomplete sweep, or an unreachable delivery daemon still\nrefuses. Silence is never evidence of death, and there is no TTL. If holder verification is\ninterrupted, the frozen operation resumes from its durable, operation-and-gate-revision-bound\nprogress after liveness is checked again. A later freeze cannot reuse that progress: the cursor\nbinds the exact op, gate revision, and holder set. Use `cotal reconcile-gate` when the boot path cannot run\n(daemon down, a non-manager endpoint, or you want to lift the freeze without starting a manager). A spawn that hits the same frozen gate names that verb in the refusal\n(`blockedOp=registration`, the holding `opId`, `remedy=cotal reconcile-gate`) instead of a\nwait-timeout: the facts were always in the manager log; they now reach the spawn caller too.\n\n## When something looks absent\n\nPermission denials are **loud, never silent**: an over-tight ACL rejects the endpoint call and\nalso shows up as a logged denial, instead of returning an empty or incomplete result that looks\nsuccessful. Check\n`.cotal/manager..log`, `.cotal/delivery..log` (one pair per space, keyed as\n[Config](config.md#project-files) describes), and `.cotal/nats.log`; `cotal status` shows\nwhat is actually running. The access rules are collected in\n[Channels & permissions](channels-and-permissions.md).\n" + "body": "# Run a mesh\n\n> **Guide** (informative) · **For:** operators · **Prereqs:** [Quickstart](getting-started.md)\n\nDay-to-day operation of a local mesh: what `cotal up` actually runs, how spawning\nresolves personas, harnesses, and models, how to reach a mesh from any directory, and the\noperator-only maintenance verbs. Every command's full flag set is in the\n[CLI reference](cli.md).\n\n## The stack\n\n`cotal up` brings up the whole local stack and bare `cotal down` stops it:\n\n- **Broker**: a local `nats-server` (logs to `.cotal/nats.log`).\n- **Delivery daemon**: the durable backstop, auth mode only\n ([what it does](delivery-daemon.md)).\n- **Manager**: a detached supervisor answering the control plane, so\n `cotal spawn --detach` and the `cotal_spawn` tool work right after `up`.\n\nThree modes:\n\n- **Default (static auth).** JWT-authed, on by default: sender authenticity and per-agent\n ACLs, enforced by the broker ([how](identity-and-auth.md)).\n- **`--user-auth --idp `.** Per-user auth: people `cotal login` once, the operator\n grants their agents on the actor ledger, and every connect is authorized live against\n that grant. Starts the space's auth service alongside the broker\n ([how](identity-and-auth.md)).\n- **`--open`.** An unauthenticated, live-only dev mesh (no auth, no delivery daemon). For\n quick local experiments.\n\nThe broker and local services bind **loopback** by default. `--host 0.0.0.0` widens the broker\nbind independently of the auth mode, so \"network-reachable\" never silently means\n\"unauthenticated\". With no explicit `--server`, `cotal up` auto-selects a free local port when\nthe default address is already held by another project; an explicit `--server` fails loud on\ncollision.\n\nA user-auth mesh can expose only its credential exchange through an operator-owned HTTPS reverse\nproxy while leaving the existing local exchange untouched:\n\n```bash\ncotal up --user-auth --idp https://idp.example/api/auth \\\n --exchange-public-port 7443 \\\n --exchange-public-url https://auth.example\n```\n\nThe public listener itself still binds `127.0.0.1:7443`; configure the proxy to terminate TLS and\nforward to it. It serves only `/health`, `/jwks`, `/exchange`, and `/.well-known/cotal-mesh` with\nthe documented methods. It needs no local file capability: the signed IdP JWT or managed-agent\nactor token is the proof, while the original loopback listener remains capability-gated. Add\n`--exchange-trusted-proxy` only when that listener is reachable exclusively through your trusted\nproxy; it keys failure throttling by the last `X-Forwarded-For` hop instead of the socket address.\nThe well-known bundle includes IdP pins and a deny-all sentinel credential, so fetch it only from\nthe configured HTTPS origin. To change these listener flags, stop and restart the mesh; a refresh\nof an already-running service does not replace its bind or proxy policy. See\n[Identity & auth](identity-and-auth.md#per-user-authentication) for the trust boundary.\n\n`cotal status` prints the detailed setup, process, registry, and live mesh status. Its Machine\nsection names the running CLI's source checkout, installed package root, or npx package root beside\nthe version. A stale Claude skills row names the installed and CLI versions it compared. `cotal\nsetup` (after the first run) prints the compact card.\n\nBefore reporting ready, the manager resolves every installed connector's declared harness\nbinaries against its own environment. A missing binary does not stop unrelated manager work: boot\ncontinues, but prints a named `connector unavailable` line and records that reason in the\nmanager's `status` response. Available connector rows record the absolute paths boot resolved.\nSpawn keeps the same pre-mint check as a backstop for connectors registered after boot.\n\nOn an authenticated manager start, unfinished static lifecycle rows reconcile while the control\nendpoint is already serving. The manager `status` response reports\nthe `staticReconciliation` state, the last sweep counts, and each failed alias with its durable\nphase and literal disposition. `cotal status --components` reports the state and per-alias failure\ndetails. A failed exact terminal is retried in the same process after 1, 5,\nand 30 seconds. Each attempt re-reads the durable slot and re-enters the same deterministic terminal\noperation; the delays only schedule work and never release the lifecycle fence.\n\nOn shutdown, the manager fences new reconciliation work and waits for an exact terminal that already\nstarted. The current serial sweep stops before its next alias, and startup cannot publish the manager\nservice after `stop()` completes.\n\nThe four-attempt budget is per manager process. An exhausted row stays held and reports\n`retry-exhausted` with the remedy to restart the manager. The next process derives a fresh budget\nfrom the still-authoritative durable row. A `recovered` row remains visible until the next static\nreconciliation sweep, then clears. This component reports reconciliation outcomes. It does not say\nwhether footprint cleanup completed independently of the terminal result; that separate durable\nprojection remains tracked by #1274.\n\nThere is no supported `cotal service install` command yet. Running the manager as a launchd agent or\nsystemd user service remains operator-managed; service installation is separate from this boot-time\ndetection behavior.\n\nStop one part without tearing down the mesh by naming its registered component: `cotal down\nmanager`, `cotal down delivery`, or `cotal down web`. Component names from installed extensions\njoin the same surface; `cotal down` with no names retains whole-stack behavior.\n\n## Remote supervised agents\n\nOn a remote user-auth mesh, foreground `cotal spawn` remains the default participant path. A\nparticipant can run detached agents only after the host advertises and operates the remote manager\nauthority service, and the participant's actor-ledger row includes `supervise`. This is not implied\nby `spawn` or `admin`.\n\nThe participant's loopback/operator exchange obtains one closed `manager-service` view for its\nordinary derived owner, a fixed server-selected manager actor, and one opaque manager instance.\nThe host, not the participant, issues the public-nkey JWT material via the replay-safe,\nlifecycle-bound prepare → activate → renew exchange. It never exports the space signer, a static\nprovisioner credential, or generic storage authority. The manager may provision only descendants\nof that same owner, with host validation at each provision.\n\nWhen the authority service, login, or renewal is unavailable, the remote manager degrades\nfail-closed: it refuses new agents, restarts, and credential replacement rather than pretending\nlocal authority exists. Existing agents remain live only while their independent credentials are\nvalid. Restore service and renew successfully before asking it to recover an agent. See\n[Identity & auth](identity-and-auth.md#remote-manager-authority) and the [CLI\nreference](cli.md#supervise).\n\n## Spawning agents\n\n```bash\ncotal spawn # foreground: your default agent, in this terminal\ncotal spawn reviewer --detach # supervised: the manager runs it in a PTY\ncotal attach --name reviewer # watch/type into a detached agent (Ctrl-] detaches)\ncotal ps # what the manager is running\ncotal stop --name reviewer # stop one\n```\n\nHow a spawn resolves:\n\n- **Persona.** A bare `cotal spawn` uses `.cotal/agents/default.md`; a positional name\n picks `.cotal/agents/.md`; `--config` takes an explicit ref or path. Set\n `COTAL_DEFAULT_PERSONA=` to change the fallback. Fields and format:\n [agent files](agent-files.md).\n- **Harness.** Resolution order is an explicit `--agent` or `cotal_spawn` `agent` argument,\n then the persona file's `agent:` pin, then the invoking caller's `COTAL_DEFAULT_AGENT`,\n then the manager's `COTAL_DEFAULT_AGENT`, then the product default (Claude). Compared in\n [Connectors](connectors.md); per-connector guides:\n [Claude](connect-claude.md) · [OpenCode](connect-opencode.md) ·\n [Hermes](connect-hermes.md) · [pi](connect-pi.md).\n- **Model.** `--model` overrides the persona file's `model:` (Claude: `opus` / `sonnet` or\n a full id; OpenCode: `provider/model`). Connectors that expose a catalog report it via\n `cotal models --agent opencode`: model ids plus available variants; pick one with\n `--model provider/model --variant high`.\n- **Tools.** A spawned agent gets only the cotal tools by default; share your own MCP\n servers deliberately with `--share-tools` ([config](config.md)).\n- **Launch options.** `--opt key=value` (repeatable) passes a native harness flag straight\n through; a persona or manifest `launchOptions:` mapping does the same declaratively (a\n `--opt` wins per key). It is a **raw passthrough**, with no allow/deny list: Claude renders\n each as `--key value` (a bare `--key` for an empty value), OpenCode merges them into its\n agent config, and Hermes has no option surface so it fails loud. The trust boundary is the\n `spawn` capability itself, not the flag set, so granting `spawn` is host-launch authority\n ([security](security.md)). A key must be a plain flag name; malformed or prototype-polluting\n keys are refused.\n\nDetach from an attached PTY with **Ctrl-]** (the agent keeps running); rebind it with\n`COTAL_DETACH_KEY=ctrl-` when it clashes with a keybinding inside the agent's TUI.\n\n**Runtimes.** The manager spawns into a **pty** it owns by default. Optional runtimes are installed\nthrough the extension surface, for example `cotal ext add @cotal-ai/orca`, then selected with\n`--runtime orca` (similarly `@cotal-ai/tmux`, `@cotal-ai/cmux`, and `@cotal-ai/herdr`). They put teammates in native\nterminal surfaces rather than manager-owned PTYs. Runtime names are open-ended and resolved from\nthe registry; a missing provider or app throws, never silently falls back\n([architecture](architecture.md)).\n\n## Mesh registry\n\n`cotal up` records each running mesh in a machine-local registry\n(`~/.cotal/meshes/space..json`, named by a case-safe hex encoding of the space: broker URL, the project root holding its creds and\npersonas, and its mode). So a bare `cotal spawn ` from *any* directory joins the\nrunning mesh with the right credentials instead of mistaking the cwd for a space:\n\n- `cotal use ` sets the default from every directory, including inside another mesh's\n project. `--space ` overrides it for one command.\n- When one broker has records for several spaces, `cotal up --space ` refreshes that named\n space.\n- With no live selected default, a project with its own `.cotal/` resolves to that project's\n mesh; otherwise one running mesh is used automatically and several are an error.\n- `cotal meshes` lists them (a `*` marks the default); `cotal down` removes the entry.\n\nThe registry stores a *path*, never a secret; trust material stays in each project's\n`.cotal/auth`. If the mesh is down or won't take your creds, spawn fails with one\nsentence, never a raw NATS trace.\n\n### Meshes you did not start here\n\nA mesh running on another machine has no `cotal up` on this one, so register it by hand:\n\n```bash\ncotal meshes add # guided: asks for the broker, probes it, offers what it finds\ncotal meshes add optiplex --server nats://100.90.12.34:4222 --root ~/meshes/optiplex \\\n --allow-unencrypted-overlay # see below: an overlay address needs this\ncotal meshes rm optiplex\n```\n\nOn a terminal, a bare `cotal meshes add` walks you through it: it probes the broker you name and\nreports whether it is open or requires credentials, offers the spaces the folder already holds\ncredentials for, and shows the record before writing it. Scripts and agents keep the flag form -\nwithout a terminal nothing prompts.\n\n`--root` is the local folder holding that mesh's `.cotal/auth` and `.cotal/agents` (its personas);\nthe mode is inferred from what that folder holds.\n\n**Know what you are copying.** For an authenticated mesh that folder carries the space's account\n**signing seed**, which is the authority to mint any identity in the space. A machine holding it\nis a certificate authority for the mesh rather than a client of it: anyone who reads it can\nimpersonate any agent, read every retained channel and DM, change ACLs, and keep issuing\nthemselves credentials. There is no per-machine revocation; undoing it means rotating the signing\nkey and re-minting every credential in the space. Copy it only to machines you would trust with\nthe whole mesh. `cotal mint` on its own does not substitute here: registering an `auth` mesh needs\nsigning material that composes, which a minted user credential is not. The\nbroker is probed before the record is written, so a bad address or a credential that mesh will not\naccept fails at registration rather than at your first `spawn` (`--force` records it without verifying,\nuseful when the mesh is simply down right now).\n\n#### Which addresses you may register\n\nRegistering a mesh is how this machine starts sending agent credentials to a broker it does not\nrun. NATS announces itself in plaintext before anyone authenticates, so an attacker on the path\ncan pose as the broker and read the credential out of the connect unless the connection\n**requires TLS**, which is recorded on the entry and enforced on every dial through it.\n\nWhat the record will require decides what you may register:\n\n- **Without required TLS**, the address is the gate: **loopback** (`127.0.0.0/8`, `::1`), or\n **your private overlay** (`100.64.0.0/10`, `fd7a:115c:a1e0::/48`) with\n `--allow-unencrypted-overlay`. The tunnel provides the protection, and this command cannot check\n its state. Hostnames are refused because the lookup would choose which machine receives your\n credentials.\n- **With required TLS**, set `--tls` or use a `tls://` URL. The recorded scheme enforces the TLS\n requirement. A **hostname or public address** is accepted because the certificate chain and\n hostname check identify the peer. A registration whose broker cannot complete the handshake\n fails unless you pass `--force`, which records the entry without verification.\n\nOrdinary private ranges like `10.x` and `192.168.x` are refused in **both** modes. A café's wifi\nis private but does not belong to you, and no public CA issues certificates for those ranges. An\naddress spelling changes nothing: `[::ffff:192.168.1.10]`, `3232235786`, `0300.0250.01.012`, and\n`192.168.257` all resolve to private addresses and receive the same refusal as the dotted form.\n`--force` exists for a mesh that is down. It never permits an unsafe credential destination.\n\n#### Registering a hosted user-auth mesh\n\nA user-auth space's IdP pins are established where the mesh runs and are never guessed. Register\none from **supplied** trust: `--user-auth-file bundle.json` (exported on the mesh's machine), or\n`--from https://…/.well-known/cotal-mesh`, which asks before it contacts the address at all,\nfetches the discovery document over HTTPS, shows you the pins, and asks again before adopting\nthem. Redirects are refused because a 302 can walk a pinned fetch down to\nplaintext or onto another host, and the pinned exchange must be an `https://` URL too. The one\nexception is an exchange on **this machine**, where nothing leaves the box: plain `http://` is\naccepted for a loopback *literal* (`127.0.0.1`, `::1`, and any spelling of them), but **not** for\n`localhost`, which a hosts entry or poisoned lookup could point elsewhere. Use the\nliteral. Registration checks that the pinned exchange\nanswers `/health` and `/jwks` as the pinned issuer. It also checks that the broker refuses a\nbare connect; that refusal is the pass. The bundle's sentinel credentials are written to a private (0600) file\nunder the entry's root; the registry itself never carries the secret.\n\n**Without required TLS**, an overlay address is **refused unless you accept the dependency\nexplicitly**, with `--allow-unencrypted-overlay`. The address is not the guarantee: it is protected\nwhile the tunnel is up, and if the tunnel is down that range is ordinary carrier-grade NAT and\nwhoever answers the dial receives your credentials. Only you can know which it is, so the command\nasks you to say so. Your acceptance is recorded on the mesh entry rather than printed and\nforgotten, and the guided form asks the same question instead of taking the flag.\n\n**With required TLS** (`--tls`, or a `tls://` URL) that consent is no longer asked for, and the\nflag is not needed: the handshake is what protects the connection, so the acceptance it stood in\nfor has been replaced by proof rather than promise. `cotal meshes add --server\nnats://100.64.0.1 --tls` registers an overlay address with no prompt, no flag and no recorded\nacceptance. This is the \"the flag disappears once the broker can be served over TLS\" case, and it\nhas now arrived.\n\nThis gate is on **registration**. `cotal join --creds --server ` deliberately takes an\nexplicit connection at face value and does not consult the registry, so it is not covered. Join\nthat way only to an address you would have registered.\n\nRecords added this way are removed only by something that names them. A mesh this machine started\ncan be dropped on a hunch, such as a failed liveness probe or a `cotal down` in its project, because\n`cotal up` writes the record straight back. One you registered by hand cannot be reconstructed, so\nnothing removes it by inference: an unreachable broker is shown as `offline` in `cotal meshes`, and\n`cotal down` / `cotal clean all` leave it alone even when `--root` pointed at the project they are\ntearing down. A `cotal up` for that space refuses outright (naming `cotal meshes rm`) unless it is\nthat same endpoint: finding a broker already answering there is a refresh that starts nothing and\nleaves the record's provenance alone, while actually starting the broker for that space, server and\nroot makes this machine the one running it, so the record becomes an ordinary local one that\n`cotal down` clears. `cotal meshes rm` drops it and re-registering with `--force` replaces it. `rm`\nonly forgets a mesh. To stop one running here, use `cotal down`.\n\n## Watching\n\n`cotal console` is the terminal view (TUI on a real terminal, plain line stream when\npiped); `cotal web` is the browser dashboard. Both are read-only observers; the\nwalkthrough is [Watch a mesh](watch-a-mesh.md).\n\n## History\n\nRetained history is operator-owned. `cotal clean history --force` purges a space's\nretained channel history; `--dms` also purges DMs (`cotal history clear` is an alias).\nIt is deliberately **not** an agent tool: agents cannot wipe the record\n([identity & auth](identity-and-auth.md)). For a **stopped** mesh, `cotal clean store\n--force` deletes the on-disk JetStream store outright, and `cotal clean all --force`\nalso resets the space identity ([CLI reference](cli.md#clean)).\n\n## Offline backup\n\nFor a coherent durable cut, preserve the whole stack first, then create the artifact while it stays\ndown:\n\n```bash\ncotal down --preserve-state\ncotal backup create ./space-backup # full by default\n# later: deliberately resume the unchanged source\ncotal up --detach\n# or, from another preserved cut, restore before the normal listener opens\ncotal up --restore ./space-backup --detach\n```\n\nUse `--store-dir` on both preservation and backup for a custom JetStream store. `registry` is the\nonly partial selection (`backup create ... --only registry`; `up --restore ... --restore-only\nregistry`). Backup never stops or restarts a mesh implicitly, never opens the original store, and\ndoes not contain credentials or trust secrets. Backup/restore in every auth mode, open included,\nuses isolated, operation-specific maintenance logins; normal agent credentials cannot enter that\nlistener. Full\nrestore requires the same space and exact current local trust continuity, recreates conservative\nconsumer checkpoints bound to their snapshot stream sequence state, and resumes retained agents under\ntheir original principals. The trust commitment includes the cryptographically validated full\noperator/system/data-account root chain as well as static/user authority state. A registry-only\nrestore completes canonical empty infrastructure but leaves retained agents stopped because their\nDM/DLV/TASK/ACL state is outside that selection. Authenticated restore validates the complete space\ntrust bundle before staging or changing the preserved store. Interrupted ordinary resume retries the\nsame durable attempt after its prior listener is stopped. Restore re-entry can recover a surviving normal listener\nonly when its attempt nonce, NATS server name, process owner, endpoint, and target-store identity all\nmatch the fsynced proof. A provably dead uncommitted owner is retired under lock and replaced with a\nfresh attempt-bound listener; an occupied foreign listener or ambiguous owner is never adopted. The\nmanager commit validates while retained cleanup is still suppressed; the CLI durably records its\nattempt-bound 64-hex token in `manager-committed` / `resume-committed` before `finalizeResume` can\nrelease suppression. A retry from either committed state goes straight to exact-token finalization;\nfailure preserves the committed gate and retained cleanup suppression. Missing commit evidence,\ninterrupted finalization, a live recorded endpoint despite missing pidfiles, or ambiguous proof fails closed. See the [CLI\nbackup and restore contract](cli.md#backups) for artifact, checkpoint, fallback,\ndisaster-consent, and degraded-recovery details.\n\n## Personas from the CLI\n\n`cotal personas` manages the local catalog offline: `list` (`--running` overlays live\nmarkers), `show `, `edit ` (re-validates on save), `new `, `rm \n--force`. The runtime write is `cotal_persona`; the runtime read is `cotal_personas`\n(list / show), both over the wire with the manager's ownership checks. Fields: [agent files](agent-files.md).\n\n## Gate recovery\n\nA manager that dies mid-registration leaves its issuance gate *frozen* under that registration\nop. The freeze is correct: it stops two incarnations serving at once. The successor now completes\nthat dead op on boot, using the same guard as [`cotal reconcile-gate`](cli.md#reconcile-gate): it\nacts only when the freeze-holder is affirmatively gone under a complete CONNZ sweep (`gone` and\n`sweepComplete=true`). If the dead op's spec write committed, it finishes that same freeze\n(promote and reopen at the committed registration revision). If the spec did not advance, it\nabort-reopens the gate (generation+1, processEpoch unchanged) and continues the normal takeover.\nA live holder, an incomplete sweep, or an unreachable delivery daemon still\nrefuses. Silence is never evidence of death, and there is no TTL. If holder verification is\ninterrupted, the frozen operation resumes from its durable, operation-and-gate-revision-bound\nprogress after liveness is checked again. A later freeze cannot reuse that progress: the cursor\nbinds the exact op, gate revision, and holder set. Use `cotal reconcile-gate` when the boot path cannot run\n(daemon down, a non-manager endpoint, or you want to lift the freeze without starting a manager). A spawn that hits the same frozen gate names that verb in the refusal\n(`blockedOp=registration`, the holding `opId`, `remedy=cotal reconcile-gate`) instead of a\nwait-timeout: the facts were always in the manager log; they now reach the spawn caller too.\n\n## When something looks absent\n\nPermission denials are **loud, never silent**: an over-tight ACL rejects the endpoint call and\nalso shows up as a logged denial, instead of returning an empty or incomplete result that looks\nsuccessful. Check\n`.cotal/manager..log`, `.cotal/delivery..log` (one pair per space, keyed as\n[Config](config.md#project-files) describes), and `.cotal/nats.log`; `cotal status` shows\nwhat is actually running. The access rules are collected in\n[Channels & permissions](channels-and-permissions.md).\n" }, { "slug": "security", diff --git a/implementations/cli/smoke/component-health.smoke.ts b/implementations/cli/smoke/component-health.smoke.ts index 439bacacb..9c880a911 100644 --- a/implementations/cli/smoke/component-health.smoke.ts +++ b/implementations/cli/smoke/component-health.smoke.ts @@ -10,7 +10,9 @@ * * This is intentionally an OPEN mesh: it removes minting from the experiment so the only changed * variable is the component's own manager control surface. The holder renews its real manager - * lease against a real JetStream broker; it is not a hand-written fake of a status reply. + * lease against a real JetStream broker; it is not a hand-written fake of a status reply. The + * verdict regression cell separately registers a real status endpoint and serves a manager-shaped + * foreign-owner reconciliation payload; the manager suite owns reconciliation-state generation. * * Run: pnpm smoke:component-health */ @@ -20,6 +22,30 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { createConnection, createServer } from "node:net"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +import { jetstreamManager } from "@nats-io/jetstream"; +import { Kvm } from "@nats-io/kv"; +import { connect } from "@nats-io/transport-node"; +import { + CotalEndpoint, + SERVICE_READY, + VOID_SCHEMA, + authorizeServeGrant, + compileContract, + contractArtifactCanonicalBytes, + contractDigest, + contractStoreContext, + createEndpointStreams, + deliveryBucket, + endpointRegistrationBarrier, + epAuthBucket, + openRecordsBucket, + provisionEndpointGateOpen, + publishContractArtifact, + registerServiceInstance, + serveEndpoint, + serveIssuanceGateKv, + writeServiceStatus, +} from "@cotal-ai/core"; import { webProbeTarget } from "../src/commands/status.js"; const WT = resolve(import.meta.dirname, "..", "..", ".."); @@ -82,6 +108,25 @@ function cli(...args: string[]) { return spawnSync(TSX, [CLI, ...args], { cwd: root, env, encoding: "utf8", timeout: 30_000 }); } +async function cliAsync(...args: string[]): Promise<{ status: number | null; stdout: string; stderr: string }> { + const child = spawn(TSX, [CLI, ...args], { cwd: root, env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout?.setEncoding("utf8"); + child.stderr?.setEncoding("utf8"); + child.stdout?.on("data", (chunk) => { stdout += chunk; }); + child.stderr?.on("data", (chunk) => { stderr += chunk; }); + const status = await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`CLI timed out: ${args.join(" ")}`)); + }, 30_000); + child.once("error", (error) => { clearTimeout(timer); reject(error); }); + child.once("exit", (code) => { clearTimeout(timer); resolve(code); }); + }); + return { status, stdout, stderr }; +} + /** The old-surface fact under comparison, excluding unrelated process-wide advisories whose * presence depends on whether this particular subprocess crossed an approximate CPU threshold. */ function oldManagerAnswer(text: string): string | undefined { @@ -132,12 +177,166 @@ async function writeDeliveryHolder(): Promise { return child; } +const SERVED_INSTANCE = "s".repeat(26); +const FOREIGN_INSTANCE = "f".repeat(26); +const FOREIGN_REMEDY = "the owning manager must reconcile or an operator must perform an explicit CAS takeover"; +const FOREIGN_FAILURE_FACT = `static alias=foreign-owned phase=active disposition=refused remedy=${FOREIGN_REMEDY}`; + +async function serveManagerWithForeignRefusal(): Promise<{ close(): Promise }> { + const nc = await connect({ servers: server }); + const kvm = new Kvm(nc); + await createEndpointStreams(await jetstreamManager(nc), kvm, SPACE); + await kvm.create(deliveryBucket(SPACE), { ttl: 10_000 }).catch(async () => { await kvm.open(deliveryBucket(SPACE)); }); + + const input = compileContract({ root: VOID_SCHEMA }); + const outputSchema = { type: "object" } as const; + const output = compileContract({ root: outputSchema }); + const document = { + urn: "ai.cotal.test.component-health-manager", + revision: 1, + attributes: [], + events: [], + commands: [{ + name: "status", + class: "ephemeral" as const, + targeted: false, + capability: "manager.read", + inputDigest: input.closureDigest, + outputDigest: output.closureDigest, + }], + }; + const documentManifest = { v: 1 as const, root: contractDigest(document), members: [] as string[] }; + const artifacts = [ + VOID_SCHEMA, + { v: 1 as const, root: contractDigest(VOID_SCHEMA), members: [] as string[] }, + outputSchema, + { v: 1 as const, root: contractDigest(outputSchema), members: [] as string[] }, + document, + documentManifest, + ]; + const artifactIndex = new Map(artifacts.map((value) => [contractDigest(value), value])); + const contractStore = await contractStoreContext(nc, SPACE); + for (const value of artifactIndex.values()) + await publishContractArtifact(contractStore, contractArtifactCanonicalBytes(value)); + + const recordsKv = await openRecordsBucket(nc, SPACE); + const authKv = await kvm.open(epAuthBucket(SPACE)); + const authority = { + authorize: (endpoint: string, owner: string) => ({ authorized: endpoint === "manager" && owner === "local", revision: 0 }), + }; + await provisionEndpointGateOpen(authKv, { + endpoint: "manager", + instanceId: SERVED_INSTANCE, + principal: "local.mgr", + }); + const barrier = endpointRegistrationBarrier(authKv, SPACE, { + endpoint: "manager", + instanceId: SERVED_INSTANCE, + opId: "r".repeat(26), + }); + const { registrationRevision } = await registerServiceInstance(recordsKv, { + space: SPACE, + spec: { endpoint: "manager", owner: "local", clusterDigests: [contractDigest(documentManifest)], protocol: { v: 1 } }, + instanceId: SERVED_INSTANCE, + registrant: { owner: "local" }, + authority, + barrier, + readClusterArtifact: (digest) => artifactIndex.get(digest), + }); + const fence = serveIssuanceGateKv(authKv, SPACE, { endpoint: "manager", instanceId: SERVED_INSTANCE }); + const observed = await fence.observe(); + assert.ok(observed, "manager service issuance gate exists after registration"); + const grant = await authorizeServeGrant(recordsKv, { + space: SPACE, + endpoint: "manager", + instanceId: SERVED_INSTANCE, + epoch: observed.processEpoch, + holder: { owner: "local" }, + authority, + readProcessEpoch: () => observed.processEpoch, + readClusterArtifact: (digest) => artifactIndex.get(digest), + }); + await writeServiceStatus(recordsKv, { + endpoint: "manager", + instanceId: SERVED_INSTANCE, + epoch: observed.processEpoch, + status: { state: SERVICE_READY, epoch: observed.processEpoch, observedSpecRevision: registrationRevision }, + readProcessEpoch: () => observed.processEpoch, + }); + const service = serveEndpoint(nc, SPACE, grant, [{ + command: "status", + contract: { input, output }, + handler: () => ({ + instanceId: SERVED_INSTANCE, + runtime: "pty", + custody: "legacy", + agentCount: 0, + uptimeMs: 1, + connectors: [], + staticReconciliation: { + state: "failed", + failures: [{ + alias: "foreign-owned", + actor: "a".repeat(26), + lifecycleUid: "u".repeat(26), + phase: "active", + attempts: 1, + maxAttempts: 3, + disposition: "refused", + lastError: `the durable slot belongs to manager instance ${FOREIGN_INSTANCE}, not ${SERVED_INSTANCE}`, + remedy: FOREIGN_REMEDY, + }], + }, + }), + }], { public: true }); + await nc.flush(); + + const health = new CotalEndpoint({ + space: SPACE, + servers: server, + channels: [], + consume: false, + registerPresence: false, + watchPresence: false, + watchChannels: false, + card: { name: "component-health-serving-manager", kind: "endpoint" }, + }); + health.on("error", (error) => console.error(`serving fixture endpoint error: ${error.message}`)); + await health.start(); + const managerLease = { + holder: health.ref().id, + instanceId: SERVED_INSTANCE, + runtime: "pty", + root, + pid: process.pid, + }; + const managerLeaseRevision = await health.acquireManagerLease(managerLease); + const deliveryLeaseRevision = await health.acquireDeliveryLease(0); + await health.markDeliveryLeaseReady(0, deliveryLeaseRevision); + writeFileSync(join(root, ".cotal", "manager.pid"), String(process.pid)); + writeFileSync(join(root, ".cotal", "delivery.pid"), String(process.pid)); + writeFileSync(join(root, ".cotal", "renewal.json"), JSON.stringify({ + ts: "2026-09-06T00:00:00.000Z", owner: "fixture", results: [], adoption: { ok: true }, + })); + + return { + close: async () => { + await service.stop(); + await health.releaseDeliveryLease(0); + await health.releaseManagerLease(SERVED_INSTANCE, managerLeaseRevision); + await health.stop().catch(() => {}); + await nc.drain().catch(() => nc.close()); + }, + }; +} + const port = await freePort(); const server = `nats://127.0.0.1:${port}`; let broker: ChildProcess | undefined; let holder: ChildProcess | undefined; let web: ChildProcess | undefined; let delivery: ChildProcess | undefined; +let servingManager: { close(): Promise } | undefined; try { broker = spawn("nats-server", ["-a", "127.0.0.1", "-p", String(port), "-js", "-sd", store], { stdio: "ignore" }); for (let i = 0; i < 100 && !(await portOpen(port)); i++) await sleep(50); @@ -166,8 +365,8 @@ try { const present = cli("status", "--components", "--space", SPACE, "--server", server); const presentText = `${present.stdout}${present.stderr}`; check("live lease-holder that never serves exits present-not-serving (2)", present.status === 2, presentText); - check("manager row names not-serving, its PID, and its unreported phase", - /manager\s+not-serving/.test(presentText) && presentText.includes(`pid ${held.pid}`) && presentText.includes("phase not reported by this manager build"), presentText); + check("manager row names not-serving, its PID, and its unreported static reconciliation", + /manager\s+not-serving/.test(presentText) && presentText.includes(`pid ${held.pid}`) && presentText.includes("static reconciliation not reported by this manager build"), presentText); check("manager row names the lease holder rather than substituting service success", /lease holder local\./.test(presentText) && presentText.includes("serve no answer"), presentText); @@ -236,9 +435,24 @@ try { delivery = undefined; rmSync(join(root, ".cotal", "delivery.pid"), { force: true }); rmSync(join(root, ".cotal", "renewal.json"), { force: true }); + await sleep(10_500); + + web = await writeWebHarness(await freePort()); + servingManager = await serveManagerWithForeignRefusal(); + const benignRefusal = await cliAsync("status", "--components", "--space", SPACE, "--server", server); + const benignRefusalText = `${benignRefusal.stdout}${benignRefusal.stderr}`; + check("benign foreign-owned reconciliation stays serving with exact failure facts", + benignRefusal.status === 0 && + /manager\s+serving/.test(benignRefusalText) && + !/manager\s+not-serving/.test(benignRefusalText) && + benignRefusalText.includes("static reconciliation failed") && + benignRefusalText.includes(FOREIGN_FAILURE_FACT) && + benignRefusalText.includes("serve reachable"), + benignRefusalText); console.log(`\nCOMPONENT HEALTH SMOKE OK ✅ (${pass} checks)`); } finally { + await servingManager?.close(); await stop(delivery); await stop(web); await stop(holder); diff --git a/implementations/cli/smoke/fixtures/component-health.mutations.json b/implementations/cli/smoke/fixtures/component-health.mutations.json index 1f8a54b24..00e6c1c50 100644 --- a/implementations/cli/smoke/fixtures/component-health.mutations.json +++ b/implementations/cli/smoke/fixtures/component-health.mutations.json @@ -3,6 +3,15 @@ "progressPattern": "\u2713", "minTicks": 13, "mutations": [ + { + "name": "a failed reconciliation state overrides a reachable manager service holding its lease", + "file": "implementations/cli/src/commands/status.ts", + "find": "return { name: \"manager\", verdict: lease ? \"serving\" : \"not-serving\", facts };", + "replace": "const reconcileFailed = reconcile?.state === \"failed\";\n return { name: \"manager\", verdict: lease && !reconcileFailed ? \"serving\" : \"not-serving\", facts };", + "expectRed": "benign foreign-owned reconciliation stays serving with exact failure facts", + "cell": "benign foreign-owned reconciliation stays serving with exact failure facts", + "afterRestore": "pnpm --filter @cotal-ai/cli build" + }, { "name": "the manager's service answer is treated as healthy even when its control rail never answered", "file": "implementations/cli/src/commands/status.ts", diff --git a/implementations/cli/src/commands/status.ts b/implementations/cli/src/commands/status.ts index 38d1f3fe1..a936417ed 100644 --- a/implementations/cli/src/commands/status.ts +++ b/implementations/cli/src/commands/status.ts @@ -607,7 +607,14 @@ async function componentEp(target: MeshTarget): Promise<{ ep: CotalEndpoint; clo async function managerServiceHealth( target: MeshTarget, auth: { creds?: string; caller: { owner: string; actor: string; uid: string } }, -): Promise<{ instanceId?: unknown; runtime?: unknown }> { +): Promise<{ + instanceId?: unknown; + runtime?: unknown; + staticReconciliation?: { + state?: unknown; + failures?: Array<{ alias?: unknown; phase?: unknown; disposition?: unknown; nextRetryAt?: unknown; remedy?: unknown }>; + }; +}> { const nc = await connect({ servers: target.server, ...standaloneConnectOpts(auth.creds ? { creds: auth.creds, tls: target.tlsRequired } : { tls: target.tlsRequired }), @@ -618,7 +625,14 @@ async function managerServiceHealth( const response = await invokeCommand(nc, target.space, service, "status", undefined, { deadlineMs: 3_000 }); if (response.reply.ok !== true) throw new EpEnvelopeError(response.reply.error?.code === "unavailable" ? "unavailable" : "failed-precondition", response.reply.error?.message ?? "manager status refused"); - return response.reply.data as { instanceId?: unknown; runtime?: unknown }; + return response.reply.data as { + instanceId?: unknown; + runtime?: unknown; + staticReconciliation?: { + state?: unknown; + failures?: Array<{ alias?: unknown; phase?: unknown; disposition?: unknown; nextRetryAt?: unknown; remedy?: unknown }>; + }; + }; } finally { await nc.drain().catch(() => nc.close()); } @@ -631,7 +645,7 @@ async function managerHealth(target: MeshTarget, context: LocalProcessContext): // permission to replace it with a network answer. Name that failed local control surface first. if (record.kind === "unattributable" || record.kind === "unknown") { facts.push(record.kind === "unattributable" ? "unattributable pidfile" : "pid liveness unestablishable"); - facts.push("phase not reported by this manager build"); + facts.push("static reconciliation not reported by this manager build"); return { name: "manager", verdict: "refused", facts }; } if (record.kind === "dead") facts.push("stale pidfile"); @@ -657,13 +671,23 @@ async function managerHealth(target: MeshTarget, context: LocalProcessContext): }); facts.push(`service instance ${served.instanceId ?? "unreported"}`); facts.push(`runtime ${served.runtime ?? "unreported"}`); - facts.push("phase not reported by this manager build"); + const reconcile = served.staticReconciliation; + if (!reconcile) { + facts.push("static reconciliation not reported by this manager build"); + } else { + facts.push(`static reconciliation ${String(reconcile.state ?? "unreported")}`); + for (const failure of reconcile.failures ?? []) { + const next = typeof failure.nextRetryAt === "string" ? ` nextRetryAt=${failure.nextRetryAt}` : ""; + const remedy = typeof failure.remedy === "string" ? ` remedy=${failure.remedy}` : ""; + facts.push(`static alias=${String(failure.alias ?? "unreported")} phase=${String(failure.phase ?? "unreported")} disposition=${String(failure.disposition ?? "unreported")}${next}${remedy}`); + } + } facts.push("serve reachable"); // A manager service without its own liveness lease is a contradicted component surface, not a // healthy one. It still reports reachability, but cannot claim the required lease holder. return { name: "manager", verdict: lease ? "serving" : "not-serving", facts }; } catch (e) { - facts.push("phase not reported by this manager build"); + facts.push("static reconciliation not reported by this manager build"); // A no-responder service rail or an absent manager registry is definitive no-service evidence. // The lease and PID answer whether that missing service belongs to an extant component (not // serving) or an absent one; any other failed probe remains a refusal. diff --git a/implementations/manager/smoke/fixtures/manager-reconcile-redrive.mutations.json b/implementations/manager/smoke/fixtures/manager-reconcile-redrive.mutations.json new file mode 100644 index 000000000..48ce59174 --- /dev/null +++ b/implementations/manager/smoke/fixtures/manager-reconcile-redrive.mutations.json @@ -0,0 +1,54 @@ +{ + "suite": "implementations/manager/smoke/manager-reconcile-redrive.smoke.ts", + "guard": "startup static reconciliation reports a failed sweep and re-drives the same durable exact terminal in process", + "command": "pnpm smoke:manager-reconcile-redrive", + "proveWith": "node scripts/mutation-proof.mjs --config implementations/manager/smoke/fixtures/manager-reconcile-redrive.mutations.json", + "why": [ + "The suite starts a real authenticated nats-server through serverConfig and an in-process Manager, then plants three durable orphan lifecycle rows.", + "The first mutation changes only the failed aggregate branch. The earliest named cell observes the completed first sweep after all three shipped terminal paths ran, so it reddens because failure became success-shaped reporting rather than because an earlier row short-circuited.", + "The second mutation removes only the retry scheduling call after the real middle-row terminal failed. Earlier aggregate, durable hold, and status cells still run; the named re-drive cell is the first one that requires a second exact terminal entry.", + "The shutdown mutations remove the accepted-sweep drain, the serial stop fence, and the start-task join. The real broker control gates the first terminal and holds service registration after startup has passed its shutdown fence, so the start-task join has its own deterministic observation before the registration gate is released." + ], + "mutations": [ + { + "name": "report a failed reconciliation sweep with the clean-success aggregate", + "file": "implementations/manager/src/manager.ts", + "find": " console.error(`! static reconcile completed: ${sweep.attempted} attempted, ${sweep.succeeded} succeeded, ${sweep.failed} failed; failed=${failed.join(\",\")}`);", + "replace": " console.error(`✓ static reconcile completed: ${sweep.attempted} attempted, ${sweep.attempted} succeeded, 0 failed`);", + "expectRed": "the first sweep is non-success and names the failed alias, phase, and disposition", + "cell": "the first sweep is non-success and names the failed alias, phase, and disposition" + }, + { + "name": "do not schedule an in-process re-drive after the exact terminal fails", + "file": "implementations/manager/src/manager.ts", + "find": " if (this.staticReconcileSweepsInFlight > 0) item.disposition = \"retry-scheduled\";\n else this.scheduleStaticReconcileRetry(key, item);", + "replace": " item.disposition = \"retry-exhausted\";\n item.remedy = \"restart this manager for a fresh per-process retry budget\";", + "expectRed": "the same manager process re-drives the failed alias to retired", + "cell": "the same manager process re-drives the failed alias to retired" + }, + { + "name": "stop returns while accepted startup reconciliation work is still running", + "file": "implementations/manager/src/manager.ts", + "find": " await this.awaitStaticReconcileDrain();\n await starting?.catch(() => {});", + "replace": "", + "expectRed": "stop waits for an accepted startup reconciliation terminal", + "cell": "stop waits for an accepted startup reconciliation terminal" + }, + { + "name": "a shutdown sweep starts the next planned alias after the accepted terminal drains", + "file": "implementations/manager/src/manager.ts", + "find": " for (const row of terminalRows) {\n if (this.staticReconcileStopping) break;\n sweep.attempted++;", + "replace": " for (const row of terminalRows) {\n sweep.attempted++;", + "expectRed": "the shutdown fence prevents the serial sweep from starting a later terminal", + "cell": "the shutdown fence prevents the serial sweep from starting a later terminal" + }, + { + "name": "stop does not join startup after draining reconciliation", + "file": "implementations/manager/src/manager.ts", + "find": " await this.awaitStaticReconcileDrain();\n await starting?.catch(() => {});", + "replace": " await this.awaitStaticReconcileDrain();", + "expectRed": "stop joins startup service registration admitted before shutdown", + "cell": "stop joins startup service registration admitted before shutdown" + } + ] +} diff --git a/implementations/manager/smoke/manager-reconcile-redrive.smoke.ts b/implementations/manager/smoke/manager-reconcile-redrive.smoke.ts new file mode 100644 index 000000000..ca6742b9b --- /dev/null +++ b/implementations/manager/smoke/manager-reconcile-redrive.smoke.ts @@ -0,0 +1,367 @@ +/* + * Startup static-lifecycle convergence (#774). + * + * A real authenticated nats-server, an in-process Manager, durable lifecycle rows, and the existing + * eviction-verdict boundary prove that one transient middle-row terminal failure is reported, held, + * and re-driven by the same process without duplicating the lifecycle or retirement operation. + * + * Run: pnpm smoke:manager-reconcile-redrive + */ +import { spawn, type ChildProcess } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createServer, type AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { connect } from "@nats-io/transport-node"; +import { Kvm } from "@nats-io/kv"; +import { + DEV_OWNER, + createSpaceAuth, + epAuthBucket, + epCall, + gateFreeze, + gateObserve, + mintCreds, + mintLifecycleUid, + newIdentity, + principalKey, + probeConnect, + recordsBucket, + registry, + serverConfig, + setupSpaceStreams, + standaloneConnectOpts, + type Connector, + type EpCaller, + type EvictionResult, + type LaunchOpts, + type LaunchSpec, + type StaticManagedSlotRow, +} from "@cotal-ai/core"; +import { authDir, saveManagerInstanceIdentity, saveSpaceAuth } from "@cotal-ai/workspace"; +import { Manager } from "../src/manager.js"; +import { MANAGER_CONTRACTS, MANAGER_ENDPOINT, type ManagerStatus } from "../src/manager-service-contract.js"; +import { + activateStaticLifecycle, + appendStaticCredentialRow, + casStaticSlot, + readStaticSlot, + recordSlotCredential, + staticLifecycleTransport, +} from "../src/static-lifecycle.js"; + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const until = async (condition: () => Promise | boolean, ms: number): Promise => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + if (await condition()) return true; + await wait(20); + } + return false; +}; +const freePort = (): Promise => new Promise((resolve, reject) => { + const server = createServer(); + server.on("error", reject); + server.listen(0, "127.0.0.1", () => { + const port = (server.address() as AddressInfo).port; + server.close(() => resolve(port)); + }); +}); +const awaitExit = (child: ChildProcess, ms = 5_000): Promise => new Promise((resolve) => { + if (child.exitCode !== null || child.signalCode !== null) return resolve(); + child.once("exit", () => resolve()); + setTimeout(resolve, ms).unref?.(); +}); + +let pass = 0; +let fail = 0; +const check = (name: string, condition: boolean, extra?: unknown): void => { + console.log(` ${condition ? "✓" : "✗ FAIL:"} ${name}${condition || extra === undefined ? "" : ` ${JSON.stringify(extra)}`}`); + if (condition) pass++; + else fail++; +}; + +const space = `reconcile-redrive-${randomUUID().slice(0, 8)}`; +const auth = await createSpaceAuth(space); +const port = await freePort(); +const servers = `nats://127.0.0.1:${port}`; +const root = mkdtempSync(join(tmpdir(), "cotal-reconcile-redrive-ws-")); +const brokerStore = mkdtempSync(join(tmpdir(), "cotal-reconcile-redrive-js-")); +const conf = join(root, "server.conf"); +const managerInstanceId = mintLifecycleUid(); +mkdirSync(join(root, ".cotal", "agents"), { recursive: true }); +saveSpaceAuth(authDir(root), auth); +saveManagerInstanceIdentity(root, space, { instanceId: managerInstanceId, serveIdentity: newIdentity() }); +writeFileSync(conf, serverConfig(auth, [auth], { transport: { kind: "plaintext" }, port, storeDir: brokerStore, host: "127.0.0.1" })); +const broker = spawn("nats-server", ["-c", conf], { stdio: "ignore" }); + +let manager: Manager | undefined; +let shutdownManager: Manager | undefined; +let observer: Awaited> | undefined; +let callerNc: Awaited> | undefined; +const logs: string[] = []; +const realError = console.error; +console.error = (...args: unknown[]): void => { + logs.push(args.map(String).join(" ")); + realError(...args); +}; + +const identities = new Map(); +const attempts = new Map(); +let firstEvictionEntered = false; +let releaseFirst!: () => void; +const firstEvictionRelease = new Promise((resolve) => { releaseFirst = resolve; }); +const seatStub = join(import.meta.dirname, "e2e-stub.mjs"); +const envJoin = (opts: LaunchOpts): Record => ({ + COTAL_SPACE: opts.space, COTAL_SERVERS: String(opts.servers ?? servers), COTAL_CREDS: String(opts.creds ?? ""), + COTAL_ID: String(opts.id ?? ""), COTAL_LIFECYCLE_UID: String(opts.lifecycleUid ?? ""), COTAL_NAME: opts.name, + PATH: process.env.PATH ?? "", +}); +const connector: Connector = { + kind: "connector", + name: "reconcile-redrive-stub", + requires: ["node"], + buildLaunch: (opts): LaunchSpec => ({ command: process.execPath, args: [seatStub], env: envJoin(opts) }), +}; +registry.register(connector); + +async function writeOrphan( + alias: string, + foreignRetirementOp = false, + ownerInstanceId = managerInstanceId, + workspaceRoot = root, +): Promise { + const identity = newIdentity(); + const uid = mintLifecycleUid(); + const principal = principalKey(DEV_OWNER, identity.id).key; + identities.set(alias, { actor: identity.id, uid, principal }); + mkdirSync(join(workspaceRoot, ".cotal", "agents"), { recursive: true }); + writeFileSync(join(workspaceRoot, ".cotal", "agents", `${alias}.md`), `---\nname: ${alias}\nrole: worker\n---\nbody\n`); + const creds = await mintCreds(auth, newIdentity(), "lifecycle-executor", { lifecycleExecutor: { owner: DEV_OWNER, actor: identity.id, lifecycleUid: uid, alias } }); + const nc = await connect({ servers, ...standaloneConnectOpts({ creds, tls: false }), maxReconnectAttempts: 0 }); + try { + const kvm = new Kvm(nc); + const transport = staticLifecycleTransport(await kvm.open(recordsBucket(space)), await kvm.open(epAuthBucket(space))); + await activateStaticLifecycle(transport, { owner: DEV_OWNER, alias, actor: identity.id, lifecycleUid: uid, managerInstance: "orphaned-process", ownerInstanceId }); + const credentialId = `cred-${alias}`; + await recordSlotCredential(transport, DEV_OWNER, alias, uid, credentialId); + await appendStaticCredentialRow(transport, { lifecycleUid: uid, credentialId, holderPrincipal: principal, exp: Math.floor(Date.now() / 1000) + 3600 }); + const slot = await readStaticSlot(transport, DEV_OWNER, alias); + await casStaticSlot(transport, { ...slot!.row, phase: "active" }, slot!.revision); + if (foreignRetirementOp) { + const gate = await gateObserve(transport, uid); + await gateFreeze(transport, { lifecycleUid: uid, revision: gate!.revision, op: { kind: "retirement", opId: "f".repeat(26) } }); + } + } finally { + await nc.drain().catch(() => nc.close()); + } +} + +async function slot(alias: string): Promise { + const records = await new Kvm(observer!).open(recordsBucket(space)); + return (await readStaticSlot(staticLifecycleTransport(records, records), DEV_OWNER, alias))?.row; +} + +async function status(caller: EpCaller): Promise { + try { + const result = await epCall(callerNc!, space, { mode: "one" }, { endpoint: MANAGER_ENDPOINT, command: "status", contract: MANAGER_CONTRACTS.status, caller }, { deadlineMs: 1_000, currentEpoch: async () => 0 }); + return result.reply.ok ? result.reply.data as ManagerStatus : undefined; + } catch { + return undefined; + } +} + +try { + let serving = false; + for (let i = 0; i < 80; i++) { + const probe = await probeConnect(servers, { timeoutMs: 300 }); + if (probe.ok || probe.reason === "auth-required") { serving = true; break; } + await wait(50); + } + check("the real authenticated broker is serving", serving); + await setupSpaceStreams({ servers, space, creds: await mintCreds(auth, newIdentity(), "provisioner") }); + for (const alias of ["orphan-first", "orphan-middle", "orphan-last"]) await writeOrphan(alias); + + observer = await connect({ servers, ...standaloneConnectOpts({ creds: await mintCreds(auth, newIdentity(), "provisioner"), tls: false }), maxReconnectAttempts: 0 }); + const callerIdentity = newIdentity(); + const caller: EpCaller = { owner: DEV_OWNER, actor: callerIdentity.id, uid: mintLifecycleUid() }; + callerNc = await connect({ servers, ...standaloneConnectOpts({ creds: await mintCreds(auth, callerIdentity, "agent", { lifecycleUid: caller.uid, endpointCapabilities: [{ endpoint: MANAGER_ENDPOINT, command: "status" }] }), tls: false }), maxReconnectAttempts: 0 }); + + manager = new Manager({ space, servers, runtime: "pty", workspaceRoot: root }); + const internals = manager as unknown as { + staticLifecycleEvict?: (principal: string) => Promise; + staticReconcileRetryDelaysMs: readonly number[]; + staticReconcileItems: Map; nextRetryAt?: string; attempts: number; disposition: string }>; + managerStatusData(): ManagerStatus; + reconcileStaticLifecycles(): Promise; + retryStaticReconcile(key: string): Promise; + }; + internals.staticReconcileRetryDelaysMs = [1_000, 80, 80]; + internals.staticLifecycleEvict = async (principal): Promise => { + const alias = [...identities].find(([, row]) => row.principal === principal)?.[0] ?? principal; + const count = (attempts.get(alias) ?? 0) + 1; + attempts.set(alias, count); + if (alias === "orphan-first" && count === 1) { + firstEvictionEntered = true; + await firstEvictionRelease; + } + const failMiddleOnce = alias === "orphan-middle" && count === 1; + const failPersistent = alias === "orphan-persistent"; + const verifiedGone = !(failMiddleOnce || failPersistent); + return { principal, kicked: verifiedGone ? 1 : 0, remaining: verifiedGone ? 0 : 1, scanComplete: true, verifiedGone }; + }; + + const starting = manager.start(); + check("startup reconciliation reached the first real terminal", await until(() => firstEvictionEntered, 20_000)); + const servingStatus = await until(async () => ["running", "retrying"].includes((await status(caller))?.staticReconciliation.state ?? ""), 20_000); + check("the manager endpoint serves while reconciliation is active", servingStatus, await status(caller)); + check("a later planned row is still active while the first terminal is paused", (await slot("orphan-last"))?.phase === "active", await slot("orphan-last")); + releaseFirst(); + await starting; + + const firstSweepDone = await until(() => logs.some((line) => /! static reconcile completed: 3 attempted, 2 succeeded, 1 failed; failed=orphan-middle:terminalizing:retry-scheduled/.test(line)), 20_000); + check("the first sweep is non-success and names the failed alias, phase, and disposition", firstSweepDone, logs.filter((line) => line.includes("static reconcile completed"))); + check("the first sweep has no N/N success-shaped progress line", !logs.some((line) => /static reconcile \d+\/\d+/.test(line)), logs.filter((line) => line.includes("static reconcile"))); + check("the later row reconciles despite the middle failure", (await slot("orphan-last"))?.phase === "retired", await slot("orphan-last")); + check("the failed alias stays durable terminalizing during the retry window", (await slot("orphan-middle"))?.phase === "terminalizing", await slot("orphan-middle")); + const failedStatus = await status(caller); + const middleFailure = failedStatus?.staticReconciliation.failures.find((row) => row.alias === "orphan-middle"); + check("served status prints retry-scheduled and the failed alias with a next retry time", failedStatus?.staticReconciliation.state === "retry-wait" && middleFailure?.disposition === "retry-scheduled" && typeof middleFailure.nextRetryAt === "string", failedStatus?.staticReconciliation); + + const blockedSpawn = await manager.startAgent({ name: "orphan-middle", agent: connector.name }); + check("the durable terminalizing row refuses same-alias spawn during the failure window", blockedSpawn.ok === false && /terminalizing/i.test(blockedSpawn.error ?? ""), blockedSpawn); + + const recovered = await until(async () => (await slot("orphan-middle"))?.phase === "retired", 5_000); + check("the same manager process re-drives the failed alias to retired", recovered && attempts.get("orphan-middle") === 2, { phase: (await slot("orphan-middle"))?.phase, attempts: attempts.get("orphan-middle") }); + const recoveredStatus = await status(caller); + const recoveredRow = recoveredStatus?.staticReconciliation.failures.find((row) => row.alias === "orphan-middle"); + check("served status prints the recovered disposition until the next sweep", recoveredStatus?.staticReconciliation.state === "recovered" && recoveredRow?.disposition === "recovered", recoveredStatus?.staticReconciliation); + + const old = identities.get("orphan-middle")!; + const lifecycleCreds = await mintCreds(auth, newIdentity(), "lifecycle-executor", { lifecycleExecutor: { owner: DEV_OWNER, actor: old.actor, lifecycleUid: old.uid, alias: "orphan-middle" } }); + const lifecycleNc = await connect({ servers, ...standaloneConnectOpts({ creds: lifecycleCreds, tls: false }), maxReconnectAttempts: 0 }); + const lifecycleKvm = new Kvm(lifecycleNc); + const oldGate = await gateObserve(staticLifecycleTransport(await lifecycleKvm.open(recordsBucket(space)), await lifecycleKvm.open(epAuthBucket(space))), old.uid); + await lifecycleNc.drain().catch(() => lifecycleNc.close()); + const expectedOp = createHash("sha256").update(`retire:${old.uid}`).digest("hex").slice(0, 26); + check("the re-drive reused one deterministic terminal operation", oldGate?.row.state === "retired" && oldGate.row.op?.opId === expectedOp, oldGate?.row); + + const respawn = await manager.startAgent({ name: "orphan-middle", agent: connector.name }); + const respawnUid = respawn.ok ? (respawn.data as { lifecycleUid: string }).lifecycleUid : undefined; + check("same-alias spawn succeeds only after retirement and mints one successor lifecycle", respawn.ok === true && respawnUid !== old.uid, respawn); + check("the predecessor lifecycle was not duplicated or replaced during re-drive", attempts.get("orphan-first") === 1 && attempts.get("orphan-middle") === 2 && attempts.get("orphan-last") === 1, Object.fromEntries(attempts)); + + // Persistent control. A fresh sweep clears the recovered transition, but keeps a failed coordinate's + // existing per-process budget. Two extra triggers join its scheduled retry rather than resetting it. + await writeOrphan("orphan-persistent"); + // The transient acceptance above already used the real timer. Keep this independent bounded and + // duplicate-trigger control manual so a loaded host cannot fire a compressed timer mid-assertion. + internals.staticReconcileRetryDelaysMs = [60_000, 60_000, 60_000]; + await internals.reconcileStaticLifecycles(); + const persistentIdentity = identities.get("orphan-persistent")!; + const persistentKey = JSON.stringify([DEV_OWNER, "orphan-persistent", persistentIdentity.uid]); + const persistentItem = internals.staticReconcileItems.get(persistentKey)!; + if (persistentItem.timer) clearTimeout(persistentItem.timer); + persistentItem.timer = undefined; + persistentItem.nextRetryAt = undefined; + await Promise.all([internals.retryStaticReconcile(persistentKey), internals.retryStaticReconcile(persistentKey)]); + check("two retry triggers join one durable re-read and exact-terminal flight", attempts.get("orphan-persistent") === 2 && persistentItem.attempts === 2, { attempts: attempts.get("orphan-persistent"), item: persistentItem }); + for (let expected = 3; expected <= 4; expected++) { + if (persistentItem.timer) clearTimeout(persistentItem.timer); + persistentItem.timer = undefined; + persistentItem.nextRetryAt = undefined; + await internals.retryStaticReconcile(persistentKey); + check(`persistent failure completed bounded attempt ${expected}/4`, attempts.get("orphan-persistent") === expected && persistentItem.attempts === expected, { attempts: attempts.get("orphan-persistent"), item: persistentItem }); + } + const exhausted = internals.managerStatusData().staticReconciliation.failures.some((row) => row.alias === "orphan-persistent" && row.disposition === "retry-exhausted"); + const exhaustedStatus = internals.managerStatusData().staticReconciliation; + const exhaustedRow = exhaustedStatus.failures.find((row) => row.alias === "orphan-persistent"); + check("persistent failure is bounded at four attempts despite duplicate retry triggers", exhausted && attempts.get("orphan-persistent") === 4, { attempts: attempts.get("orphan-persistent"), exhaustedStatus }); + check("status prints retry-exhausted with no next retry and the restart disposition", exhaustedStatus.state === "failed" && exhaustedRow?.disposition === "retry-exhausted" && exhaustedRow.nextRetryAt === undefined && /restart this manager for a fresh per-process retry budget/.test(exhaustedRow.remedy ?? ""), exhaustedStatus); + check("the prior recovered transition clears on the next sweep", !exhaustedStatus.failures.some((row) => row.alias === "orphan-middle"), exhaustedStatus); + const attemptsAtExhaustion = attempts.get("orphan-persistent"); + await wait(250); + check("an exhausted item does not busy-loop", attempts.get("orphan-persistent") === attemptsAtExhaustion, Object.fromEntries(attempts)); + await internals.reconcileStaticLifecycles(); + check("a later sweep in the same process does not reset an exhausted retry budget", attempts.get("orphan-persistent") === attemptsAtExhaustion, Object.fromEntries(attempts)); + check("the exhausted durable alias remains terminalizing for a fresh manager process to re-plan", (await slot("orphan-persistent"))?.phase === "terminalizing", await slot("orphan-persistent")); + + await writeOrphan("orphan-foreign", true); + await internals.reconcileStaticLifecycles(); + const foreignRow = internals.managerStatusData().staticReconciliation.failures.find((row) => row.alias === "orphan-foreign"); + check("a foreign frozen retirement is refused literally and never reaches eviction", foreignRow?.disposition === "refused-foreign" && attempts.get("orphan-foreign") === undefined, { foreignRow, attempts: Object.fromEntries(attempts) }); + + // Shutdown control. A second logical manager can coexist in this space, which isolates its owned + // rows from the main redrive scenario above. Gate its first exact terminal, request stop, then + // release it. The registration gate holds startup after it has passed the shutdown fence, so + // stop() must join that admitted startup task after draining the terminal. The serial sweep must + // not start the later row, and no service registration may survive the completed shutdown. + const shutdownRoot = join(root, "shutdown-manager"); + const shutdownInstanceId = mintLifecycleUid(); + mkdirSync(join(shutdownRoot, ".cotal", "agents"), { recursive: true }); + saveSpaceAuth(authDir(shutdownRoot), auth); + saveManagerInstanceIdentity(shutdownRoot, space, { instanceId: shutdownInstanceId, serveIdentity: newIdentity() }); + await writeOrphan("shutdown-first", false, shutdownInstanceId, shutdownRoot); + await writeOrphan("shutdown-last", false, shutdownInstanceId, shutdownRoot); + let shutdownFirstEntered = false; + let releaseShutdownFirst!: () => void; + const shutdownFirstRelease = new Promise((resolve) => { releaseShutdownFirst = resolve; }); + shutdownManager = new Manager({ space, servers, runtime: "pty", workspaceRoot: shutdownRoot }); + const shutdownInternals = shutdownManager as unknown as { + staticLifecycleEvict?: (principal: string) => Promise; + serviceServe?: unknown; + registerManagerService(): Promise; + reconcileStaticLifecycles(): Promise; + }; + let shutdownRegistrationEntered = false; + let releaseShutdownRegistration!: () => void; + const shutdownRegistrationRelease = new Promise((resolve) => { releaseShutdownRegistration = resolve; }); + const registerShutdownManagerService = shutdownInternals.registerManagerService.bind(shutdownManager); + shutdownInternals.registerManagerService = async () => { + shutdownRegistrationEntered = true; + await shutdownRegistrationRelease; + await registerShutdownManagerService(); + }; + shutdownInternals.staticLifecycleEvict = async (principal): Promise => { + const alias = [...identities].find(([, row]) => row.principal === principal)?.[0] ?? principal; + attempts.set(alias, (attempts.get(alias) ?? 0) + 1); + if (alias === "shutdown-first") { + shutdownFirstEntered = true; + await shutdownFirstRelease; + } + return { principal, kicked: 1, remaining: 0, scanComplete: true, verifiedGone: true }; + }; + const shutdownStarting = shutdownManager.start(); + check("shutdown control reached its first accepted exact terminal", await until(() => shutdownFirstEntered, 20_000)); + check("shutdown control reached service registration after the startup fence", await until(() => shutdownRegistrationEntered, 20_000)); + let shutdownSettled = false; + const shutdownStopping = shutdownManager.stop().then(() => { shutdownSettled = true; }); + await wait(150); + check("stop waits for an accepted startup reconciliation terminal", shutdownSettled === false); + releaseShutdownFirst(); + check("stop joins startup service registration admitted before shutdown", !(await until(() => shutdownSettled, 1_000))); + releaseShutdownRegistration(); + await Promise.allSettled([shutdownStarting, shutdownStopping]); + check("the shutdown fence prevents the serial sweep from starting a later terminal", attempts.get("shutdown-first") === 1 && attempts.get("shutdown-last") === undefined, Object.fromEntries(attempts)); + check("startup cannot publish the manager service after stop completes", shutdownInternals.serviceServe === undefined, { registered: shutdownInternals.serviceServe !== undefined }); + await shutdownInternals.reconcileStaticLifecycles(); + check("shutdown refuses a new static reconciliation sweep", attempts.get("shutdown-last") === undefined, Object.fromEntries(attempts)); + shutdownManager = undefined; +} finally { + console.error = realError; + await shutdownManager?.stop().catch(() => {}); + await manager?.stop().catch(() => {}); + await callerNc?.drain().catch(() => callerNc?.close()); + await observer?.drain().catch(() => observer?.close()); + broker.kill("SIGTERM"); + await awaitExit(broker); + rmSync(root, { recursive: true, force: true }); + rmSync(brokerStore, { recursive: true, force: true }); +} + +if (fail) { + console.log(`MANAGER RECONCILE REDRIVE FAILED (${fail} failures, ${pass} passed)`); + process.exit(1); +} +console.log(`MANAGER RECONCILE REDRIVE OK (${pass} checks)`); diff --git a/implementations/manager/smoke/manager-service-invoke.smoke.ts b/implementations/manager/smoke/manager-service-invoke.smoke.ts index 7261e9367..e59c4a4e7 100644 --- a/implementations/manager/smoke/manager-service-invoke.smoke.ts +++ b/implementations/manager/smoke/manager-service-invoke.smoke.ts @@ -110,6 +110,7 @@ try { console.log("1. resolveService: describe + fetch + recompile the full surface (no hand-imported schemas)"); const service = await resolveService(nc, space, MANAGER_ENDPOINT, caller, { deadlineMs: 10_000 }); + check("static reconciliation status advances the manager cluster revision", shipped.revision === 13, shipped); check("the resolved surface matches the shipped cluster document", service.commands.size === shipped.commandCount && shipped.names.every((n) => service.commands.has(n)) && service.commands.has("status") && service.commands.has("spawn") && service.commands.has("despawn"), [...service.commands.keys()].sort()); const statusCmd = service.commands.get("status")!; check("status resolved: untargeted, read capability, recompiled contracts carry closure digests", diff --git a/implementations/manager/smoke/mutations/reconcile-startup.json b/implementations/manager/smoke/mutations/reconcile-startup.json index 41864fc1b..1bce1fb80 100644 --- a/implementations/manager/smoke/mutations/reconcile-startup.json +++ b/implementations/manager/smoke/mutations/reconcile-startup.json @@ -21,8 +21,8 @@ { "name": "serve last again, recreating startup-wide manager unavailability", "file": "implementations/manager/src/manager.ts", - "find": " const startupReconcile = this.auth && !this.userMode ? this.reconcileStaticLifecycles() : undefined;\n if (startupReconcile)\n void startupReconcile.catch((e) => console.error(`! ${STARTUP_RECONCILING}: ${(e as Error).message} - a later manager start retries any unfinished terminal`));", - "replace": " if (this.auth && !this.userMode) await this.reconcileStaticLifecycles();", + "find": " const startupReconcile = this.auth && !this.userMode ? this.reconcileStaticLifecycles() : undefined;\n if (startupReconcile)", + "replace": " const startupReconcile = this.auth && !this.userMode ? await this.reconcileStaticLifecycles() : undefined;\n if (startupReconcile)", "expectRed": "status serves while later orphan rows remain ACTIVE", "cell": "status serves while later orphan rows remain ACTIVE", "afterRestore": "pnpm --filter @cotal-ai/manager... build" diff --git a/implementations/manager/smoke/mutations/shipped-surface-pin.json b/implementations/manager/smoke/mutations/shipped-surface-pin.json index c9f11da57..0ad76fa82 100644 --- a/implementations/manager/smoke/mutations/shipped-surface-pin.json +++ b/implementations/manager/smoke/mutations/shipped-surface-pin.json @@ -8,7 +8,9 @@ "defect class: correct the day it is written, wrong after the next served command.", "", "M1 puts the restated 18 back into managerShippedSurface. The invoke cell compares live resolve", - "size to that count, so it goes red on the named mismatch. Unique code-only return, not a comment." + "size to that count, so it goes red on the named mismatch. Unique code-only return, not a comment.", + "M2 restores revision 12 after the status output digest changed. The named revision cell runs after", + "the service has published and a generic client has resolved the actual shipped document." ], "mutations": [ { @@ -18,6 +20,14 @@ "replace": " return { revision: document.revision, commandCount: 18, names };", "expectRed": "the resolved surface matches the shipped cluster document", "cell": "the resolved surface matches the shipped cluster document" + }, + { + "name": "M2 keep revision 12 after static reconciliation changes status output", + "file": "implementations/manager/src/manager-service-contract.ts", + "find": " revision: 13,", + "replace": " revision: 12,", + "expectRed": "static reconciliation status advances the manager cluster revision", + "cell": "static reconciliation status advances the manager cluster revision" } ] } diff --git a/implementations/manager/src/manager-service-contract.ts b/implementations/manager/src/manager-service-contract.ts index 2c6895d6d..afaacdfd9 100644 --- a/implementations/manager/src/manager-service-contract.ts +++ b/implementations/manager/src/manager-service-contract.ts @@ -57,7 +57,7 @@ export const MANAGER_CLUSTER_URN = "ai.cotal.manager"; const STATUS_OUTPUT_SCHEMA = { type: "object", additionalProperties: false, - required: ["instanceId", "runtime", "custody", "agentCount", "uptimeMs", "connectors"], + required: ["instanceId", "runtime", "custody", "agentCount", "uptimeMs", "connectors", "staticReconciliation"], properties: { /** The manager's stable service instance id (its per-process incarnation uid). */ instanceId: { type: "string" }, @@ -84,6 +84,49 @@ const STATUS_OUTPUT_SCHEMA = { }, }, }, + /** Current startup/post-adoption static lifecycle convergence state. */ + staticReconciliation: { + type: "object", + additionalProperties: false, + required: ["state", "failures"], + properties: { + state: { enum: ["idle", "running", "retry-wait", "retrying", "recovered", "failed"] }, + lastSweep: { + type: "object", + additionalProperties: false, + required: ["kind", "startedAt", "attempted", "succeeded", "failed"], + properties: { + kind: { enum: ["startup", "post-adoption"] }, + startedAt: { type: "string" }, + completedAt: { type: "string" }, + attempted: { type: "integer", minimum: 0 }, + succeeded: { type: "integer", minimum: 0 }, + failed: { type: "integer", minimum: 0 }, + }, + }, + failures: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["alias", "actor", "lifecycleUid", "phase", "attempts", "maxAttempts", "disposition"], + properties: { + alias: { type: "string" }, + actor: { type: "string" }, + lifecycleUid: { type: "string" }, + phase: { enum: ["provisioning", "active", "terminalizing", "retired"] }, + attempts: { type: "integer", minimum: 0 }, + maxAttempts: { type: "integer", minimum: 1 }, + disposition: { enum: ["retry-scheduled", "retrying", "recovered", "refused", "refused-foreign", "retry-exhausted"] }, + lastError: { type: "string" }, + nextRetryAt: { type: "string" }, + recoveredAt: { type: "string" }, + remedy: { type: "string" }, + }, + }, + }, + }, + }, }, } as const; @@ -95,6 +138,36 @@ export interface ManagerStatus { agentCount: number; uptimeMs: number; connectors: ManagerConnectorStatus[]; + staticReconciliation: ManagerStaticReconciliationStatus; +} + +export interface ManagerStaticReconciliationSweep { + kind: "startup" | "post-adoption"; + startedAt: string; + completedAt?: string; + attempted: number; + succeeded: number; + failed: number; +} + +export interface ManagerStaticReconciliationFailure { + alias: string; + actor: string; + lifecycleUid: string; + phase: "provisioning" | "active" | "terminalizing" | "retired"; + attempts: number; + maxAttempts: number; + disposition: "retry-scheduled" | "retrying" | "recovered" | "refused" | "refused-foreign" | "retry-exhausted"; + lastError?: string; + nextRetryAt?: string; + recoveredAt?: string; + remedy?: string; +} + +export interface ManagerStaticReconciliationStatus { + state: "idle" | "running" | "retry-wait" | "retrying" | "recovered" | "failed"; + lastSweep?: ManagerStaticReconciliationSweep; + failures: ManagerStaticReconciliationFailure[]; } export interface ManagerConnectorStatus { @@ -690,7 +763,10 @@ export const MANAGER_STATUS_CONTRACT: { input: CompiledContract; output: Compile * * 12 = the workflow-run family (`run-start`, `run-resume`, `run-answer`, `run-status`, `run-ps`, * SPEC 14.3): the manager hosts a run's driver and serves its operator surface. NEW SERVED - * COMMANDS are what a revision is for, and five of them cannot fold into 11. */ + * COMMANDS are what a revision is for, and five of them cannot fold into 11. + * + * 13 = manager `status` adds static reconciliation state. Its output digest changed, so cached + * revision-12 descriptions cannot name the new required output contract. */ export function managerClusterDocument(): { urn: string; revision: number; @@ -708,7 +784,7 @@ export function managerClusterDocument(): { } { return { urn: MANAGER_CLUSTER_URN, - revision: 12, + revision: 13, attributes: [], events: [], commands: ROWS.map((r) => ({ diff --git a/implementations/manager/src/manager.ts b/implementations/manager/src/manager.ts index 91fbd9797..133fca446 100644 --- a/implementations/manager/src/manager.ts +++ b/implementations/manager/src/manager.ts @@ -142,6 +142,7 @@ import { submissionFingerprint, EpEnvelopeError, lifecycleBlocked, + lifecycleBlockedFrom, renderLifecycleBlocked, type EpCommandDef, type EpServeContext, @@ -151,7 +152,7 @@ import { type Identity, type ServiceNameAuthority, } from "@cotal-ai/core"; -import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues, type ManagerConnectorStatus, type ManagerStatus } from "./manager-service-contract.js"; +import { MANAGER_ENDPOINT, managerClusterArtifacts, managerCommandDefs, managerContractArtifactValues, type ManagerConnectorStatus, type ManagerStaticReconciliationFailure, type ManagerStaticReconciliationStatus, type ManagerStaticReconciliationSweep, type ManagerStatus } from "./manager-service-contract.js"; import type { NatsConnection, Subscription } from "@nats-io/transport-node"; import type { KV } from "@nats-io/kv"; import { @@ -232,6 +233,19 @@ export function credRenewIntervalMs(ttlSeconds: number): number { * these aliases must wait until THAT alias's exact-op terminal attempt returns rather than racing * a reuse. */ const STARTUP_RECONCILING = "startup static lifecycle reconciliation"; +/** One initial terminal entry plus three re-drives. The short first retry absorbs a one-shot + * broker/eviction blip; the wider later spacing avoids hammering a persistently unavailable + * authority. The finite ~36s window is intentionally not a liveness inference: exhausting it + * leaves the durable alias held and requires a later manager start. */ +const STATIC_RECONCILE_RETRY_DELAYS_MS = [1_000, 5_000, 30_000] as const; +const STATIC_RECONCILE_MAX_ATTEMPTS = STATIC_RECONCILE_RETRY_DELAYS_MS.length + 1; + +type StaticReconcileItem = ManagerStaticReconciliationFailure & { + owner: string; + timer?: ReturnType; + flight?: Promise; + retryFlight?: Promise; +}; /** The STABLE retirement opId for one lifecycle (#29 piece 3): deterministic from the uid, so a * despawn retry, a same-name-spawn nudge, and the auth service's boot resume all drive the SAME @@ -971,6 +985,19 @@ export class Manager { /** Static aliases whose startup exact-op terminal is still in flight. Scoped per alias: control * serves during the sweep, but no caller can reuse or attach an alias the sweep still owns. */ private readonly reconcilingAliases = new Set(); + /** Current failed/recovered static reconciliation coordinates. This is an operator snapshot, not + * lifecycle authority: every re-drive re-reads broker KV and re-runs planStaticSlotResume. */ + private readonly staticReconcileItems = new Map(); + private staticReconcileLastSweep?: ManagerStaticReconciliationSweep; + private staticReconcileSweepsInFlight = 0; + private staticReconcileDrainWaiters: Array<() => void> = []; + private staticReconcileStopping = false; + /** The complete boot task. stop() joins it after fencing static reconciliation, so a registration + * already past an earlier shutdown check cannot finish after stop() returns. */ + private startTask?: Promise; + /** Held as an instance field so broker-owning smokes can compress the schedule without changing + * production semantics. It is never a fence expiry: the durable non-retired row stays authoritative. */ + private staticReconcileRetryDelaysMs: readonly number[] = STATIC_RECONCILE_RETRY_DELAYS_MS; /** SINGLE-FLIGHT guard for {@link deprovision} (INT-2/C): one in-flight teardown per * (name, lifecycleUid). The detached freeSlot teardown and every same-name-spawn nudge that * re-drives it JOIN one promise instead of launching a SECOND, concurrent teardown. Without it, @@ -1120,7 +1147,19 @@ export class Manager { `refused outright and writes no creds file.`; } - async start(): Promise { + start(): Promise { + if (this.startTask) return this.startTask; + const task = this.runStart(); + let wrapped!: Promise; + wrapped = task.finally(() => { + if (this.startTask === wrapped) this.startTask = undefined; + }); + this.startTask = wrapped; + return wrapped; + } + + private async runStart(): Promise { + this.staticReconcileStopping = false; await this.inspectConnectorsAtBoot(); await this.attach.start(); // In auth mode the manager is just another user in the space's account — it mints @@ -1252,7 +1291,7 @@ export class Manager { // attach THAT alias until its terminal attempt returns. const startupReconcile = this.auth && !this.userMode ? this.reconcileStaticLifecycles() : undefined; if (startupReconcile) - void startupReconcile.catch((e) => console.error(`! ${STARTUP_RECONCILING}: ${(e as Error).message} - a later manager start retries any unfinished terminal`)); + void startupReconcile.catch((e) => console.error(`! ${STARTUP_RECONCILING}: ${(e as Error).message} - no per-alias retry could be planned; a later manager start re-reads unfinished durable terminals`)); // P2 item 1 (1d): the manager serves NO ctl tiers - its whole control surface is the v0.4 // service endpoint registered below. The old three-tier rail (self/manager/admin) is deleted; // `ctl.delivery`/`ctl.delivery-admin` (the delivery daemon) and `ctl.auth-admin` (the auth @@ -1268,6 +1307,10 @@ export class Manager { this.credRenewTimer = setInterval(() => { void this.renewDaemonCreds(); }, credRenewIntervalMs(STANDING_RENEWABLE_TTL_SEC)); this.credRenewTimer.unref?.(); } + // stop() fences before it waits for an accepted startup reconciliation terminal. Once that + // terminal drains, start() and stop() resume from the same await boundary; start must observe + // the fence before it can publish a fresh service registration after shutdown. + if (this.staticReconcileStopping) return; // P2 item 1: register the manager as an ordinary v0.4 `service` endpoint (SPEC §13.7/§13.9) // and serve its typed command surface on the ep rails - since 1d the ONLY control door, in // EVERY mesh mode. Static + user meshes mint the scoped executor + endpoint-serve credential; @@ -1275,6 +1318,7 @@ export class Manager { // (there is no credential system - the broker enforces nothing, matching the old open-mesh ctl // trust). Fail-loud: a manager that cannot register does not start half-registered. await this.registerManagerService(); + if (this.staticReconcileStopping) return; // P2 item 2: stand up the standing goal-writer connection for spawn-as-action — AFTER // registration (it writes this endpoint's goal facts/records), disjoint from the serve cred. await this.startGoalWriter(); @@ -1910,6 +1954,15 @@ export class Manager { } async stop(): Promise { + this.staticReconcileStopping = true; + const starting = this.startTask; + for (const item of this.staticReconcileItems.values()) { + if (item.timer) clearTimeout(item.timer); + item.timer = undefined; + item.nextRetryAt = undefined; + } + await this.awaitStaticReconcileDrain(); + await starting?.catch(() => {}); if (this.leaseTimer) clearInterval(this.leaseTimer); if (this.credRenewTimer) clearInterval(this.credRenewTimer); if (this.sessionKeyRenewTimer) clearInterval(this.sessionKeyRenewTimer); @@ -2137,7 +2190,7 @@ export class Manager { // retiredPrincipals, so a copied JWT is refused). Best-effort + loud: a sweep failure must // not fail the finalize (the next non-resume boot re-drives it), but it is never swallowed. if (this.auth && !this.userMode) - await this.reconcileStaticLifecycles(true).catch((e) => console.error(`! post-resume static reconcile: ${(e as Error).message} - a durable active orphan may still wedge its alias until the next non-resume restart`)); + await this.reconcileStaticLifecycles(true).catch((e) => console.error(`! post-resume static reconcile: ${(e as Error).message} - no per-alias retry could be planned; a later manager start re-reads unfinished durable terminals`)); this.resumeFinalized = true; this.resumeRequired = false; return { ok: true, data: { attemptId: args.attemptId, state: "active" } }; @@ -5144,6 +5197,26 @@ export class Manager { agentCount: this.agents.size, uptimeMs: Date.now() - this.startedAtMs, connectors: this.connectorStatuses.map((row) => ({ ...row, binaries: { ...row.binaries } })), + staticReconciliation: this.staticReconciliationStatus(), + }; + } + + /** Render the current operator snapshot. Recovered rows live until the next sweep starts, then + * clear, so recovery is visible without becoming stale process-lifetime history. */ + private staticReconciliationStatus(): ManagerStaticReconciliationStatus { + const failures = [...this.staticReconcileItems.values()] + .map(({ owner: _owner, timer: _timer, flight: _flight, retryFlight: _retryFlight, ...item }) => ({ ...item })) + .sort((a, b) => a.alias.localeCompare(b.alias)); + const state: ManagerStaticReconciliationStatus["state"] = + failures.some((item) => item.disposition === "retrying") ? "retrying" : + failures.some((item) => item.disposition === "retry-scheduled") ? "retry-wait" : + failures.some((item) => item.disposition === "refused" || item.disposition === "refused-foreign" || item.disposition === "retry-exhausted") ? "failed" : + this.staticReconcileSweepsInFlight > 0 ? "running" : + failures.some((item) => item.disposition === "recovered") ? "recovered" : "idle"; + return { + state, + ...(this.staticReconcileLastSweep ? { lastSweep: { ...this.staticReconcileLastSweep } } : {}), + failures, }; } @@ -6616,7 +6689,7 @@ export class Manager { * clears (ABA-guarded by uid). A PRE-UNIT-B lifecycle (no slot row — spawned before the * durable registry existed) has nothing to terminalize: its footprint teardown runs directly * and the hold clears, the honest upgrade path. */ - private async driveStaticRetirement(a: { id: string; name: string; lifecycleUid: string; secretPaths?: ManagedAgent["secretPaths"]; staticCredentialRenewal?: Promise }): Promise { + private async driveStaticRetirement(a: { id: string; name: string; lifecycleUid: string; secretPaths?: ManagedAgent["secretPaths"]; staticCredentialRenewal?: Promise }, surfaceFailure = false): Promise { // A renewal that published its flight before the synchronous terminal latch is accepted work. // Drain it before the durable terminal enumerates credential ids and before cleanup deletes its // material; a failed renewal must not block retirement because it may still have staged an id. @@ -6664,6 +6737,7 @@ export class Manager { if (h && h.lifecycleUid === a.lifecycleUid) h.lastError = `the static retirement did not complete (${(e as Error).message}); the name stays held - a same-name spawn retries the same terminal (op ${opId})`; console.error(`static retirement ${a.name} (${a.id}): ${(e as Error).message}`); + if (surfaceFailure) throw e; } } @@ -6721,68 +6795,333 @@ export class Manager { * slots while a resume is still pending (adoption runs after it); the POST-ADOPTION sweep * (`postAdoption=true`, inside finalizeResume while `resumeRequired` still fences ordinary * spawns) terminalizes any active slot the resume did not claim. */ - private async reconcileStaticLifecycles(postAdoption = false): Promise { - if (!this.auth) return; + private staticReconcileKey(row: Pick): string { + return JSON.stringify([row.owner, row.alias, row.lifecycleUid]); + } + + /** Read one retry coordinate from the durable slot store. The status snapshot never authorizes a + * re-drive: every timer comes through this fresh read before the planner and exact terminal. */ + private async readStaticReconcileSlot(alias: string): Promise { + if (!this.auth) return undefined; const identity = newIdentity(); const creds = await mintCreds(this.auth, identity, "provisioner"); - const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 }); - const slotRows: StaticManagedSlotRow[] = []; + const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, ...standaloneConnectOpts({ creds, tls: false }), maxReconnectAttempts: 0 }); try { - const jsm = await jetstreamManager(nc); - const kvm = new Kvm(nc); - await ensureAuthorityStores(jsm, kvm, this.space); - const recordsKv = await kvm.open(recordsBucket(this.space)); - const t = staticLifecycleTransport(recordsKv, recordsKv /* auth reads unused in the sweep */); - const keys = await recordsKv.keys(`${STATIC_SLOT_PREFIX}.${DEV_OWNER}.>`); - const aliases: string[] = []; - for await (const k of keys) aliases.push(k.split(".").slice(2).join(".")); - for (const alias of aliases) { - const slot = await readStaticSlot(t, DEV_OWNER, alias); - if (slot !== undefined) slotRows.push(slot.row); - } + const recordsKv = await new Kvm(nc).open(recordsBucket(this.space)); + return (await readStaticSlot(staticLifecycleTransport(recordsKv, recordsKv), DEV_OWNER, alias))?.row; } finally { await nc.drain().catch(() => nc.close()); } - const terminalRows: StaticManagedSlotRow[] = []; - for (const row of slotRows) { - if (row.phase === "retired") { - // A retirement is a GLOBAL refusal fact — seed the F5 index for EVERY retired incarnation - // regardless of which instance owned it, so a sibling-retired incarnation's copied credential - // is refused at this control surface too. Ownership gates only the DESTRUCTIVE sweep below. - this.retiredPrincipals.add(principalKey(row.owner, row.actor).key); - continue; + } + + private scheduleStaticReconcileRetry(key: string, item: StaticReconcileItem): void { + if (this.staticReconcileStopping || item.timer || item.attempts >= item.maxAttempts) return; + const delay = this.staticReconcileRetryDelaysMs[item.attempts - 1]; + if (delay === undefined) return; + const nextRetryAtMs = Date.now() + delay; + item.disposition = "retry-scheduled"; + item.nextRetryAt = new Date(nextRetryAtMs).toISOString(); + item.timer = setTimeout(() => { + item.timer = undefined; + item.nextRetryAt = undefined; + const run = async (): Promise => { + // The timer may fire while the failed attempt is still unwinding, especially in compressed + // smoke schedules. Join that flight before consuming another budget entry; never overlap it. + if (item.flight) await item.flight; + await this.retryStaticReconcile(key); + }; + void run().catch((error) => { + item.lastError = `retry driver failed before the exact terminal: ${(error as Error).message}`; + item.disposition = "retry-exhausted"; + item.remedy = "restart this manager for a fresh per-process retry budget"; + console.error(`! static reconcile retry-exhausted alias=${item.alias} phase=${item.phase} uid=${item.lifecycleUid}: ${item.lastError}; NEXT: restart this manager for a fresh per-process retry budget`); + }); + }, delay); + item.timer.unref?.(); + console.error(`! static reconcile retry-scheduled alias=${item.alias} phase=${item.phase} uid=${item.lifecycleUid} attempt=${item.attempts + 1}/${item.maxAttempts} nextRetryAt=${item.nextRetryAt}`); + } + + private retryStaticReconcile(key: string): Promise { + const item = this.staticReconcileItems.get(key); + if (!item || this.staticReconcileStopping || item.disposition === "recovered") return Promise.resolve(); + if (item.retryFlight) return item.retryFlight; + const flight = this.driveStaticReconcileRetry(key, item); + let wrapped!: Promise; + wrapped = flight.finally(() => { + if (item.retryFlight === wrapped) item.retryFlight = undefined; + }); + item.retryFlight = wrapped; + return wrapped; + } + + private async driveStaticReconcileRetry(key: string, item: StaticReconcileItem): Promise { + if (item.flight) await item.flight; + if (this.staticReconcileStopping || item.disposition === "recovered") return; + // A retry trigger consumes one finite budget entry even if the authoritative re-read itself + // fails. Otherwise a broker outage before the planner could reschedule forever without ever + // incrementing `attempts`, violating the bounded-retry guarantee. + item.attempts++; + item.disposition = "retrying"; + item.remedy = undefined; + console.error(`static reconcile retrying alias=${item.alias} phase=${item.phase} uid=${item.lifecycleUid} attempt=${item.attempts}/${item.maxAttempts}`); + let row: StaticManagedSlotRow | undefined; + try { + row = await this.readStaticReconcileSlot(item.alias); + } catch (error) { + item.lastError = `durable slot re-read failed: ${(error as Error).message}`; + if (item.attempts >= item.maxAttempts) { + item.disposition = "retry-exhausted"; + item.remedy = "restart this manager for a fresh per-process retry budget"; + console.error(`! static reconcile retry-exhausted alias=${item.alias} phase=${item.phase} uid=${item.lifecycleUid}: ${item.lastError}; NEXT: restart this manager for a fresh per-process retry budget`); + } else { + this.scheduleStaticReconcileRetry(key, item); } - // 3b-2 RECONCILE OWNERSHIP (multi-manager-per-space): a manager adjudicates ONLY the non-retired - // rows THIS logical instance owns. A SIBLING manager's active/provisioning row is LEFT UNTOUCHED — - // sweep-terminalizing it would destroy the sibling's live agent (the historical all-agents-kill - // hazard, now cross-instance). A legacy row (pre-3b-2, no owner recorded) predates multi-manager, - // so this manager is its legitimate single-manager-past successor and reconciles it. An orphaned - // sibling row is reclaimed only by an explicit operator CAS takeover (ruling 1), never here. - if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId) continue; - // ADOPTION is genuine membership: a slot backed by a live managed agent THIS process owns - // at the SAME uid is never an orphan (empty at boot; exactly the adopted set at the - // post-adoption sweep — the fix for the F3 resume hole). - const live = this.agents.get(row.alias); - const adopted = live !== undefined && live.lifecycleUid === row.lifecycleUid; - // Boot sweep with a resume PENDING: an active slot may yet be adopted (the resume path runs - // AFTER this boot sweep), so DEFER it — the post-adoption sweep terminalizes any the resume - // did not claim. provisioning/terminalizing NEVER defer (they are crashed operations, never - // an agent to adopt). At `postAdoption` (or a non-resume boot) nothing defers. - if (!postAdoption && row.phase === "active" && !adopted && this.resumeRequired) continue; - if (planStaticSlotResume(row, adopted) !== "none") terminalRows.push(row); + return; + } + if (row === undefined || row.owner !== item.owner || row.actor !== item.actor || row.lifecycleUid !== item.lifecycleUid) { + item.disposition = "refused"; + item.remedy = "inspect the durable slot; reconciliation will not cross incarnations"; + item.lastError = row === undefined + ? "the durable slot vanished; reconciliation never acts from the in-memory snapshot" + : `the durable slot moved to actor ${row.actor} uid ${row.lifecycleUid}; reconciliation never crosses incarnations`; + console.error(`! static reconcile refused alias=${item.alias} phase=${item.phase} uid=${item.lifecycleUid}: ${item.lastError}`); + return; + } + item.phase = row.phase; + if (row.phase === "retired") { + item.disposition = "recovered"; + item.recoveredAt = new Date().toISOString(); + item.lastError = undefined; + console.error(`✓ static reconcile recovered alias=${item.alias} phase=retired uid=${item.lifecycleUid} attempts=${item.attempts}/${item.maxAttempts}`); + return; } - // Mark the complete planned set before the first terminal awaits. The manager may already be - // serving by now; this synchronous handoff prevents a spawn from slipping between an alias's - // discovery and its later serial exact-op terminal. - for (const row of terminalRows) this.reconcilingAliases.add(row.alias); - for (let index = 0; index < terminalRows.length; index++) { - const row = terminalRows[index]!; + if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId) { + item.disposition = "refused"; + item.remedy = "the owning manager must reconcile or an operator must perform an explicit CAS takeover"; + item.lastError = `the durable slot belongs to manager instance ${row.ownerInstanceId}, not ${this.managerInstanceId}`; + console.error(`! static reconcile refused alias=${item.alias} phase=${item.phase} uid=${item.lifecycleUid}: ${item.lastError}`); + return; + } + const live = this.agents.get(row.alias); + const adopted = live !== undefined && live.lifecycleUid === row.lifecycleUid; + if (planStaticSlotResume(row, adopted) !== "drive-terminal") { + item.disposition = "refused"; + item.remedy = adopted ? "leave the adopted live lifecycle running" : "inspect the fresh durable slot state"; + item.lastError = adopted + ? "the durable lifecycle is now backed by an adopted live managed agent" + : `the fresh durable phase ${row.phase} is not terminal-plannable`; + console.error(`! static reconcile refused alias=${item.alias} phase=${item.phase} uid=${item.lifecycleUid}: ${item.lastError}`); + return; + } + await this.attemptStaticReconcile(key, item, row, false); + } + + /** Enter one exact terminal at most once per coordinate. Sweep and timer triggers join the same + * promise. A settled failed item keeps its original per-process budget; a restart builds a fresh + * map from the still-authoritative durable row. */ + private attemptStaticReconcile(key: string, item: StaticReconcileItem, row: StaticManagedSlotRow, countAttempt = true): Promise { + if (item.flight) return item.flight; + if (item.timer) return Promise.resolve(false); + // A timer/retry driver increments before its mandatory durable re-read, then calls with + // countAttempt=false. Let that already-consumed budget entry enter the terminal. Only a fresh + // sweep entry (countAttempt=true) is refused once the per-process budget is exhausted. + if (countAttempt && item.attempts >= item.maxAttempts) return Promise.resolve(false); + const flight = (async (): Promise => { + if (countAttempt) item.attempts++; + item.phase = row.phase; + item.disposition = "retrying"; + item.nextRetryAt = undefined; + this.reconcilingAliases.add(row.alias); + console.error(`static reconcile terminal alias=${row.alias} phase=${row.phase} uid=${row.lifecycleUid} attempt=${item.attempts}/${item.maxAttempts}`); try { - console.error(`static reconcile ${index + 1}/${terminalRows.length} via ${this.servers ?? DEFAULT_SERVER}: ${row.alias} is ${row.phase} with no live managed owner${postAdoption ? " after resume adoption" : ""} - driving its exact-op terminal (uid ${row.lifecycleUid})`); - await this.driveStaticRetirement({ id: row.actor, name: row.alias, lifecycleUid: row.lifecycleUid }); + await this.driveStaticRetirement({ id: row.actor, name: row.alias, lifecycleUid: row.lifecycleUid }, true); + if (item.attempts === 1) { + this.staticReconcileItems.delete(key); + } else { + item.phase = "retired"; + item.disposition = "recovered"; + item.recoveredAt = new Date().toISOString(); + item.lastError = undefined; + console.error(`✓ static reconcile recovered alias=${item.alias} phase=retired uid=${item.lifecycleUid} attempts=${item.attempts}/${item.maxAttempts}`); + } + return true; + } catch (error) { + item.lastError = (error as Error).message; + try { + const fresh = await this.readStaticReconcileSlot(item.alias); + if (fresh?.lifecycleUid === item.lifecycleUid && fresh.actor === item.actor) item.phase = fresh.phase; + } catch { /* the terminal error remains primary; the next retry re-reads authoritatively */ } + const blocked = lifecycleBlockedFrom(error); + const ownOp = retireOpId(item.lifecycleUid); + const foreign = blocked !== undefined && (blocked.blockedOp !== "retirement" || blocked.opId !== ownOp); + if (foreign) { + item.disposition = "refused-foreign"; + item.remedy = blocked?.remedy ?? "resolve the foreign durable operation; it is never force-cleared"; + console.error(`! static reconcile refused-foreign alias=${item.alias} phase=${item.phase} uid=${item.lifecycleUid}: ${item.lastError}`); + } else if (item.attempts >= item.maxAttempts) { + item.disposition = "retry-exhausted"; + item.remedy = "restart this manager for a fresh per-process retry budget"; + console.error(`! static reconcile retry-exhausted alias=${item.alias} phase=${item.phase} uid=${item.lifecycleUid}: ${item.lastError}; NEXT: restart this manager for a fresh per-process retry budget`); + } else { + // The initial serial sweep must publish its complete failed aggregate before any timer can + // re-enter a middle row. Mark it now; the sweep arms the timer after its last planned row. + if (this.staticReconcileSweepsInFlight > 0) item.disposition = "retry-scheduled"; + else this.scheduleStaticReconcileRetry(key, item); + } + return false; } finally { this.reconcilingAliases.delete(row.alias); } + })(); + item.flight = flight.finally(() => { + if (item.flight === wrapped) item.flight = undefined; + }); + const wrapped = item.flight; + return wrapped; + } + + private releaseStaticReconcileSweep(): void { + this.staticReconcileSweepsInFlight--; + if (this.staticReconcileSweepsInFlight !== 0) return; + const waiters = this.staticReconcileDrainWaiters; + this.staticReconcileDrainWaiters = []; + for (const wake of waiters) wake(); + } + + /** Shutdown accepts no new static reconciliation work, then drains every sweep and exact-terminal + * flight that crossed the fence before stop(). A sweep can discover several rows before its first + * await, so the serial loop also consults the fence and never starts a later row after shutdown. */ + private async awaitStaticReconcileDrain(): Promise { + while (true) { + if (this.staticReconcileSweepsInFlight > 0) + await new Promise((resolve) => this.staticReconcileDrainWaiters.push(resolve)); + const flights: Array | Promise> = []; + for (const item of this.staticReconcileItems.values()) { + if (item.flight) flights.push(item.flight); + if (item.retryFlight) flights.push(item.retryFlight); + } + if (flights.length === 0 && this.staticReconcileSweepsInFlight === 0) return; + await Promise.allSettled(flights); + } + } + + private async reconcileStaticLifecycles(postAdoption = false): Promise { + if (!this.auth || this.staticReconcileStopping) return; + // Admission is synchronous with the shutdown fence. Once this increments, stop() must drain the + // whole accepted sweep even if it is still minting its short-lived provisioner credential. + this.staticReconcileSweepsInFlight++; + try { + // Recovered is a transition report, not permanent history. The next sweep clears it before + // deriving a fresh plan; failed coordinates keep their per-process budget and are joined. + for (const [key, item] of this.staticReconcileItems) { + if (item.disposition === "recovered") this.staticReconcileItems.delete(key); + } + const sweep: ManagerStaticReconciliationSweep = { + kind: postAdoption ? "post-adoption" : "startup", + startedAt: new Date().toISOString(), + attempted: 0, + succeeded: 0, + failed: 0, + }; + this.staticReconcileLastSweep = sweep; + const identity = newIdentity(); + const creds = await mintCreds(this.auth, identity, "provisioner"); + const nc = await connect({ servers: this.servers ?? DEFAULT_SERVER, ...standaloneConnectOpts({ creds, /* not yet wired to a recorded transport */ tls: false }), maxReconnectAttempts: 0 }); + const slotRows: StaticManagedSlotRow[] = []; + try { + const jsm = await jetstreamManager(nc); + const kvm = new Kvm(nc); + await ensureAuthorityStores(jsm, kvm, this.space); + const recordsKv = await kvm.open(recordsBucket(this.space)); + const t = staticLifecycleTransport(recordsKv, recordsKv /* auth reads unused in the sweep */); + const keys = await recordsKv.keys(`${STATIC_SLOT_PREFIX}.${DEV_OWNER}.>`); + const aliases: string[] = []; + for await (const k of keys) aliases.push(k.split(".").slice(2).join(".")); + for (const alias of aliases) { + const slot = await readStaticSlot(t, DEV_OWNER, alias); + if (slot !== undefined) slotRows.push(slot.row); + } + } catch (error) { + sweep.completedAt = new Date().toISOString(); + throw error; + } finally { + await nc.drain().catch(() => nc.close()); + } + const terminalRows: StaticManagedSlotRow[] = []; + for (const row of slotRows) { + if (row.phase === "retired") { + // A retirement is a GLOBAL refusal fact — seed the F5 index for EVERY retired incarnation + // regardless of which instance owned it, so a sibling-retired incarnation's copied credential + // is refused at this control surface too. Ownership gates only the DESTRUCTIVE sweep below. + this.retiredPrincipals.add(principalKey(row.owner, row.actor).key); + continue; + } + // 3b-2 RECONCILE OWNERSHIP (multi-manager-per-space): a manager adjudicates ONLY the non-retired + // rows THIS logical instance owns. A SIBLING manager's active/provisioning row is LEFT UNTOUCHED — + // sweep-terminalizing it would destroy the sibling's live agent (the historical all-agents-kill + // hazard, now cross-instance). A legacy row (pre-3b-2, no owner recorded) predates multi-manager, + // so this manager is its legitimate single-manager-past successor and reconciles it. An orphaned + // sibling row is reclaimed only by an explicit operator CAS takeover (ruling 1), never here. + if (row.ownerInstanceId !== undefined && row.ownerInstanceId !== this.managerInstanceId) continue; + // ADOPTION is genuine membership: a slot backed by a live managed agent THIS process owns + // at the SAME uid is never an orphan (empty at boot; exactly the adopted set at the + // post-adoption sweep — the fix for the F3 resume hole). + const live = this.agents.get(row.alias); + const adopted = live !== undefined && live.lifecycleUid === row.lifecycleUid; + // Boot sweep with a resume PENDING: an active slot may yet be adopted (the resume path runs + // AFTER this boot sweep), so DEFER it — the post-adoption sweep terminalizes any the resume + // did not claim. provisioning/terminalizing NEVER defer (they are crashed operations, never + // an agent to adopt). At `postAdoption` (or a non-resume boot) nothing defers. + if (!postAdoption && row.phase === "active" && !adopted && this.resumeRequired) continue; + if (planStaticSlotResume(row, adopted) !== "none") terminalRows.push(row); + } + try { + // Mark the complete planned set before the first terminal awaits. A timer owns only its own + // alias, but the initial serial sweep must close every discovery-to-await gap up front. + for (const row of terminalRows) this.reconcilingAliases.add(row.alias); + for (const row of terminalRows) { + if (this.staticReconcileStopping) break; + sweep.attempted++; + const key = this.staticReconcileKey(row); + let item = this.staticReconcileItems.get(key); + if (!item) { + item = { + owner: row.owner, + alias: row.alias, + actor: row.actor, + lifecycleUid: row.lifecycleUid, + phase: row.phase, + attempts: 0, + maxAttempts: STATIC_RECONCILE_MAX_ATTEMPTS, + disposition: "retrying", + }; + this.staticReconcileItems.set(key, item); + } + const succeeded = await this.attemptStaticReconcile(key, item, row); + if (succeeded) sweep.succeeded++; + else sweep.failed++; + this.reconcilingAliases.delete(row.alias); + } + sweep.completedAt = new Date().toISOString(); + for (const row of terminalRows) { + const key = this.staticReconcileKey(row); + const item = this.staticReconcileItems.get(key); + if (item?.disposition === "retry-scheduled" && !item.timer) this.scheduleStaticReconcileRetry(key, item); + } + const failed = terminalRows.flatMap((row) => { + const item = this.staticReconcileItems.get(this.staticReconcileKey(row)); + return item && item.disposition !== "recovered" ? [`${item.alias}:${item.phase}:${item.disposition}`] : []; + }); + if (sweep.failed === 0) { + console.error(`✓ static reconcile completed: ${sweep.attempted} attempted, ${sweep.succeeded} succeeded, 0 failed`); + } else { + console.error(`! static reconcile completed: ${sweep.attempted} attempted, ${sweep.succeeded} succeeded, ${sweep.failed} failed; failed=${failed.join(",")}`); + } + } finally { + for (const row of terminalRows) this.reconcilingAliases.delete(row.alias); + } + } finally { + this.releaseStaticReconcileSweep(); } } diff --git a/package.json b/package.json index cc1ddb6a4..2bbe71863 100644 --- a/package.json +++ b/package.json @@ -395,6 +395,7 @@ "smoke:goal-sibling-race": "tsx implementations/manager/smoke/goal-sibling-race.smoke.ts", "smoke:manager-reconcile-ownership": "tsx implementations/manager/smoke/manager-reconcile-ownership.smoke.ts", "smoke:manager-reconcile-startup": "tsx implementations/manager/smoke/manager-reconcile-startup.smoke.ts", + "smoke:manager-reconcile-redrive": "tsx implementations/manager/smoke/manager-reconcile-redrive.smoke.ts", "smoke:manager-name-claim": "tsx implementations/manager/smoke/manager-name-claim.smoke.ts", "smoke:manager-on-route": "tsx implementations/manager/smoke/manager-on-route.smoke.ts", "smoke:manager-scatter": "tsx implementations/manager/smoke/manager-scatter.smoke.ts",