diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index db9b175..b8f676d 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -17,10 +17,10 @@ jobs: benchmark: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -30,7 +30,7 @@ jobs: pip install -e ".[dev]" - name: Restore benchmark history - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: tests/benchmarks/results/ key: benchmark-history-${{ runner.os }} @@ -74,7 +74,7 @@ jobs: - name: Upload benchmark results if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: benchmark-results path: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d47e0a6..75ea9b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,10 @@ jobs: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} @@ -51,7 +51,7 @@ jobs: - name: Upload coverage artifact if: matrix.python-version == '3.12' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: coverage-report path: coverage.xml @@ -59,10 +59,10 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -78,10 +78,10 @@ jobs: typecheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -97,10 +97,10 @@ jobs: security: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -123,10 +123,10 @@ jobs: dependency-audit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -144,12 +144,12 @@ jobs: secrets-scan: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 - name: TruffleHog secrets scan - uses: trufflesecurity/trufflehog@37b77001d0174ebec2fcca2bd83ff83a6d45a3ab # v3.95.3 + uses: trufflesecurity/trufflehog@6961f2bace57ab32b23b3ba40f8f420f6bc7e004 # v3.93.3 with: extra_args: --only-verified @@ -158,9 +158,9 @@ jobs: # between .pre-commit-config.yaml and CI (e.g. mypy version skew). runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install pre-commit diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9b65168..67416e4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Fetch upstream sha512 + compare to pin run: ./scripts/check-gvisor-pin.sh --fetch @@ -74,10 +74,10 @@ jobs: timeout-minutes: 90 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python 3.12 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -104,7 +104,7 @@ jobs: - name: Install addons run: | - cp src/addons/*.py ~/.brig/cells/addons/ + cp src/brig/warden_addons/*.py ~/.brig/cells/addons/ cat > ~/.brig/cells/network-policy.json << 'EOF' { "allow": [ @@ -124,7 +124,7 @@ jobs: EOF - name: Restore VM cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 id: vm-cache with: path: ~/.lima/cell/ @@ -325,7 +325,7 @@ jobs: - name: Upload logs if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: e2e-logs path: /tmp/e2e-logs/ diff --git a/.github/workflows/fresh-install.yml b/.github/workflows/fresh-install.yml index 2b243c6..9155f7f 100644 --- a/.github/workflows/fresh-install.yml +++ b/.github/workflows/fresh-install.yml @@ -60,10 +60,10 @@ jobs: runs-on: macos-15 timeout-minutes: 30 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Set up Python 3.12 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -95,7 +95,7 @@ jobs: - name: Upload logs if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: fresh-install-logs path: /tmp/fresh-install-logs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 94f7a7d..c599de6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,48 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Security + +- **Cell network is verified `--internal` before reuse.** `observe()` recorded only whether a `brig-` network existed, and `plan_run` skipped `CREATE_NETWORK` (the sole `--internal` site) on the reuse path — so a leftover/tampered/operator-made non-internal network of the same name was silently adopted, giving the cell off-segment routes (invariant 1) and a path around Warden. The state dir / VM network set is untrusted (invariant 4), so the reconciler now inspects the network and **fails closed** if it isn't internal. +- **Blocked-request paths are redacted in the `ctx.log` sink too.** The `BLOCKED:` (enforce) and `INGRESS MISS:` (ingress) log lines went to warden's container stdout with only control-character stripping, so a secret in the path/query (`?api_key=…`) on a *blocked* request landed verbatim in `podman logs warden`. Both now run the path through the shared `redact_path` redactor, closing the one sink the redaction pipeline missed. +- Query-value redaction broadened: the sensitive-param allowlist gains `session`/`sig`/`signature`/`code`/`refresh_token`/`id_token`/`sas`, and **any** query value that classifies as a secret segment (token-shaped / high-entropy) is now masked regardless of its parameter name. +- `server_connect` re-checks the upstream port against the egress allowlist (80/443) as defense-in-depth, so the connect-time guard is self-sufficient even if a future flow type reaches it without the request-time port check (legitimate non-80/443 destinations — host_service rewrites, ingress, declared TCP host_services — stay exempt). +- The proxy-env override guard now strips/normalizes the env key before comparison, and `_v_env` rejects keys that aren't valid POSIX environment names — so a whitespace-padded ` http_proxy` can't slip past the guard protecting the Warden choke point. +- `mount_root_slug` is derived from the realpath everywhere (validation, lima.yaml render, reconciler bind), so two symlinked `mount_roots` can't collide at one `/mnt/host/` with no validation error (which would shadow one host tree with another). +- `_v_workdir` rejects `..`/`.` segments and doubled slashes, matching the other in-cell path validators. +- Path filters (`policy.allow` with `paths:`) match on the path **without** its query string, so an intended endpoint scope isn't affected by query content; the glob-over-path semantics are now documented. +- `brig system down` self-heals orphaned subnet allocations (frees `/24`s whose podman network is gone, then sweeps their ingress routes) instead of leaking them until a manual `prune`; fails safe if the network list can't be read. +- Connect-time destination-IP validation closes the SSRF / DNS-rebinding gap on the egress paths the response-only check missed. `enforce.server_connect` resolves the destination and refuses (`data.server.error`) any MITM flow that resolves into a blocked range *before* the request is forwarded; `enforce.tls_clienthello` resolves the SNI and refuses to flip TLS passthrough into a blocked range (a raw-TCP passthrough tunnel produces no HTTP response, so the old `responseheaders` re-check never covered it). Warden's own routing — host_service rewrites (→ host IP) and ingress reverse-proxy (→ cell IP) — is exempt. +- `brig run --profile untrusted` now enforces the untrusted-profile guards on the CLI path. The profile name was not recorded where validation could see it, so a cell launched with `--profile untrusted -f cell.yaml` could declare `host_sockets`, `tls_passthrough`, or `host_services` unchecked. (The SDK path was already correct.) +- Image references beginning with `-` are rejected on the cell-yaml `image:` and SDK `image=` paths, and `podman run` now gets a `--` end-of-options marker before the image. An `image:` value like `--runtime=runc` (silent gVisor downgrade), `--privileged`, or `-v /:/host` can no longer be parsed by podman as a flag. +- `verify_gvisor_runtime` reads the named runtime (top-level `OCIRuntime`) instead of `HostConfig.Runtime` (the OCI category, always `"oci"`). `brig system verify` can now actually distinguish a `runsc` cell from a `crun` downgrade (invariant 5) instead of false-failing every cell. +- `workspace_mount` rejects non-normalized paths (doubled/leading/trailing slashes, `.` segments). A value like `/run//host` previously slipped past the lexical forbidden-prefix check while the kernel collapsed it back onto a brig-internal mount root. +- `BLOCKED_NETWORKS` adds the IPv4 6to4-relay anycast (`192.88.99.0/24`) and IETF protocol-assignment / NAT64 well-known (`192.0.0.0/24`) ranges, for parity with the existing IPv6 tunnel prefixes. +- The `host_socket` bridge writes the realpath validated at check time into the launchd plist instead of re-resolving at write time, closing a validate→freeze TOCTOU window. +- Per-cell policy read-modify-write is serialized under one exclusive lock (`mutate_cell_policy`), so a concurrent `brig policy set` racing `brig run`'s policy re-sync can no longer silently drop an update. +- `image_digest` is matched at the exact per-algorithm hex length (sha256/384/512); cell-name / secret-name / domain / memory validators anchor with `\Z` so a trailing newline is rejected. + +### Changed + +- Version bumped to **0.4.0** (new invariants 11–13 + removed `brig cell trace`); `pyproject.toml` and `brig.config.VERSION` are now guarded against drift by a test. +- The pinned mitmproxy base image has a single source of truth (`warden.proxy.BASE_IMAGE`), imported by `brig image warmup`; a test keeps the warden Dockerfile `FROM`/`LABEL` in lockstep, matching the gVisor/collector pin discipline. +- Invariant 6 docs realigned with the code: `proxy-external` admits brig infrastructure (warden **and** the OTel collector `brig-otel`, per `INFRA_CONTAINER_NAMES`); no cell can reach that network. +- **Fail-closed behavior changes** (operators take note): `brig cell start` now refuses to start a digest-pinned cell whose container reports an empty image digest (was: started unverified); a cell that declares `ingress` now fails and rolls back if its container can't be inspected or has no IP at start (was: started with the route silently unregistered); TLS passthrough is refused (falls through to MITM) when the host resolves into a blocked range. +- `brig system verify` no longer reports airgapped (`--network none`) cells as single-homing violations, and now flags network members that lack a name instead of silently skipping them. +- The warden addons moved from the loose top-level `src/addons/` into the brig package as shipped data (`src/brig/warden_addons/`). They now ship in the wheel (declared as `brig` package-data) and `brig.ops.addon_deploy` resolves them via `importlib.resources`, so the deployed data plane is resolvable in any install layout, not just an editable checkout. The addons stay flat-loaded by mitmproxy (no `__init__.py`); the deploy target (`~/.brig/cells/addons` → `/addons`) is unchanged. + +### Removed + +- `brig cell trace` and the OTel **traces** pipeline. No addon ever emitted a span (the tracer/exporter were initialized but unused), so the collector's traces lane and `traces.jsonl` were always empty and the command always returned no data. Removed the command, the unused `TracerProvider`/`OTLPSpanExporter` setup in `otel_export.py`, the `traces` pipeline + `file/traces` exporter in the collector config, `brig/observability/traces.py`, and the stale docs claim that spans are attached per request. Metrics and logs (the signals that actually flow) are unaffected. + +### Fixed + +- **Documentation accuracy (quality audit).** `docs/learning/concepts.md` claimed the `dev` profile disables gVisor (it can't — gVisor is mandatory, invariant 5) and documented a `--sanitize` / `--allow-scripts` / `--allow-office` flag family and blocklist that don't exist (export sanitization is unconditional; the real blocklist is `UNSAFE_EXTENSIONS`). Also corrected: `security.md` "proxy listens ONLY on 8080" (ingress adds 8443, TCP host_services add their ports) and the `tls_mode=mitm` scope (OTel records, not the JSONL flow log); `INVARIANTS.md` symbol `_register_cell_ingress` → `register_ingress_for`; `warden reload` addon credit; `secrets add --force` flag; `sdk.py` `list_sync` docstring. +- More actionable errors: "Invalid cell name" now states the allowed format; `Could not parse …` raises suggest `brig system doctor`; the TCP-restart prompt fails fast with a `--yes` hint on non-interactive callers instead of an EOF-driven abort. +- **TLS passthrough (invariant 11) now actually engages.** `tls_clienthello` set `client_conn.tls_passthrough = True`, an attribute the warden image's mitmproxy (10.1.1) does not read — so passthrough was a silent no-op: every host was MITM'd, and a host that refuses mitmproxy's relayed handshake simply failed. Warden now sets `data.ignore_connection = True`, the switch the TLS layer actually reads. Verified e2e (manual, against the pinned image): a host in `tls_passthrough` receives its real upstream certificate (warden tunnels raw, no decryption), while an allowlisted-but-not-passthrough host still receives warden's MITM cert. The gating is preserved (allow-coverage, SNI/CONNECT match, resolved-IP guard). NOTE: a true passthrough tunnel produces no mitmproxy flow, so the `warden_passthrough_*` otel counters and the `tcp_*` hooks do NOT fire for it — the only passthrough audit is the connection-level `PASSTHROUGH: cell=… sni=…` log line. See `docs/INVARIANTS.md` invariant 11. +- `scripts/local-smoke-test.sh` updated to the 0.3.0 noun-verb CLI (`brig system up/verify`, `brig cell list/inspect/exec/stop/rm`); it had broken against the restructured surface. +- Atomic file writes `fsync` the parent directory after rename, so a crash immediately after the rename can't lose the directory entry. + ## [0.3.1] - 2026-05-26 ### Fixed @@ -75,7 +117,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Authenticated ingress reverse proxy through Warden: external requests to `https://warden:8443/{cell}/{prefix}/...` route to a cell-internal port after Bearer-token auth. Salted SHA-256 token hashing, constant-time comparison, per-IP auth-failure rate limiting. - WebSocket passthrough through Warden (logged but not re-policy-checked once the upgrade has been allowed). -- `brig doctor` — deep environment check (PATH, Lima VM state, addon presence, directory permissions, port collisions). Complements the lighter `brig health`. +- `brig doctor` — deep environment check (PATH, Lima VM state, addon presence, directory permissions, port collisions). Complements the lighter `--quick` check. - `brig policy test ` — host-side passthrough that runs the same allow/deny logic warden uses. - `brig events --follow` — block and tail new lifecycle events. - `brig network --blocked` — filter to only the requests warden blocked, with reason. @@ -119,8 +161,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `brig policy rm `: drops a cell's per-cell policy override (the cell falls back to the global policy on the next request). Wired up an existing-but-unreachable `delete_cell_policy()` function that already had tests. -- `brig policy set --host-service `: per-cell ACL grant. Previously `--host-service` only worked for the global policy (`name:port` form), so the H1 per-cell ACL field could only be set by hand-editing JSON. Now: global takes `name:port` (declares warden's forward target), per-cell takes a bare `name` (grants the cell access to a globally-declared service). The CLI rejects each form in the wrong context with a clear suggestion. +- `brig policy rm `: drops a cell's per-cell policy; the cell then blocks all egress (default-deny) on the next request. Wired up an existing-but-unreachable `delete_cell_policy()` function that already had tests. - The `log_compaction` block in `docs/examples/network-policy.example.json` (no consumer). ### Moved diff --git a/CLAUDE.md b/CLAUDE.md index b514af0..11ea60f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -149,17 +149,19 @@ src/ │ ├── ops/ # Logging, cache, rate limiting, history, atomic │ ├── workspace/ # Cell workspace file ops (cp in/out, sanitize) │ ├── vm/ # Lima shell wrapper + VM config template -│ └── commands/ # Thin CLI handlers (one file per command group) -├── warden/ # Proxy manager (lifecycle, policy, health, reconcile, logs) -└── addons/ # mitmproxy addons mounted into warden container: - # _common (shared helpers), _policy (policy data - # structures), enforce, logger, ops, ingress, - # notifier +│ ├── commands/ # Thin CLI handlers (one file per command group) +│ └── warden_addons/ # mitmproxy addons (brig package-data, NOT a submodule): +│ # _common (shared helpers), _policy (policy data +│ # structures), enforce, logger, ops, ingress, +│ # notifier +└── warden/ # Proxy manager (lifecycle, policy, health, reconcile, logs) ``` -Addons run inside the warden container with their own Python env. They can -import sibling addons (e.g. `from _common import ...`) but cannot import -`brig.*`. +Addons ship as brig package-data and are synced into the warden container by +`brig system up`. They run inside the container with their own Python env, +flat-loaded by mitmproxy: they import sibling addons (e.g. `from _common import +...`) but cannot import `brig.*`. They are data, not an importable `brig` +submodule (no `__init__.py`), which is what keeps the flat imports valid. ### Data (`~/.brig/`) @@ -167,8 +169,10 @@ import sibling addons (e.g. `from _common import ...`) but cannot import ~/.brig/ ├── lima.yaml # VM config ├── cells/ # Cell definitions -│ ├── network-policy.json # Warden allowlist -│ └── addons/ # Warden addons +│ ├── network-policy.json # Process-wide warden settings (rate limits, log +│ │ # filter, policy trace, notifications) — NOT egress +│ │ # allow/deny, which is per-cell +│ └── addons/ # Warden addons (deployed copy of brig/warden_addons) ├── secrets/ # One file per secret └── state/ # Cell workspaces and logs └── system/ # Subnet allocator state (subnets.json) diff --git a/Makefile b/Makefile index e38a634..350ddcf 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: setup test check smoke clean reset help up down bench +.PHONY: setup test check smoke redteam clean reset help up down bench help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}' @@ -18,15 +18,17 @@ setup: .venv ## Install brig, create VM, start everything limactl start brig; \ fi ./scripts/provision-vm.sh - uv run brig up + @echo "Building the warden image inside the VM (one-time, ~1-2 min)..." + ./scripts/build-warden-image.sh + uv run brig system up @echo "" @echo "Ready. Try: uv run brig run alpine echo hello" up: ## Start VM + warden (if already set up) - uv run brig up + uv run brig system up down: ## Stop all cells + warden - uv run brig down + uv run brig system down # --- Testing --- @@ -41,6 +43,9 @@ check: ## Run full CI checks locally smoke: ## Run end-to-end smoke test (requires VM) ./scripts/local-smoke-test.sh +redteam: ## Run the containment red-team (Tier-1, requires VM up) + ./tests/test_containment_e2e.sh + bench: ## Run benchmarks uv run pytest tests/benchmarks/ -m bench --benchmark-enable -q @@ -72,14 +77,15 @@ pin-gvisor: ## Fetch + write gVisor sha512s into scripts/provision-vm.sh (run on _copy-addons: @mkdir -p ~/.brig/cells/addons @chmod 0700 ~/.brig/cells/addons - @# Copy every *.py in src/addons/ — simpler than maintaining a + @# Copy every *.py in src/brig/warden_addons/ — simpler than maintaining a @# split between "required" and "optional" sets that drifts from - @# what warden actually loads (audit M7). cp without -r since the - @# dir is flat; explicit error if there's nothing to copy. - @if ! ls src/addons/*.py >/dev/null 2>&1; then \ - echo "ERROR: no addon .py files in src/addons/"; exit 1; \ + @# what warden actually loads. cp without -r since the dir is flat; explicit + @# error if there's nothing to copy. (Ongoing drift is handled by + @# `brig system up`; this stays for first-boot + seccomp staging.) + @if ! ls src/brig/warden_addons/*.py >/dev/null 2>&1; then \ + echo "ERROR: no addon .py files in src/brig/warden_addons/"; exit 1; \ fi - @cp src/addons/*.py ~/.brig/cells/addons/ + @cp src/brig/warden_addons/*.py ~/.brig/cells/addons/ @# Seccomp profiles are referenced by reconciler.build_run_command as @# /cells/seccomp/.json inside the VM (the host's ~/.brig/cells @# is mounted at /cells). Without these, --seccomp-profile fails to diff --git a/README.md b/README.md index 46dcc24..adfaf04 100644 --- a/README.md +++ b/README.md @@ -105,22 +105,34 @@ Policy commands always operate against a named cell; the cell yaml is the source brig system up # start everything (VM + warden) brig system down # stop everything brig system down --vm # also stop the VM -brig system verify # check all 12 security invariants +brig system verify # check the runtime-verifiable invariants (6 of 12) brig system doctor --quick # system health check brig cell diagnose mycell # debug a specific cell ``` ## Network Policy -Default policy (`~/.brig/cells/network-policy.json`) allows pypi, github, npm: +Egress allow/deny is **per-cell** and **default-deny**: a cell with no policy +reaches nothing. Rules come from the cell's `policy:` block (or a trust +profile) and can be edited live with `brig policy set `: + +```yaml +# in a cell yaml +policy: + allow: + - pypi.org + - "*.pythonhosted.org" + - github.com + - api.github.com + deny: + - "*.ngrok.io" +``` + +`~/.brig/cells/network-policy.json` carries only process-wide operational +settings — it holds **no** allow/deny rules: ```json { - "allow": [ - "pypi.org", "*.pythonhosted.org", "github.com", - "api.github.com", "*.githubusercontent.com", "registry.npmjs.org" - ], - "deny": [], "rate_limits": {"default": {"rate": 100, "burst": 500}} } ``` @@ -134,7 +146,10 @@ Default policy (`~/.brig/cells/network-policy.json`) allows pypi, github, npm: | Per-cell networks | No lateral movement between cells | | Warden proxy | Egress filtering, logging, rate limiting | -12 security invariants, all tested. Run `brig system verify` to check. +12 security invariants. 6 are runtime-verifiable via `brig system verify`; the +rest have automated tests, except invariant 3 (secrets are observable, not +preventable) which is a design property rather than a testable check. See +`docs/INVARIANTS.md` for the full coverage ledger. ## Development @@ -155,7 +170,7 @@ make bench # benchmarks - [Cell Definition Reference](docs/design/cell-definition.md) - [Architecture](docs/design/architecture.md) - [Security Design](docs/design/security.md) — and the [supply-chain notes](docs/design/supply-chain.md) -- [SDK Specification](docs/sdk-spec.md) +- [SDK Specification](docs/reference/sdk.md) - [Brig CLI Reference](docs/reference/brig-cli.md) - [Warden CLI Reference](docs/reference/warden-cli.md) - [Cell Metadata Reference](docs/reference/cell-metadata.md) — `/run/brig/cell.json` schema and workspace-passthrough security model diff --git a/docs/INVARIANTS.md b/docs/INVARIANTS.md index 101093d..3cb51c9 100644 --- a/docs/INVARIANTS.md +++ b/docs/INVARIANTS.md @@ -1,6 +1,6 @@ # Brig Security Invariants — Living Ledger -The 12 invariants from [docs/design/security.md](design/security.md) are the +The 13 invariants from [docs/design/security.md](design/security.md) are the thing this project exists to uphold. This document is the single source of truth for **which tests prove them** and **which CI lane runs those tests**. @@ -11,8 +11,12 @@ to that invariant's row — don't leave the coverage implicit. ## How to read this - **Unit** — runs on every PR under `.github/workflows/ci.yml` on Ubuntu. No VM. -- **E2E** — runs on PRs touching `src/**`, `tests/**`, or CI config, under - `.github/workflows/e2e.yml` on macos-15 with a real Lima VM + podman + gVisor. +- **E2E** — the `.github/workflows/e2e.yml` lane (real Lima VM + podman + gVisor + on macos-15) is **gated on nested virtualization** (`kern.hv_support == 1`), + which GitHub-hosted runners do NOT provide — so it runs **manually / on + dispatch / on a self-hosted runner only**, NOT automatically on PRs. Rows + that list an "E2E test" name the test that exists; "CI" states what actually + runs in hosted CI. - **Code location** — where the invariant is enforced in production code. ## Invariant ledger @@ -22,32 +26,37 @@ to that invariant's row — don't leave the coverage implicit. | Surface | Location | |---|---| | Enforcement | `src/brig/security/verify.py:verify_network_isolation`; per-cell `--internal` network created in `src/brig/cell/reconciler.py` | -| Cell-def guard | `src/brig/cell/spec.py:validate_cell_definition` rejects `network: proxy-external` and non-default/none values | +| Cell-def guard | `src/brig/cell/validators.py:_v_network` rejects `network: proxy-external` and non-default/none values | | Unit test | `tests/test_cell_spec.py::TestValidateCellDefinition::test_network_proxy_external_rejected` | | Unit test | `tests/test_cell_spec.py::TestValidateCellDefinition::test_network_list_rejected` | | Unit test | `tests/test_security_verify.py::TestVerifyNetworkIsolation` | | E2E test | `tests/test_vm_foundation.sh` — east-west ping from cell A to cell B must fail | -| CI | Unit + E2E | +| CI | Unit (the E2E `.sh` lane is gated on nested-virt and does NOT run on GitHub-hosted CI — manual/dispatch only) | ### 2. Proxy Cannot Be Abused as Gateway | Surface | Location | |---|---| -| Enforcement | `src/addons/enforce.py` — port allowlist (80/443), literal-IP block, RFC1918/CGNAT/etc block at request + http_connect + responseheaders | -| Blocklist source | `src/addons/_common.py:BLOCKED_NETWORKS` — single source shared by enforce + notifier | -| DNS rebinding defense | `responseheaders` re-checks resolved IP (`flow.server_conn.peername`) against `BLOCKED_NETWORKS`. Skip is **gated on `flow.metadata["host_service"]` / `["ingress_route"]`** (populated by `_handle_host_service` / ingress.py), not on a `(ip, port)` tuple — a tuple skip would let a DNS-rebinding allowlisted domain reach a host service. The check used to ALSO run in `server_connected`, but `data.server.close()` no longer exists on mitmproxy >= 10 (raised AttributeError, masked the would-be block) AND `data.flow` was None there so exemptions didn't fire either. Aitelier diagnosed; check now lives only in `responseheaders` where metadata is populated. | +| Enforcement | `src/brig/warden_addons/enforce.py` — port allowlist (80/443), literal-IP block, RFC1918/CGNAT/etc block at request + http_connect + server_connect + responseheaders | +| Blocklist source | `src/brig/warden_addons/_common.py:BLOCKED_NETWORKS` — single source shared by enforce + notifier | +| DNS rebinding (layered) | Request-time checks see the cell-supplied host *name*; the resolved upstream IP is only known at/after connect, so three hooks cover the resolved IP across flow types (below). | +| DNS rebinding · MITM connect | `server_connect` resolves the destination and refuses (`data.server.error`) any flow resolving into `BLOCKED_NETWORKS`, before the request is forwarded. Exempts warden's own routing: host_service rewrite (→ host IP) and ingress reverse-proxy (client on :8443 → cell IP). | +| DNS rebinding · passthrough | `tls_clienthello` resolves the SNI and refuses to flip TLS passthrough into a blocked range — a raw-TCP passthrough tunnel produces no HTTP response, so `responseheaders` never re-checks it. Refusal falls through to MITM, which fails closed. | +| DNS rebinding · MITM response | `responseheaders` re-checks `flow.server_conn.peername`, skip-gated on `flow.metadata["host_service"]` / `["ingress_route"]` — NOT on a `(ip, port)` tuple, which would let a rebinding allowlisted domain reach a host service. | +| DNS rebinding · rejected approach | `server_connected` was tried and abandoned: `data.server.close()` is gone on mitmproxy ≥ 10 and `data.flow` was None there (exemptions couldn't fire). `server_connect` is the workable connect-time hook. | | Host header smuggling | `_host_header_mismatches` in `request()` and `http_connect()`. Multi-colon strings validated via `ipaddress.ip_address` so non-IPv6 inputs like `example.com:80:extra` don't silently get treated as bare IPv6. | -| Host services | `.host.brig` virtual domains are an intentional, scoped relaxation of this invariant. The cell yaml declares `host_services: [{name, port}]`; that declaration is the sole grant. Warden reads the port from the cell's own per-cell policy file (there is no separate global registry). Cells without `host_services` in yaml have no host-service access. Unknown `.host.brig` domains are blocked. The `untrusted` profile rejects `host_services` at parse time. See `_handle_host_service()` in `src/addons/enforce.py`. | -| Ingress | Warden ingress (port 8443) allows authenticated inbound traffic to cells. enforce.py blocks unhandled ingress requests (fail closed). CONNECT is blocked entirely on the ingress port. See `src/addons/ingress.py`. | +| Host services | `.host.brig` virtual domains are an intentional, scoped relaxation of this invariant. The cell yaml declares `host_services: [{name, port}]`; that declaration is the sole grant. Warden reads the port from the cell's own per-cell policy file (there is no separate global registry). Cells without `host_services` in yaml have no host-service access. Unknown `.host.brig` domains are blocked. The `untrusted` profile rejects `host_services` at parse time. See `_handle_host_service()` in `src/brig/warden_addons/enforce.py`. | +| Ingress | Warden ingress (port 8443) routes declared inbound traffic to cells. Per-route auth is operator policy: `auth: token` (default — brig is the Bearer-token gate) or `auth: none` (transparent pass-through; the cell's app is the gate). A route missing `auth` is treated as `token` (fail-secure); `auth: none` is rejected on the `untrusted` profile and audited at run time (`ingress_unauthenticated` lifecycle event + operator NOTE). enforce.py blocks unhandled ingress requests (fail closed). CONNECT is blocked entirely on the ingress port. See `src/brig/warden_addons/ingress.py`. | | Unit test | `tests/test_addons_ops.py` — token bucket rate limiting | -| Unit test | `tests/test_ingress.py` — ingress routing, token auth, cell IP validation (rejects `.0` / `.1` / `.255`), rate limiting | +| Unit test | `tests/test_ingress.py` — ingress routing, token auth, rate limiting. Cell-IP validation: `TestIngressAddonCellIpValidation::test_in_range_reserved_host_octets_rejected` feeds in-subnet `10.60.x.0/.1/.255` so the reserved-host-octet gate (`host_octet < 2`, `.255`) is actually exercised, plus `test_invalid_cell_ip_rejected` for out-of-range IPs | +| Unit test | `tests/test_addon_common.py::TestBlockedNetworks` — every `BLOCKED_NETWORKS` class has a representative blocked address (RFC1918, CGNAT, link-local, IPv4-mapped, IPv6 NAT64/6to4/discard, IPv4 6to4-relay/NAT64-WKA, alternate IPv4 encodings) so a deleted/mistyped CIDR fails CI | | Unit test | `tests/test_security_audit.py::TestResponseHeadersDnsRebinding` — RFC1918, localhost, link-local, IPv4-mapped-IPv6, IPv6 link-local; flow-metadata-gated host-service / ingress-route skip; a naked tuple match must NOT bypass | | Unit test | `tests/test_security_audit.py::TestConnectMethodEnforcement` — CONNECT to disallowed port / internal IP / literal IP / disallowed domain / ingress port | | Unit test | `tests/test_security_audit.py::TestNormalizeHostspecRobustness` — bracketed IPv6, multi-colon non-IPv6 strings | | Unit test | `tests/test_security_audit.py::TestHandleHostService` — per-cell ACL: cell with no per-cell policy is blocked; cell whose policy doesn't list the service is blocked | | Unit test | `tests/test_addon_common.py::TestBlockedNetworks` — covers every entry in the SSRF blocklist | | E2E test | `tests/test_proxy_policy.sh` tests 7-11 — asserted via JSONL log entries | -| CI | Unit + E2E | +| CI | Unit (the E2E `.sh` lane is gated on nested-virt and does NOT run on GitHub-hosted CI — manual/dispatch only) | ### 3. Secrets Are Observable, Not Preventable @@ -62,29 +71,31 @@ proxy logs, not prevention. | Path validation | `src/brig/security/secrets.py:validate_secret_path` — resolve + relative_to | | Unit test | `tests/test_security_secrets.py::TestValidateSecretPath` — legit file, symlink escape, double-hop symlink, nonexistent secret | | E2E test | `tests/test_hardening.sh` | -| CI | Unit + E2E | +| CI | Unit (the E2E `.sh` lane is gated on nested-virt and does NOT run on GitHub-hosted CI — manual/dispatch only) | ### 5. gVisor Must Be Active (no silent downgrade) | Surface | Location | |---|---| | Enforcement | `src/brig/cell/reconciler.py:build_run_command` hardcodes `--runtime runsc` | -| Verify check | `src/brig/security/verify.py:verify_gvisor_runtime` | +| Verify check | `src/brig/security/verify.py:verify_gvisor_runtime` — reads the top-level `OCIRuntime` (named runtime, `runsc`/`crun`), NOT `HostConfig.Runtime` (the OCI category, always `"oci"`, which cannot tell runsc from a crun downgrade) | | Unit test | `tests/test_cell_reconciler.py::TestBuildRunCommand::test_runtime_always_runsc` | | Unit test | `tests/test_security_verify.py::TestVerifyGvisorRuntime::test_runtime_downgrade` | +| Unit test | `tests/test_security_verify.py::TestVerifyGvisorRuntime::test_real_podman_inspect_shape` — drives the check with a real `podman inspect` fixture so the field name can't silently regress | | E2E test | `tests/test_cell_lifecycle.sh` — dmesg grep for "Starting gVisor" | -| CI | Unit + E2E | +| CI | Unit (the E2E `.sh` lane is gated on nested-virt and does NOT run on GitHub-hosted CI — manual/dispatch only) | ### 6. Only Infrastructure Containers May Attach to proxy-external | Surface | Location | |---|---| -| Cell-def guard | `src/brig/cell/spec.py:validate_cell_definition` rejects `network: proxy-external` | -| Verify check | `src/brig/security/verify.py:verify_proxy_network` | +| Allowlist | `src/brig/config.py:INFRA_CONTAINER_NAMES` — the enforced set: warden + the OTel collector `brig-otel` (both brig-managed infra; no cell can reach proxy-external) | +| Cell-def guard | `src/brig/cell/validators.py:_v_network` rejects `network: proxy-external` | +| Verify check | `src/brig/security/verify.py:verify_proxy_network` (membership ⊆ `INFRA_CONTAINER_NAMES`) | | Unit test | `tests/test_cell_spec.py::TestValidateCellDefinition::test_network_proxy_external_rejected` | | Unit test | `tests/test_cell_spec.py::TestValidateCellDefinition::test_network_arbitrary_rejected` | | Unit test | `tests/test_security_verify.py::TestVerifyProxyNetwork` | -| CI | Unit + E2E | +| CI | Unit (the E2E `.sh` lane is gated on nested-virt and does NOT run on GitHub-hosted CI — manual/dispatch only) | ### 7. No Privileged Services on Cell Networks @@ -94,17 +105,17 @@ proxy logs, not prevention. | Unit test | `tests/test_security_verify.py::TestVerifyCellNetworkMembers::test_foreign_container` | | Unit test | `tests/test_security_verify.py::TestVerifyCellNetworkMembers::test_only_warden_and_cell` | | E2E test | `tests/test_invariants_7_8.sh` — attaches a foreign container to a cell's network and asserts `brig system verify` detects it | -| CI | Unit + E2E | +| CI | Unit (the E2E `.sh` lane is gated on nested-virt and does NOT run on GitHub-hosted CI — manual/dispatch only) | ### 8. Cells Must Be Single-Homed | Surface | Location | |---|---| -| Cell-def guard | `src/brig/cell/spec.py:validate_cell_definition` rejects list network values | +| Cell-def guard | `src/brig/cell/validators.py:_v_network` rejects list network values | | Verify check | `src/brig/security/verify.py:verify_single_homed` | | Unit test | `tests/test_cell_spec.py::TestValidateCellDefinition::test_network_list_rejected` | | Unit test | `tests/test_security_verify.py::TestVerifySingleHomed::test_multi_homed` | -| CI | Unit + E2E | +| CI | Unit (the E2E `.sh` lane is gated on nested-virt and does NOT run on GitHub-hosted CI — manual/dispatch only) | ### 9. Proxy Must Be Running Before Cells Start @@ -113,7 +124,7 @@ proxy logs, not prevention. | Enforcement | `src/brig/cell/lifecycle.py:run_cell` checks `proxy_running()` early | | Unit test | `tests/test_cell_lifecycle.py::TestRunCell::test_invariant_9_proxy_must_be_running` | | E2E test | `tests/test_cell_lifecycle.sh` | -| CI | Unit + E2E | +| CI | Unit (the E2E `.sh` lane is gated on nested-virt and does NOT run on GitHub-hosted CI — manual/dispatch only) | ### 10. host_sockets Bypass Warden by Design @@ -138,7 +149,7 @@ The invariant we DO uphold: | Surface | Location | |---|---| -| Parse-time guards | `src/brig/cell/spec.py:_v_host_socket_entry`, `_v_host_sockets` | +| Parse-time guards | `src/brig/cell/validators.py:_v_host_socket_entry`, `_v_host_sockets` | | Engine denylist | `src/brig/config.py:HOST_SOCKET_ENGINE_DENYLIST` | | Runtime TOCTOU | `src/brig/cell/reconciler.py:_attach_host_sockets` (lstat, S_ISSOCK) | | Bridge defense | `src/brig/cell/host_sockets_bridge.py:_validate_target` | @@ -146,7 +157,7 @@ The invariant we DO uphold: | Banner | `src/brig/cell/lifecycle.py:run_cell` prints NOTE on cells with sockets | | Unit tests | `tests/test_host_sockets_spec.py` (19 cases) | | Unit tests | `tests/test_reconciler_host_sockets.py` (7 cases) | -| Unit tests | `tests/test_host_sockets_bridge.py` (9 cases) | +| Unit tests | `tests/test_host_sockets_bridge.py` (12 cases) | | Unit tests | `tests/test_metadata_host_sockets.py` (3 cases) | | CI | Unit | @@ -178,34 +189,48 @@ The invariants we DO uphold for passthrough hosts: - **SNI in the client hello must match the CONNECT host** (Phase 2). Otherwise a malicious cell could CONNECT to allowed-host:443 then send SNI=attacker.com to abuse Warden as a generic TLS tunnel. - - **Audit log entries are tls_mode-tagged.** Passthrough records carry - SNI + bytes + duration; per-URL/body attributes are absent **by - construction** (Warden never decrypted them). Operators can grep for - `tls_mode=passthrough` to enumerate uninspected flows (Phase 3). + - **Passthrough connections are audited at the TLS handshake.** Warden + logs `PASSTHROUGH: cell= sni=` in `tls_clienthello` when it + engages passthrough, so the cell + destination of every uninspected flow + is recorded. Per-URL/body/byte detail is absent **by construction** — + Warden never decrypts, and a true (ignore_connection) passthrough tunnel + produces no mitmproxy flow, so the tcp_*/byte hooks don't fire. - **Untrusted profile cannot declare passthrough** (Phase 1 follow-up). The trade-off requires informed operator consent; untrusted profiles don't get to make that choice. | Surface | Location | |---|---| -| Cell-def guard | `src/brig/cell/spec.py:_v_policy` — cross-field check that every `tls_passthrough` host appears in `allow`; rejects passthrough under the untrusted profile | +| Cell-def guard | `src/brig/cell/validators.py:_v_policy` — cross-field check that every `tls_passthrough` host appears in `allow`; rejects passthrough under the untrusted profile | | Spec field | `src/brig/cell/spec.py:CellSpec.policy_passthrough_tls` | -| YAML flattening | `src/brig/commands/lifecycle_cmd.py` (`policy.tls_passthrough` → `policy_passthrough_tls`) | +| YAML flattening | `src/brig/commands/lifecycle_run.py` (`policy.tls_passthrough` → `policy_passthrough_tls`) | | Profile propagation | `src/brig/cell/profiles.py` (profile-level `policy.tls_passthrough` prepends to cell's list) | -| Per-cell policy write | `src/brig/commands/lifecycle_cmd.py:_sync_cell_policy` writes `tls_passthrough` to `.json` | -| Policy class | `src/addons/_policy.py:Policy.is_passthrough` — defense-in-depth: a host must match BOTH passthrough rules AND allow rules (a tampered policy file can't opt a host out of MITM without allow coverage) | -| Addon hook | `src/addons/enforce.py:tls_clienthello` — reads SNI, flips `client_conn.tls_passthrough` when matched, blocks SNI/CONNECT mismatches | -| OTel counters | `src/addons/otel_export.py:tcp_start/tcp_message/tcp_end` — `warden_passthrough_connections_total`, `warden_passthrough_bytes_total{direction}`, `warden_passthrough_duration_ms` | -| Log shape | `src/addons/otel_export.py` tags MITM records with `tls_mode=mitm`, passthrough records with `tls_mode=passthrough`. Passthrough records omit method/path/status BY CONSTRUCTION (warden never saw them) | -| CLI rendering | `src/brig/commands/network_cmd.py:_print_network_line` renders passthrough lines as `PASSTHROUGH (NB in / NB out)` — visually distinct from `OUT:` and `INGRESS:` | +| Per-cell policy write | `src/brig/commands/lifecycle_run.py:_sync_cell_policy` writes `tls_passthrough` to `.json` | +| Policy class | `src/brig/warden_addons/_policy.py:Policy.is_passthrough` — defense-in-depth: a host must match BOTH passthrough rules AND allow rules (a tampered policy file can't opt a host out of MITM without allow coverage) | +| Addon hook | `src/brig/warden_addons/enforce.py:tls_clienthello` — reads SNI, sets `data.ignore_connection = True` (the passthrough switch mitmproxy's TLS layer reads) when the SNI matches BOTH allow and passthrough, blocks SNI/CONNECT mismatches, and refuses passthrough whose SNI resolves into a blocked IP range | +| Passthrough audit | The connection-level `PASSTHROUGH: cell=… sni=…` log line in `tls_clienthello`. NOTE: a true (ignore_connection) passthrough tunnel produces no TCPFlow, so `otel_export.tcp_start/message/end` do NOT fire — the `warden_passthrough_*` counters are not emitted today. Byte/duration metering would require a flow-bearing relay (e.g. a `next_layer`-installed TCPLayer); tracked separately. | +| Log shape | `src/brig/warden_addons/otel_export.py` tags MITM records with `tls_mode=mitm`. (Passthrough flows don't reach the otel hooks — see Passthrough audit above.) | +| CLI rendering | `src/brig/commands/network_cmd.py:_print_network_line` renders passthrough lines as `PASSTHROUGH: ` — visually distinct from `OUT:` and `INGRESS:` | | Unit tests | `tests/test_cell_spec.py::TestValidateCellDefinition::test_policy_tls_passthrough_*` (4 cases) | -| Unit tests | `tests/test_passthrough_tls.py` (10 cases — `is_passthrough` defense-in-depth, wildcard semantics, untrusted-profile rejection, per-cell-policy persistence, CLI render) | +| Unit tests | `tests/test_passthrough_tls.py` — `is_passthrough` defense-in-depth, wildcard semantics, untrusted-profile rejection, per-cell-policy persistence, CLI render, and `tls_clienthello` engaging passthrough via `ignore_connection` (asserts the real switch, not a no-op attr) incl. SNI/CONNECT-mismatch and blocked-IP-resolution refusal | | Unit tests | `tests/test_cell_profiles.py::test_policy_tls_passthrough_propagates_from_profile` | | CI | Unit | +**Verified e2e** (manual, against the pinned warden image, mitmproxy 10.1.1): + + - A cell with `example.com` in `allow` + `tls_passthrough` receives + example.com's **real upstream certificate** (Cloudflare) — warden tunnels + the TLS raw, no decryption. A cell with `example.com` in `allow` only + receives **warden's MITM cert** (`mitmproxy`). This confirms + `data.ignore_connection` engages passthrough exactly for opted-in hosts. + Cert issuer is the reliable signal; warden stdout is buffered and a busybox + `wget` (alpine) can't complete the handshake — use a real TLS client. + **Not yet landed** (tracked separately): - - E2E: `tests/test_passthrough_tls.sh` against a Cloudflare-fronted host (e.g. chatgpt.com), validating both the handshake-survives path and the SNI/CONNECT-mismatch rejection. + - Automated E2E (`tests/test_passthrough_tls.sh`) using a real TLS client + (python `ssl`, not busybox `wget`) that inspects the peer cert issuer to + assert passthrough-vs-MITM, plus the SNI/CONNECT-mismatch rejection. ### 12. Warden CA Auto-Mount Is Per-Cell, Re-Extracted From Container, Opt-Out-Able @@ -235,7 +260,7 @@ The invariants we DO uphold: request.** `warden start` polls `/var/lib/warden/mitmproxy-state/ mitmproxy-ca-cert.pem` for up to 30s after the container is healthy and refuses to declare warden ready until the cert exists. Cells - that race a fresh `brig up` can no longer get an empty / missing + that race a fresh `brig system up` can no longer get an empty / missing bundle. (Aitelier diagnosed all three of: lazy CA gen, root-owned tmpfs, and sh-c bypassing auto-sudo. Each is structurally fixed.) - **Bundle staged inside the VM, not on macOS.** Lima's `/state` virtio @@ -254,10 +279,48 @@ The invariants we DO uphold: | Surface | Location | |---|---| | Spec field | `src/brig/cell/spec.py:CellSpec.trust_warden_ca` (default True) | -| Validator | `src/brig/cell/spec.py:_v_trust_warden_ca` | +| Validator | `src/brig/cell/validators.py:_v_trust_warden_ca` | | Staging | `src/brig/cell/ca_bundle.py:stage_bundle` (extracts + concats + atomic mv) | | Reconciler | `src/brig/cell/reconciler.py` PODMAN_RUN action invokes stage_bundle; `build_run_command` adds `--volume` and `-e SSL_CERT_FILE=...` when applicable | -| Unit tests | `tests/test_warden_ca_mount.py` (8 cases) | +| Unit tests | `tests/test_warden_ca_mount.py` (9 cases) | +| CI | Unit | + +### 13. Scoped Host Mounts Are Opt-In, Bounded, and Bypass Warden by Design + +A `supervised`/`dev` cell may bind-mount an operator-chosen host directory into +itself via `mounts:` (ro default / rw opt-in), bounded by the VM-level +`mount_roots` allowlist. The bytes flow between the cell and the host files +directly — Warden does not mediate them (same trade-off as host_sockets). + +The invariants we DO uphold: + + - Opt-in per cell yaml (no default access); the `untrusted` profile rejects + `mounts:` at parse time. + - `host_path` realpath must resolve under a declared `mount_roots` entry — a + cell cannot reach host trees the operator did not allowlist, and the VM's + host exposure is bounded to those roots. + - A cell-created symlink inside the mount cannot escape the subtree to a VM + path: container mount-namespace isolation makes it dangle (verified under + runsc AND crun — runtime-independent, not reliant on gVisor; see + docs/design/mount-symlink-hardening.md). + - `mount_point` cannot shadow a system path or the cell's `/work` (parse-time). + - Every attach is audited (`log_lifecycle("mount_attach", ...)`) and the cell + banner states Warden does not see these bytes. + - Residual risk is HOST-SIDE: a cell can plant a symlink pointing out of the + shared folder that a host consumer might follow. `brig cell mount-scan` + reports/quarantines such symlinks; the consumer must treat cell-written + files as untrusted. brig sandboxes the cell's execution, not the fate of + files it may write (confused-deputy boundary). + +| Surface | Location | +|---|---| +| Parse-time guards | `src/brig/cell/validators.py:_v_mount_entry`, `_v_mounts` | +| Root allowlist | `src/brig/config.py:mount_roots()`, `validate_mount_roots()`, `src/brig/vm/lima_mounts.py` (lima.yaml managed block) | +| Runtime translation + containment recheck | `src/brig/cell/reconciler.py:_attach_mounts`, `_mount_bind_arg` | +| Profile gate | `_v_mounts` rejects on `untrusted` (via `_profile_is_untrusted`) | +| Audit + banner | `src/brig/cell/lifecycle.py:run_cell` emits `mount_attach` | +| Host-side symlink guard | `src/brig/workspace/workspace.py:find_escaping_symlinks`; `brig cell mount-scan` | +| Unit tests | `tests/test_mounts_spec.py` (22), `tests/test_reconciler_mounts.py` (8), `tests/test_lima_mounts.py` (13), `tests/test_mount_scan.py` (7), `tests/test_mount_roots_validation.py` (17) | | CI | Unit | ## State-consistency invariants @@ -274,7 +337,8 @@ The invariants we DO uphold: | Surface | Location | |---|---| -| Enforcement | `src/addons/enforce.py:PolicyEnforcer.load()` calls `_reload_policy(strict=True)` | +| Enforcement | `src/brig/warden_addons/enforce.py:PolicyEnforcer.load()` calls `_reload_policy(strict=True)` | +| Unit test | `tests/test_security_audit.py::TestStrictPolicyLoadFailsClosed` — strict load raises on missing/malformed `/policy.json`; non-strict reload swallows a bad edit and keeps last-good | | CI | Unit | ## Adversarial tests @@ -284,13 +348,13 @@ The invariants we DO uphold: | `--env HTTP_PROXY=attacker` to bypass warden | `test_cell_reconciler.py::TestBuildRunCommand::test_all_proxy_env_names_rejected` | | Symlink in secrets dir escaping | `test_security_secrets.py::TestValidateSecretPath::test_symlink_escaping_rejected` | | Double-hop symlink in secrets | `test_security_secrets.py::TestValidateSecretPath::test_double_hop_symlink_rejected` | -| Symlink at ingress token read site | `src/brig/cell/lifecycle.py:_register_cell_ingress` calls `validate_secret_path` (covered by the symlink-escape tests above, applied at the read site) | +| Symlink at ingress token read site | `src/brig/cell/lifecycle.py:register_ingress_for` calls `validate_secret_path` (covered by the symlink-escape tests above, applied at the read site) | | Concurrent allocator race — 50 threads | `test_network_subnet.py::TestConcurrentAllocation::test_concurrent_allocate_no_duplicates` | | Cell def with `network: proxy-external` | `test_cell_spec.py::TestValidateCellDefinition::test_network_proxy_external_rejected` | | DNS rebinding to host-service tuple | `test_security_audit.py::TestResponseHeadersDnsRebinding::test_does_not_skip_without_metadata` — naked (ip,port) match must not bypass | | Webhook redirect to internal host | `notifier.py` urllib fallback uses a redirect-disabling opener; urllib3 path uses `assert_hostname` and `cert_reqs=CERT_REQUIRED` | | Cell with deny-all reaching host service | `test_security_audit.py::TestHandleHostService::test_no_cell_policy_blocked` | -| Ingress route pointing at warden gateway IP | `src/addons/ingress.py` `_reload_routes` rejects host octets `< 2` | +| Ingress route pointing at warden gateway IP | `src/brig/warden_addons/ingress.py` `_reload_routes` rejects host octets `< 2` | ## CI wiring diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a47f8cf..b6df5ef 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -32,6 +32,64 @@ cloud. **Effort:** High. Abstract `vm_run()` to support direct execution. Handle path mapping differences. Test on multiple distros. +### Cell substrate: Apple Containerization (VM-per-cell) +**Status:** Deferred — evaluate as Apple's framework matures (macOS 26+). + +Replace (or offer alongside) Lima-VM + gVisor with Apple's Containerization +framework, which runs each container in its **own** lightweight VM via +Virtualization.framework. Brig's value — the Warden egress choke point, per-cell +policy, secrets, audit, invariants — is independent of the substrate and sits on +top unchanged; Apple's `container` is a runtime primitive, not a competitor. + +Wins: per-cell isolation becomes **hypervisor-enforced**. Today the Lima VM is +the only hard boundary and gVisor is explicitly *not* one (defense-in-depth), so +no-east-west / single-homing are enforced by network topology; VM-per-cell makes +them hardware-enforced. Plus sub-second starts and the end of the +image-store-on-one-shared-VM-disk class of problems (and gVisor provisioning). + +Cost / open questions: +- **Network choke point.** Warden is today the sole egress path *inside* brig's + single VM. With per-container VMs, Warden must be inserted in front of each + cell-VM's networking (vmnet / per-container IP) so it stays mandatory and + un-bypassable — the real re-architecture. +- **Invariant model.** The gVisor-centric invariants (5 = gVisor active; the + shared-VM east-west rules) get re-expressed for a per-VM-boundary world — + mostly a strengthening, but the ledger and `brig verify` need rework. +- OCI-compatible (image build/run carry over) but Swift/macOS-only and newer + (container networking was limited on macOS 15, improved on 26 / Tahoe). + +**Trigger:** Apple's framework is stable enough to depend on AND you want +stronger per-cell isolation, faster starts, or to stop maintaining Lima+gVisor. +Overlaps with "Brig on Linux" — both are substrate abstractions; do the +`vm_run()` / runtime abstraction once and back it with either. + +**Effort:** High. Substrate swap behind the reconciler, plus a genuine re-think +of the network choke point and the invariant ledger. + +### Host-side process hardening (macOS seatbelt) +**Status:** Deferred — defense-in-depth for the untrusted-state-dir boundary. + +Confine brig's *macOS-side* processes with seatbelt (App Sandbox / +`sandbox-exec` profiles): the host_socket bridges, `brig cell mount-scan`, and +any host tooling that touches cell-influenced files (invariant 4 — the state dir +is untrusted). A profile limiting each helper's filesystem reach and denying +unexpected network/exec shrinks the blast radius if a cell plants content a +host-side helper later consumes (the confused-deputy boundary `mount-scan` +already guards). Pairs with — does not replace — the VM boundary. + +Scope notes: +- Warden and cells run *inside* the VM, so seatbelt doesn't apply there — this is + purely host-side helper hardening. +- `sandbox-exec` is deprecated-but-present; the App Sandbox entitlement path is + the supported route for a signed binary. Pick per brig's distribution model. + +**Trigger:** You're hardening the host-side confused-deputy surface (e.g. an +operator process auto-consuming files from a cell's rw mount) and want +OS-enforced confinement beyond `mount-scan` + consumer discipline. + +**Effort:** Medium. Per-helper profiles + testing they don't break legitimate +access; more if it needs re-signing / entitlements. + ### Dispatcher integration **Status:** Deferred — dispatcher's job, not Brig's. @@ -134,6 +192,70 @@ workload itself might be malicious, not just the network. **Effort:** Low for the CLI; the addon side has prior art that can be rewritten in ~200 lines. +### Adversarial-agent lab (containment CTF + jailbreak-under-observation) +**Status:** Deferred — the deterministic floor already exists as a test. + +Brig's reason to exist is "run untrusted/agent code and *control + observe* it," +so the natural validation is to attack it with a real agent. Two experiments, +building on `tests/test_containment_e2e.sh` (`make redteam`) and the canary item: + +1. **Containment CTF (agent-driven).** An LLM agent with shell+network tools and + an explicit "capture the flags by any means" prompt, against tiered flags + mapped to the invariants: Tier-1 (egress bypass / SSRF / east-west / host + secret — a break here is a real failure), Tier-2 (gVisor->VM, expected-possible + defense-in-depth), Tier-3 (VM->macOS, the only hard boundary). `make redteam` + is the deterministic Tier-1 floor; this is the agent-driven, higher-tier + escalation. Score on **two** axes: prevention *and* detection (every attempt + must show in Warden logs / `mount-scan` / audit — "contained but invisible" is + a partial fail). +2. **Jailbreak-under-observation.** Run a jailbreak suite against an agent *inside* + a cell and let it be fully compromised — brig's job isn't to stop the jailbreak + (it's not an LLM-safety layer) but to guarantee zero blast radius and a complete + forensic trail of what a jailbroken agent *attempts*. Turns brig into a safe lab + for prompt-injection / agent-red-team research — a capability the bare runtime + (or Apple's `container`) can't offer. + +**Trigger:** You want to validate containment against a real adversary (not just +the scripted probe), or you're doing agent-safety research and need a contained, +observable harness. Pairs with the canary item (flags = canaries) and would +exercise the `novel_allowed` egress-novelty detection if that lands. + +**Effort:** Medium. The harness is mostly orchestration (agent + flag scoring + +log-scraping the two axes); the deterministic core is already the e2e test. + +### VM + warden autostart at login (consumer feedback: aitelier) +**Status:** Deferred — `restart: always` covers cells, not the VM itself. + +`restart: always` re-launches cells on `brig system up`, but the VM + warden +don't come up at host login, so after a reboot something must run +`brig system up` first. Consumers self-heal with a wrapper (aitelier's +`scripts/lib.sh`). An optional, off-by-default brig launchd/login agent would +bring the VM + warden up at login so `restart: always` cells return unattended. + +**Trigger:** A consumer wants a truly always-on brig-hosted service without a +wrapper script. **Effort:** Medium (a launchd plist + install/uninstall flow). + +### Per-cell credential rotation without restart (consumer feedback: hermes) +**Status:** Deferred — needs a design pass. + +Secret files (OAuth tokens) are read at cell start; a rotated value needs a +cell restart to take effect. Cells holding hourly-rotating OAuth tokens want the +new value picked up live (re-read the secret mount, or signal the cell). + +**Trigger:** A cell's credential rotates faster than its acceptable restart +cadence. **Effort:** Medium — depends on how the in-cell app consumes the secret. + +### Inter-cell routing primitive (`cell_services`) (consumer feedback: hermes) +**Status:** Deferred — host_services + ingress cover most shapes today. + +A sanctioned cell→cell address path (without violating no-east-west) for +multi-cell architectures where one cell must call another. Would route through +warden like host_services, keeping the choke point. Overlaps "Cell groups / +compose" below. + +**Trigger:** A multi-cell workload that genuinely can't be expressed via +host_services or ingress. **Effort:** High — new routing path + policy surface. + ## Nice-to-have ### `brig watch` — live TUI dashboard @@ -173,6 +295,43 @@ services (e.g., MCP server as a separate cell that the cell talks to directly). **Effort:** High. New spec format, dependency ordering, shared lifecycle. +### Writable non-`/tmp` HOME mount (consumer feedback: aitelier) +**Status:** Deferred — minor; non-fatal today. + +The read-only rootfs (writable only at `/work`, `/tmp`, `/run`) pushes cell +authors to put `HOME` under `/tmp`; some CLIs balk (codex refuses to create +helper binaries under a `/tmp` HOME — non-fatal warning, but a stricter CLI +could fail). A brig-provided small per-cell writable HOME tmpfs *outside* `/tmp` +(e.g. `/home/`) would let HOME-sensitive CLIs behave without a hand-rolled +mount. **Trigger:** a cell's CLI hard-fails on a `/tmp` HOME. **Effort:** Low +(one more tmpfs mount + a default HOME). + +### `brig system doctor`: stale cell `HTTP_PROXY` check (consumer feedback: hermes) +**Status:** Deferred — edge; the `restart: always` recreate path already avoids it. + +A cell's `HTTP_PROXY` is baked at creation to warden's then-current per-cell-net +IP; if that IP churns (a full `system down/up`), an already-created cell keeps +the stale address and `brig cell start` reuses it (only `rm --keep-workspace` + +`run` recovers). Have `doctor` flag "cell HTTP_PROXY ≠ current warden IP" with +the recreate fix. **Trigger:** warden IP churn strands a started cell. +**Effort:** Low (one doctor check). + +### Cross-source audit query (consumer feedback: hermes) +**Status:** Deferred — needs a correlation-id contract. + +`brig cell audit --since 1h` joining warden's network log with the +consumer's own run/event log by correlation id, for one timeline across the +proxy boundary. **Trigger:** incident triage that spans warden + the app. +**Effort:** Medium — requires a shared correlation id end to end. + +### Mount-side `nosymfollow` (consumer feedback: hermes) +**Status:** Deferred — blocked on runtime support. + +If podman 5.x / an alternate runtime exposes `nosymfollow` for bind mounts, use +it to kill the host-side symlink confused-deputy at the mount layer, retiring +the consumer-side `safe_open` / `mount-scan` discipline. **Trigger:** the runtime +exposes it. **Effort:** Low once available (a mount flag) — until then, N/A. + ## Out of scope ### Multi-tenant Warden diff --git a/docs/plans/0.3-validation-plan.md b/docs/archive/0.3-validation-plan.md similarity index 98% rename from docs/plans/0.3-validation-plan.md rename to docs/archive/0.3-validation-plan.md index 5c2ade3..a937b53 100644 --- a/docs/plans/0.3-validation-plan.md +++ b/docs/archive/0.3-validation-plan.md @@ -315,11 +315,11 @@ Source: audit code-quality finding §6. `brig system history` overlap heavily. Two dispatchers (`brig system preflight`, `warden preflight`) do the same thing. -Smaller scope for 0.3: deprecate `brig health` in favor of `brig system doctor --quick`. +Smaller scope for 0.3: deprecate `brig system doctor --quick` in favor of `brig system doctor --quick`. Print a deprecation note for one release; remove in 0.4. **Proof:** -1. `brig health` prints "Deprecated; use `brig system doctor --quick`" to stderr. +1. `brig system doctor --quick` prints "Deprecated; use `brig system doctor --quick`" to stderr. 2. `brig system doctor --quick` exists and matches old `health` output shape. 3. `tests/test_cmd_handlers.py` covers both. diff --git a/docs/design/architecture.md b/docs/design/architecture.md index a5a9f09..01fe2d7 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -1,5 +1,10 @@ # Brig Architecture +Component-level breakdown for contributors: how the pieces wire +together, network topology, deployment shape. For a user-facing +explanation of *why* each layer exists and what it protects you +from, see [`learning/concepts.md`](../learning/concepts.md). + ## Overview **Brig** is a secure, observable harness for running untrusted code on macOS. It provides VM-isolated containers (hardware boundary at the Lima VM) with controlled network egress and full observability. @@ -156,7 +161,7 @@ If you need true air-gap isolation, disable all network egress or use a dedicate **Key points:** - Each cell gets its own `--internal` network (created by `brig run`) -- Proxy joins each cell's network (so DNS name `proxy` resolves) +- Proxy joins each cell's network; `brig run` injects the proxy's per-cell IP into the cell's `HTTP(S)_PROXY` environment - No shared network = no east-west traffic by topology - No iptables rules needed = no chain ordering bugs - IPv6 disabled = simpler security model @@ -268,7 +273,7 @@ Runs inside Lima. Routes and logs all cell traffic. Cell containers are attached only to a per-cell **internal** Podman network (created with `--internal`). These networks have **no route/NAT to the VM's egress interface**. The only component with internet access is the **proxy**, which is the **only** container attached to: -- each `cell-*` internal network (to accept proxied requests), and +- each `brig-` internal network (to accept proxied requests), and - the `proxy-external` network (for outbound access). Additionally, the Lima VM applies a **fail-closed** firewall on traffic forwarded from the `proxy-external` CIDR to the VM egress interface. @@ -278,5 +283,5 @@ Additionally, the Lima VM applies a **fail-closed** firewall on traffic forwarde ## DNS Model - Cells do not perform external DNS lookups directly. External name resolution happens inside the proxy as part of egress. -- Cells only need internal DNS to resolve the proxy hostname (`proxy`) on their private network. Podman's internal DNS handles this automatically. +- Cells reach the proxy via the per-cell IP injected into `HTTP(S)_PROXY` at run time (the proxy container is named `warden`); no internal DNS name for the proxy is created or required. - **DNS over HTTPS (DoH):** If a cell attempts DoH to an allowed domain (e.g., `cloudflare-dns.com`), the request will succeed but is **visible in proxy logs** as HTTPS traffic to that domain. diff --git a/docs/design/cell-definition.md b/docs/design/cell-definition.md index 4d71376..e6e6e13 100644 --- a/docs/design/cell-definition.md +++ b/docs/design/cell-definition.md @@ -92,6 +92,7 @@ timeout: "30m" # Auto-kill after duration (30s, 5m, 2h, 1d) # Workspace workspace_quota: "500m" # Max workspace size +workspace_mount: /work # Mount point for the cell's writable workspace (default /work) # Cell rootfs writability. Default false (safe). When false, brig runs the # cell with --read-only rootfs + sized tmpfs at /tmp (64m) and /run (16m). @@ -105,9 +106,30 @@ workspace_quota: "500m" # Max workspace size # run untrusted code. writable_rootfs: false +# Seccomp profile (optional). Filename of a profile staged under +# ~/.brig/cells/seccomp/ (e.g. default.json); must be a bare filename, never a +# path or "unconfined". Omit to use the container runtime's default profile. +seccomp_profile: default.json + # Execution mode detach: false # Run in background +# Restart policy (default: no). "always" re-launches the cell on +# `brig system up` whenever its container is gone — e.g. a VM restart drops +# every container. brig persists the full spec at run time to replay it. +# An *exited* cell (still present, e.g. `brig cell stop`) is left alone, but a +# VM restart drops the container, so a stopped restart:always cell DOES +# relaunch on the next up; use `brig cell rm` to keep one down for good. +restart: always + +# Process user (podman --user; uid[:gid] or name[:group]). Default: the image's +# USER. gVisor presents virtiofs host mounts (and /work) as owned by 0:0 inside +# the cell, so a non-root cell can't own a rw `mounts:` dir — set user: "0" to +# run the cell as root and let it fully read/write/rewrite the mount. Writes +# still land owned by the operator on macOS, so readback is unaffected. Running +# as root *inside* the gVisor+VM sandbox is not a host-privilege change. +user: "0" + # Working directory inside container (default: /work) workdir: /app @@ -116,18 +138,31 @@ labels: team: platform purpose: scraping -# Ingress — authenticated reverse proxy for inbound traffic (optional) -# Warden routes external requests to cell-internal ports by path prefix. -# All ingress is token-authenticated and logged. +# Ingress — reverse proxy for inbound traffic (optional). Warden routes +# external requests to cell-internal ports by path prefix; all routes are logged. +# +# auth: token (default) — brig is the gate. The bearer token comes from a secret +# named `-ingress-token` (preferred) or `ingress-token` (fallback); +# register it before `brig run`, e.g.: +# openssl rand -hex 32 | brig secrets add -ingress-token - +# A cell that declares auth: token with no matching secret fails to start. +# auth: none — transparent pass-through: brig does NOT authenticate; the cell's +# own app is the gate (the Authorization header and ?query are passed through +# untouched). Use for services that authenticate themselves or for browser/ +# Electron WebSocket clients that can't send an Authorization header. The +# trade-off is explicit, mirroring tls_passthrough: you give up brig's +# perimeter gate for that route. Rejected on the `untrusted` profile and +# announced at run time (audit event + operator NOTE). A route that omits +# `auth` defaults to token (fail-secure). ingress: - name: api # Alphanumeric + hyphens, max 31 chars port: 8642 # Cell-internal port (1-65535) path_prefix: /api # Route prefix (must start with /) - auth: token # Auth method (only "token" supported) - - name: webhooks - port: 8642 - path_prefix: /webhooks - auth: token + auth: token # "token" (brig gates) or "none" (app gates) + - name: dashboard # e.g. a self-authenticating UI with WS + port: 9119 + path_prefix: /dashboard + auth: none # Host services — forwarding from cell to host port, through Warden. # Declaration here is the grant; no separate registration step. @@ -155,19 +190,27 @@ host_sockets: host_path: /tmp/postgres.sock mount_point: /run/host/postgres.sock mode: rw + +# Host mounts — bind-mount a host directory into the cell, bounded by the +# VM-level `mount_roots` allowlist. ro by default; rw needs `user: "0"` (above). +# Warden does not mediate these bytes. supervised/dev only; untrusted rejects. +# See "Host Mounts" below and invariant 13. +mounts: + - {name: repo, host_path: /Users/you/work/repo, mount_point: /workspace, mode: rw} ``` ## Ingress Cells have no inbound connectivity by default. Declaring `ingress` enables -authenticated reverse proxying through Warden. +reverse proxying through Warden — gated by a token (`auth: token`, default) or +transparent (`auth: none`, the app authenticates itself). External requests reach the cell via: ``` https://warden:8443/{cell_name}/{path_prefix}/... ``` -Requirements: +Requirements (`auth: token` routes — `auth: none` routes have none): - Each request must include `Authorization: Bearer ` - Token is stored as a Brig secret with a strict naming convention: - **Preferred:** `-ingress-token` — per-cell token, rotate @@ -185,7 +228,10 @@ Requirements: Security properties: - Opt-in (zero inbound unless declared) -- Token-authenticated with salted SHA-256 hashing +- `auth: token` (default) gates with a salted-SHA-256 Bearer token; `auth: none` + is transparent pass-through (the cell's app is the gate) — opt-in, rejected on + the untrusted profile, audited at run time. A route omitting `auth` defaults to + token (fail-secure). - Path-scoped (only declared prefixes are routed) - All inbound traffic logged - CONNECT tunneling blocked on ingress port @@ -386,6 +432,40 @@ unix sockets — bypass-of-warden is total there (see invariant 10). TCP host_services preserve warden in the path for audit + connection limits. +## Host Mounts + +Bind-mount an operator-chosen host directory into the cell (ro default, rw +opt-in). For a sandboxed agent that edits a real host folder in place. See +`docs/design/host-mounts.md`. + +```yaml +mounts: + - {name: repo, host_path: /Users/you/work/repo, mount_point: /workspace, mode: rw} + - {name: refdata, host_path: /Users/you/work/corpus, mount_point: /data} # mode: ro default +``` + +Prerequisite — declare the allowlisted root(s) once (VM-level; a VM recreate +applies the change): + +``` +brig config set mount_roots /Users/you/work +``` + +Rules: +- `host_path` realpath must resolve under a configured `mount_roots` entry and + be an existing directory. +- `mount_roots` may not be `/`, `$HOME`, `~/.ssh`/`~/.aws`/`~/.gnupg`/…, `~/.brig`, + `/etc`, or an ancestor/descendant of those; each root's basename (its + `/mnt/host/`) must be unique. +- `mount_point` must be absolute, not shadow a system path or the cell's + workspace mount (`/work`). +- Maximum 8 mounts per cell. +- The `untrusted` profile rejects `mounts:` at parse time. +- **Warden does not see these bytes** (like `host_sockets`). The cell can't + symlink-escape the subtree (mount-namespace isolation), but it *can* plant a + symlink a host consumer might follow — scan with `brig cell mount-scan ` + before consuming cell-written files, and treat them as untrusted. + ## Examples ### Minimal diff --git a/docs/design/host-mounts.md b/docs/design/host-mounts.md new file mode 100644 index 0000000..789f080 --- /dev/null +++ b/docs/design/host-mounts.md @@ -0,0 +1,249 @@ +# Design: Scoped Host-Directory Mounts (`mounts:`) + +**Status:** Implemented (ro + rw). The original premise — that rw was unsafe +until in-VM `nosymfollow` hardening — was **disproved empirically** (see +[Symlink reality](#symlink-reality-empirically-verified)): container +mount-namespace isolation already prevents a cell from following a symlink out +of the mounted subtree to a VM path, under both `runsc` and `crun`. The real +residual symlink risk is **host-side** and is the consumer's responsibility, +backed by a brig host-side scanner. See `docs/design/mount-symlink-hardening.md`. + +## Motivation + +Today the only host↔cell filesystem bridge is the managed `/work` workspace +(`~/.brig/state//workspace`). Editing a host directory that lives elsewhere +means cloning it into `/work` and syncing changes back — a clone/sync dance. + +Concrete driver: a hermes agent keeps a directory of working files and delegates +work on those files to aitelier, which runs the work inside a brig cell. The cell +needs **read-write** access to that shared folder so changes land in place. + +`mounts:` lets a non-`untrusted` cell bind-mount a single operator-chosen host +directory (ro by default, rw opt-in), keeping the existing isolation guarantee for +everything outside the mounted subtree. + +## Threat-model framing (read this first) + +This feature changes brig's job, for the affected cell, from *"isolate untrusted +code"* to *"let a semi-trusted agent edit a shared folder in place."* That is why +it is a `supervised`/`dev` capability and the `untrusted` profile rejects it — the +same rule already applied to `host_sockets`, `host_services`, and TLS passthrough. + +Two distinct risks, with different owners: + +1. **brig's boundary** (brig's job): a cell must not reach *outside* the mounted + subtree. Two sub-cases, both handled: (a) **widening the VM's host view** is + bounded by the `mount_roots` allowlist; (b) **symlink-escape from inside the + cell** is blocked by container mount-namespace isolation — a symlink in the + mount that targets an unmounted VM path simply dangles (verified under runsc + *and* crun; see [Symlink reality](#symlink-reality-empirically-verified)). +2. **Confused deputy** (the operator's job — brig *cannot* solve this): the cell is + an untrusted *writer*; whatever trusted host process consumes the files it wrote + is a *reader* outside the sandbox. **brig sandboxes the cell's execution, not the + fate of files it is permitted to write.** If the consumer executes / sources / + builds-from those files without its own review or sandbox, the sandbox is + bypassed by the data flow, not by a breakout. This risk is identical for a rw + mount, an overlay export, or a git push-back — it is inherent to "untrusted-cell + output feeds a trusted consumer," not to the mount mechanism. + +The cell-to-cell story is unchanged: cells still cannot read each other (invariant +1 is network isolation; separate workspaces/sandboxes). A shared mount is a +**deliberate filesystem channel between exactly the cells granted it** — two cells +pointed at the same rw host dir can pass data through it by design. Only grant a +shared mount to cells that are allowed to see each other's data. + +## Design + +### Cell yaml — `mounts:` (per-cell) + +Shape and `mode` values are identical to `host_sockets`, on purpose: + +```yaml +# user: "0" — a rw mount the cell must own needs a root cell (gVisor presents +# virtiofs mounts as 0:0; see "Ownership & writability" below). ro needs nothing. +user: "0" +mounts: + - { name: hermes-files, host_path: /Users/d0c/work/hermes/files, mount_point: /workspace, mode: rw } + - { name: refdata, host_path: /Users/d0c/work/corpus, mount_point: /data, mode: ro } +``` + +- `name` — audit label; unique within the cell. +- `host_path` — absolute, normalized; its **realpath must resolve under one of the + declared `mount_roots`** and be an existing directory. +- `mount_point` — where it appears in the cell. +- `mode` — `ro` (default) | `rw`. Per-entry; this is the rw opt-in. + +### `mount_roots` — VM-level allowlist (list) + +``` +brig config set mount_roots /Users/d0c/work,/Users/d0c/code # one-time; VM restart to apply +``` + +`mount_roots` is the set of host trees the VM is *ever* willing to expose to cells. +It is **VM-level, not per-cell** — for two reasons: + +1. **Lima mounts are static and VM-wide.** A cell runs *inside* the VM, so a host + file is only visible to a cell if it is mounted into the VM, and Lima mounts are + fixed at VM start and shared by every cell. "Which host trees may enter the VM" + is therefore unavoidably a VM fact, not a per-cell one. +2. **It is the security bound.** An operator-declared allowlist, set once and + auditable, prevents a less-careful (or agent-authored) cell yaml from exposing an + arbitrary host tree (`/`, `~/.ssh`, …) to the VM. Without it the cell yaml would + be the sole gate, leaving only a leaky denylist. This is consistent with + invariant 4 (the macOS state dir / specs are untrusted). + +The per-cell thing — *what this cell mounts* — stays in the cell yaml (`mounts:`). +The VM-level thing — *which roots may ever be mounted* — stays in config. + +### Lima mount mechanics + +Each declared root `R` is statically Lima-mounted at `/mnt/host/` (slug = +sanitized basename; validation rejects two roots with colliding slugs). Adding or +changing `mount_roots` rewrites both the template and the live instance `lima.yaml` +and is applied by a **lossless** VM restart (`brig system down --vm && brig system +up`): vz re-applies the virtiofs mounts on start and the podman image/container +store survives — only `limactl delete` would wipe it. No per-cell VM reconfiguration. + +`HostPaths.MOUNT_ROOTS` (list) and `VMPaths.MOUNTS_DIR` (`/mnt/host`) are added to +`config.py`. If `mount_roots` is empty, any cell declaring `mounts:` is rejected at +validation. Default: empty (opt-in). + +### Ownership & writability of rw mounts (gVisor reality) + +gVisor's gofer presents virtiofs-backed files (host `mounts:` **and** the managed +`/work` workspace) as owned by **`0:0` inside the cell**, and that ownership can't +be changed from the guest — `chown`, podman `:U`, and `:idmap` are all no-ops or +unsupported on virtiofs under runsc (empirically verified). Consequences: + +- A virtiofs rw mount is fully owned/writable only by a cell running as **root + (uid 0)** — it's the owner of the `0:0`-squashed tree. A non-root cell is + permanently "other": it may write only what the dir mode grants and can't + rewrite files or chmod. This is why a non-root image (e.g. one whose `USER` is + not 0) hits `EACCES` writing a dir it conceptually owns. +- Regardless of the cell's uid, files it writes land **owned by the operator on + macOS** (virtiofs maps guest writes back to the VM/host user), so host-side + readback is never the problem. + +So to host-mount a directory the cell must *own* (HERMES_HOME, app state, a build +output dir), run that cell as root with the cell-yaml `user: "0"`. Running as root +*inside* the gVisor+VM sandbox is not a host-privilege change — the VM is the only +hard boundary and the cell is untrusted regardless of its internal uid. `ro` +reference mounts need none of this. + +### Validation — `_v_mounts` / `_v_mount_entry` (`validators.py`) + +Mirrors `_v_host_socket_entry`; deltas: + +- `name`: required, `MOUNT_NAME_PATTERN`, unique per cell. +- `host_path`: required, absolute, normalized, no `..`. `realpath(host_path)` must + (a) exist, (b) be a directory, (c) live under `realpath(R)` for some `R` in + `mount_roots` (`startswith(R + "/")`, same idiom as the host_socket realpath + guard). Rejected if `mount_roots` is empty. +- `mount_point`: required, absolute, normalized — validated by + `_v_cell_mount_point` (mirrors, kept separate from, `_v_workspace_mount`): + forbidden prefixes (`/proc`, `/sys`, `/dev`, `/etc`, `/run/secrets`, + `/run/host`, `/run/brig`, not `/`); no `:` (the podman `-v` separator); must + not equal *or* be an ancestor/descendant of the cell's `workspace_mount` + (default `/work`); deduped. +- `mode`: `ro` | `rw` (`MOUNT_MODES`). +- Cross-field: **rejected on the `untrusted` profile** via `_profile_is_untrusted()`. +- `MAX_MOUNTS_PER_CELL` cap (`config.py`). +- Register `"mounts": _v_mounts`; add the `if "mounts" in cell_def` dispatch in + `validate_cell_definition`. + +### Reconciler — `_attach_mounts(spec, cmd)` (`reconciler.py`) + +Mirrors `_attach_host_sockets`. Per entry, in `build_run_command`: + +1. Re-resolve `realpath(host_path)`, re-confirm it is under one of the `mount_roots` + (runtime check is the real boundary; TOCTOU note as in the socket path). +2. Translate to the VM path: `/mnt/host//`. +3. Emit `-v {vm_path}:{mount_point}:{mode}`. +4. `log_lifecycle("mount_attach", spec.name, {name, mount_point, mode})` and a loud + `info()` banner — the same "Warden does not see this" disclaimer host_sockets + prints, adapted: *"cell '' has a host mount at → + ; the cell reads/writes these host files directly."* + +### Profile gating + +`untrusted` rejects `mounts:` at parse time (identical to `host_sockets` / +`host_services` / passthrough). `supervised` / `dev` allow it. Per-entry `mode` is +the rw opt-in; no separate global kill-switch. + +## Symlink reality (empirically verified) + +The original design gated rw on in-VM `nosymfollow` hardening, on the theory +that a runtime symlink in the mount could escape the subtree. **On-VM testing +disproved that for brig's stack** (see `docs/design/mount-symlink-hardening.md` +for the full results): + +- A cell creating `ln -s /workspace/x` and reading `x` got + **"No such file or directory"** — under **both `runsc` and `crun`**. Container + **mount-namespace isolation** means the only host-backed path the cell sees is + the mounted subtree; a symlink to any other VM path simply dangles. This is + *runtime-independent* — it does not rely on gVisor. + +So no in-VM symlink hardening is needed for brig's boundary. The reconciler binds +the **realpath** of `host_path` (collapsing symlink components in the declared +path; runtime containment re-checked at attach), and that is sufficient +in-cell. + +**The real residual symlink risk is host-side.** The cell has rw, so it can write +`folder/evil → /Users/you/.ssh/id_rsa` (a symlink's target is just a stored +string — it need not exist in the cell's namespace). That symlink lands in the +shared folder on macOS, where the path *is* resolvable. If a trusted host +consumer (hermes, your editor) follows it, it escapes the folder. No in-VM +mechanism touches this — it is part of the confused-deputy boundary and is the +**consumer's** responsibility. brig ships a host-side mitigation: `brig cell +mount-scan ` walks the declared mount dirs and reports/quarantines symlinks +whose realpath escapes the dir (reusing the copy-out sanitizer's logic). + +## Isolation summary + +| Axis | Guarantee | +|---|---| +| Network (cell↔cell) | Unchanged — invariant 1, no east-west. | +| Filesystem (cell↔cell) | Separate `/work` + rootfs; no sharing **unless** the operator mounts the same host dir into both cells (then they share *that dir only*). | +| Cell↔host (outside mount) | Cell confined to its rootfs + `/work` + declared mounts; nothing else on the host is reachable. | +| Cell↔host (inside mount) | Cell reads/writes the mounted subtree directly; Warden does not mediate these bytes. | +| VM host exposure | Bounded to the declared `mount_roots`; nothing else enters the VM. | + +## INVARIANTS ledger row (added to docs/INVARIANTS.md on enablement) + +The invariant text + surface table now live in `docs/INVARIANTS.md` (invariant +13). Summary: opt-in per cell yaml; `untrusted` profile rejects; `host_path` +realpath must resolve under a declared `mount_roots` entry; `mount_point` can't +shadow system paths or `/work`; cell-side escape is blocked by mount-namespace +isolation; the host-side symlink risk is mitigated by `brig cell mount-scan` and +is otherwise the consumer's responsibility; every attach is audited + bannered; +Warden does not mediate the bytes. + +## Implementation plan (TDD; each step ships green) + +1. **Config + Lima:** `MOUNT_ROOTS` (list) in `config.py`, `brig config set + mount_roots`, lima.yaml mount generation, `VMPaths.MOUNTS_DIR`. Tests: config + parse, slug uniqueness, lima rendering. +2. **Spec + validation:** `CellSpec.mounts`, `_v_mounts`/`_v_mount_entry`, dispatch. + Tests: `tests/test_mounts_spec.py` (paths, modes, root-containment, untrusted + reject, shadow/`/work`, cap, `mount_roots`-empty reject). +3. **Reconciler (ro first):** `_attach_mounts` realpath→VM translation, `-v ...:ro`, + audit + banner. Tests: `tests/test_reconciler_mounts.py` (argv shape, translation). +4. **Enable rw:** mounts (ro+rw) are unconditionally enabled; `_attach_mounts` + emits the binds for both modes; on-VM verification confirmed no in-VM symlink + hardening is required. +5. **Host-side scanner:** `brig cell mount-scan ` — reports/quarantines + symlinks whose realpath escapes a mounted dir (the host-side residual-risk + mitigation). SDK `run_sync(mounts=...)` param. INVARIANTS row. +6. **Docs:** `cell-definition.md` `mounts:` reference; `brig-cli.md` `config set + mount_roots` + `cell mount-scan`; export/inspect round-trip. + +## Open assumptions + +- Directories only (no single-file mounts) in v1. +- `mount_roots` is a list of absolute, existing dirs; obvious-catastrophe roots + (`/`, `$HOME` itself, `~/.brig`, `~/.ssh`) are rejected even though operator-set. + The floor (`validate_mount_roots`) compares by real path and on-disk identity, + so a symlink, a realpath alias (`/etc` → `/private/etc`), or a case variant on a + case-insensitive filesystem (`~/.SSH` == `~/.ssh`) can't dodge it. +- Changing `mount_roots` requires a lossless VM restart (`brig system down --vm && + brig system up`); the image store is preserved (only `limactl delete` wipes it). diff --git a/docs/design/implementation.md b/docs/design/implementation.md index 015bf2e..0d1c0b1 100644 --- a/docs/design/implementation.md +++ b/docs/design/implementation.md @@ -29,7 +29,7 @@ src/ security/ secrets.py # Secret path validation image.py # Image signature verification - verify.py # All 9 invariant verification checks + verify.py # Security-invariant verification checks (verify_* fns) ops/ logging.py # Canonical logging (ONE copy) @@ -44,8 +44,16 @@ src/ workspace/ workspace.py # File copy with sanitization + quarantine + observability/ + collector.py # OTel collector container lifecycle (brig-otel) + collector_config.yaml # Collector pipeline config + promql.py # PromQL query helpers + stats.py # Stats aggregation for `brig system stats` + commands/ - lifecycle_cmd.py # run/stop/kill/rm/list/exec/shell/etc. + lifecycle_run.py # brig run + its arg-parse/merge/diagnose helpers + lifecycle_inspect.py # list/inspect/files/read/logs/cp/top/diff/stats/export/ingress/preflight + lifecycle_control.py # stop/kill/rm/start/restart/pause/unpause/wait/rename/exec/shell/attach system_cmd.py # init/verify/doctor/preflight/metrics/prune/history/stats policy_cmd.py # policy show/set secrets_cmd.py # secrets list/add/rm @@ -57,21 +65,24 @@ src/ sdk.py # Programmatic API (execute_sync for agents) + warden_addons/ # mitmproxy addons — brig package-data, flat-loaded + _common.py # Shared: BLOCKED_NETWORKS, SubnetResolver, atomic_write_json + _policy.py # Policy data structures (PolicyRule, DomainTrie, Policy) + _log_writer.py # Batched JSONL log writer with rotation + _notifier_state.py # Webhook URL SSRF resolution/state for notifier + enforce.py # Policy enforcer addon (uses _policy) + logger.py # Request logging (JSONL per cell) + ops.py # Merged: metrics + rate limiting + health endpoint + ingress.py # Ingress reverse proxy (port 8443; auth: token|none) + notifier.py # Webhook notifications + otel_export.py # OpenTelemetry metrics + log records export + warden/ cli.py # Warden CLI entry point proxy.py # Container lifecycle (start/stop/status) reconcile.py # Subnet state reconciliation health.py # Health checks logs.py # Log management (prune) - - addons/ - _common.py # Shared: BLOCKED_NETWORKS, SubnetResolver, atomic_write_json - _policy.py # Policy data structures (PolicyRule, DomainTrie, Policy) - enforce.py # Policy enforcer addon (uses _policy) - logger.py # Request logging (JSONL per cell) - ops.py # Merged: metrics + rate limiting + health endpoint - ingress.py # Authenticated reverse proxy (port 8443) - notifier.py # Webhook notifications ``` ## Declarative Reconciler @@ -93,6 +104,7 @@ and the Lima VM. | File | Location | Purpose | |------|----------|---------| | `subnets.json` | `~/.brig/state/system/` | Subnet allocation | -| `network-policy.json` | `~/.brig/cells/` | Global egress policy | +| `network-policy.json` | `~/.brig/cells/` | Process-wide operational settings (rate limits, log filter, policy tracing) — no allow/deny | +| `policies/.json` | `~/.brig/state/system/policies/` | Per-cell egress allow/deny | | `operations.jsonl` | `~/.brig/state/system/` | Command audit log | | `lifecycle.jsonl` | `~/.brig/state/system/` | Cell lifecycle events | diff --git a/docs/design/mount-symlink-hardening.md b/docs/design/mount-symlink-hardening.md new file mode 100644 index 0000000..55248cc --- /dev/null +++ b/docs/design/mount-symlink-hardening.md @@ -0,0 +1,71 @@ +# Mount Symlink Behavior — Findings & Decision + +**Status:** Resolved by on-VM testing. Supersedes the earlier "rw is gated on +in-VM `nosymfollow`" premise, which was **wrong** for brig's stack. See +[host-mounts.md](host-mounts.md). + +## The question + +`mounts:` gives a cell rw access to a host directory. Can a symlink created +inside the mounted subtree be used to read/write **outside** it? + +## What we tested (live brig VM, kernel 6.8, podman 4.9.3) + +A VM-only secret was placed *outside* the subtree; a cell created a symlink to +it inside its mount and tried to read through it: + +| Scenario | Result | +|---|---| +| `runsc` (gVisor): `ln -s /workspace/x; cat x` (abs + rel climb) | **"No such file or directory"** — no leak | +| `crun` (gVisor OFF): same | **"No such file or directory"** — no leak | +| Kernel `nosymfollow` bind remount: `cat` a symlink on it | `ELOOP` — primitive works, available if ever needed | + +## Conclusion + +**Container mount-namespace isolation already blocks the cell→VM symlink +escape, and it is runtime-independent** (runsc *and* crun). The only host-backed +path a cell sees is the mounted subtree; a symlink targeting any other VM path +isn't present in the cell's namespace, so it dangles. **No in-VM `nosymfollow` +hardening is required** for brig's boundary. + +- `nosymfollow` on the VM-side mount would only be defense-in-depth against a + hypothetical *runtime/gofer bug* that followed symlinks host-side. The kernel + primitive works (verified) if we ever want that belt-and-suspenders, but it is + **not** a precondition for shipping `mounts:`. We do not add it now (keep the + bind simple; the boundary holds without it). +- The reconciler still binds the **realpath** of `host_path` (collapses symlink + components in the *declared* path) and re-checks containment at attach. + +## Where the real residual symlink risk is: host-side + +The cell has rw, so it can write a symlink whose target string is an absolute +**macOS** path — `folder/evil → /Users/you/.ssh/id_rsa` — or a relative climb. +A symlink's target is just stored text; the cell doesn't need the path to exist +in its namespace to create it. That symlink lands in the shared folder on macOS +via virtiofs, where the path **is** resolvable. If a trusted host consumer +(hermes, an editor, a build) reads the folder and **follows** the symlink, it +escapes the folder — into the SSH key, etc. + +No in-VM mechanism touches this; it's the macOS side. It is the same +confused-deputy boundary already documented: the consumer must treat the folder +— **symlinks included** — as untrusted. + +## Mitigation (shipped) + +1. **`brig cell mount-scan `** — walks each declared mount's host dir and + reports symlinks whose realpath escapes the dir; `--quarantine` removes them. + Reuses the copy-out sanitizer's escaping-symlink logic + (`workspace._sanitize_tree`). The operator/hermes runs it before consuming + cell output. +2. **Guidance for consumers:** read cell-written files with `O_NOFOLLOW` / reject + symlinks resolving outside the folder. brig sandboxes the cell's *execution*, + not the *fate of files it may write* — running the consume step in its own + cell is the strongest option. + +## Residual / out of scope + +- A gVisor escape inside the VM is in-model (defense-in-depth, not a boundary) — + unchanged by this feature. +- Host-side TOCTOU (consumer races the scanner) — `mount-scan` is a + point-in-time check; for adversarial timing, the consumer's own `O_NOFOLLOW` + discipline is the real control. diff --git a/docs/design/security.md b/docs/design/security.md index bc53d5b..c8b0d57 100644 --- a/docs/design/security.md +++ b/docs/design/security.md @@ -56,7 +56,7 @@ podman network rm "brig-${CELL_NAME}" **Rules:** -1. Proxy listens ONLY on port 8080 on the internal interface +1. Warden's egress proxy listens on port 8080 on the internal interface. (Ingress adds 8443, and each declared `protocol: tcp` host_service adds its port — all on warden's internal IP, gated by `enforce.py`'s per-cell policy.) 2. CONNECT method allowed ONLY to port 443 (HTTPS) 3. CONNECT to literal IPs is blocked (domain allowlist only) 4. CONNECT to RFC1918/link-local/localhost/CGNAT is blocked @@ -181,31 +181,34 @@ Measured on Apple Silicon (Lima VZ, steady-state inside running container): **Key insight:** gVisor has zero overhead for pure compute. The 3x cost applies only to syscalls. Network I/O through the Warden proxy is unaffected because it flows through the cell's network stack, not the filesystem. -#### Choosing a Runtime +#### Choosing a Profile -| Profile | Runtime | Threat Model | Use Case | -|---------|---------|-------------|----------| -| `untrusted` | gVisor | Unknown/hostile code | Running submissions from strangers | -| `supervised` | gVisor | Semi-trusted, defense-in-depth | AI agents, CI runners | -| `dev` | crun + seccomp | Your own code | Development, fast iteration | -| `airgapped` | gVisor | No network, maximum isolation | Offline compute | -| `honeypot` | gVisor | Adversarial; capture telemetry | Studying malware behaviour | +| Profile | Threat Model | Use Case | +|---------|-------------|----------| +| `untrusted` | Unknown/hostile code | Running submissions from strangers | +| `supervised` | Semi-trusted, defense-in-depth | AI agents, CI runners | +| `dev` | Your own code | Development, fast iteration | +| `airgapped` | No network, maximum isolation | Offline compute | +| `honeypot` | Adversarial; capture telemetry | Studying malware behaviour | -Use gVisor when the code is untrusted and you want protection against unknown kernel exploits. Use crun when the code is trusted and syscall-heavy performance matters — the VM + seccomp + network isolation still provide strong protection. +All profiles run on gVisor (`runsc`): invariant 5 forbids a silent runtime +downgrade, so the reconciler hardcodes `--runtime runsc` and profiles cannot +override it. Profiles differ in resource limits, seccomp, and network defaults +— not in the kernel-isolation boundary. --- ### Invariant 6: Only Infrastructure Containers May Attach to `proxy-external` -**Rule:** Only `warden` may attach to `proxy-external`. No cell containers. +**Rule:** Only brig's own infrastructure containers may attach to `proxy-external` — `warden` and the OTel collector (`brig-otel`). No cell containers. (The enforced allowlist is `INFRA_CONTAINER_NAMES` in `src/brig/config.py`; `verify_proxy_network` checks membership.) -**Why this matters:** The `proxy-external` network has outbound internet access. Any container attached to it inherits the VM-level egress rules but bypasses per-cell isolation. +**Why this matters:** The `proxy-external` network has outbound internet access. Any container attached to it inherits the VM-level egress rules but bypasses per-cell isolation. The collector qualifies as infrastructure: it is brig-managed and pinned, runs no untrusted workload, is on this network only so warden can resolve its OTLP endpoint by name, and **no cell can reach it** — cells live on their own `brig-` `--internal` networks with no route to `proxy-external`. **Enforcement:** ```bash # Verify only infrastructure containers are on proxy-external: -ALLOWED="warden" +ALLOWED="warden brig-otel" containers=$(podman network inspect proxy-external --format '{{range .Containers}}{{.Name}} {{end}}') for c in $containers; do if ! echo "$ALLOWED" | grep -qw "$c"; then @@ -312,7 +315,7 @@ For an agent runtime holding the operator's provider credentials (Claude OAuth, 3. At runtime, Warden's `Policy.is_passthrough` re-checks both lists — a tampered policy file that lists a host *only* in `tls_passthrough` cannot bypass MITM (defense in depth against invariant 4: state dir untrusted). 4. SNI in the client hello must match the CONNECT host. Otherwise a malicious cell could CONNECT to allowed-host:443 and SNI=attacker.com to abuse Warden as a generic tunnel. 5. Untrusted profile cannot declare passthrough. Adversarial cells must remain inspectable. -6. Audit log entries are tagged `tls_mode=mitm` or `tls_mode=passthrough`. Passthrough entries omit method/path/status BY CONSTRUCTION (Warden never decrypted them). `brig cell network` renders passthrough lines as `PASSTHROUGH (NB in / NB out)`; `brig system stats` shows `PT/CONN` and `PT/BYTES` columns. +6. Passthrough connections are audited at the TLS handshake via the connection-level `PASSTHROUGH: cell=… sni=…` log line (`enforce.py:tls_clienthello`). Method/path/status/bytes are absent BY CONSTRUCTION — Warden never decrypts, and a true (`ignore_connection`) passthrough tunnel produces no mitmproxy flow, so the per-byte/connection `tcp_*` hooks and `warden_passthrough_*` counters never fire. MITM OTel records are tagged `tls_mode=mitm` (the JSONL flow log written by `logger.py` carries no `tls_mode` field). **Cell yaml shape:** @@ -335,11 +338,11 @@ Two lists, not one with attributes, so `grep -l tls_passthrough cells/*.yaml` an **Rule:** Cells with `trust_warden_ca: true` (the default) get a combined system+Warden CA bundle bind-mounted at `/run/brig/ca-bundle.crt`, with `SSL_CERT_FILE` / `REQUESTS_CA_BUNDLE` / `CURL_CA_BUNDLE` / `NODE_EXTRA_CA_CERTS` auto-exported. Bundle is staged from the live Warden container at every cell start — not cached on macOS. -**Why this matters:** Without auto-mount, every brig-cell consumer rediscovers a manual workaround (extract CA → concat onto system roots → export the four env vars). Auto-mount also lets brig handle CA rotation transparently: every cell start re-extracts the current Warden CA, so a `brig system restart` (which can rotate the CA) doesn't leave cells with stale trust. +**Why this matters:** Without auto-mount, every brig-cell consumer rediscovers a manual workaround (extract CA → concat onto system roots → export the four env vars). Auto-mount also lets brig handle CA rotation transparently: every cell start re-extracts the current Warden CA, so a `brig system down && brig system up` (which can rotate the CA) doesn't leave cells with stale trust. **Sub-rules brig enforces** (`docs/INVARIANTS.md` invariant 12): -1. **Bundle source-of-truth is `/var/lib/warden/mitmproxy-state/`** on the VM — a persistent dir owned by uid 1000 (mitmproxy user). Bind-mounted into Warden so mitmproxy can write the CA. Brig reads via plain `cat` — no `podman exec` (audit eliminated that surface; see commits 51082fd, 6b51eed). +1. **Bundle source-of-truth is `/var/lib/warden/mitmproxy-state/`** on the VM — a persistent dir owned by uid 1000 (mitmproxy user). Bind-mounted into Warden so mitmproxy can write the CA. Brig reads via plain `cat` — no `podman exec` (that surface was eliminated). 2. **CA generated eagerly at `warden start`.** Polls for the cert file up to 30s after container is healthy; refuses to declare Warden ready until it exists. Cells racing a fresh `brig system up` can no longer get an empty bundle. 3. **Bundle staged inside the VM, not on macOS.** Lives at `/state//ca-bundle.crt`; the VM is the trust boundary (invariant 4 keeps macOS state untrusted). 4. **Cell-side mount is read-only.** Compromised cell can't tamper with its own trust store. @@ -349,6 +352,25 @@ Two lists, not one with attributes, so `grep -l tls_passthrough cells/*.yaml` an --- +### Invariant 13: Scoped Host Mounts Are Opt-In, Bounded, and Bypass Warden by Design + +**Rule:** A `supervised`/`dev` cell may bind-mount an operator-chosen host directory into itself via `mounts:` (read-only by default, `rw` opt-in), bounded by the VM-level `mount_roots` allowlist. Like `host_sockets`, the bytes flow between the cell and the host files directly — Warden does not mediate them. + +**Why this matters:** Agents frequently need a real working tree (a repo to edit, a dataset to read) that's larger or more persistent than `/work`, and copying everything in and out is slow and loses edits. `mounts:` trades audit visibility over those files for direct, persistent access — with the host exposure bounded to roots the operator explicitly allowlisted at the VM level. + +**Sub-rules brig enforces** (`docs/INVARIANTS.md` invariant 13): + +1. **Opt-in per cell yaml.** No default access; the `untrusted` profile rejects `mounts:` at parse time. +2. **Bounded by `mount_roots`.** A mount's `host_path` realpath must resolve under a declared VM-level `mount_roots` entry — a cell cannot reach host trees the operator did not allowlist. Re-resolved and re-checked at attach time (TOCTOU-safe), not just at parse time. +3. **`mount_point` cannot shadow** a system path or the cell's `/work` (parse-time). +4. **Symlinks inside the mount cannot escape** the subtree to a VM path — container mount-namespace isolation makes such a symlink dangle (verified under runsc AND crun, so it does not rely on gVisor; see `docs/design/mount-symlink-hardening.md`). +5. **Every attach is audited** via `log_lifecycle("mount_attach", …)`, and the cell startup banner states Warden does not see these bytes. +6. **Residual risk is host-side, by design.** A cell can plant a symlink pointing out of the shared folder that a *host* consumer might follow (confused-deputy). `brig cell mount-scan` reports/quarantines such symlinks; consumers must treat cell-written files as untrusted. Brig sandboxes the cell's execution, not the fate of files it writes. + +`mount_roots` is empty by default — the whole feature is off until an operator opts the VM in. See `docs/design/host-mounts.md` for the full design. + +--- + ## Verification Tests Run these to verify isolation is working correctly. @@ -443,43 +465,46 @@ echo "Exit code: $?" # MUST be non-zero brig cell kill cell-a && brig cell rm cell-a ``` -### Test 9: Proxy Only Exposes Port 8080 +### Test 9: Proxy Exposes Only Expected Ports ```bash brig run --name test-portscan --rm -- sh -c ' apk add --no-cache nmap >/dev/null 2>&1 - nmap -p 1-65535 proxy --open -T4 2>/dev/null | grep "^[0-9]" + nmap -p 1-65535 warden --open -T4 2>/dev/null | grep "^[0-9]" ' -# Should output ONLY: 8080/tcp open http-proxy +# For a cell with no ingress and no TCP host_services: ONLY 8080/tcp (http-proxy). +# A cell that declares ingress also sees 8443; a `protocol: tcp` host_service also +# sees its declared port (e.g. 5432). All listeners are gated by enforce.py policy. ``` ### Test 10: Proxy Is Not a Gateway ```bash # Try CONNECT to non-443 port (should be rejected) -brig run --name test --rm -- curl -s -x http://proxy:8080 http://example.com:8080/ +brig run --name test --rm -- curl -s -x http://warden:8080 http://example.com:8080/ # Expected: 403 Forbidden # Try CONNECT to literal IP (should be rejected) -brig run --name test --rm -- curl -s -x http://proxy:8080 https://142.250.80.46/ +brig run --name test --rm -- curl -s -x http://warden:8080 https://142.250.80.46/ # Expected: 403 Forbidden # Try CONNECT to internal range (should be rejected) -brig run --name test --rm -- curl -s -x http://proxy:8080 http://10.50.0.1/ +brig run --name test --rm -- curl -s -x http://warden:8080 http://10.50.0.1/ # Expected: 403 Forbidden ``` ### Test 11: Default Runtime Check ```bash -# Run without specifying runtime (should use gVisor) +# Run without specifying a profile (should use gVisor) brig run --name test-default --rm -- cat /proc/version -# Should show gVisor +# Should contain "gvisor" -# Verify the only way to opt out of gVisor is to pick a non-gVisor profile. -# `brig run` has no `--runtime` flag — runtime is set by the profile. -brig run --name test --profile dev -- echo hello # Uses crun. -brig run --name test --profile untrusted -- echo hello # Uses gVisor. +# Runtime is fixed at runsc and is NOT profile-selectable (invariant 5): +# the reconciler hardcodes `--runtime runsc` and `brig run` has no +# `--runtime` flag. Every profile runs on gVisor. +brig run --name test-dev --rm --profile dev -- cat /proc/version # contains "gvisor" +brig run --name test-untrusted --rm --profile untrusted -- cat /proc/version # contains "gvisor" ``` ### Test 12: IPv6 Is Disabled @@ -584,19 +609,18 @@ brig cell kill test-identity && brig cell rm test-identity ### Soft Reset (restart cells and proxy) ```bash -brig cell stop --all -warden restart -brig cell start --all +brig system down # stop all cells + warden +brig system up # restart warden +brig cell start # restart each cell you need ``` ### Hard Reset (kill everything, keep state) ```bash -brig cell kill --all -warden stop +brig system down # stop all cells + warden limactl shell --workdir / brig -- sudo 'for net in $(podman network ls -q | grep "^brig-"); do podman network rm "$net" 2>/dev/null; done' brig system up -brig cell start --all +brig cell start # repeat for each cell to restart ``` ### VM Restart (preserves macOS state) @@ -604,7 +628,7 @@ brig cell start --all ```bash limactl stop brig && limactl start brig # Proxy starts automatically via systemd -brig cell start --all +brig cell start # repeat for each cell to restart ``` ### VM Recreate (clean slate VM, preserves macOS state) @@ -612,7 +636,7 @@ brig cell start --all ```bash limactl delete brig && make setup # All VM state destroyed and rebuilt -brig cell start --all +brig cell start # repeat for each cell to restart ``` ### Full Reset (destroy everything) @@ -628,11 +652,11 @@ limactl delete brig && make setup After any recovery: ```bash -# Quick smoke test -brig run --name recovery-test --rm -- curl -m 5 https://httpbin.org/ip +# Quick smoke test (egress is default-deny — allow the test host) +brig run --name recovery-test --rm --policy-allow httpbin.org -- curl -m 5 https://httpbin.org/ip -# Full verification suite -brig test isolation +# Re-check the runtime-verifiable invariants +brig system verify ``` --- @@ -655,16 +679,6 @@ security: network: none # No network at all ``` -### Egress Rate Limits - -```yaml -# Future: in network-policy.json -rate_limits: - per_cell: - requests_per_minute: 1000 - bytes_per_minute: 100MB -``` - ### Per-Cell Proxy ```yaml diff --git a/docs/design/supply-chain.md b/docs/design/supply-chain.md index 821489f..7514c67 100644 --- a/docs/design/supply-chain.md +++ b/docs/design/supply-chain.md @@ -29,34 +29,38 @@ How we keep Brig's dependencies, container images, and CI infrastructure trustwo ## Container image maintenance -The mitmproxy container image is pinned by digest: +Warden runs a **custom** image (mitmproxy + the OpenTelemetry SDK), built +inside the VM so wheels match the runtime arch. `src/warden/proxy.py` holds: ```python -IMAGE = "docker.io/mitmproxy/mitmproxy@sha256:39ef4ec..." +WARDEN_IMAGE_TAG = "..." # localhost/brig-warden: +WARDEN_IMAGE_DIGEST = "sha256:..." # pin verified at start (_verify_warden_image) +BASE_IMAGE = "docker.io/mitmproxy/mitmproxy@sha256:..." # no-OTel fallback only ``` -To update: +`_warden_image()` runs `localhost/brig-warden:` whenever +`WARDEN_IMAGE_DIGEST` is pinned (the normal case); `BASE_IMAGE` is only the +fallback when no custom image is pinned. **Patching `BASE_IMAGE` alone does +NOT change the launched image.** To update: -1. Pull the new image: `podman pull docker.io/mitmproxy/mitmproxy:latest` -2. Get the digest: `podman inspect --format '{{.Digest}}' docker.io/mitmproxy/mitmproxy:latest` -3. Update `src/warden/proxy.py` with the new digest -4. Run E2E tests: `make e2e` -5. Commit with the mitmproxy version in the message +1. Bump `BASE_IMAGE` and/or the OTel SDK version in `src/warden/image/Dockerfile`. +2. Rebuild + re-pin: `./scripts/build-warden-image.sh` — it builds inside the + VM and rewrites `WARDEN_IMAGE_TAG` + `WARDEN_IMAGE_DIGEST` in `proxy.py`. +3. Run E2E tests: `make e2e` +4. Commit the one-line digest update with the version in the message. ## gVisor (runsc) bumps -`scripts/provision-vm.sh` pins `GVISOR_RELEASE` and a per-arch sha512. To bump: +`GVISOR_RELEASE` + a per-arch sha512 are pinned in **two** files that must +stay in sync — `scripts/provision-vm.sh` and `src/brig/vm/lima.yaml.template` +— and `scripts/check-gvisor-pin.sh` (wired into CI) hard-fails on drift. Use +the canonical updater, which writes both: 1. Pick a release from . -2. Fetch the sha512 from the release page (don't compute it from the same source you're pulling the binary from). -3. Update `GVISOR_RELEASE` and the matching `GVISOR_SHA512_BY_ARCH` entry in `scripts/provision-vm.sh`. -4. Verify locally before merging: - -```bash -curl -fsSL "https://storage.googleapis.com/gvisor/releases/release/${RELEASE}/${ARCH}/runsc" \ - | sha512sum -# Expected sha512 must match the value you put in the script. -``` +2. `make pin-gvisor` (or `./scripts/pin-gvisor.sh `) — fetches the + per-arch sha512 from the release and rewrites both files. +3. Verify: `./scripts/check-gvisor-pin.sh` (the same check CI runs). +4. Commit; do not hand-edit only one of the two files (CI will reject it). ## Responding to CVEs diff --git a/docs/examples/README.md b/docs/examples/README.md new file mode 100644 index 0000000..449bc2d --- /dev/null +++ b/docs/examples/README.md @@ -0,0 +1,11 @@ +# Examples + +Drop-in config snippets referenced from the learning + design docs. + +| File | Purpose | Referenced by | +|---|---|---| +| `network-policy.example.json` | Example of `~/.brig/cells/network-policy.json` — the process-wide operational config: `rate_limits` / `log_filter` / `log_quota` / `policy_trace` / `notifications`. Egress allow/deny is per-cell, not here. | [`docs/learning/concepts.md`](../learning/concepts.md), [`docs/reference/addons.md`](../reference/addons.md) | +| `brig-logrotate.conf` | Logrotate config for `~/.brig/state//network/*.jsonl` on macOS/Linux hosts that wire up logrotate. | [`docs/learning/troubleshooting.md`](../learning/troubleshooting.md) | + +Add your own examples here; cross-link from wherever the doc points the +reader at a concrete config file. diff --git a/docs/examples/network-policy.example.json b/docs/examples/network-policy.example.json index 12eb7e2..7cfc522 100644 --- a/docs/examples/network-policy.example.json +++ b/docs/examples/network-policy.example.json @@ -1,30 +1,4 @@ { - "allow": [ - "example.com", - "*.example.com", - "api.github.com", - "*.githubusercontent.com", - { - "domain": "api.openai.com", - "paths": ["/v1/*"], - "methods": ["POST"] - }, - { - "domain": "registry.npmjs.org", - "paths": ["/*"], - "methods": ["GET"] - } - ], - "deny": [ - "evil.com", - "*.malware.com" - ], - "cells": { - "trusted-cell": { - "allow": ["internal.corp.com"], - "deny": [] - } - }, "rate_limits": { "default": { "rate": 100, @@ -43,10 +17,11 @@ "min_status": 0, "sample_rate": 1.0 }, - "log_retention": { - "max_age_days": 7, - "max_size_mb": 500, - "compress_after_days": 1 + "log_quota": { + "max_size_mb": 500 + }, + "policy_trace": { + "enabled": false }, "notifications": { "webhook_url": "", diff --git a/docs/learning/concepts.md b/docs/learning/concepts.md index 679c505..dce1383 100644 --- a/docs/learning/concepts.md +++ b/docs/learning/concepts.md @@ -1,6 +1,10 @@ # Concepts -Deep dives into how Brig works. +User-facing explanations of *why* Brig is built the way it is — what +each layer protects you from, what it doesn't, and what you should +expect when you use it. For the component-level breakdown (how the +pieces wire together, network topology, deployment), see +[`design/architecture.md`](../design/architecture.md). ## Lima VM as Security Boundary @@ -75,13 +79,11 @@ Some programs don't work with gVisor: - Some network tools - Programs with specific kernel dependencies -If you must run without gVisor (not recommended), use the `dev` profile, -which keeps gVisor off by default but applies the same network/policy -gating: - -```bash -brig run --name test --profile dev your-image -- your-command -``` +gVisor is **mandatory** for cells — no flag, profile, or yaml field disables it +(invariant 5: gVisor must be active, no silent downgrade). The `dev` profile only +raises resource limits (memory / cpus / pids); it does not change the runtime. If +a workload genuinely can't run under gVisor's syscall surface, it can't run as a +brig cell — that's the security boundary, not a setting to turn off. --- @@ -155,20 +157,26 @@ Even if a cell ignores `HTTP_PROXY` environment variables, it still can't reach ### Policy Enforcement +Egress allow/deny is **per-cell** and **default-deny** — a cell with no policy +reaches nothing. Rules live in the cell's `policy:` block (or a trust profile) +and can be edited live with `brig policy set `: + ```yaml -# network-policy.json -allow: - - pypi.org - - "*.pythonhosted.org" - - github.com - - api.openai.com - -deny: - - pastebin.com - - "*.ngrok.io" +# in a cell yaml +policy: + allow: + - pypi.org + - "*.pythonhosted.org" + - github.com + - api.openai.com + deny: + - pastebin.com + - "*.ngrok.io" ``` -Requests to non-allowed domains return 403. +Requests to non-allowed domains return 403. The process-wide +`~/.brig/cells/network-policy.json` holds only operational settings (rate +limits, log filtering, policy tracing) — no allow/deny rules. ### Hot Reload @@ -308,19 +316,23 @@ The `~/.brig/state/` directory contains untrusted output from cells. brig cell files my-cell ``` -2. Export with sanitization: +2. Export safely — sanitization is automatic on every `cell cp` out of a cell; + there is no flag to enable or disable it: ```bash - brig cell cp --sanitize my-cell:/work/report.html ./report.html + brig cell cp my-cell:/work/report.html ./report.html ``` -3. Never run from state directory directly +3. Never run files directly from the state directory + +### What export sanitization does -### What Gets Blocked by `--sanitize` +Every file copied out of a cell gets a macOS **quarantine xattr** (Gatekeeper +treats it as downloaded from an untrusted source). Files whose extension is in +the unsafe-executable set additionally have their **execute bits stripped**. +Nothing is dropped — you always get the file; brig just makes cell-written files +non-executable and marks them untrusted. -| Type | Examples | Action | -|------|----------|--------| -| Executables | `.app`, `.command`, `.exe` | Blocked | -| Scripts | `.sh`, `.py`, `.js` | Blocked unless `--allow-scripts` | -| Office docs | `.docx`, `.pdf` | Blocked unless `--allow-office` | -| Data files | `.json`, `.csv`, `.txt` | Allowed | -| Images | `.png`, `.jpg` | Allowed | +| Type | Examples | On export | +|------|----------|-----------| +| Unsafe executables | `.app`, `.command`, `.scpt`, `.dmg`, `.pkg`, `.webloc`, `.jar`, `.exe`, `.bat`, `.cmd`, `.msi`, `.vbs`, `.ps1` | quarantined + execute bits removed | +| Everything else | `.py`, `.sh`, `.pdf`, `.docx`, `.json`, `.csv`, `.png` | quarantined (copied as-is) | diff --git a/docs/learning/host-an-agent.md b/docs/learning/host-an-agent.md index ef5e2be..2a9e227 100644 --- a/docs/learning/host-an-agent.md +++ b/docs/learning/host-an-agent.md @@ -2,15 +2,14 @@ This guide walks through running an agent inside a brig cell so it can talk to a local model API server on the macOS host through the Warden proxy. -We'll use the [the cell Agent](https://github.com/NousResearch/my-cell-agent) -(NousResearch) as the example, but the same pattern works for any -container that needs cell-isolated egress plus a route back to a service -on the host. +We'll use the Hermes agent (NousResearch) as the example, but the same +pattern works for any container that needs cell-isolated egress plus a +route back to a service on the host. -> **Production-shaped reference**: `cells//` in this repo is the -> canonical worked example — a real `Containerfile`, a `my-cell.yaml` +> **Production-shaped reference**: `cells/hermes/` in this repo is the +> canonical worked example — a real `Containerfile`, a `hermes.yaml` > cell spec, a brig-aware entrypoint, and a phase-by-phase validation -> plan at `cells//VALIDATION.md`. Start from there and adapt; the +> plan at `cells/hermes/VALIDATION.md`. Start from there and adapt; the > generic walk-through below covers the same pattern with placeholder > names. @@ -119,7 +118,7 @@ the cell's name in the warden log file inside the VM. ```bash brig cell stop my-agent brig cell rm my-agent -brig policy rm my-agent # remove per-cell ACL (drops back to global) +brig policy rm my-agent # remove per-cell policy (cell then blocks all egress) brig secrets rm my-api-key ``` diff --git a/docs/learning/quickstart.md b/docs/learning/quickstart.md index 8a5e482..7bde8d9 100644 --- a/docs/learning/quickstart.md +++ b/docs/learning/quickstart.md @@ -33,12 +33,20 @@ egress filtered through the Warden proxy. The cell name is auto-generated. ## 3. Run a Named Cell ```bash -brig run --name my-cell -d python:3.12 python -c " -import urllib.request +brig run --name my-cell --policy-allow pypi.org -d python:3.12 python -c " +import time, urllib.request print(urllib.request.urlopen('https://pypi.org').status) +while True: time.sleep(60) " ``` +Egress is **default-deny**: without `--policy-allow pypi.org` the request is +blocked (403) and the cell crashes before it can sleep. The trailing +`while True: time.sleep(60)` keeps the cell running so +`brig cell exec / files / logs` work. Without it the cell exits +within a second of the urlopen call and the inspection commands below +fail with "cell not running." + Check it: ```bash diff --git a/docs/learning/troubleshooting.md b/docs/learning/troubleshooting.md index 17aadf9..fc4b40f 100644 --- a/docs/learning/troubleshooting.md +++ b/docs/learning/troubleshooting.md @@ -40,13 +40,13 @@ Common causes, in order of likelihood: 1. **Addons missing.** If `~/.brig/cells/addons/enforce.py` doesn't exist: ```bash - make _copy-addons + brig system up # syncs addons from the installed package and (re)starts warden ``` + (`make _copy-addons` does the same one-shot staging if you only want to copy.) -2. **Policy file is malformed.** Validate it: - ```bash - limactl shell brig -- warden policy validate - ``` +2. **Policy file is malformed.** `warden start` refuses to boot on an + unparseable `network-policy.json` — check the JSON syntax of + `~/.brig/cells/network-policy.json`. 3. **State drift.** The subnet allocator state and podman's actual networks disagree (e.g. you `podman rm`'d a network outside brig): @@ -68,7 +68,9 @@ brig cell network --blocked Shows the most recent blocked requests with the block reason inline. Common reasons: -- `not in allowlist` — domain isn't in `network-policy.json`'s `allow` list. +- `not in allowlist` — domain isn't in the cell's per-cell `allow` list (or the + cell has no policy at all, which is default-deny). Edit with `brig policy set + --allow `. - `denied by rule: ` — explicit deny rule matched. - `host header mismatch` — the cell tried to set a Host header that disagrees with the URL it's connecting to (smuggling defense). @@ -81,7 +83,7 @@ Common reasons: To test a domain without sending real traffic: ```bash -brig policy test --path /api +brig policy test --path /api ``` ## Disk space @@ -166,9 +168,11 @@ Inspect the file directly: ```bash brig cell exec -- cat /var/log/myapp.log -brig cell read /var/log/myapp.log ``` +(`brig cell read` only reaches files under the cell's workspace mount, not +arbitrary container paths like `/var/log` — use `exec ... cat` for those.) + For long-running interactive cells, write app logs to stdout (most runtimes have a flag for this) so `brig cell logs -f ` works. diff --git a/docs/learning/writing-a-cell.md b/docs/learning/writing-a-cell.md index 24767e4..4d5f5d6 100644 --- a/docs/learning/writing-a-cell.md +++ b/docs/learning/writing-a-cell.md @@ -92,7 +92,7 @@ time. ## Accepting inbound HTTP from the host Declare an `ingress` entry to make a cell-internal port reachable -from outside the VM through warden's authenticated reverse proxy on +from outside the VM through warden's reverse proxy on `https://warden:8443///...`. ```yaml @@ -108,6 +108,10 @@ ingress: `auth: token` requires a secret named `-ingress-token` (or `ingress-token` as a fallback). Add it with `brig secrets add agent-ingress-token`. +Use `auth: none` instead for a service that authenticates itself (or a browser +WebSocket client that can't send an `Authorization` header) — brig then proxies +transparently and the app is the gate. `auth: none` isn't allowed on the +`untrusted` profile. ## Egress policy diff --git a/docs/reference/addons.md b/docs/reference/addons.md index 6b7d37c..595c39a 100644 --- a/docs/reference/addons.md +++ b/docs/reference/addons.md @@ -4,8 +4,14 @@ Warden's behavior is composed from mitmproxy addons mounted into the warden container. This page lists each addon, what it does, whether it's required, and how to configure it. -Addons live in `src/addons/` (host) and are copied to `~/.brig/cells/addons/` -by `make _copy-addons`. They run inside the warden container at `/addons/`. +Addons live in `src/brig/warden_addons/`, shipped with brig as package-data (so +they're present in any install, editable or wheel). `brig system up` syncs them +into `~/.brig/cells/addons/` — and bounces warden if any changed — so the +deployed copy can't silently drift from the installed package; `make _copy-addons` +does the same one-shot staging at first setup. They run inside the warden +container at `/addons/`, flat-loaded by mitmproxy (each addon imports its +siblings as flat modules, e.g. `from _common import ...`), which is why they are +data rather than an importable `brig` submodule. ## Required addons @@ -62,20 +68,22 @@ A `tcp_start` hook gates per-cell access for raw TCP `host_services` - Skips TLS-passthrough flows (different mechanism). - Fail-closed on any unexpected mitmproxy API shape. -OTel passthrough metrics emitted via `otel_export.py` `tcp_start` / -`tcp_message` / `tcp_end` hooks: - -| Metric | Cardinality | -|---|---| -| `warden_passthrough_connections_total{cell,host}` | one counter per (cell, SNI) | -| `warden_passthrough_bytes_total{cell,host,direction}` | direction ∈ {in, out} | -| `warden_passthrough_duration_ms{cell,host}` | histogram | +**NOTE — passthrough emits no per-byte/connection metrics.** TLS passthrough +engages via `data.ignore_connection`, which makes mitmproxy build an *ignored* +TCP layer with no flow object — so `otel_export.py`'s `tcp_start` / +`tcp_message` / `tcp_end` hooks never fire for passthrough, and the +`warden_passthrough_*` counters they define are not produced. The only +passthrough audit is the connection-level `PASSTHROUGH: cell=… sni=…` log line +emitted by `enforce.py:tls_clienthello`. (The hooks remain for a possible +future flow-bearing relay; see `docs/INVARIANTS.md` invariant 11.) These surface in `brig system stats` as `PT/CONN` / `PT/IN` / `PT/OUT` columns (the `PT/*` callout appears when any cell had passthrough connections). -Configured via `~/.brig/cells/network-policy.json`: +Allow/deny and `host_services` are **per-cell** — each cell's policy lives in +`~/.brig/state/system/policies/.json` (managed via `brig policy set +`). A cell with no policy file is blocked (default deny): ```json { @@ -85,8 +93,8 @@ Configured via `~/.brig/cells/network-policy.json`: } ``` -Per-cell overrides go in `~/.brig/state/system/policies/.json` (managed -via `brig policy set `). +The process-wide `~/.brig/cells/network-policy.json` carries no allow/deny — +the enforce addon reads only its `policy_trace` block from there. ### `logger.py` @@ -125,16 +133,18 @@ Loaded only if the file exists in `~/.brig/cells/addons/`. ### `ingress.py` -Authenticated reverse proxy on port 8443. Routes external requests to a -cell-internal port: +Reverse proxy on port 8443. Routes external requests to a cell-internal port: ``` GET https://warden:8443/{cell}/{prefix}/... → http://{cell_ip}:{cell_port}/... ``` -Auth: salted SHA-256 token compared with `hmac.compare_digest`. Per-IP auth -failure rate limit (10 failures per minute, LRU-evicted at 10k tracked IPs). +Auth is per-route: `auth: token` (default) compares a salted SHA-256 Bearer +token with `hmac.compare_digest` (per-IP failure rate limit: 10/min, LRU-evicted +at 10k tracked IPs); `auth: none` is transparent pass-through — no token check, +`Authorization` forwarded untouched for the cell's app to authenticate. A route +missing `auth` is treated as `token` (fail-secure). Routes are written by `brig run` based on the cell's `ingress` field; tokens are read from `~/.brig/secrets/-ingress-token` (or generic @@ -151,6 +161,37 @@ a CA bundle. The urllib fallback path explicitly disables HTTP redirects. Circuit breaker (configurable failure threshold / recovery timeout) + exponential backoff retries + dead-letter queue for failed notifications. +**`novel_allowed` — first-seen detection on allow-listed hosts.** Blocked-alerting +catches "tried somewhere new"; this catches "used an allowed destination a new +way" — where telemetry/exfil hide, since they ride a host you *had* to allow and +are never blocked. On each allowed response it keys `(cell, host, +path-template)` (high-cardinality segments — ids/hashes/uuids/tokens — collapse +to `{id}`; query stripped) and alerts the first time a key is seen. A separate +`suspicious_query` signal flags an over-long query string on an already-known +path (the exfil channel path-novelty can't see); it reports the query *length*, +never its content. The baseline is seeded from the cell's existing +`/var/log/brig/network/.jsonl` on load, so a warden restart doesn't replay +known paths as novel. Default off; opt in per cell and **run `dry_run` first** to +confirm the baseline + ignore-lists are tight before enabling delivery: + +```json +"notifications": { + "webhook_url": "https://example.com/webhook", + "novel_allowed": { + "enabled": true, + "cells": ["sandbox-agent", "hermes"], + "ignore_hosts": ["pypi.org"], + "ignore_paths": ["^/v1/acp/"], + "dry_run": false, + "max_query_len": 512 + } +} +``` + +Limits: `tls_passthrough` hosts expose SNI only (no path), so novelty there is +host-level (== the allowlist); and data smuggled in a POST *body* on a known path +is out of scope (path-template + query-length don't see it). + ## How addons load The warden container is started with one `-s /addons/.py` per addon by diff --git a/docs/reference/brig-cli.md b/docs/reference/brig-cli.md index 8f22b7d..46f2c32 100644 --- a/docs/reference/brig-cli.md +++ b/docs/reference/brig-cli.md @@ -2,7 +2,7 @@ A working reference for the `brig` command. Companion to [`warden-cli.md`](warden-cli.md) for the proxy lifecycle commands. For -the SDK that wraps these, see [`sdk-spec.md`](../sdk-spec.md). +the SDK that wraps these, see [`sdk.md`](sdk.md). `brig --version` prints the installed version. `brig --help` lists every subcommand. `--debug` adds verbose logs; `--quiet` suppresses info-level @@ -13,8 +13,8 @@ output; `--no-color` disables ANSI colors. | Command | What it does | |---|---| | `brig run [cmd...]` | Create + start a new cell. See [brig run flags](#brig-run-flags) for the full flag list. Flags must precede the image — `brig run` rejects flag-after-image with an explanatory error. | -| `brig cell list [--format=table\|wide\|json]` | List cells. `wide` adds CREATED, NETWORK columns. | -| `brig cell inspect ` | Raw podman inspect JSON. | +| `brig cell list [--format=table\|wide\|json]` | List cells. `wide` adds CREATED, NETWORK columns. Aliases: `brig cell ls`, top-level `brig ps`. | +| `brig cell inspect ` | Raw podman inspect JSON. Alias: `brig cell status `. | | `brig cell diagnose ` | Per-cell state summary (status, runtime, networks). | | `brig cell stop ` | SIGTERM with 10s grace. | | `brig cell kill ` | SIGKILL. | @@ -29,8 +29,8 @@ output; `--no-color` disables ANSI colors. | `brig cell shell ` | Open `/bin/sh` inside the cell. | | `brig cell exec [-i] -- cmd...` | Run a one-off command inside the cell. | | `brig cell read ` | Stream a workspace file to stdout. Race-free; refuses symlinks. Safer than `brig cell cp` for plain reads. | -| `brig cell trace ` | Render a request trace from the OTel collector by trace_id (or prefix; first match wins). | -| `brig cell export ` | Print the cell's YAML definition (round-trips via `brig run --file -`). | +| `brig cell export ` | Print the cell's YAML definition. Round-trip with `brig cell export > cell.yaml && brig run --file cell.yaml` (no stdin: `--file` takes a path). | +| `brig cell mount-scan [--quarantine]` | Scan a cell's host `mounts:` for symlinks whose realpath escapes the mounted dir (a cell can plant one pointing out of a shared rw mount). Reports them (exit 1 if any); `--quarantine` removes them. Run before a host process consumes cell-written files. | ### brig run flags @@ -73,6 +73,7 @@ output; `--no-color` disables ANSI colors. |---|---| | `brig cell network [--tail N] [--blocked] [--otel]` | Warden's per-cell request log. `--blocked` filters to only the requests warden denied, with the block reason on the same line. `--otel` reads from the OTel collector's flow store instead of the per-cell JSONL file (needed when the collector has retention older than the JSONL rotation). | | `brig cell events [cell] [--tail N] [-f]` | Lifecycle events stream. `-f` follows; default is one-shot. | +| `brig cell ingress ` | Show the reachable ingress URL(s) for a cell, the token-secret name to use in the `Authorization: Bearer` header, and whether the cell currently has a registered route. | | `brig system history [--tail N] [--cell ]` | Operations history (every `brig` invocation with exit code + duration). | | `brig policy show ` | Print the cell's policy. Cells with no policy file show the empty default (cell blocks all egress). | | `brig policy set --allow DOMAIN [...] --deny DOMAIN [...]` | Extend a cell's allow/deny lists. Use `--remove-allow` / `--remove-deny` to drop. To declare host_services, edit the cell yaml. | @@ -84,7 +85,7 @@ output; `--no-color` disables ANSI colors. | Command | What it does | |---|---| | `brig secrets list` | List secrets + their mount paths and env var names. | -| `brig secrets add [--value V \| --from-file F]` | Add a secret. Falls back to interactive `getpass` prompt if stdin is a TTY, else pipes from stdin. | +| `brig secrets add [--value V \| --from-file F] [--force]` | Add a secret. Falls back to interactive `getpass` prompt if stdin is a TTY, else pipes from stdin. `--force` overwrites an existing secret (otherwise adding a name that exists is refused). | | `brig secrets rm [-y\|--yes]` | Delete a secret. **Requires `--yes` for non-interactive use** (refuses to delete in scripts/CI without explicit confirmation); interactive shells get a `[y/N]` prompt. | ## Images @@ -93,18 +94,18 @@ output; `--no-color` disables ANSI colors. |---|---| | `brig image build [--tag TAG] [--file PATH] [--build-arg K=V] [--use-warden]` | Build a container image from a directory inside the VM. Auto-derives tag from the dir basename if `--tag` omitted; auto-detects `Containerfile`/`Dockerfile` if `--file` omitted. `--use-warden` routes the build's HTTP(S) traffic through warden — injects `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` build args and mounts the warden CA at `/etc/ssl/certs/warden-ca.crt`. Same policy applies as runtime. | | `brig image pull ` | Pull + cache an image. | -| `brig image warmup [--profile NAME]` | Pre-pull images for a profile. | -| `brig image verify [--key KEY \| --keyless]` | Verify a cosign signature. Cosign is a hard requirement (no `podman trust` fallback). | +| `brig image warmup` | Pre-pull the pinned warden/mitmproxy base image into the VM cache. | +| `brig image verify [--key KEY \| --keyless]` | Verify a cosign signature. Cosign is a hard requirement (no `podman trust` fallback). Advisory only — `--keyless` without an identity/issuer constraint confirms the image is signed, not who signed it; run-time integrity comes from digest pinning. | ## System / lifecycle | Command | What it does | |---|---| -| `brig system up` | Initialize `~/.brig` if needed, create Lima VM if missing, start it if stopped, start warden. The "make it work" command. | +| `brig system up` | Initialize `~/.brig` if needed, create Lima VM if missing, start it if stopped, start warden, then re-launch any gone `restart: always` cells. The "make it work" command. | | `brig system down [--vm]` | Stop all cells + warden. `--vm` also stops the Lima VM. | | `brig system init` | Bootstrap `~/.brig`, set 0700 perms on `secrets`/`addons`/`state/system`, write default policy. Idempotent. | | `brig system profiles` | List trust profiles (`untrusted`, `supervised`, `dev`, `airgapped`, `honeypot`). | -| `brig system doctor --quick [--format=table\|json]` | Lightweight health check: proxy running + VM reachable. | +| `brig system doctor --quick` | Lightweight health check: proxy running + VM reachable. | | `brig system doctor` | Deep environment check: tooling on PATH, Lima VM state, addon presence, directory permissions, network policy parses, warden running. Prints a checklist with fix suggestions on each failure. Use this before filing a bug. | | `brig system verify` | Run security invariant checks (`verify_all` in `brig.security.verify`). | | `brig system preflight` | Reconcile subnet state with podman networks; report drift. | @@ -119,6 +120,7 @@ output; `--no-color` disables ANSI colors. |---|---| | `brig config show [key]` | Print config. Dot paths supported (`brig config show operation_logging.level`). | | `brig config set ` | Set a config value. Dot paths create intermediate dicts. Values are parsed as JSON if possible, else stored as a string. | +| `brig config set mount_roots ` | Declare the host trees that cells may bind-mount via `mounts:` (VM-level allowlist). Rejects `/`, `$HOME`, secret dirs, `~/.brig`, `/etc`, non-dirs, and slug collisions — comparing by real path + on-disk identity, so symlinks, realpath aliases (`/etc` → `/private/etc`), and case variants (`~/.SSH`) can't slip past. Accepts a JSON list or a comma-separated string. A lossless VM restart applies it (`brig system down --vm && brig system up`; images preserved). See `docs/design/host-mounts.md`. | | `brig config reset` | Restore defaults. | ## Common workflows @@ -132,7 +134,7 @@ brig run alpine echo hello **Debugging blocked requests:** ```bash brig cell network --blocked -brig policy test # try a domain without sending real traffic +brig policy test # try a domain without sending real traffic ``` **Recovering from a confused state:** diff --git a/docs/reference/cell-metadata.md b/docs/reference/cell-metadata.md index 9e14223..5ba7382 100644 --- a/docs/reference/cell-metadata.md +++ b/docs/reference/cell-metadata.md @@ -10,11 +10,11 @@ The pattern mirrors Kubernetes' downward API and cloud instance metadata: brig writes a small JSON file on the host, podman bind-mounts it read-only into the cell. The cell can read but cannot modify it. -## Schema (v2) +## Schema (v3) ```json { - "version": 2, + "version": 3, "name": "my-cell", "started_at": "2026-05-18T17:30:00Z", "workspace": { @@ -24,6 +24,7 @@ it read-only into the cell. The cell can read but cannot modify it. "ingress": [ {"name": "api", "port": 8000, "path_prefix": "/api", "auth": "token"} ], + "image_digest": "sha256:abc...", "policy": { "host_services": ["model"] } @@ -32,13 +33,30 @@ it read-only into the cell. The cell can read but cannot modify it. | Field | Type | Notes | |---|---|---| -| `version` | int | Schema version. Currently `2`. Bumps on breaking shape changes. | +| `version` | int | Schema version. Currently `3`. Bumps on shape changes. | | `name` | string | Cell name, matches `--name` / yaml `name:`. | | `started_at` | string | RFC 3339 UTC timestamp of cell creation. | | `workspace.mount_point` | string | Path inside the cell, default `/work`, overridable via `workspace_mount` in the cell spec. | | `policy.host_services` | string[] | Per-cell host-service ACL — the names of host services this cell may reach. Ports live in the per-cell policy file on disk; metadata exposes names only. | | `host_sockets[]` | `[{name, mount_point}]` | Unix sockets bind-mounted into the cell from the host (host_path is intentionally omitted). | -| `ingress[]` | `[{name, port, path_prefix, auth}]` | Ingress endpoints the cell publishes through warden's `:8443` reverse proxy. The bearer token itself is never stored here — it lives in `~/.brig/secrets/-ingress-token`. `brig cell start` uses this list to replay route registration with a freshly-inspected cell IP after a `brig system down` / `up` cycle. | +| `ingress[]` | `[{name, port, path_prefix, auth}]` | Ingress endpoints the cell publishes through warden's `:8443` reverse proxy. `auth` is `token` (default; the bearer token is never stored here — it lives in `~/.brig/secrets/-ingress-token`) or `none` (transparent pass-through, no token). `brig cell start` uses this list to replay route registration with a freshly-inspected cell IP after a `brig system down` / `up` cycle. | +| `image_digest` | string? | Optional. Set when the cell was created with a pinned digest. `brig cell start` re-verifies the container's current image digest against this value before letting the cell start. | + +### What changed in v3 + +v2 → v3 added two optional fields: + +- `ingress` — lets `brig cell start` replay route registration without + the original yaml after a `brig system down`/`up` cycle. No secrets + land here; the bearer token still lives in the secrets directory. +- `image_digest` — lets `brig cell start` re-verify the digest pin + before letting the container start, closing the + `podman commit + restart` operator-side bypass. + +Both fields are additive; a v2 reader sees them as unknown keys and +ignores them. Bumping to v3 reflects that the writer's output shape +changed, so a future reader checking `version >= 3` can rely on +`image_digest` / `ingress` semantics. ### What changed in v2 diff --git a/docs/reference/observability.md b/docs/reference/observability.md index b8f192f..627f63b 100644 --- a/docs/reference/observability.md +++ b/docs/reference/observability.md @@ -2,9 +2,8 @@ Brig ships with an OpenTelemetry collector that runs as a sibling container to warden inside the Lima VM. Warden's `otel_export` addon -pushes metrics, traces, and logs to it; the brig CLI reads them back -out for `brig system stats`, `brig cell trace`, and -`brig cell network --otel`. +pushes metrics and logs to it; the brig CLI reads them back out for +`brig system stats` and `brig cell network --otel`. ## The collector @@ -36,18 +35,6 @@ brig system metrics # raw Prometheus-format counters requests, and TLS-mode (mitm vs passthrough) totals per cell. It exits non-zero with a clear message if the collector isn't running. -## Reading traces - -Warden's mitmproxy addon attaches OTel spans to each request flow. -The CLI looks up a single trace by ID: - -```bash -brig cell trace # or a unique prefix; first match wins -``` - -The trace ID typically shows up in the warden log lines (`trace_id=...`) -that `brig cell network` and `brig cell logs` already print. - ## Reading flows `brig cell network ` defaults to the per-cell JSONL file @@ -65,19 +52,18 @@ The two sources agree on recent flows; the OTel side retains longer ## Where the raw data lives -The collector writes traces and logs to rotated JSONL files inside -the VM at `/var/lib/otel/`: +The collector writes logs to a rotated JSONL file inside the VM at +`/var/lib/otel/`: | Signal | Path | Rotation | |---|---|---| -| Traces | `/var/lib/otel/traces.jsonl` | 100 MB × 3 backups, 7-day max-age | | Logs | `/var/lib/otel/logs.jsonl` | 100 MB × 5 backups, 7-day max-age | | Metrics | (in-memory; exposed at `:9464/metrics`) | n/a | -To peek at the raw files from the host: +To peek at the raw file from the host: ```bash -limactl shell brig -- sudo cat /var/lib/otel/traces.jsonl | tail -50 +limactl shell brig -- sudo cat /var/lib/otel/logs.jsonl | tail -50 ``` The configuration template is at diff --git a/docs/reference/sdk.md b/docs/reference/sdk.md new file mode 100644 index 0000000..8fbca0a --- /dev/null +++ b/docs/reference/sdk.md @@ -0,0 +1,277 @@ +# Brig SDK Specification + +Version 0.4.0 | Python 3.10+ + +The SDK calls Brig's domain modules directly (no CLI shell-out). Every +method has an `async` and `_sync` variant; the async path just offloads +to `asyncio.to_thread`. + +## Import + +```python +from brig.sdk import Brig, Cell, BrigError +# or +from brig import Brig, Cell, BrigError +``` + +## Exception Hierarchy + +``` +BrigError(Exception) +├── CellNotFoundError # Cell does not exist +├── ImageVerificationError # Image digest mismatch or verification failure +├── ProfileError # Unknown or invalid trust profile +└── SecretNotFoundError # Referenced secret file not found +``` + +All exceptions carry: +- `message: str` — Human-readable error description. +- `returncode: int` — Exit code (default `1`). +- `stderr: str` — Error details. +- `suggestion: str | None` — Suggested fix. + +## Dataclasses + +### `CellInfo` + +Returned by `Brig.list_cells()` / `Brig.list_sync()`. + +| Field | Type | Description | +|----------|-------|---------------------| +| `name` | `str` | Cell name | +| `status` | `str` | Cell status | +| `image` | `str` | Container image | + +### `CellRunResult` + +Returned by `Brig.execute()` / `Brig.execute_sync()`. + +| Field | Type | Description | +|-------------|--------|--------------------------| +| `name` | `str` | Cell name | +| `exit_code` | `int` | Process exit code (`-1` if wait itself failed) | +| `stdout` | `str` | Combined stdout/stderr (podman merges them) | +| `stderr` | `str` | Always `""` — see note on `stdout` | +| `success` | `bool` | True if `exit_code == 0` | + +### `WardenStatus` + +Returned by `WardenHandle.status()`. + +| Field | Type | Description | +|-----------|-------------|--------------------------| +| `running` | `bool` | Whether Warden is active | +| `networks`| `list[str]` | Connected networks | + +## `Brig` Class + +### Constructor + +```python +Brig() +``` + +No parameters. The SDK calls domain modules directly — it does not shell +out to the CLI binary. + +### `Brig.run()` / `Brig.run_sync()` + +```python +def run_sync( + name: str, + image: str, + command: list[str] | None = None, + env: list[str] | None = None, + secrets: list[str] | None = None, + memory: str = "2g", + cpus: str = "2", + pids_limit: int = 512, + network: str = "default", + profile: str | None = None, + detach: bool = True, + timeout: str | None = None, + labels: list[str] | None = None, + host_sockets: list[dict] | None = None, + host_services: list[dict] | None = None, + mounts: list[dict] | None = None, # {name, host_path, mount_point, mode?} — bounded by mount_roots + ingress: list[dict] | None = None, + policy_allow: list[str] | None = None, + policy_deny: list[str] | None = None, + policy_passthrough_tls: list[str] | None = None, + image_digest: str | None = None, + trust_warden_ca: bool = True, + workdir: str | None = None, + workspace_quota: str | None = None, + workspace_mount: str = "/work", + writable_rootfs: bool = False, + seccomp_profile: str | None = None, + restart: str = "no", # "always" re-launches on `brig system up` + user: str | None = None, # podman --user; "0" to own a rw mounts: dir +) -> Cell +``` + +Creates and starts a new cell. Returns a `Cell` handle. + +Field semantics match the cell yaml — see +[`docs/design/cell-definition.md`](../design/cell-definition.md) for +the full schema (validation rules, expected shapes for `host_sockets`, +`host_services`, `ingress`, `policy_*`). `validate_cell_definition` +runs against the SDK args before the cell starts, so the same +untrusted-profile guards and SSRF wildcard checks that apply to +`brig run --file` apply here. + +Enforces: +- Invariant 9: Proxy must be running (unless `network="none"`). +- Cell name validation against `CELL_NAME_PATTERN`. +- Profile precedence: profile defaults < explicit args. + +### `Brig.execute()` / `Brig.execute_sync()` — Agent API + +Single-call method: runs code, waits for completion, collects output, +removes the cell. Designed for agents that don't need a long-lived +handle. + +```python +def execute_sync( + image: str, + command: list[str], + name: str | None = None, # auto-generated if omitted + timeout: str = "5m", + network: str = "default", # or "none" for airgap + env: list[str] | None = None, + secrets: list[str] | None = None, + profile: str | None = None, +) -> CellRunResult +``` + +Returns `CellRunResult(name, exit_code, stdout, stderr, success)`. The +cell is always removed in `finally:` so a wait timeout doesn't leak it. +`stdout` carries the merged stream; `stderr` is always `""` because +`podman logs` does not separate streams. + +`execute_sync` is intentionally a narrower surface than `run_sync` — +the long list of advanced fields (host_sockets, ingress, policy_*) is +not exposed because the typical agent use case doesn't need them. Drop +to `run_sync` when you do. + +### `Brig.list_cells()` / `Brig.list_sync()` + +```python +def list_sync() -> list[CellInfo] +``` + +Returns information about all existing cells (running and stopped). +The method is `list_cells()`, not `list()` — there is no `list()` +method. + +### `Brig.cell()` + +```python +def cell(name: str) -> Cell +``` + +Get a handle to an existing cell. Raises `CellNotFoundError` if it +doesn't exist. + +### `Brig.warden` + +A `WardenHandle` for proxy management: + +```python +b = Brig() +b.warden.start() # Start the proxy +b.warden.stop() # Stop the proxy +b.warden.status() # -> WardenStatus +``` + +## `Cell` Class + +Handle to a running or completed cell. Every method has both async and +sync variants (e.g., `wait()` / `wait_sync()`). + +| Method | Returns | Description | +|---------------------|---------|------------------------------------------| +| `wait(timeout=None)`| `int` | Block until cell exits; returns exit code, or `-1` if the wait itself failed | +| `stop()` | `None` | Gracefully stop | +| `kill()` | `None` | Immediately kill | +| `rm(force=False)` | `None` | Remove cell and resources | +| `logs(tail=None)` | `str` | Combined container logs (podman merges streams) | +| `is_alive()` | `bool` | Whether the cell's container is running | +| `copy_in(src, dst)` | `None` | Copy a host file into the cell workspace | +| `copy_out(src, dst)`| `None` | Copy a cell workspace file to the host (sanitized) | + +`wait_sync` returns `-1` when the wait *itself* failed (subprocess +error, timeout, unparseable podman status), distinct from a real +non-zero cell exit code. The agent path (`execute_sync`) surfaces +this `-1` in `CellRunResult.exit_code`. + +## Examples + +### Agent use (single call) + +```python +from brig import Brig + +b = Brig() +result = b.execute_sync( + "python:3.12", + ["python", "-c", "print('hello from sandbox')"], + timeout="30s", + network="none", # airgap +) +print(result.exit_code) # 0 +print(result.stdout) # "hello from sandbox\n" +# Cell is automatically cleaned up. +``` + +### Long-running cell with file I/O + +```python +from brig import Brig + +b = Brig() +cell = b.run_sync( + name="scraper", + image="python:3.12", + command=["python", "scrape.py"], + profile="supervised", + secrets=["api-key"], +) + +cell.copy_in("./urls.txt", "/work/urls.txt") +exit_code = cell.wait_sync(timeout=300) +cell.copy_out("results.json", "./results.json") +cell.rm_sync() +``` + +### Cell with HTTP host service + ingress + +```python +from brig import Brig + +b = Brig() +cell = b.run_sync( + name="webapp", + image="localhost/webapp:latest", + image_digest="sha256:abc123...", # pin to a specific build + host_services=[ + {"name": "db", "port": 5432, "protocol": "tcp"}, + ], + ingress=[ + {"name": "api", "port": 8000, + "path_prefix": "/api", "auth": "token"}, + ], + policy_allow=["api.github.com", "*.example.com"], +) +``` + +## Architecture + +The SDK calls domain modules directly (`brig.cell.lifecycle`, +`brig.cell.reconciler`, `brig.network.subnet`, etc.) rather than +shelling out to the CLI. All podman commands are routed through +`brig.vm.shell.vm_run()` which wraps them in `limactl shell brig --`. + +The same security enforcement applies: gVisor runtime is hardcoded, +proxy env vars cannot be overridden, cell names are validated, and the +proxy must be running before cells start. diff --git a/docs/reference/warden-cli.md b/docs/reference/warden-cli.md index 546cd3a..2839ef9 100644 --- a/docs/reference/warden-cli.md +++ b/docs/reference/warden-cli.md @@ -13,7 +13,7 @@ directly via `limactl shell brig -- warden `. | `warden stop` | Sends SIGTERM (10s grace), then removes the container. Idempotent. | | `warden restart` | `stop` then `start`. | | `warden status` | Prints `running` / `not running` and the list of cell networks the proxy is attached to. | -| `warden reload` | Sends SIGHUP to mitmproxy. The `enforce` and `logger` addons hot-reload `network-policy.json`, `subnet-map.json`, and the per-cell policy directory on receipt. Quicker than a restart. | +| `warden reload` | Sends SIGHUP to mitmproxy. `enforce` hot-reloads `network-policy.json`, the per-cell policy directory, and `subnet-map.json`; `logger` reloads its log filter (from `network-policy.json`) and `subnet-map.json`. Quicker than a restart. | | `warden preflight` | Reconciles the subnet allocator state file with podman's actual networks. Reports missing networks, orphaned subnets, and inconsistencies — without making any changes. Run this when warden won't start. | ## Health & logs @@ -26,12 +26,9 @@ directly via `limactl shell brig -- warden `. ## Policy -| Command | What it does | -|---|---| -| `warden policy validate [path]` | Loads a policy JSON/YAML file and reports parse errors and rule problems (invalid domains, suspicious patterns). Useful for linting per-cell policies before they're written. | -| `warden policy test [--path /...] [--method GET]` | Runs the proxy's allow/deny matcher against the policy at ``. From the host the equivalent for a deployed cell is `brig policy test `. | - -For the host-side `brig` command, see [`brig-cli.md`](brig-cli.md). +Network policy is enforced **per cell** — there is no global allow/deny list. +Author and inspect a cell's policy from the host with `brig policy …` (see +[`brig-cli.md`](brig-cli.md)); the warden CLI does not have policy subcommands. ## Common workflows @@ -61,9 +58,6 @@ limactl shell brig -- warden restart # Preflight reports state inconsistencies without changing anything. limactl shell brig -- warden preflight -# Check the policy file parses. -limactl shell brig -- warden policy validate - # Last resort: tail container logs while attempting to start. limactl shell brig -- warden logs & limactl shell brig -- warden start diff --git a/docs/sdk-spec.md b/docs/sdk-spec.md deleted file mode 100644 index 28a4efc..0000000 --- a/docs/sdk-spec.md +++ /dev/null @@ -1,219 +0,0 @@ -# Brig SDK Specification - -Version 0.3.0 | Python 3.10+ - -## Import - -```python -from brig.sdk import Brig, Cell, BrigError -# or -from brig import Brig, Cell, BrigError -``` - -## Exception Hierarchy - -``` -BrigError(Exception) -├── CellNotFoundError # Cell does not exist -├── ImageVerificationError # Image digest mismatch or verification failure -├── ProfileError # Unknown or invalid trust profile -└── SecretNotFoundError # Referenced secret file not found -``` - -All exceptions carry: -- `message: str` — Human-readable error description. -- `returncode: int` — Exit code (default `1`). -- `stderr: str` — Error details. -- `suggestion: str | None` — Suggested fix. - -## Dataclasses - -### `CellInfo` - -Returned by `Brig.list_cells()` / `Brig.list_sync()`. - -| Field | Type | Description | -|----------|-------|---------------------| -| `name` | `str` | Cell name | -| `status` | `str` | Cell status | -| `image` | `str` | Container image | - -### `CellRunResult` - -Returned by `Brig.execute()` / `Brig.execute_sync()`. - -| Field | Type | Description | -|-------------|--------|--------------------------| -| `name` | `str` | Cell name | -| `exit_code` | `int` | Process exit code | -| `stdout` | `str` | Standard output | -| `stderr` | `str` | Standard error | -| `success` | `bool` | True if exit_code == 0 | - -### `WardenStatus` - -Returned by `WardenHandle.status()`. - -| Field | Type | Description | -|-----------|-------------|--------------------------| -| `running` | `bool` | Whether Warden is active | -| `networks`| `list[str]` | Connected networks | - -## `Brig` Class - -### Constructor - -```python -Brig() -``` - -No parameters. The SDK calls domain modules directly — it does not shell out to the CLI binary. - -### `Brig.run()` / `Brig.run_sync()` - -```python -async def run( - name: str, - image: str, - command: list[str] | None = None, - env: list[str] | None = None, - secrets: list[str] | None = None, - memory: str = "2g", - cpus: str = "2", - pids_limit: int = 512, - network: str = "default", - profile: str | None = None, - detach: bool = True, - timeout: str | None = None, - labels: list[str] | None = None, -) -> Cell -``` - -Creates and starts a new cell. Returns a `Cell` handle. - -Enforces: -- Invariant 9: Proxy must be running (unless airgapped). -- Rate limiting. -- Cell name validation against `CELL_NAME_PATTERN`. - -### `Brig.list_cells()` / `Brig.list_sync()` - -```python -async def list_cells() -> list[CellInfo] -def list_sync() -> list[CellInfo] -``` - -Returns information about all existing cells (running and stopped). Note: -the method is `list_cells()`, not `list()` — there is no `list()` method. - -### `Brig.cell()` - -```python -def cell(name: str) -> Cell -``` - -Get a handle to an existing cell. Raises `CellNotFoundError` if it doesn't exist. - -### `Brig.execute()` / `Brig.execute_sync()` — Agent API - -Single-call method: runs code, waits for completion, collects output, cleans up. - -```python -async def execute( - image: str, - command: list[str], - name: str | None = None, # auto-generated if omitted - timeout: str = "5m", - network: str = "default", # or "none" for air-gap - env: list[str] | None = None, - secrets: list[str] | None = None, - profile: str | None = None, -) -> CellRunResult -``` - -Returns `CellRunResult(name, exit_code, stdout, stderr, success)`. -Cell is automatically removed after execution. - -### `Brig.warden` - -A `WardenHandle` for proxy management: - -```python -b = Brig() -b.warden.start() # Start the proxy -b.warden.stop() # Stop the proxy -b.warden.status() # -> WardenStatus -``` - -## `Cell` Class - -### Methods - -All methods have both async and sync variants (e.g., `wait()` / `wait_sync()`). - -| Method | Returns | Description | -|---------------|-------------|--------------------------------| -| `wait()` | `int` | Block until cell exits, return exit code | -| `stop()` | `None` | Gracefully stop | -| `kill()` | `None` | Immediately kill | -| `rm(force=False)` | `None` | Remove cell and resources | -| `logs(tail=None)` | `str` | Get container logs | -| `is_alive()` | `bool` | Check if still running | -| `copy_in(src, dst)` | `None` | Copy file into cell workspace | -| `copy_out(src, dst)` | `None` | Copy file from cell (sanitized) | - -## Examples - -### Agent use (single call) - -```python -from brig import Brig - -b = Brig() -result = b.execute_sync( - "python:3.12", - ["python", "-c", "print('hello from sandbox')"], - timeout="30s", - network="none", # air-gapped -) -print(result.exit_code) # 0 -print(result.stdout) # "hello from sandbox\n" -# Cell is automatically cleaned up. -``` - -### Long-running cell with file I/O - -```python -from brig import Brig - -b = Brig() -cell = b.run_sync( - name="scraper", - image="python:3.12", - command=["python", "scrape.py"], - profile="supervised", - secrets=["api-key"], -) - -# Copy input file into cell. -cell.copy_in("./urls.txt", "scraper:/work/urls.txt") - -# Wait for completion. -exit_code = cell.wait_sync(timeout=300) - -# Copy results out (sanitized — unsafe extensions blocked, quarantine applied). -cell.copy_out("scraper:/work/results.json", "./results.json") - -# Clean up. -cell.rm_sync() -``` - -## Architecture - -The SDK calls domain modules directly (`brig.cell.lifecycle`, `brig.cell.reconciler`, -`brig.network.subnet`, etc.) rather than shelling out to the CLI. All podman commands -are routed through `brig.vm.shell.vm_run()` which wraps them in `limactl shell brig --`. - -The same security enforcement applies: gVisor runtime is hardcoded, proxy env vars -cannot be overridden, cell names are validated, and the proxy must be running before -cells start. diff --git a/pyproject.toml b/pyproject.toml index e5e2495..6f2fb30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "brig" -version = "0.3.1" +version = "0.4.0" description = "Secure workload harness for running untrusted code on macOS" readme = "README.md" license = {text = "MIT"} @@ -43,7 +43,14 @@ dev = [ # (enforce, ingress, logger, notifier, ops) are the most security- # critical code in the tree; mocking mitmproxy's API surface means # an upstream signature change only surfaces in E2E. - "mitmproxy>=10.0", + # + # PINNED to the 10.x major to match the warden image (mitmproxy 10.1.1, + # pinned by digest in src/warden/image/Dockerfile). Unbounded `>=10.0` + # floated to 12.x, where the addon API differs materially (no ctx.log, no + # tls_passthrough, ignore_connection semantics) — so CI was exercising the + # security-critical addons against semantics production never runs. Bump + # this in lockstep with the Dockerfile base. + "mitmproxy>=10.1.1,<11", ] [project.scripts] @@ -60,14 +67,19 @@ package-dir = {"" = "src"} [tool.setuptools.packages.find] where = ["src"] -# addons/ is intentionally NOT installed as a Python package — those files -# are mounted into the warden container via `make _copy-addons` and have -# no __init__.py. They are sibling scripts, not a brig submodule. +# brig/warden_addons/ is NOT an importable package (no __init__) — the addons +# are flat-loaded by mitmproxy inside the warden container and import each other +# as flat modules (`from _common import ...`). They ship as brig package-data +# (below) so they're resolvable in any install (editable AND wheel) and synced +# into the container by `brig system up`. Excluded from `find` for the same +# reason (no __init__). include = ["brig*", "warden*"] [tool.setuptools.package-data] "brig.vm" = ["*.template"] "brig.observability" = ["*.yaml"] +# The warden data plane — shipped as data, deployed flat into the container. +"brig" = ["warden_addons/*.py"] [tool.ruff] line-length = 100 @@ -77,7 +89,7 @@ target-version = "py310" select = ["E", "F", "W"] # F401 is intentionally NOT in this global ignore — orphan imports # should be flagged. Re-exports opt in per-file via `# noqa: F401` -# (see e.g. src/addons/enforce.py's re-exports for tests and external +# (see e.g. src/brig/warden_addons/enforce.py's re-exports for tests and external # callers). ignore = ["E402", "E501", "F405"] @@ -119,12 +131,20 @@ omit = [ ] [tool.mypy] +# Type-check brig/ and warden/. Addons run inside the warden container +# with their own Python environment and depend on mitmproxy's stubs, +# which use newer syntax than our `python_version` floor — they're +# excluded here and covered by their unit tests instead. python_version = "3.10" -warn_return_any = false +# Resolve intra-repo `brig.*` / `warden.*` imports as packages. Without this, +# ignore_missing_imports treats every cross-module call as Any and triggers +# spurious warn_return_any errors at `return other_module_fn()` sites. +mypy_path = "src" +warn_return_any = true warn_unused_configs = true ignore_missing_imports = true -check_untyped_defs = false -exclude = ["tests/"] +check_untyped_defs = true +exclude = ["tests/", "src/brig/warden_addons/"] [tool.coverage.report] exclude_lines = [ diff --git a/scripts/build-warden-image.sh b/scripts/build-warden-image.sh index 12270a9..ea1f091 100755 --- a/scripts/build-warden-image.sh +++ b/scripts/build-warden-image.sh @@ -1,6 +1,6 @@ #!/bin/bash # build-warden-image.sh — build the warden image inside the brig VM and -# pin its sha256 in src/warden/proxy.py:IMAGE_DIGEST. +# pin its sha256 in src/warden/proxy.py:WARDEN_IMAGE_DIGEST. # # Run this when WARDEN_IMAGE_TAG or the OTel SDK version changes. # Operator commits the resulting one-line digest update. diff --git a/scripts/check-coverage-per-module.py b/scripts/check-coverage-per-module.py index 71bdc5e..9741ee0 100755 --- a/scripts/check-coverage-per-module.py +++ b/scripts/check-coverage-per-module.py @@ -34,7 +34,7 @@ # (commented in the right column). # # Path current → gate audit goal -# addons/enforce.py ~58% 55.0% 90% +# brig/warden_addons/enforce.py ~58% 55.0% 90% # brig/security/ ~83% 80.0% 90% # brig/cell/reconciler.py ~81% 78.0% 85% # @@ -42,7 +42,7 @@ # an immediate test-write demand. The security gate stays at 80 because # the slow-marked tests skewed local coverage above what CI sees. THRESHOLDS: list[tuple[str, float]] = [ - ("addons/enforce.py", 55.0), + ("brig/warden_addons/enforce.py", 55.0), ("brig/security/", 80.0), ("brig/cell/reconciler.py", 78.0), ] diff --git a/scripts/local-smoke-test.sh b/scripts/local-smoke-test.sh index f1b0e31..0500232 100755 --- a/scripts/local-smoke-test.sh +++ b/scripts/local-smoke-test.sh @@ -136,7 +136,7 @@ fi if limactl shell --workdir / brig -- sudo test -x /usr/local/bin/runsc; then pass "gVisor (runsc) installed in VM" else - fail "gVisor not found in VM (provision VM with: limactl delete brig && make vm)" + fail "gVisor not found in VM (re-provision with: limactl delete brig && make setup)" fi # --------------------------------------------------------------------------- @@ -149,9 +149,9 @@ WARDEN_STATUS=$(limactl shell --workdir / brig -- sudo podman inspect warden --f if [ "$WARDEN_STATUS" = "running" ]; then pass "Warden proxy is running" else - echo " Warden not running, attempting start via brig up..." - if $BRIG up 2>/dev/null; then - pass "Warden started (via brig up)" + echo " Warden not running, attempting start via brig system up..." + if $BRIG system up 2>/dev/null; then + pass "Warden started (via brig system up)" else fail "Warden failed to start — run: make up" fi @@ -172,17 +172,17 @@ else fail "brig run" fi -# Test 2: brig list. -LIST_OUT=$($BRIG list 2>/dev/null) +# Test 2: brig cell list. +LIST_OUT=$($BRIG cell list 2>/dev/null) if echo "$LIST_OUT" | grep -q "$CELL_NAME"; then - pass "brig list shows cell" + pass "brig cell list shows cell" else - fail "brig list does not show cell (output: $LIST_OUT)" + fail "brig cell list does not show cell (output: $LIST_OUT)" fi -# Test 3: brig inspect. -if $BRIG inspect "$CELL_NAME" >/dev/null 2>&1; then - pass "brig inspect" +# Test 3: brig cell inspect. +if $BRIG cell inspect "$CELL_NAME" >/dev/null 2>&1; then + pass "brig cell inspect" else fail "brig inspect" fi @@ -220,30 +220,30 @@ else fail "proxy env vars not set: $HTTP_PROXY" fi -# Test 7: brig exec. -EXEC_OUT=$($BRIG exec "$CELL_NAME" echo "hello from cell" 2>/dev/null) +# Test 7: brig cell exec. +EXEC_OUT=$($BRIG cell exec "$CELL_NAME" echo "hello from cell" 2>/dev/null) if echo "$EXEC_OUT" | grep -q "hello from cell"; then - pass "brig exec" + pass "brig cell exec" else - fail "brig exec output: $EXEC_OUT" + fail "brig cell exec output: $EXEC_OUT" fi -# Test 8: brig stop. -if $BRIG stop "$CELL_NAME" 2>/dev/null; then - pass "brig stop" +# Test 8: brig cell stop. +if $BRIG cell stop "$CELL_NAME" 2>/dev/null; then + pass "brig cell stop" else - fail "brig stop" + fail "brig cell stop" fi -# Test 9: brig rm. -if $BRIG rm "$CELL_NAME" 2>/dev/null; then - pass "brig rm" +# Test 9: brig cell rm. +if $BRIG cell rm "$CELL_NAME" 2>/dev/null; then + pass "brig cell rm" else - fail "brig rm" + fail "brig cell rm" fi # Test 10: Verify cleanup. -if ! $BRIG list 2>/dev/null | grep -q "$CELL_NAME"; then +if ! $BRIG cell list 2>/dev/null | grep -q "$CELL_NAME"; then pass "cell removed from list" else fail "cell still in list after rm" @@ -251,11 +251,11 @@ fi # --------------------------------------------------------------------------- info "" -info "Phase 6: brig verify (security invariants)" +info "Phase 6: brig system verify (security invariants)" # --------------------------------------------------------------------------- -if $BRIG verify 2>&1; then - pass "brig verify — all invariants" +if $BRIG system verify 2>&1; then + pass "brig system verify — all invariants" else fail "brig verify reported issues" fi @@ -272,7 +272,7 @@ if $BRIG run --name "$AIR_NAME" --network none alpine echo "isolated" 2>&1; then else fail "airgapped cell failed" fi -$BRIG rm -f "$AIR_NAME" 2>/dev/null +$BRIG cell rm -f "$AIR_NAME" 2>/dev/null # --------------------------------------------------------------------------- info "" @@ -289,7 +289,7 @@ if $BRIG run --name "$PROF_NAME" --profile untrusted -d alpine sleep 10 2>&1; th else fail "memory limit: $MEM (expected 536870912 for 512m)" fi - $BRIG rm -f "$PROF_NAME" 2>/dev/null + $BRIG cell rm -f "$PROF_NAME" 2>/dev/null else fail "profile-based run" fi @@ -323,7 +323,7 @@ if $BRIG run --name "$POLICY_NAME" -d alpine sleep 30 2>&1; then fail "proxy blocked allowed domain github.com" fi - $BRIG rm -f "$POLICY_NAME" 2>/dev/null + $BRIG cell rm -f "$POLICY_NAME" 2>/dev/null else fail "policy test cell failed to start" fi diff --git a/src/addons/_common.py b/src/addons/_common.py deleted file mode 100644 index 0e5ede7..0000000 --- a/src/addons/_common.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -Shared helpers for warden addons. - -Mounted alongside enforce.py / logger.py / notifier.py inside the warden -container. Addons import via `from _common import ...`. - -Contents: - - BLOCKED_NETWORKS: SSRF blocklist (RFC1918, localhost, CGNAT, link-local, - benchmarking, multicast, IPv4-mapped-IPv6, etc.). Single source of truth. - - SubnetResolver: subnet-map.json loader + cached IP -> cell resolver. - - atomic_write_json: tempfile + fsync + rename helper. -""" - -from __future__ import annotations - -import ipaddress -import json -import os -import tempfile -from pathlib import Path -from typing import Optional - - -# Networks that an egress request must never resolve to. Mirrors the blocklist -# in src/brig/network/validation.py for the host-side validators. -BLOCKED_NETWORKS = [ - # IPv4 private + reserved. - ipaddress.ip_network("10.0.0.0/8"), - ipaddress.ip_network("172.16.0.0/12"), - ipaddress.ip_network("192.168.0.0/16"), - ipaddress.ip_network("127.0.0.0/8"), - ipaddress.ip_network("169.254.0.0/16"), - ipaddress.ip_network("100.64.0.0/10"), - ipaddress.ip_network("198.18.0.0/15"), - ipaddress.ip_network("240.0.0.0/4"), - ipaddress.ip_network("0.0.0.0/8"), - ipaddress.ip_network("224.0.0.0/4"), - # IPv6 equivalents. - ipaddress.ip_network("::1/128"), - ipaddress.ip_network("fc00::/7"), - ipaddress.ip_network("fe80::/10"), - # IPv4-mapped IPv6 (covers all IPv4 ranges above when expressed as ::ffff:a.b.c.d). - ipaddress.ip_network("::ffff:0:0/96"), - ipaddress.ip_network("2001:db8::/32"), - ipaddress.ip_network("ff00::/8"), - # Additional IPv6 ranges that can encapsulate or route to private - # address space and let an attacker tunnel around the basic RFC1918 - # blocklist. - # - 64:ff9b::/96 RFC6052 well-known NAT64 prefix (can map to v4) - # - 100::/64 RFC6666 discard-only address block - # - 2002::/16 RFC3056 6to4, can encapsulate RFC1918 v4 - ipaddress.ip_network("64:ff9b::/96"), - ipaddress.ip_network("100::/64"), - ipaddress.ip_network("2002::/16"), -] - - -def is_blocked_ip(ip_str: str) -> bool: - """Return True if ip_str parses to an IP inside BLOCKED_NETWORKS. - - Returns False (not blocked) on parse error so callers can apply their own - fail-closed policy at the call site. - """ - try: - addr = ip_str[1:-1] if ip_str.startswith("[") and ip_str.endswith("]") else ip_str - ip = ipaddress.ip_address(addr) - except ValueError: - return False - return any(ip in net for net in BLOCKED_NETWORKS) - - -class SubnetResolver: - """Loads subnet-map.json and resolves client IPs to cell names. - - Single instance per addon. Caches mtime so repeated calls without file - changes are cheap. Builds an O(1) index keyed by the top 24 bits of the - network address for /24 IPv4 subnets (the common case); falls back to a - linear scan for non-/24 or IPv6 entries. - """ - - def __init__(self, subnet_map_file: Path): - self.subnet_map_file = subnet_map_file - self.subnet_map: dict[str, str] = {} - self._index: dict[int, str] = {} - self._mtime = 0.0 - - def reload(self) -> bool: - """Reload the subnet map if the file has changed. Returns True if reloaded.""" - try: - if not self.subnet_map_file.exists(): - return False - mtime = self.subnet_map_file.stat().st_mtime - if mtime == self._mtime: - return False - with open(self.subnet_map_file, "r") as f: - data = json.load(f) - if not isinstance(data, dict): - return False - self.subnet_map = data - self._mtime = mtime - self._build_index() - return True - except (json.JSONDecodeError, IOError, OSError): - return False - - def _build_index(self) -> None: - """Build O(1) lookup index for /24 IPv4 subnets keyed by top 24 bits.""" - index: dict[int, str] = {} - for subnet_str, cell_name in self.subnet_map.items(): - try: - net = ipaddress.ip_network(subnet_str, strict=False) - except ValueError: - continue - if net.version == 4 and net.prefixlen == 24: - prefix = int(net.network_address) >> 8 - index[prefix] = cell_name - self._index = index - - def get_cell_name(self, client_ip: str) -> Optional[str]: - """Resolve a client IP to its cell name. Returns None if unknown.""" - try: - ip = ipaddress.ip_address(client_ip) - except ValueError: - return None - if isinstance(ip, ipaddress.IPv4Address) and self._index: - cell = self._index.get(int(ip) >> 8) - if cell is not None: - return cell - for subnet_str, cell_name in self.subnet_map.items(): - try: - net = ipaddress.ip_network(subnet_str, strict=False) - except ValueError: - continue - if ip in net: - return cell_name - return None - - -def atomic_write_json(path: Path, data, indent: Optional[int] = 2) -> None: - """Write JSON atomically: tempfile in same dir + fsync + rename. - - POSIX rename is atomic for paths in the same filesystem, so readers either - see the old file or the new one — never a half-written file. The fsync - before rename guarantees the new bytes hit disk before the directory entry - flips. - """ - path.parent.mkdir(parents=True, exist_ok=True) - fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp") - try: - with os.fdopen(fd, "w") as f: - json.dump(data, f, indent=indent) - f.flush() - os.fsync(f.fileno()) - os.rename(tmp_path, str(path)) - except Exception: - try: - os.unlink(tmp_path) - except OSError: - pass - raise diff --git a/src/brig/cell/ca_bundle.py b/src/brig/cell/ca_bundle.py index 30dc5fd..4416e81 100644 --- a/src/brig/cell/ca_bundle.py +++ b/src/brig/cell/ca_bundle.py @@ -4,8 +4,7 @@ Without this, every cell-image author has to extract Warden's CA, mount it as a secret, append it onto /etc/ssl/certs, and export the assorted SSL_CERT_FILE / REQUESTS_CA_BUNDLE / CURL_CA_BUNDLE / NODE_EXTRA_CA_CERTS -env vars themselves. Aitelier flagged that as their #1 adoption ask — -every consumer was rediscovering the workaround. +env vars themselves. Staging it once here removes that per-image setup. Threat model: @@ -40,10 +39,8 @@ # Warden's CA cert lives in a persistent bind-mounted dir inside the # VM (see src/warden/proxy.py VM_WARDEN_STATE_DIR). brig reads it from -# the VM filesystem directly — no `podman exec`. The previous design -# went through `podman exec warden cat ...` and hit three compounding -# bugs aitelier diagnosed (sh -c skipping auto-sudo, lazy CA gen, root- -# owned tmpfs). The current design eliminates all three by structure: +# the VM filesystem directly — no `podman exec` — which avoids three +# failure modes by structure: # - direct filesystem read uses vm_run([cat, ...]) which is on the # sudo whitelist # - eager CA gen at `warden start` (proxy.py:_ensure_warden_ca_exists) @@ -74,7 +71,7 @@ def stage_bundle(cell_name: str) -> None: never sees a torn write. Raises BrigError if warden's CA file is missing — that means `brig system up` hasn't run yet (or warden failed to generate its cert at start, which would have already - surfaced as a `brig up` failure). + surfaced as a `brig system up` failure). """ bundle = vm_bundle_path(cell_name) tmp = bundle.with_suffix(".crt.tmp") @@ -84,7 +81,7 @@ def stage_bundle(cell_name: str) -> None: raise BrigError( f"Warden CA cert is missing at {VM_WARDEN_CA_FILE}", suggestion=( - "Bring warden up first: brig up\n" + "Bring warden up first: brig system up\n" f" (Cell '{cell_name}' aborted before any state was created.)" ), ) @@ -106,7 +103,7 @@ def stage_bundle(cell_name: str) -> None: f"chmod 0644 {shlex.quote(str(tmp))}; " f"mv {shlex.quote(str(tmp))} {shlex.quote(str(bundle))}" ) - result = vm_run(["sudo", "sh", "-c", script], timeout=15) + result = vm_run(["sh", "-c", script], timeout=15, sudo=True) if result.returncode != 0: # BrigError (not RuntimeError) so the operator gets the # standard suggestion-line affordance brig uses everywhere diff --git a/src/brig/cell/host_sockets_bridge.py b/src/brig/cell/host_sockets_bridge.py index 18b9244..ef017fa 100644 --- a/src/brig/cell/host_sockets_bridge.py +++ b/src/brig/cell/host_sockets_bridge.py @@ -12,7 +12,7 @@ - `brig run` calls start_cell_bridges(spec.name, spec.host_sockets) BEFORE the reconciler attempts the cell start. The runtime check in the reconciler then sees the bridge socket and proceeds. - - `brig stop` / `brig rm` calls stop_cell_bridges(spec.name) to + - `brig cell stop` / `brig cell rm` calls stop_cell_bridges(spec.name) to bootout the launchd jobs and remove plist files. Threat surface: socat now sits on the trust path between cell and host @@ -129,9 +129,13 @@ def esc(s: str) -> str: """ -def _validate_target(host_path: str) -> None: +def _validate_target(host_path: str) -> str: """Runtime guard: reject engine sockets and non-socket targets. + Returns the validated canonical realpath so the caller can freeze THAT + exact value into the launchd plist — recomputing realpath at write time + would reopen a TOCTOU window between validation and freeze. + Defense in depth: - Realpath-resolve to defeat symlinks at ANY level (parent dir symlinks would otherwise sneak past a leaf-only lstat check). @@ -170,14 +174,12 @@ def _validate_target(host_path: str) -> None: f"host_socket target {host_path} is a symlink — " f"refusing to bridge through a symlink", ) - # Realpath canonicalizes the entire ancestor chain — any symlink - # at any level gets resolved here, so a post-validation swap of a - # parent directory can't redirect us to attacker-controlled - # storage. We then stat (NOT lstat) the realpath: realpath already - # collapsed any symlinks; we just want S_ISSOCK on the canonical - # path. A normal /tmp → /private/tmp Mac-ism is fine; what we - # reject is a final-target that isn't actually a socket. - real = os.path.realpath(host_path) + # Reuse the single `real` resolved above for the socket check — resolving + # a second time would let the denylist decision and the frozen/returned + # value diverge if the filesystem changed between the two calls. We stat + # (NOT lstat) the canonical path: realpath already collapsed any symlinks; + # we just want S_ISSOCK on it. A normal /tmp → /private/tmp Mac-ism is + # fine; what we reject is a final target that isn't actually a socket. try: real_st = os.stat(real) except FileNotFoundError: @@ -190,6 +192,15 @@ def _validate_target(host_path: str) -> None: f"host_socket target {host_path} is not a unix socket " f"(realpath={real}, mode={oct(real_st.st_mode)})", ) + # socat treats ',' as its address-option separator, so a comma in the + # canonical path would be parsed as filename + options in the generated + # `UNIX-CONNECT:` plist line. Refuse it (XML-escaping doesn't help). + if "," in real: + raise BrigError( + f"host_socket target realpath {real} contains a comma, which " + f"socat would misparse as an option separator — rename the path", + ) + return real def start_cell_bridges( @@ -213,9 +224,12 @@ def start_cell_bridges( suggestion="brew install socat", ) - # Validate every target up front so we don't half-bridge. + # Validate every target up front so we don't half-bridge, capturing the + # canonical realpath each one validated to (keyed by socket name) so the + # write loop freezes the exact validated path rather than re-resolving. + validated_real: dict[str, str] = {} for entry in host_sockets: - _validate_target(entry["host_path"]) + validated_real[entry["name"]] = _validate_target(entry["host_path"]) bridge_dir = _bridge_dir_for_cell(cell_name) bridge_dir.mkdir(parents=True, exist_ok=True) @@ -225,7 +239,6 @@ def start_cell_bridges( try: for entry in host_sockets: sock_name = entry["name"] - target = entry["host_path"] label = _label_for(cell_name, sock_name) bridge = _bridge_path(cell_name, sock_name) plist = _plist_path(cell_name, sock_name) @@ -236,12 +249,11 @@ def start_cell_bridges( bridge.unlink() # Freeze the connect target by writing the realpath into the - # plist instead of the literal path. _validate_target() above - # already rejected leaf symlinks and ancestor-chain redirects, - # but socat would otherwise follow any post-validation symlink - # swap. Baking the canonical path into the launchd config closes - # that window — socat connects to a fixed inode chain. - target_real = os.path.realpath(target) + # plist instead of the literal path. Use the exact value that + # _validate_target() resolved-and-checked above; re-resolving here + # would reopen the validate→freeze TOCTOU a swapped symlink could + # exploit. socat then connects to a fixed canonical path. + target_real = validated_real[sock_name] xml = generate_plist( label=label, socat_bin=socat, bridge_path=str(bridge), target_path=target_real, @@ -265,6 +277,12 @@ def start_cell_bridges( f"{res.stderr.strip()}", ) + # Track the label as soon as the job is loaded — BEFORE waiting for + # the socket. The job has KeepAlive=true and respawns socat, so a + # readiness timeout below must still reach the rollback boot-out; + # appending only after _wait_for_socket would leak the live job. + started.append(label) + if not _wait_for_socket(bridge): raise BrigError( f"bridge socket for '{sock_name}' did not appear at " @@ -272,8 +290,7 @@ def start_cell_bridges( f"check /tmp/{label}.err.log", ) - started.append(label) - info(f"host_socket bridge started: {sock_name} → {target}") + info(f"host_socket bridge started: {sock_name} → {target_real}") except Exception: for label in started: # Best-effort cleanup; ignore errors here so the original diff --git a/src/brig/cell/lifecycle.py b/src/brig/cell/lifecycle.py index 80f0e04..ecadb41 100644 --- a/src/brig/cell/lifecycle.py +++ b/src/brig/cell/lifecycle.py @@ -9,7 +9,7 @@ from __future__ import annotations import re -from typing import Callable +from typing import Any, Callable from brig.cell.reconciler import ( Action, @@ -25,19 +25,10 @@ from brig.errors import BrigError from brig.ops.history import log_lifecycle, log_operation, log_policy_change from brig.ops.logging import debug, info -from brig.ops.ratelimit import check_rate_limit +from brig.ops.ratelimit import check_rate_limit, record_rate_limit from brig.vm.shell import vm_run -def _register_cell_ingress(spec: CellSpec, result: ReconcileResult) -> None: - """Register ingress routes for a newly started cell. - - Reads the cell's IP from podman inspect, reads the auth token from - the mounted ingress-token secret, and registers routes. - """ - register_ingress_for(spec.name, spec.ingress) - - def register_ingress_for(cell_name: str, ingress_spec: list[dict]) -> None: """Inspect `cell_name`, look up the ingress token, and register routes. @@ -46,11 +37,42 @@ def register_ingress_for(cell_name: str, ingress_spec: list[dict]) -> None: Raises BrigError if ingress is declared but the auth token is missing or empty — registering would-be-rejected routes is worse than failing loudly. + + The replay path reads `ingress_spec` from `cell-metadata.json`, which + invariant 4 names as untrusted. Each entry is re-run through the + same per-entry validator that gates cell yaml at parse time before + any persistence happens. """ if not ingress_spec: return - from brig.config import HostPaths + from brig.cell.validators import _v_ingress_entry + from brig.config import MAX_INGRESS_PER_CELL + errors: list[str] = [] + # The replay path (cell start) reads ingress from untrusted + # cell-metadata.json (invariant 4); apply the same count cap that + # _v_ingress enforces at parse time, not just the per-entry shape check. + if len(ingress_spec) > MAX_INGRESS_PER_CELL: + errors.append( + f"too many ingress entries ({len(ingress_spec)}), " + f"max {MAX_INGRESS_PER_CELL}" + ) + seen_names: set = set() + seen_prefixes: set = set() + for i, entry in enumerate(ingress_spec): + errors.extend(_v_ingress_entry(i, entry, seen_names, seen_prefixes, "")) + if errors: + raise BrigError( + f"Refusing to register ingress for '{cell_name}': " + f"invalid entry shape — " + "; ".join(errors), + suggestion=( + f"~/.brig/state/{cell_name}/cell-metadata.json may have been " + f"hand-edited or corrupted. Re-create the cell from yaml: " + f"brig cell rm {cell_name} && brig run --file " + ), + ) + + from brig.config import HostPaths, INGRESS_TOKEN_MIN_LEN from brig.network.ingress import register_ingress from brig.security.secrets import validate_secret_path @@ -58,8 +80,12 @@ def register_ingress_for(cell_name: str, ingress_spec: list[dict]) -> None: from brig.cell.reconciler import _podman_inspect_json container_info = _podman_inspect_json(cn) if not container_info: - debug(f"Could not inspect {cn} for ingress registration") - return + raise BrigError( + f"Cannot register ingress for '{cell_name}': container {cn} " + f"is not inspectable. Declared ingress routes would be silently " + f"unregistered.", + suggestion=f"brig cell logs {cell_name} # check why the cell isn't up", + ) cell_ip = ( container_info.get("NetworkSettings", {}) .get("Networks", {}) @@ -67,38 +93,48 @@ def register_ingress_for(cell_name: str, ingress_spec: list[dict]) -> None: .get("IPAddress", "") ) if not cell_ip: - debug("Could not determine cell IP for ingress registration") - return + raise BrigError( + f"Cannot register ingress for '{cell_name}': cell IP could not " + f"be determined. Declared ingress routes would be silently " + f"unregistered.", + suggestion=f"brig cell logs {cell_name}", + ) - token_name = f"{cell_name}-ingress-token" - try: - token_path = validate_secret_path(token_name, HostPaths.SECRETS_DIR) - except (ValueError, FileNotFoundError): + # A token is needed only if at least one route is auth: token. An + # all-`auth: none` cell (transparent pass-through) requires no secret. + needs_token = any(e.get("auth") == "token" for e in ingress_spec) + auth_token: str | None = None + if needs_token: + token_name = f"{cell_name}-ingress-token" try: - token_path = validate_secret_path("ingress-token", HostPaths.SECRETS_DIR) + token_path = validate_secret_path(token_name, HostPaths.SECRETS_DIR) except (ValueError, FileNotFoundError): + try: + token_path = validate_secret_path("ingress-token", HostPaths.SECRETS_DIR) + except (ValueError, FileNotFoundError): + raise BrigError( + f"Cell '{cell_name}' declares ingress with auth: token " + f"but no token secret exists. Ingress would register " + f"routes that reject every request.", + suggestion=( + f"Create the token (32+ random chars), then re-run:\n" + f" openssl rand -hex 32 | brig secrets add {token_name} -\n" + f" brig cell rm {cell_name} && brig run --file " + ), + ) + + auth_token = token_path.read_text().strip() + if not auth_token: raise BrigError( - f"Cell '{cell_name}' declares ingress with auth: token " - f"but no token secret exists. Ingress would register " - f"routes that reject every request.", - suggestion=( - f"Create the token (32+ random chars), then re-run:\n" - f" openssl rand -hex 32 | brig secrets add {token_name} -\n" - f" brig cell rm {cell_name} && brig run --file " - ), + f"Ingress token for '{cell_name}' is empty", + suggestion=f"openssl rand -hex 32 | brig secrets add {token_name} -", + ) + if len(auth_token) < INGRESS_TOKEN_MIN_LEN: + raise BrigError( + f"Ingress token for '{cell_name}' is too short " + f"({len(auth_token)} chars); minimum is {INGRESS_TOKEN_MIN_LEN}", + suggestion=f"openssl rand -hex 32 | brig secrets add {token_name} -", ) - - auth_token = token_path.read_text().strip() - if not auth_token: - raise BrigError( - f"Ingress token for '{cell_name}' is empty", - suggestion=f"openssl rand -hex 32 | brig secrets add {token_name} -", - ) - if len(auth_token) < 32: - info( - f"WARNING: Ingress token for '{cell_name}' is short. " - f"Use at least 32 characters." - ) register_ingress(cell_name, cell_ip, ingress_spec, auth_token) @@ -118,8 +154,8 @@ def _container_name_from_entry(entry: dict) -> str: """ names = entry.get("Names", "") if isinstance(names, list): - return names[0] if names else "" - return names + return str(names[0]) if names else "" + return str(names) def list_cell_containers(*, include_stopped: bool = True) -> list[tuple[str, dict]]: @@ -162,7 +198,11 @@ def list_cell_containers(*, include_stopped: bool = True) -> list[tuple[str, dic return out -_DIGEST_PATTERN = re.compile(r"^sha(?:256|384|512):[0-9a-fA-F]{64,}$") +# Exact hex length per algorithm — an open-ended {64,} would accept an +# over-long sha256 or a wrong-length sha384/sha512 as well-formed. +_DIGEST_PATTERN = re.compile( + r"^sha(?:256:[0-9a-fA-F]{64}|384:[0-9a-fA-F]{96}|512:[0-9a-fA-F]{128})\Z" +) def _apply_image_digest_pin(spec: CellSpec) -> None: @@ -197,9 +237,59 @@ def _apply_image_digest_pin(spec: CellSpec) -> None: spec.image = f"{spec.image}@{digest}" +def sync_cell_policy(spec: CellSpec) -> None: + """Write the cell's allow / deny / host_services / tls_passthrough to its + per-cell policy file (`.json`). Replace semantics — the spec is the + source of truth. Skips the write when the on-disk policy already matches + so idempotent re-runs don't churn mtime (which would trigger a warden + reload). Called by run_cell so BOTH the CLI and the SDK persist policy — + without this, a cell launched via the SDK has no per-cell policy file and + warden default-denies all its egress. + """ + from brig.policy.policy import mutate_cell_policy + + desired = { + "allow": list(spec.policy_allow or []), + "deny": list(spec.policy_deny or []), + "host_services": list(spec.host_services or []), + "tls_passthrough": list(spec.policy_passthrough_tls or []), + } + current_holder: dict[str, Any] = {} + + def _mutate(existing: dict[str, Any] | None) -> dict[str, Any] | None: + existing = existing or {} + current = { + "allow": existing.get("allow", []), + "deny": existing.get("deny", []), + "host_services": existing.get("host_services", []), + "tls_passthrough": existing.get("tls_passthrough", []), + } + current_holder.update(current) + if desired == current: + return None # No change — skip the write. + merged = dict(existing) + merged.update(desired) + return merged + + written = mutate_cell_policy(spec.name, _mutate) + if written is None: + return + current = current_holder + + def _names(items: Any) -> set[str]: + return {e["name"] for e in items if isinstance(e, dict) and "name" in e} + added = _names(desired["host_services"]) - _names(current["host_services"]) + removed = _names(current["host_services"]) - _names(desired["host_services"]) + for n in sorted(added): + info(f"host_service granted: {spec.name} → {n} (from cell yaml)") + for n in sorted(removed): + info(f"host_service revoked: {spec.name} → {n} (no longer in cell yaml)") + + def run_cell( spec: CellSpec, proxy_check: Callable[[], bool] = _default_proxy_check, + count_against_rate_limit: bool = True, ) -> ReconcileResult: """Run a new cell. @@ -207,6 +297,9 @@ def run_cell( spec: Cell specification. proxy_check: Callable returning True if proxy is running. Injected for testability; defaults to the real proxy_running(). + count_against_rate_limit: when False, skip the creation rate limiter. + Set by restore (replaying already-authorized restart:always cells), + which would otherwise throttle past the limit and strand the rest. Enforces: - Invariant 9: proxy must be running (unless airgapped) @@ -219,15 +312,24 @@ def run_cell( if not spec.is_airgapped and not proxy_check(): raise BrigError( "Warden proxy is not running", - suggestion="Start with: brig up", + suggestion="Start with: brig system up", ) - if not check_rate_limit(): + if count_against_rate_limit and not check_rate_limit(): raise BrigError( "Rate limit exceeded: too many cells created recently", suggestion="Wait a moment and try again, or increase the rate limit", ) + # Persist the cell's per-cell policy before it starts so warden enforces + # the intended allow/deny (not default-deny). Done here, not in the CLI, + # so SDK-launched cells get it too. Track whether a policy already existed + # so a failed reconcile can clean up a file THIS run created without + # deleting a legitimate policy on an idempotent re-run of a live cell. + from brig.policy.policy import load_cell_policy + policy_preexisted = load_cell_policy(spec.name) is not None + sync_cell_policy(spec) + actual = observe(spec.name) if actual.exists and actual.running: raise BrigError( @@ -256,24 +358,25 @@ def run_cell( from brig.cell.host_sockets_bridge import start_cell_bridges start_cell_bridges(spec.name, spec.host_sockets) + def _rollback_reconcile_side_effects() -> None: + # apply() rolls back its own network/subnet/podman actions, but bridges + # and the pre-written policy file are ours to clean up. + if spec.host_sockets: + from brig.cell.host_sockets_bridge import stop_cell_bridges + stop_cell_bridges(spec.name) + if not policy_preexisted: + from brig.policy.policy import delete_cell_policy + delete_cell_policy(spec.name) + debug(f"Reconciliation plan: {[a.type.name for a in actions]}") try: result = apply(actions) except Exception: - # apply() rolled back its own actions, but it doesn't know about - # bridges. Tear them down so we don't leak a socat process for - # a cell that never started. - if spec.host_sockets: - from brig.cell.host_sockets_bridge import stop_cell_bridges - stop_cell_bridges(spec.name) + _rollback_reconcile_side_effects() raise if not result.success: - # Same rollback as the exception path: apply()'s _rollback handles - # network/subnet/podman actions, but bridges are ours to clean up. - if spec.host_sockets: - from brig.cell.host_sockets_bridge import stop_cell_bridges - stop_cell_bridges(spec.name) + _rollback_reconcile_side_effects() failed = result.actions_failed[0] if result.actions_failed else (None, "unknown") raise BrigError(f"Failed to start cell '{spec.name}': {failed[1]}") @@ -292,7 +395,22 @@ def run_cell( ) # Register ingress routes if the cell has ingress endpoints. if spec.ingress: - _register_cell_ingress(spec, result) + register_ingress_for(spec.name, spec.ingress) + # An auth: none route removes brig's perimeter gate — the cell's + # app is the authenticator. Surface it loudly (audit + operator + # NOTE), the way mounts/host_sockets announce their bypasses. + open_routes = [e for e in spec.ingress if e.get("auth") == "none"] + for e in open_routes: + log_lifecycle( + "ingress_unauthenticated", spec.name, + details={"route": e.get("name"), "path_prefix": e.get("path_prefix")}, + ) + if open_routes: + info( + f"NOTE: cell '{spec.name}' has {len(open_routes)} ingress " + f"route(s) with auth: none — brig does NOT authenticate " + f"these; the cell's app must be the gate." + ) # Audit any host_sockets that were mounted. The bytes flowing # over these sockets bypass Warden, so the attach event is the # only thing we can record — make sure it's loud. @@ -308,32 +426,103 @@ def run_cell( f"NOTE: cell '{spec.name}' has {len(spec.host_sockets)} " f"host_sockets — Warden does not see traffic over these." ) + # Audit host-directory mounts. The cell reads/writes these host files + # directly (Warden not in the path); a rw mount lets it modify files a + # host process later consumes — surface it loudly. + if spec.mounts: + for entry in spec.mounts: + log_lifecycle( + "mount_attach", spec.name, + details={"mount": entry["name"], + "host_path": entry["host_path"], + "mount_point": entry["mount_point"], + "mode": entry.get("mode", "ro")}, + ) + rw = [m for m in spec.mounts if m.get("mode") == "rw"] + info( + f"NOTE: cell '{spec.name}' has {len(spec.mounts)} host mount(s) " + f"({len(rw)} rw) — Warden does not see these bytes. Treat files " + f"the cell writes as untrusted; scan with: brig cell mount-scan " + f"{spec.name}" + ) info(f"Cell '{spec.name}' started") except BrigError: # Roll the cell back — container is running, but post-start - # config failed. Better to fail clean than ship a half-cell. + # config failed. Preserve the workspace: a post-start failure + # is brig's fault, not the user's, and if the cell name was + # reused for an already-populated workspace (the + # exists-but-stopped recovery path), wiping it would destroy + # the user's data. info(f"post-start failure for '{spec.name}'; rolling back cell") try: - rm_cell(spec.name, force=True) + rm_cell(spec.name, force=True, keep_workspace=True) except Exception as cleanup_err: debug(f"rollback rm_cell failed: {cleanup_err}") raise + # Persist the spec for restart:always cells so `brig system up` can + # re-launch them after a VM restart. Best-effort — a persistence failure + # must not fail an otherwise-healthy cell. + try: + from brig.cell.metadata import write_cell_spec + write_cell_spec(spec) + except Exception as e: + debug(f"failed to persist restart spec for '{spec.name}': {e}") + + # The cell is fully up and configured — count it against the creation + # quota now, so no-ops (already running, empty plan), rolled-back + # reconciles, AND rolled-back post-start failures never burned a slot. + if count_against_rate_limit: + record_rate_limit() return result +def restore_persisted_cells() -> None: + """Re-launch `restart: always` cells whose container is gone, on + `brig system up` (once warden is up). + + A cell is restored only when its container no longer exists. An *exited* + cell (still present — e.g. an explicit `brig cell stop`) is left alone. + Note a VM restart drops every container, so a stopped restart:always cell + DOES relaunch on the next up; use `brig cell rm` to keep one down for good. + + The persisted spec is re-validated before launch (the state dir is + untrusted, invariant 4) and replayed without counting against the creation + rate limit (these cells were already authorized). + """ + import dataclasses + from brig.cell.metadata import restorable_cell_specs + from brig.cell.spec import validate_cell_definition + + valid = {f.name for f in dataclasses.fields(CellSpec)} + for raw in restorable_cell_specs(): + name = raw.get("name") + if not name or observe(name).exists: + continue + errs = validate_cell_definition(raw) + if errs: + info(f" (warn) skipping restore of '{name}': invalid persisted spec: {errs[0]}") + continue + info(f"Restoring cell '{name}' (restart: always)...") + try: + spec = CellSpec(**{k: v for k, v in raw.items() if k in valid}) + run_cell(spec, count_against_rate_limit=False) + except Exception as e: + info(f" (warn) could not restore cell '{name}': {e}") + + def stop_cell(cell_name: str) -> None: """Gracefully stop a running cell.""" actual = observe(cell_name) if not actual.exists: raise BrigError( f"Cell '{cell_name}' does not exist", - suggestion="Use 'brig list' to see available cells", + suggestion="Use 'brig cell list' to see available cells", ) if not actual.running: raise BrigError( f"Cell '{cell_name}' is not running", - suggestion=f"Use 'brig start {cell_name}' to start it", + suggestion=f"Use 'brig cell start {cell_name}' to start it", ) # Deregister ingress routes before stopping (prevents stale routes @@ -364,7 +553,7 @@ def kill_cell(cell_name: str) -> None: if not actual.exists: raise BrigError( f"Cell '{cell_name}' does not exist", - suggestion="Use 'brig list' to see available cells", + suggestion="Use 'brig cell list' to see available cells", ) # Deregister ingress routes before killing. @@ -375,7 +564,16 @@ def kill_cell(cell_name: str) -> None: from brig.cell.host_sockets_bridge import stop_cell_bridges stop_cell_bridges(cell_name) - actions = [Action(ActionType.PODMAN_KILL, cell_name)] if actual.running else [] + # observe() reports a paused container as running=False, so gate on the + # actual status too — otherwise killing a paused cell would no-op, log a + # false "kill" success, and leave it up with its egress wiring stripped. + # podman refuses to SIGKILL a paused container, so unpause it first. + should_kill = actual.running or actual.status == "paused" + if should_kill and actual.status == "paused": + from brig.config import container_name + vm_run(["podman", "unpause", container_name(cell_name)]) + + actions = [Action(ActionType.PODMAN_KILL, cell_name)] if should_kill else [] result = apply(actions) if result.success: @@ -395,23 +593,29 @@ def rm_cell( callers that want to extract files first should `brig cell cp` them out before calling rm. - Why default-delete (was leave-on-disk in earlier versions): the - workspace can contain cell-controlled content including symlinks - pointing at host files. If a new cell takes the same name later, it - inherits the prior cell's planted bait. Cleaning by default closes - that re-use foot-gun; users who need the data ask for it. + Why default-delete: the workspace can contain cell-controlled content + including symlinks pointing at host files. If a new cell takes the same + name later, it inherits the prior cell's planted bait. Cleaning by + default closes that re-use foot-gun; users who need the data ask for it. """ + # Validate before any path construction: rm_cell does a destructive + # rmtree of ~/.brig/state//, so gate on the name pattern (forbids + # '/' and '..') rather than relying on a downstream existence check. + from brig.config import CELL_NAME_PATTERN + if not CELL_NAME_PATTERN.match(cell_name): + raise BrigError(f"Invalid cell name: '{cell_name}'") + actual = observe(cell_name) if not actual.exists and not actual.network_exists: raise BrigError( f"Cell '{cell_name}' does not exist", - suggestion="Use 'brig list' to see available cells", + suggestion="Use 'brig cell list' to see available cells", ) if actual.running and not force: raise BrigError( f"Cell '{cell_name}' is running. Stop it first or use --force.", - suggestion=f"brig stop {cell_name} OR brig rm -f {cell_name}", + suggestion=f"brig cell stop {cell_name} OR brig cell rm -f {cell_name}", ) # Deregister ingress routes before destroying the cell. @@ -429,6 +633,18 @@ def rm_cell( failed = result.actions_failed[0] if result.actions_failed else (None, "unknown") raise BrigError(f"Failed to remove cell '{cell_name}': {failed[1]}") + # Delete the per-cell policy file so a future cell reusing this name + # doesn't inherit the removed cell's allow/deny (the workspace + subnet + # are freed by plan_destroy; the policy file lives separately). + from brig.policy.policy import delete_cell_policy + delete_cell_policy(cell_name) + + # Drop the persisted restart spec unconditionally — even with + # keep_workspace — so a removed cell can't be resurrected by restart:always + # on the next `brig system up` (the spec is a sibling of the workspace). + from brig.cell.metadata import remove_cell_spec + remove_cell_spec(cell_name) + # Clean up host-side per-cell state (workspace + metadata file). # Deletion is destructive but matches the principle that `rm` should # leave nothing behind for the next cell with the same name to diff --git a/src/brig/cell/metadata.py b/src/brig/cell/metadata.py index ab83e14..d63b13b 100644 --- a/src/brig/cell/metadata.py +++ b/src/brig/cell/metadata.py @@ -10,19 +10,30 @@ into the cell at `/run/brig/cell.json`. The cell can read but not modify it. -Schema (v2): +Schema (v3): { - "version": 2, + "version": 3, "name": "", "started_at": "", "workspace": { "mount_point": "/work" }, + "host_sockets": [{"name": ..., "mount_point": ...}, ...], + "ingress": [{"name": ..., "port": int, + "path_prefix": ..., "auth": "token"}, ...], + "image_digest": "sha256:..." // optional; only when pinned "policy": { - "host_services": ["", ...] // per-cell ACL (may be empty) + "host_services": ["", ...] // per-cell ACL } } +What changed since v2: + - `ingress` added so `brig cell start` can replay route registration + without the original yaml. No secrets land here. + - `image_digest` added so `brig cell start` can re-verify the pinned + digest before letting the container start. + Both fields are optional and additive — v2 readers ignore them. + Why v2 dropped `workspace.host_path`: publishing the absolute host path made it trivial for a consumer to do plain `open(host_path)`, which follows any symlink the cell planted in its workspace — letting @@ -50,7 +61,7 @@ from brig.ops.atomic import atomic_write_json from brig.policy.policy import load_cell_policy -SCHEMA_VERSION = 2 +SCHEMA_VERSION = 3 IN_CELL_PATH = "/run/brig/cell.json" @@ -66,12 +77,63 @@ def _vm_metadata_path(cell_name: str) -> Path: return VMPaths.STATE_DIR / cell_name / "cell-metadata.json" +def _host_spec_path(cell_name: str) -> Path: + """Where brig persists a `restart: always` cell's full spec, so + `brig system up` can replay it after a VM restart drops the container.""" + return HostPaths.STATE_DIR / cell_name / "cell-spec.json" + + +def write_cell_spec(spec: Any) -> None: + """Persist a cell's full spec when restart == "always" (else remove any + stale copy). Mode 0600 since env values may carry sensitive data.""" + import dataclasses + if getattr(spec, "restart", "no") != "always": + remove_cell_spec(spec.name) + return + atomic_write_json(_host_spec_path(spec.name), dataclasses.asdict(spec), mode=0o600) + + +def remove_cell_spec(cell_name: str) -> None: + """Delete a cell's persisted restart spec so it can't be resurrected by + restart:always on the next `brig system up`.""" + try: + _host_spec_path(cell_name).unlink() + except FileNotFoundError: + pass + + +def read_cell_spec(cell_name: str) -> dict[str, Any] | None: + """Read a persisted cell spec, or None if absent/unreadable.""" + import json as _json + try: + data = _json.loads(_host_spec_path(cell_name).read_text()) + except (FileNotFoundError, ValueError, OSError): + return None + return data if isinstance(data, dict) else None + + +def restorable_cell_specs() -> list[dict[str, Any]]: + """All persisted specs whose restart policy is "always".""" + out: list[dict[str, Any]] = [] + base = HostPaths.STATE_DIR + if not base.is_dir(): + return out + for entry in sorted(base.iterdir()): + if not entry.is_dir(): + continue + spec = read_cell_spec(entry.name) + if spec and spec.get("restart") == "always": + out.append(spec) + return out + + def build_metadata( cell_name: str, workspace_mount: str, started_at: datetime | None = None, host_sockets: list[dict[str, Any]] | None = None, ingress: list[dict[str, Any]] | None = None, + image_digest: str | None = None, ) -> dict[str, Any]: """Compose the cell metadata payload. Pure function for testability. @@ -121,23 +183,29 @@ def build_metadata( "host_services": _per_cell_host_services(cell_name), }, } + if image_digest: + payload["image_digest"] = image_digest return payload def _per_cell_host_services(cell_name: str) -> list[str]: - """Read the cell's per-cell policy and return its host_services ACL. + """Read the cell's per-cell policy and return its host_services ACL + as a list of names. - Returns an empty list if no per-cell policy exists (the default — - cells have no host-service access until granted). + The on-disk shape is `[{name, port, protocol}, ...]` — the in-cell + metadata only exposes the names (no port leak into the cell's view + of its own ACL). Returns empty list if no per-cell policy exists. """ policy = load_cell_policy(cell_name) if not policy: return [] - services = policy.get("host_services", []) - # The per-cell shape stores bare names (strings). Defensively filter to - # strings only in case someone hand-edited the file with the global - # shape (dicts with name+port). - return [s for s in services if isinstance(s, str)] + names: list[str] = [] + for s in policy.get("host_services", []): + if isinstance(s, dict) and isinstance(s.get("name"), str): + names.append(s["name"]) + elif isinstance(s, str): + names.append(s) + return names def refresh_metadata_if_present(cell_name: str) -> Path | None: @@ -149,9 +217,9 @@ def refresh_metadata_if_present(cell_name: str) -> Path | None: no metadata file (cell was never started, or was removed). Bind mounts are fixed at podman-create time, so the cell's running - container can't pick up a new workspace_mount — we preserve whatever - was originally set. The policy field is the only one that can drift - out of sync; that's what this fixes. + container can't pick up a new workspace_mount — the original is + preserved. The policy field is the only one that can drift out of + sync, so it is re-synced here. """ import json as _json existing = _host_metadata_path(cell_name) @@ -172,6 +240,7 @@ def refresh_metadata_if_present(cell_name: str) -> Path | None: return write_metadata( cell_name, workspace_mount, host_sockets=prior_sockets, ingress=prior_ingress, + image_digest=read_image_digest(cell_name), ) @@ -195,11 +264,43 @@ def read_ingress(cell_name: str) -> list[dict[str, Any]]: ] +def read_host_sockets(cell_name: str) -> list[dict[str, Any]]: + """Return the cell's stored host_sockets entries from cell-metadata.json. + + Only the cell-visible projection (name, mount_point) is stored — host_path + is deliberately omitted (the file is mounted into the untrusted cell), so + bridges cannot be reconstructed from this alone. Used to detect that a + cell declares host_sockets, not to recreate them. + """ + import json as _json + try: + payload = _json.loads(_host_metadata_path(cell_name).read_text()) + except (FileNotFoundError, _json.JSONDecodeError, OSError): + return [] + return [e for e in (payload.get("host_sockets", []) or []) if isinstance(e, dict)] + + +def read_image_digest(cell_name: str) -> str | None: + """Return the cell's stored image_digest from cell-metadata.json. + + None when the cell was created without a pinned digest, the file + predates the field, or is unreadable. + """ + import json as _json + try: + payload = _json.loads(_host_metadata_path(cell_name).read_text()) + except (FileNotFoundError, _json.JSONDecodeError, OSError): + return None + val = payload.get("image_digest") + return val if isinstance(val, str) and val else None + + def write_metadata( cell_name: str, workspace_mount: str, host_sockets: list[dict[str, Any]] | None = None, ingress: list[dict[str, Any]] | None = None, + image_digest: str | None = None, ) -> Path: """Write the cell's metadata JSON to the host path. Returns the path. @@ -214,6 +315,7 @@ def write_metadata( payload = build_metadata( cell_name, workspace_mount, host_sockets=host_sockets, ingress=ingress, + image_digest=image_digest, ) target = _host_metadata_path(cell_name) atomic_write_json(target, payload, mode=0o644) diff --git a/src/brig/cell/profiles.py b/src/brig/cell/profiles.py index b6a1b4e..d7998e9 100644 --- a/src/brig/cell/profiles.py +++ b/src/brig/cell/profiles.py @@ -18,9 +18,8 @@ # Built-in profiles. Each is a partial cell definition (no name/image/command). # # `runtime` is intentionally omitted: invariant 5 requires gVisor (`runsc`) -# and the reconciler hardcodes `--runtime runsc`. The field used to be set -# here but was never propagated by apply_profile() — keeping it would have -# implied configurability that doesn't exist. +# and the reconciler hardcodes `--runtime runsc`, so a profile-level runtime +# would imply a configurability that does not exist. BUILTIN_PROFILES: dict[str, dict[str, Any]] = { "untrusted": { "memory": "512m", @@ -65,18 +64,29 @@ def load_profile(name: str, profiles_dir: Path = PROFILES_DIR) -> dict[str, Any]: """Load a trust profile by name. - Checks user profiles directory first, then built-in profiles. - Raises ValueError if not found. + Built-in profile names are reserved — a user profile file with the + same name is refused outright so it can't silently shadow the + builtin. This is load-bearing for `untrusted`, where the untrusted + cell's adversarial-code guarantees come from the profile NAME being + `untrusted`; if a user-defined file at ~/.brig/profiles/untrusted.yaml + were loaded instead, the operator could remove the host_sockets / + host_services / tls_passthrough rejections by editing one file. + + Raises ValueError if not found or if a user file shadows a builtin. """ - # User-defined profiles. for ext in (".yaml", ".yml", ".json"): user_profile = profiles_dir / f"{name}{ext}" if user_profile.exists(): + if name in BUILTIN_PROFILES: + raise ValueError( + f"User profile {user_profile} shadows the built-in " + f"'{name}' profile, which is not allowed. Rename " + f"the file to a different profile name." + ) debug(f"Loading user profile: {user_profile}") from brig.cell.spec import load_cell_definition return load_cell_definition(str(user_profile)) - # Built-in profiles. if name in BUILTIN_PROFILES: debug(f"Loading built-in profile: {name}") return BUILTIN_PROFILES[name].copy() diff --git a/src/brig/cell/reconciler.py b/src/brig/cell/reconciler.py index 8065fe0..eea8b9e 100644 --- a/src/brig/cell/reconciler.py +++ b/src/brig/cell/reconciler.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import re from dataclasses import dataclass, field from enum import Enum, auto from pathlib import Path @@ -25,7 +26,7 @@ ) from brig.cell.metadata import IN_CELL_PATH, vm_source_path, write_metadata from brig.cell.spec import CellSpec -from brig.config import HostPaths, PROXY_NAME, RUNTIME, VMPaths, container_name +from brig.config import HostPaths, PROXY_NAME, PROXY_PORT, RUNTIME, VMPaths, container_name from brig.errors import BrigError from brig.ops.logging import debug from brig.security.secrets import validate_secret_path @@ -63,9 +64,9 @@ class CellState: container_name: str = "" network_name: str = "" network_exists: bool = False + network_internal: bool = False proxy_connected: bool = False proxy_ip: str = "" - runtime: str = "" status: str = "" @@ -90,9 +91,33 @@ def _podman_inspect_json(name: str) -> dict | None: return None try: info = json.loads(result.stdout) - return info[0] if isinstance(info, list) else info except json.JSONDecodeError: return None + if isinstance(info, list): + info = info[0] if info else None + return info if isinstance(info, dict) else None + + +def _network_internal(name: str) -> bool: + """True if the named podman network exists AND is --internal (no egress). + + Cell networks must be internal — that is what delivers east-west isolation + (invariant 1) and keeps Warden the only egress path. The VM's network set is + untrusted (invariant 4), so a pre-existing same-named network is verified, + not assumed safe. + """ + result = _run_cmd(["podman", "network", "inspect", name]) + if result.returncode != 0: + return False + try: + data = json.loads(result.stdout) + except json.JSONDecodeError: + return False + if isinstance(data, list): + data = data[0] if data else None + if not isinstance(data, dict): + return False + return bool(data.get("internal", data.get("Internal", False))) def observe(cell_name: str) -> CellState: @@ -107,12 +132,12 @@ def observe(cell_name: str) -> CellState: state.exists = True state.status = info.get("State", {}).get("Status", "") state.running = state.status == "running" - state.runtime = info.get("HostConfig", {}).get("Runtime", "") result = _run_cmd(["podman", "network", "exists", name]) state.network_exists = result.returncode == 0 if state.network_exists: + state.network_internal = _network_internal(name) proxy_info = _podman_inspect_json(PROXY_NAME) if proxy_info: proxy_networks = proxy_info.get("NetworkSettings", {}).get("Networks", {}) @@ -136,6 +161,20 @@ def plan_run(spec: CellSpec, actual: CellState) -> list[Action]: if actual.exists and actual.running: return [] + # Fail closed: never adopt a pre-existing same-named network that isn't + # --internal. CREATE_NETWORK (the only place --internal is applied) is + # skipped on the reuse path below, so a leftover/tampered/operator-made + # non-internal `brig-` network would otherwise give the cell + # off-segment routes (invariant 1) and a path around Warden. The VM network + # set is untrusted (invariant 4), so verify rather than assume. + if actual.network_exists and not actual.network_internal: + raise BrigError( + f"Refusing to start '{spec.name}': network '{actual.network_name}' " + f"already exists but is not --internal. A non-internal cell network " + f"would break east-west isolation and bypass Warden. Remove it first: " + f"brig system down && podman network rm {actual.network_name}" + ) + # Stopped container from a previous run — clean up first. if actual.exists and not actual.running: actions.append(Action(ActionType.PODMAN_RM, spec.name)) @@ -233,10 +272,10 @@ def build_run_command(spec: CellSpec, proxy_ip: str | None) -> list[str]: ) cmd.extend([ "--network", name, - "-e", f"http_proxy=http://{proxy_ip}:8080", - "-e", f"https_proxy=http://{proxy_ip}:8080", - "-e", f"HTTP_PROXY=http://{proxy_ip}:8080", - "-e", f"HTTPS_PROXY=http://{proxy_ip}:8080", + "-e", f"http_proxy=http://{proxy_ip}:{PROXY_PORT}", + "-e", f"https_proxy=http://{proxy_ip}:{PROXY_PORT}", + "-e", f"HTTP_PROXY=http://{proxy_ip}:{PROXY_PORT}", + "-e", f"HTTPS_PROXY=http://{proxy_ip}:{PROXY_PORT}", "-e", "no_proxy=localhost,127.0.0.1", ]) @@ -285,13 +324,16 @@ def build_run_command(spec: CellSpec, proxy_ip: str | None) -> list[str]: cmd.append("--rm") for env in spec.env: - env_key = env.split("=", 1)[0].lower() if "=" in env else env.lower() + env_key = (env.split("=", 1)[0] if "=" in env else env).strip().lower() if env_key in _PROXY_ENV_NAMES: raise BrigError( f"Cannot override proxy environment variable: {env.split('=', 1)[0]}" ) cmd.extend(["-e", env]) + if spec.user: + cmd.extend(["--user", spec.user]) + workspace_dir = VMPaths.STATE_DIR / spec.name / "workspace" cmd.extend([ "-v", f"{workspace_dir}:{spec.workspace_mount}:rw", @@ -325,14 +367,22 @@ def build_run_command(spec: CellSpec, proxy_ip: str | None) -> list[str]: for secret_name in spec.secrets: resolved = secrets_dir / secret_name cmd.extend(["-v", f"{resolved}:/run/secrets/{secret_name}:ro"]) - env_name = secret_name.rsplit(".", 1)[0].upper().replace("-", "_") + "_FILE" + # Strip a trailing extension, then map any non-alphanumeric (dots, + # dashes) to '_' so the result is a valid POSIX env name — + # `api.key.prod` -> `API_KEY_FILE`, not the unusable `API.KEY_FILE`. + env_base = secret_name.rsplit(".", 1)[0] + env_name = re.sub(r"[^A-Za-z0-9]", "_", env_base).upper() + "_FILE" cmd.extend(["-e", f"{env_name}=/run/secrets/{secret_name}"]) if spec.workdir: cmd.extend(["--workdir", spec.workdir]) _attach_host_sockets(spec, cmd) + _attach_mounts(spec, cmd) + # End-of-options marker so an image reference (or command token) that + # begins with '-' can never be parsed by podman as a flag. + cmd.append("--") cmd.append(spec.image) cmd.extend(spec.command) @@ -357,12 +407,21 @@ def _attach_host_sockets(spec: CellSpec, cmd: list[str]) -> None: # Per-cell bridge dir so two cells declaring the same physical host # service (e.g. both want /tmp/postgres.sock) each get their own # bridge socket. No reference counting; bridges live with the cell. - bridge_dir = Path(str(VMPaths.HOST_SOCKETS_DIR)) / spec.name + # + # CRITICAL: this code runs in the HOST (macOS) process, but the `-v` + # mount source must be the VM-namespace path (podman runs inside the VM + # and sees the bridge dir at /state/..., not ~/.brig/...). So validate the + # HOST path (the same inode, shared into the VM via virtio-fs) and emit the + # VM path as the mount source. Validating the VM path here would lstat a + # nonexistent /state on macOS — breaking the feature and leaving the real + # bridge socket unchecked. + host_bridge_dir = HostPaths.HOST_SOCKETS_DIR / spec.name + vm_bridge_dir = Path(str(VMPaths.HOST_SOCKETS_DIR)) / spec.name for entry in spec.host_sockets: name = entry["name"] mount_point = entry["mount_point"] mode = entry.get("mode", "ro") - source = bridge_dir / f"{name}.sock" + source = host_bridge_dir / f"{name}.sock" # lstat (NOT stat) so symlinks don't get followed silently. # Symlinks AT the bridge path OR anywhere in its ancestor @@ -378,7 +437,7 @@ def _attach_host_sockets(spec: CellSpec, cmd: list[str]) -> None: except FileNotFoundError: raise BrigError( f"host_socket '{name}': bridge socket not found at {source}", - suggestion="Is the launchd bridge running? Try: brig up", + suggestion="Is the launchd bridge running? Try: brig system up", ) if _stat.S_ISLNK(st.st_mode): raise BrigError( @@ -398,14 +457,92 @@ def _attach_host_sockets(spec: CellSpec, cmd: list[str]) -> None: # (resolve both sides; macOS /tmp → /private/tmp makes a raw # comparison falsely flag every real path as escaping). real_source = _os.path.realpath(str(source)) - real_root = _os.path.realpath(str(bridge_dir)) + real_root = _os.path.realpath(str(host_bridge_dir)) if not real_source.startswith(real_root + "/"): raise BrigError( f"host_socket '{name}': bridge socket realpath {real_source} " f"escapes bridge dir {real_root}" ) - cmd.extend(["-v", f"{source}:{mount_point}:{mode}"]) + # Validation passed against the host path; mount the VM-namespace path. + cmd.extend(["-v", f"{vm_bridge_dir / f'{name}.sock'}:{mount_point}:{mode}"]) + + +def _resolved_mount_roots() -> list[tuple[str, str]]: + """[(realpath_root, slug), ...] for the configured mount_roots. + + Fails closed (BrigError) if the configured roots don't validate or two + roots share a slug — independent of the lima-render guard, so the bind + can't resolve against the wrong (but still allowlisted) tree under drift. + """ + import os.path as _ospath + from brig.config import mount_root_slug, mount_roots, validate_mount_roots + roots = mount_roots() + errs = validate_mount_roots(roots) + if errs: + raise BrigError("Invalid mount_roots: " + "; ".join(errs)) + return [(_ospath.realpath(r.rstrip("/")), mount_root_slug(r)) for r in roots] + + +def _verify_mount_sources_for_run(spec: CellSpec) -> None: + """Refuse to start a cell whose mount source isn't a live VM mount. + + mount_roots changed without a VM recreate => /mnt/host/ is absent; + podman -v would auto-create an empty dir and the cell would silently mount + VM-local storage instead of the host files. Fail closed (mirrors the + host_socket bridge guard). + """ + if not spec.mounts: + return + roots = _resolved_mount_roots() + for entry in spec.mounts: + vm_src = _mount_bind_arg(entry, roots).rsplit(":", 2)[0] + r = _run_cmd(["test", "-d", vm_src]) + if r.returncode != 0: + raise BrigError( + f"mount source {vm_src} is not present in the VM — mount_roots " + f"likely changed without recreating the VM.", + suggestion="brig system down --vm, then `limactl delete brig`, " + "then brig system up", + ) + + +def _mount_bind_arg(entry: dict, roots: list[tuple[str, str]]) -> str: + """Translate a validated `mounts:` entry to a podman `-v` value + (`::`). + + Re-resolves host_path's realpath and re-confirms containment under a + configured root — the runtime check is the real boundary (invariant 4: + the cell yaml is untrusted), not the parse-time validator. + """ + import os.path as _ospath + real = _ospath.realpath(entry["host_path"]) + for root_real, slug in roots: + if real == root_real or real.startswith(root_real + "/"): + vm_path = VMPaths.MOUNTS_DIR / slug + rel = _ospath.relpath(real, root_real) + if rel != ".": + vm_path = vm_path / rel + return f"{vm_path}:{entry['mount_point']}:{entry.get('mode', 'ro')}" + raise BrigError( + f"mount host_path {real} is not under any configured mount_roots" + ) + + +def _attach_mounts(spec: CellSpec, cmd: list[str]) -> None: + """Append `--volume` args for each declared `mounts:` entry. + + A cell-created symlink in the mount can't escape the subtree to a VM path + (container mount-namespace isolation — verified runtime-independent; see + docs/design/mount-symlink-hardening.md), so no in-VM symlink hardening is + applied here. The host-side symlink risk is mitigated by `brig cell + mount-scan`. We bind the realpath and re-check containment per entry. + """ + if not spec.mounts: + return + roots = _resolved_mount_roots() + for entry in spec.mounts: + cmd.extend(["-v", _mount_bind_arg(entry, roots)]) _ROLLBACK_MAP = { @@ -481,6 +618,7 @@ def _execute_action(action: Action, result: ReconcileResult) -> None: # so unit tests can drive build_run_command without needing real # files on disk. _verify_secrets_for_run(spec) + _verify_mount_sources_for_run(spec) # Ensure workspace directory exists inside the VM before podman mounts it. workspace = VMPaths.STATE_DIR / spec.name / "workspace" _run_cmd(["mkdir", "-p", str(workspace)]) @@ -488,7 +626,8 @@ def _execute_action(action: Action, result: ReconcileResult) -> None: # podman creates the read-only bind mount at /run/brig/cell.json. write_metadata(spec.name, spec.workspace_mount, host_sockets=spec.host_sockets, - ingress=spec.ingress) + ingress=spec.ingress, + image_digest=spec.image_digest) # Stage the combined CA bundle inside the VM so HTTPS clients in # the cell trust Warden's MITM cert. Re-extracted from Warden # every start so a CA rotation doesn't leave cells with stale @@ -513,7 +652,7 @@ def _execute_action(action: Action, result: ReconcileResult) -> None: f"Could not determine proxy IP on network {name}", suggestion=( "Warden may not be connected to the cell network. " - "Try: brig up" + "Try: brig system up" ), ) diff --git a/src/brig/cell/spec.py b/src/brig/cell/spec.py index 6f4d92b..2919929 100644 --- a/src/brig/cell/spec.py +++ b/src/brig/cell/spec.py @@ -35,8 +35,10 @@ def parse_size(size_str: str) -> int: raise ValueError("Empty size string") if size_str[-1] in _SIZE_UNITS: try: + # OverflowError: float('inf')/'1e400' make int() overflow; the + # callers' contract is ValueError, so normalize it here. return int(float(size_str[:-1]) * _SIZE_UNITS[size_str[-1]]) - except ValueError: + except (ValueError, OverflowError): raise ValueError(f"Invalid size: {size_str}") try: return int(size_str) @@ -119,6 +121,12 @@ class CellSpec: # Declaring in yaml IS the grant — there is no separate global # registry. See cell.validators._v_host_services. host_services: list[dict[str, Any]] = field(default_factory=list) + # mounts — bind-mount an operator-chosen host directory into the cell. + # Each entry: {name, host_path, mount_point, mode?}. host_path must + # resolve under a declared config.mount_roots() entry; ro default, rw + # opt-in. Bypasses Warden by design; rejected on the untrusted profile. + # See cell.validators._v_mounts and docs/design/host-mounts.md. + mounts: list[dict[str, Any]] = field(default_factory=list) # Trust Warden's MITM CA out of the box. When true (default), brig # stages a combined bundle (system roots + Warden CA) inside the VM # and mounts it at /run/brig/ca-bundle.crt, plus sets SSL_CERT_FILE / @@ -127,6 +135,20 @@ class CellSpec: # for cells with strict cert pinning or that manage their own trust # store. See brig.cell.ca_bundle. trust_warden_ca: bool = True + # restart — "no" (default) or "always". An "always" cell is re-launched by + # `brig system up` whenever its container is gone (e.g. a VM restart drops + # every container). brig persists the full spec at run time to replay it. + # An *exited* cell (still present, e.g. `brig cell stop`) is left alone — but + # a VM restart drops the container, so a stopped restart:always cell DOES + # relaunch on the next up; use `brig cell rm` to keep one down for good. + restart: str = "no" + # user — podman --user (uid[:gid] or name[:group]); default is the image's + # USER. gVisor presents virtiofs host mounts (and /work) as owned by 0:0 + # inside the cell, so only a cell running as root (user: "0") can fully own + # a rw `mounts:` dir (read/write/rewrite). Writes still land owned by the + # operator on macOS, so readback is unaffected. Running as root *inside* the + # gVisor+VM sandbox is not a host-privilege change (the VM is the boundary). + user: str | None = None def __post_init__(self) -> None: """Validate inputs at construction time — the system boundary. diff --git a/src/brig/cell/validators.py b/src/brig/cell/validators.py index 0384fe2..accb64b 100644 --- a/src/brig/cell/validators.py +++ b/src/brig/cell/validators.py @@ -9,6 +9,7 @@ from __future__ import annotations +import math import re from typing import Any @@ -26,7 +27,10 @@ MAX_HOST_SERVICES_PER_CELL, MAX_HOST_SOCKETS_PER_CELL, MAX_INGRESS_PER_CELL, + MAX_MOUNTS_PER_CELL, MEMORY_PATTERN, + MOUNT_MODES, + MOUNT_NAME_PATTERN, SECRET_NAME_PATTERN, ) from brig.network.validation import is_suspicious_domain @@ -43,6 +47,15 @@ def _v_name(value: Any, context: str) -> list[str]: def _v_image(value: Any, context: str) -> list[str]: if not isinstance(value, str) or not value: return [f"'image' must be a non-empty string{context}"] + # A leading '-' lets the image reference be parsed by `podman run` as a + # flag (e.g. --runtime=runc downgrades gVisor, --privileged, -v /:/host), + # since the image is a positional argument. The CLI positional is guarded + # separately; this closes the yaml `image:` and SDK `image` paths. + if value.startswith("-"): + return [ + f"'image' must not start with '-' (would be parsed as a " + f"podman flag){context}" + ] return [] @@ -54,6 +67,11 @@ def _v_command(value: Any, context: str) -> list[str]: return [] +# POSIX environment variable name. Anchored so a padded/odd key (e.g. ' http_proxy') +# can't slip past the reconciler's proxy-override guard, which compares exact names. +_ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + def _v_env(value: Any, context: str) -> list[str]: errors: list[str] = [] if isinstance(value, dict): @@ -61,11 +79,17 @@ def _v_env(value: Any, context: str) -> list[str]: if not isinstance(k, str) or not isinstance(v, (str, int, float, bool)): errors.append(f"'env' keys must be strings, values must be primitives{context}") break + if not _ENV_NAME_RE.match(k): + errors.append(f"'env' key {k!r} is not a valid environment variable name{context}") + break elif isinstance(value, list): for item in value: if not isinstance(item, str) or "=" not in item: errors.append(f"'env' list items must be 'KEY=value' strings{context}") break + if not _ENV_NAME_RE.match(item.split("=", 1)[0]): + errors.append(f"'env' entry {item!r} has an invalid variable name{context}") + break else: errors.append(f"'env' must be a dict or list{context}") return errors @@ -108,16 +132,28 @@ def _v_memory(value: Any, context: str) -> list[str]: mem = str(value) if not re.match(MEMORY_PATTERN, mem): return [f"Invalid 'memory' format '{mem}': use format like '512m', '2g'{context}"] + # MEMORY_PATTERN matches "0"/"0g", but `podman run --memory 0` means + # UNLIMITED — silently defeating the profile caps. Require a positive size + # (mirrors _v_cpus rejecting <= 0). + from brig.cell.spec import parse_size + if parse_size(mem) <= 0: + return [f"'memory' must be greater than 0{context}"] return [] def _v_cpus(value: Any, context: str) -> list[str]: - if not isinstance(value, (int, float, str)): + # bool is an int subclass; reject it explicitly so True/False don't + # slip through as 1.0/0.0. + if isinstance(value, bool) or not isinstance(value, (int, float, str)): return [f"'cpus' must be a number or string{context}"] try: - float(value) + cpus = float(value) except ValueError: return [f"'cpus' must be a valid number{context}"] + # float() accepts 'inf'/'nan'; reject them and non-positive values + # rather than passing them to `podman run --cpus`, which fails opaquely. + if not math.isfinite(cpus) or cpus <= 0: + return [f"'cpus' must be a positive, finite number{context}"] return [] @@ -188,6 +224,27 @@ def _v_network(value: Any, context: str) -> list[str]: return [] +def _v_restart(value: Any, context: str) -> list[str]: + """Restart policy — only "no" (default) or "always".""" + if not isinstance(value, str) or value not in ("no", "always"): + return [f"'restart' must be 'no' or 'always'{context}"] + return [] + + +# podman --user: uid[:gid] or name[:group]. Restrict to predictable, shell-safe +# tokens (alphanumeric/_/-/., optional :group) — it's passed as a single podman +# arg, but reject anything unexpected at the boundary anyway. +_USER_PATTERN = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,31}(:[A-Za-z0-9_][A-Za-z0-9_.-]{0,31})?\Z") + + +def _v_user(value: Any, context: str) -> list[str]: + """Cell process user (podman --user) — uid[:gid] or name[:group].""" + if not isinstance(value, str) or not _USER_PATTERN.match(value): + return [f"'user' must be uid[:gid] or name[:group] (e.g. '0' or " + f"'1000:1000'){context}"] + return [] + + def _v_workspace_quota(value: Any, context: str) -> list[str]: from brig.cell.spec import parse_size if not isinstance(value, str): @@ -228,6 +285,19 @@ def _v_workspace_mount(value: Any, context: str) -> list[str]: return [f"'workspace_mount' must be an absolute path{context}"] if ".." in value.split("/"): return [f"'workspace_mount' must not contain '..'{context}"] + # Reject non-normalized paths (doubled/trailing slashes, '.' segments). + # The forbidden-prefix check below is lexical, so without this a value + # like '/run//host' slips past it while the kernel collapses it back to + # the protected '/run/host' at mount time. + # `//` is checked explicitly: os.path.normpath preserves a leading + # double slash (POSIX), so the normpath comparison alone would miss + # '//etc' even though the kernel collapses it to '/etc'. + import os.path as _ospath + if "//" in value or _ospath.normpath(value) != value: + return [ + f"'workspace_mount' must be a normalized path (no '.', repeated " + f"slashes, or trailing slash){context}" + ] if value == "/": return [f"'workspace_mount' must not be '/' (shadows rootfs){context}"] forbidden_prefixes = ( @@ -256,6 +326,82 @@ def _v_detach(value: Any, context: str) -> list[str]: return [] +def _v_labels(value: Any, context: str) -> list[str]: + if value is None: + return [] + # Accept the dict form (cell yaml) or the flat list of 'KEY=value' + # strings (the form CellSpec stores), matching how `env` is handled. + if isinstance(value, dict): + for k, v in value.items(): + if not isinstance(k, str) or not isinstance(v, (str, int, float, bool)): + return [f"'labels' keys must be strings, values must be primitives{context}"] + return [] + if isinstance(value, list): + for item in value: + if not isinstance(item, str) or "=" not in item: + return [f"'labels' list items must be 'KEY=value' strings{context}"] + return [] + return [f"'labels' must be a dict or list{context}"] + + +def _v_timeout(value: Any, context: str) -> list[str]: + if value is None: + return [] + from brig.cell.spec import parse_duration + if not isinstance(value, str): + return [f"'timeout' must be a string like '30s', '5m', '1h'{context}"] + seconds = parse_duration(value) + if seconds is None: + return [f"Invalid 'timeout' value '{value}': use format like '30s', '5m', '1h'{context}"] + # podman reads `--timeout 0` as "no timeout", silently disabling the cap; + # require a positive duration so the field always means what it says. + if seconds <= 0: + return [f"'timeout' must be greater than 0{context}"] + return [] + + +def _v_seccomp_profile(value: Any, context: str) -> list[str]: + # Mirrors the reconciler's runtime guard so a bad value fails at parse + # time. The profile is a filename resolved under ~/.brig/cells/seccomp/; + # a path or 'unconfined' would escape that directory or disable seccomp. + if value is None: + return [] + if not isinstance(value, str) or not value: + return [f"'seccomp_profile' must be a non-empty string{context}"] + if value.lower() == "unconfined": + return [f"'seccomp_profile' must not be 'unconfined' (disables seccomp){context}"] + if "/" in value or ".." in value: + return [f"'seccomp_profile' must be a filename, not a path{context}"] + return [] + + +def _v_workdir(value: Any, context: str) -> list[str]: + if value is None: + return [] + if not isinstance(value, str) or not value: + return [f"'workdir' must be a non-empty string{context}"] + if not value.startswith("/"): + return [f"'workdir' must be an absolute path{context}"] + parts = value.split("/") + if ".." in parts or "." in parts: + return [f"'workdir' must not contain '.' or '..' path segments{context}"] + if "//" in value: + return [f"'workdir' must not contain doubled slashes{context}"] + return [] + + +def _v_image_digest(value: Any, context: str) -> list[str]: + if value is None: + return [] + from brig.cell.lifecycle import _DIGEST_PATTERN + if not isinstance(value, str) or not _DIGEST_PATTERN.match(value.strip()): + return [ + f"Invalid 'image_digest' value: must be sha256:<64-hex> " + f"(or sha384/sha512){context}" + ] + return [] + + def _v_host_socket_entry( i: int, entry: Any, seen_names: set, seen_mounts: set, context: str, ) -> list[str]: @@ -444,14 +590,21 @@ def _v_host_service_entry( f"'host_services[{i}].protocol' must be 'http' or 'tcp'{context}" ) - # Reject TCP services on warden's reserved ports. + # TCP host_services become bound reverse-proxy listeners inside the warden + # container, which runs as the non-root `mitmproxy` user under --cap-drop + # ALL. A privileged port (<1024) can't be bound there and crashes the + # single mitmdump process — warden then fails to come up for ANY cell. + # HTTP services are virtual-domain rewrites, not bound listeners, so this + # only applies to TCP. if protocol == "tcp" and isinstance(port, int): - try: - from warden.proxy import WARDEN_RESERVED_PORTS - reserved = WARDEN_RESERVED_PORTS - except ImportError: - reserved = frozenset({8080, 8443}) - if port in reserved: + if port < 1024: + errors.append( + f"'host_services[{i}].port' {port} is privileged (<1024); the " + f"non-root warden process can't bind it. Use a port >= 1024" + f"{context}" + ) + from warden.proxy import WARDEN_RESERVED_PORTS + if port in WARDEN_RESERVED_PORTS: errors.append( f"'host_services[{i}].port' {port} is reserved by " f"warden (HTTP proxy / ingress); pick a different " @@ -488,6 +641,152 @@ def _v_host_services(value: Any, cell_def: dict, context: str) -> list[str]: return errors +# In-cell mount points that must never be shadowed by a `mounts:` entry — +# same set the workspace_mount validator protects. +_MOUNT_FORBIDDEN_PREFIXES = ( + "/proc", "/sys", "/dev", "/run/secrets", "/etc", "/run/host", "/run/brig", +) + + +def _v_cell_mount_point(value: Any, field: str, workspace_mount: str, + seen_mounts: set, context: str) -> list[str]: + """Validate an in-cell mount point: absolute, normalized, not shadowing a + system path or the cell's workspace mount, unique. Mirrors the + `_v_workspace_mount` checks (kept separate to avoid touching that path).""" + import os.path as _ospath + if not value or not isinstance(value, str): + return [f"'{field}' is required and must be a string{context}"] + if not value.startswith("/"): + return [f"'{field}' must be an absolute path{context}"] + if ".." in value.split("/"): + return [f"'{field}' must not contain '..'{context}"] + if ":" in value: + # ':' is the podman -v field separator — a mount point containing it + # would be misparsed into source/options. + return [f"'{field}' must not contain ':'{context}"] + if "//" in value or _ospath.normpath(value) != value: + return [f"'{field}' must be a normalized path (no '.', repeated or " + f"trailing slashes){context}"] + if value == "/": + return [f"'{field}' must not be '/' (shadows rootfs){context}"] + for p in _MOUNT_FORBIDDEN_PREFIXES: + if value == p or value.startswith(p + "/"): + return [f"'{field}' must not shadow {p}{context}"] + ws = _ospath.normpath(workspace_mount or "/work") + nv = _ospath.normpath(value) + # Reject equality AND either ancestor relationship — a mount at /work/data + # or at /work (when ws=/work/data) masks the workspace via mount-over-mount. + if nv == ws or nv.startswith(ws + "/") or ws.startswith(nv + "/"): + return [f"'{field}' must not shadow the workspace mount " + f"({workspace_mount or '/work'}){context}"] + norm = _ospath.normpath(value) + if norm in seen_mounts: + return [f"Duplicate mounts mount_point '{value}'{context}"] + seen_mounts.add(norm) + return [] + + +def _v_mount_entry(i: int, entry: Any, seen_names: set, seen_mounts: set, + roots: list[str], workspace_mount: str, + context: str) -> list[str]: + """Validate one `mounts:` entry. Mirrors `_v_host_socket_entry`. + + `roots` are the realpath-canonicalized, configured mount_roots; a + host_path whose realpath isn't under one of them is refused (the + allowlist is the boundary on the cell-yaml -> host-path edge). + """ + if not isinstance(entry, dict): + return [f"'mounts[{i}]' must be a dict{context}"] + import os.path as _ospath + errors: list[str] = [] + + name = entry.get("name") + if not name or not isinstance(name, str): + errors.append(f"'mounts[{i}].name' is required and must be a string{context}") + elif not MOUNT_NAME_PATTERN.match(name): + errors.append( + f"'mounts[{i}].name' must be lowercase alphanumeric with hyphens, " + f"max 31 chars{context}" + ) + elif name in seen_names: + errors.append(f"Duplicate mounts name '{name}'{context}") + else: + seen_names.add(name) + + host_path = entry.get("host_path") + if not host_path or not isinstance(host_path, str): + errors.append(f"'mounts[{i}].host_path' is required and must be a string{context}") + elif not host_path.startswith("/"): + errors.append(f"'mounts[{i}].host_path' must be absolute{context}") + elif ".." in host_path.split("/"): + errors.append(f"'mounts[{i}].host_path' must not contain '..'{context}") + elif not roots: + errors.append( + f"'mounts' requires mount_roots to be configured first: " + f"brig config set mount_roots {context}" + ) + else: + real = _ospath.realpath(host_path) + if not _ospath.isdir(real): + errors.append(f"'mounts[{i}].host_path' must be an existing directory{context}") + elif not any(real == r or real.startswith(r + "/") for r in roots): + errors.append( + f"'mounts[{i}].host_path' ({real}) must resolve under a " + f"configured mount_roots entry{context}" + ) + + errors.extend(_v_cell_mount_point( + entry.get("mount_point"), f"mounts[{i}].mount_point", + workspace_mount, seen_mounts, context, + )) + + mode = entry.get("mode", "ro") + if not isinstance(mode, str) or mode not in MOUNT_MODES: + errors.append( + f"'mounts[{i}].mode' must be one of: {', '.join(sorted(MOUNT_MODES))}{context}" + ) + + return errors + + +def _v_mounts(value: Any, cell_def: dict, context: str) -> list[str]: + """Validate the cell's `mounts:` list as a whole.""" + if not isinstance(value, list): + return [f"'mounts' must be a list{context}"] + + errors: list[str] = [] + if len(value) > MAX_MOUNTS_PER_CELL: + errors.append( + f"Too many mounts entries ({len(value)}), max {MAX_MOUNTS_PER_CELL}{context}" + ) + + if value and _profile_is_untrusted(cell_def.get("profile")): + errors.append( + f"'mounts' is not allowed with the untrusted profile — untrusted " + f"cells must not have host-directory mounts{context}" + ) + + import os.path as _ospath + from brig.config import mount_roots, validate_mount_roots + configured = mount_roots() + # The roots themselves must be safe (not '/', $HOME, ~/.ssh, …; unique + # slugs) — host_path-under-a-root is necessary but not sufficient. + if value: + errors.extend(f"{e}{context}" for e in validate_mount_roots(configured)) + roots = [_ospath.realpath(r.rstrip("/")) for r in configured] + workspace_mount = cell_def.get("workspace_mount", "/work") + + seen_names: set = set() + seen_mounts: set = set() + for i, entry in enumerate(value): + errors.extend( + _v_mount_entry(i, entry, seen_names, seen_mounts, roots, + workspace_mount, context) + ) + + return errors + + def _v_ingress_entry(i: int, entry: Any, seen_names: set, seen_prefixes: set, context: str) -> list[str]: """Validate one ingress entry. Mutates seen_* sets to track duplicates.""" @@ -563,6 +862,16 @@ def _v_ingress(value: Any, cell_def: dict, context: str) -> list[str]: net = cell_def.get("network", "default") if isinstance(net, str) and net == "none" and value: errors.append(f"'ingress' cannot be used with network='none' (airgapped){context}") + + # auth: none removes brig's perimeter gate — an untrusted cell must not be + # openly reachable without it (consistent with host_sockets / tls_passthrough). + if _profile_is_untrusted(cell_def.get("profile")): + for i, entry in enumerate(value): + if isinstance(entry, dict) and entry.get("auth") == "none": + errors.append( + f"'ingress[{i}].auth: none' is not allowed with the untrusted " + f"profile — untrusted cells must keep brig's token gate{context}" + ) return errors @@ -581,6 +890,13 @@ def _v_ingress(value: Any, cell_def: dict, context: str) -> list[str]: "writable_rootfs": _v_writable_rootfs, "trust_warden_ca": _v_trust_warden_ca, "detach": _v_detach, + "labels": _v_labels, + "timeout": _v_timeout, + "seccomp_profile": _v_seccomp_profile, + "workdir": _v_workdir, + "image_digest": _v_image_digest, + "restart": _v_restart, + "user": _v_user, } @@ -597,6 +913,13 @@ def validate_cell_definition(cell_def: dict[str, Any], file_path: str = "") -> l if "name" not in cell_def: errors.append(f"'name' is required{context}") + # Accept either the yaml-style nested `policy: {allow, deny, + # tls_passthrough}` or the flat `policy_allow / policy_deny / + # policy_passthrough_tls` form the SDK and CLI use. Fold flat into + # nested once here so _v_policy has exactly one shape to reason + # about — keeps the SDK from having to know the nested layout. + cell_def = _fold_flat_policy_into_nested(cell_def) + for field_name, validator in _SIMPLE_VALIDATORS.items(): if field_name in cell_def: errors.extend(validator(cell_def[field_name], context)) @@ -613,4 +936,33 @@ def validate_cell_definition(cell_def: dict[str, Any], file_path: str = "") -> l if "host_services" in cell_def: errors.extend(_v_host_services(cell_def["host_services"], cell_def, context)) + if "mounts" in cell_def: + errors.extend(_v_mounts(cell_def["mounts"], cell_def, context)) + return errors + + +def _fold_flat_policy_into_nested(cell_def: dict[str, Any]) -> dict[str, Any]: + """Merge `policy_allow / policy_deny / policy_passthrough_tls` keys + into a single `policy: {allow, deny, tls_passthrough}` dict. + + Returns a shallow copy when any folding happened; otherwise the + original dict is returned unchanged (cheap for the common + yaml-path). The flat form is what CellSpec stores and what the SDK + accepts; the nested form is what cell yaml uses and what + `_v_policy` reads. + """ + flat_keys = ("policy_allow", "policy_deny", "policy_passthrough_tls") + if not any(cell_def.get(k) for k in flat_keys): + return cell_def + out = dict(cell_def) + existing = out.get("policy") or {} + nested = dict(existing) if isinstance(existing, dict) else {} + for flat, target in (("policy_allow", "allow"), + ("policy_deny", "deny"), + ("policy_passthrough_tls", "tls_passthrough")): + if out.get(flat): + nested.setdefault(target, []) + nested[target] = list(nested[target]) + list(out[flat]) + out["policy"] = nested + return out diff --git a/src/brig/cli.py b/src/brig/cli.py index bababbe..18187a9 100644 --- a/src/brig/cli.py +++ b/src/brig/cli.py @@ -37,6 +37,11 @@ _HOST_ONLY_IMAGE = frozenset({"verify"}) +# `cell` verb aliases → canonical verb. argparse stores the literal alias the +# user typed, so dispatch (keyed on the canonical verb) normalizes via this map. +_CELL_VERB_ALIASES = {"ls": "list", "status": "inspect"} + + def _build_parser() -> argparse.ArgumentParser: """Build the argument parser for all brig commands.""" parser = argparse.ArgumentParser( @@ -58,6 +63,10 @@ def _build_parser() -> argparse.ArgumentParser: _add_image_group(sub) _add_system_group(sub) + # Top-level `brig ps` — docker-style shortcut for `brig cell list`. + p_ps = sub.add_parser("ps", help="List all cells (alias for `cell list`)") + p_ps.add_argument("--format", choices=["table", "wide", "json"], default="table") + # External groups registered by their command modules. from brig.commands import secrets_cmd, policy_cmd, config_cmd secrets_cmd.register_parser(sub) @@ -135,8 +144,11 @@ def _add_cell_group(sub: argparse._SubParsersAction) -> None: "top": "Show processes in cell", "diff": "Show filesystem changes since image base", } + # Convenience aliases for near-universal names (argparse normalizes the + # stored verb back to the primary, so dispatch needs no extra entries). + _ALIASES = {"inspect": ["status"]} for name, help_text in _ONE_ARG.items(): - p = cs.add_parser(name, help=help_text) + p = cs.add_parser(name, aliases=_ALIASES.get(name, []), help=help_text) p.add_argument("name", help="Cell name") p_rm = cs.add_parser( @@ -163,13 +175,26 @@ def _add_cell_group(sub: argparse._SubParsersAction) -> None: p_rename.add_argument("old_name", help="Current name") p_rename.add_argument("new_name", help="New name") - p_exec = cs.add_parser("exec", help="Execute command in cell") + p_exec = cs.add_parser( + "exec", help="Execute command in cell", + epilog="Flags (-i) must come before the cell name; everything after " + "the name is the command. Use '--' to pass flags to the command: " + "brig cell exec -i mycell -- ls -la", + ) p_exec.add_argument("name", help="Cell name") p_exec.add_argument("-i", "--interactive", action="store_true", help="Interactive mode") p_exec.add_argument("exec_cmd", nargs=argparse.REMAINDER, metavar="command", - help="Command to execute") + help="Command to execute (everything after the cell name)") - p_list = cs.add_parser("list", help="List all cells") + p_mscan = cs.add_parser( + "mount-scan", + help="Scan a cell's host mounts for symlinks escaping the mounted dir", + ) + p_mscan.add_argument("name", help="Cell name") + p_mscan.add_argument("--quarantine", action="store_true", + help="Remove escaping symlinks instead of only reporting") + + p_list = cs.add_parser("list", aliases=["ls"], help="List all cells") p_list.add_argument("--format", choices=["table", "wide", "json"], default="table") p_files = cs.add_parser("files", help="List workspace contents") @@ -183,12 +208,6 @@ def _add_cell_group(sub: argparse._SubParsersAction) -> None: p_read.add_argument("name", help="Cell name") p_read.add_argument("path", help="Relative path inside the workspace") - p_trace = cs.add_parser( - "trace", - help="Show a request trace from the OTel collector by trace_id", - ) - p_trace.add_argument("trace_id", help="Trace ID (or prefix; first match wins)") - p_logs = cs.add_parser("logs", help="View cell logs") p_logs.add_argument("name", help="Cell name") p_logs.add_argument("-f", "--follow", action="store_true", help="Follow log output") @@ -216,6 +235,12 @@ def _add_cell_group(sub: argparse._SubParsersAction) -> None: p_events.add_argument("-f", "--follow", action="store_true", help="Block and print new events as they arrive") + p_ingress = cs.add_parser( + "ingress", + help="Show reachable ingress URLs (and the token secret) for a cell", + ) + p_ingress.add_argument("name", help="Cell name") + def _add_image_group(sub: argparse._SubParsersAction) -> None: """`brig image ` — image build / pull / verify.""" @@ -250,10 +275,17 @@ def _add_image_group(sub: argparse._SubParsersAction) -> None: p_verify = isub.add_parser("verify", help="Verify image signature (cosign)") p_verify.add_argument("image", help="Image to verify") p_verify.add_argument("--key", help="Cosign public key") - p_verify.add_argument("--keyless", action="store_true", help="Keyless verification") + p_verify.add_argument( + "--keyless", action="store_true", + help="Keyless (Fulcio) verification — advisory only: without an " + "identity/issuer constraint it confirms the image is signed, not who " + "signed it", + ) - p_warmup = isub.add_parser("warmup", help="Pre-pull images for a profile") - p_warmup.add_argument("--profile", help="Profile name") + isub.add_parser( + "warmup", + help="Pre-pull the warden proxy base image into the VM cache", + ) def _add_system_group(sub: argparse._SubParsersAction) -> None: @@ -270,8 +302,7 @@ def _add_system_group(sub: argparse._SubParsersAction) -> None: ss.add_parser("metrics", help="Output Prometheus metrics") ss.add_parser("stats", help="Per-cell summary from the OTel collector") - p_verify = ss.add_parser("verify", help="Verify security invariants") - p_verify.add_argument("--fix", action="store_true", help="Auto-fix issues") + ss.add_parser("verify", help="Verify security invariants") p_doctor = ss.add_parser( "doctor", help="Check environment and report fixable issues", @@ -356,12 +387,6 @@ def _stats_dispatch(args): return cmd_stats(args) -def _trace_dispatch(args): - """Lazy import for the same reason as _stats_dispatch.""" - from brig.observability.traces import cmd_trace - return cmd_trace(args) - - def main() -> None: """CLI entry point.""" parser = _build_parser() @@ -400,7 +425,8 @@ def main() -> None: sys.exit(1) from brig.commands import ( - lifecycle_cmd, system_cmd, policy_cmd, + lifecycle_run, lifecycle_inspect, lifecycle_control, + system_cmd, policy_cmd, network_cmd, config_cmd, image_cmd, convenience_cmd, secrets_cmd, watchdog_cmd, ) @@ -410,35 +436,37 @@ def main() -> None: # argparse `set_defaults(func=...)`) so command-module imports stay # lazy — `brig --version` / `brig --help` don't pay the import cost. dispatch: dict = { - "run": lifecycle_cmd.cmd_run, + "run": lifecycle_run.cmd_run, + "ps": lifecycle_inspect.cmd_list, # cell * - ("cell", "list"): lifecycle_cmd.cmd_list, - ("cell", "inspect"): lifecycle_cmd.cmd_inspect, - ("cell", "export"): lifecycle_cmd.cmd_export, - ("cell", "stop"): lifecycle_cmd.cmd_stop, - ("cell", "kill"): lifecycle_cmd.cmd_kill, - ("cell", "start"): lifecycle_cmd.cmd_start, - ("cell", "restart"): lifecycle_cmd.cmd_restart, - ("cell", "pause"): lifecycle_cmd.cmd_pause, - ("cell", "unpause"): lifecycle_cmd.cmd_unpause, - ("cell", "attach"): lifecycle_cmd.cmd_attach, - ("cell", "shell"): lifecycle_cmd.cmd_shell, - ("cell", "wait"): lifecycle_cmd.cmd_wait, - ("cell", "rm"): lifecycle_cmd.cmd_rm, - ("cell", "preflight"): lifecycle_cmd.cmd_preflight, - ("cell", "rename"): lifecycle_cmd.cmd_rename, - ("cell", "exec"): lifecycle_cmd.cmd_exec, - ("cell", "files"): lifecycle_cmd.cmd_files, - ("cell", "read"): lifecycle_cmd.cmd_read, - ("cell", "trace"): _trace_dispatch, - ("cell", "logs"): lifecycle_cmd.cmd_logs, - ("cell", "top"): lifecycle_cmd.cmd_top, - ("cell", "diff"): lifecycle_cmd.cmd_diff, - ("cell", "stats"): lifecycle_cmd.cmd_stats, - ("cell", "cp"): lifecycle_cmd.cmd_cp, + ("cell", "list"): lifecycle_inspect.cmd_list, + ("cell", "inspect"): lifecycle_inspect.cmd_inspect, + ("cell", "export"): lifecycle_inspect.cmd_export, + ("cell", "mount-scan"): lifecycle_inspect.cmd_mount_scan, + ("cell", "stop"): lifecycle_control.cmd_stop, + ("cell", "kill"): lifecycle_control.cmd_kill, + ("cell", "start"): lifecycle_control.cmd_start, + ("cell", "restart"): lifecycle_control.cmd_restart, + ("cell", "pause"): lifecycle_control.cmd_pause, + ("cell", "unpause"): lifecycle_control.cmd_unpause, + ("cell", "attach"): lifecycle_control.cmd_attach, + ("cell", "shell"): lifecycle_control.cmd_shell, + ("cell", "wait"): lifecycle_control.cmd_wait, + ("cell", "rm"): lifecycle_control.cmd_rm, + ("cell", "preflight"): lifecycle_inspect.cmd_preflight, + ("cell", "rename"): lifecycle_control.cmd_rename, + ("cell", "exec"): lifecycle_control.cmd_exec, + ("cell", "files"): lifecycle_inspect.cmd_files, + ("cell", "read"): lifecycle_inspect.cmd_read, + ("cell", "logs"): lifecycle_inspect.cmd_logs, + ("cell", "top"): lifecycle_inspect.cmd_top, + ("cell", "diff"): lifecycle_inspect.cmd_diff, + ("cell", "stats"): lifecycle_inspect.cmd_stats, + ("cell", "cp"): lifecycle_inspect.cmd_cp, ("cell", "network"): network_cmd.cmd_network, ("cell", "events"): network_cmd.cmd_events, + ("cell", "ingress"): lifecycle_inspect.cmd_ingress, ("cell", "diagnose"): system_cmd.cmd_diagnose, # image * @@ -465,6 +493,10 @@ def main() -> None: if args.command in {"cell", "image", "system"}: sub_attr = f"{args.command}_command" sub_value = getattr(args, sub_attr) + # argparse stores the literal alias (e.g. "ls"); map it to the + # canonical verb the dispatch table is keyed on. + if args.command == "cell": + sub_value = _CELL_VERB_ALIASES.get(sub_value, sub_value) cmd_func = dispatch.get((args.command, sub_value)) cmd_name = f"{args.command}.{sub_value}" elif args.command == "policy": diff --git a/src/brig/commands/config_cmd.py b/src/brig/commands/config_cmd.py index 6a1bed7..0f06cec 100644 --- a/src/brig/commands/config_cmd.py +++ b/src/brig/commands/config_cmd.py @@ -5,7 +5,7 @@ from __future__ import annotations import json -from typing import Any +import argparse from brig.config import CONFIG_FILE from brig.errors import BrigError @@ -25,7 +25,7 @@ def register_parser(sub) -> None: s.add_parser("reset", help="Reset to defaults") -def cmd_config_show(args: Any) -> int: +def cmd_config_show(args: argparse.Namespace) -> int: """Handle `brig config show`.""" try: with open(CONFIG_FILE) as f: @@ -52,7 +52,7 @@ def cmd_config_show(args: Any) -> int: return 0 -def cmd_config_set(args: Any) -> int: +def cmd_config_set(args: argparse.Namespace) -> int: """Handle `brig config set`.""" try: with open(CONFIG_FILE) as f: @@ -71,19 +71,49 @@ def cmd_config_set(args: Any) -> int: # Dot-path set (e.g. "operation_logging.level"). parts = args.key.split(".") target = config - for part in parts[:-1]: + for i, part in enumerate(parts[:-1]): if part not in target: target[part] = {} + elif not isinstance(target[part], dict): + # A prior scalar set at this key blocks descending into it; raise a + # clear error instead of a TypeError on item assignment below. + conflict = ".".join(parts[: i + 1]) + raise BrigError( + f"Cannot set '{args.key}': '{conflict}' is already a value, " + f"not a section", + suggestion=f"brig config set {conflict} # overwrite it, OR " + f"pick a different key", + ) target = target[part] target[parts[-1]] = value + # mount_roots feeds lima.yaml + grants cells host access — vet it now so the + # operator gets immediate feedback instead of a later cell/VM failure. + if args.key == "mount_roots": + from brig.config import validate_mount_roots + # Mirror config.mount_roots()'s coercion: a list, or a comma-separated + # string. Any other JSON type is a usage error, not a list of paths. + if isinstance(value, list): + raw = value + elif isinstance(value, str): + raw = value.split(",") + else: + raise BrigError( + "mount_roots must be a list or a comma-separated string of " + "absolute paths" + ) + roots = [p.strip() for p in raw if isinstance(p, str) and p.strip()] + errs = validate_mount_roots(roots) + if errs: + raise BrigError("Invalid mount_roots:\n - " + "\n - ".join(errs)) + atomic_write_json(CONFIG_FILE, config) info(f"Set {args.key} = {args.value}") return 0 -def cmd_config_reset(args: Any) -> int: +def cmd_config_reset(args: argparse.Namespace) -> int: """Handle `brig config reset`.""" default = { "operation_logging": { diff --git a/src/brig/commands/convenience_cmd.py b/src/brig/commands/convenience_cmd.py index 5ed4f40..013dc52 100644 --- a/src/brig/commands/convenience_cmd.py +++ b/src/brig/commands/convenience_cmd.py @@ -1,5 +1,5 @@ """ -Convenience commands: brig up, brig down, brig profiles. +Convenience commands: brig system up, brig system down, brig system profiles. These reduce the multi-step setup to single commands. """ @@ -7,39 +7,56 @@ from __future__ import annotations import subprocess -from typing import Any +import argparse from brig.config import VM_NAME, HostPaths from brig.errors import BrigError -from brig.ops.logging import debug, output +from brig.ops.logging import info, output -def cmd_up(args: Any) -> int: - """Handle `brig up` — ensure VM + warden are running. +def cmd_up(args: argparse.Namespace) -> int: + """Handle `brig system up` — ensure VM + warden are running. Steps: - 1. Run brig init if ~/.brig doesn't exist. + 1. Run brig system init if ~/.brig doesn't exist. 2. Create Lima VM if it doesn't exist. 3. Start Lima VM if not running. 4. Start warden if not running. """ # Step 1: init. if not HostPaths.BRIG_HOME.exists(): - output("Initializing brig...") + info("Initializing brig...") from brig.commands.system_cmd import cmd_init - import types - cmd_init(types.SimpleNamespace()) + cmd_init(argparse.Namespace()) from brig.vm.shell import vm_exists, vm_running as check_vm + # Keep the VM's mount_roots block in sync with config — both the template + # (used on VM create) and the live instance config (read on `limactl start`), + # so a restart applies a mount_roots change. The drift check below confirms + # against the running VM. + from brig.vm.lima_mounts import sync_all_lima_mount_roots + try: + sync_all_lima_mount_roots() + except BrigError as e: + output(f" (warn) {e} — cells using `mounts:` may not start until the VM " + f"config's brig:mount_roots markers are repaired.") + + # Keep the deployed warden addons in lockstep with the installed package, so + # editing an addon can't leave warden running stale code. If anything + # changed, warden is reloaded below (mitmproxy doesn't reliably hot-reload + # sibling helper modules). + from brig.ops.addon_deploy import sync_addons + addons_changed = sync_addons() + # Step 2: create VM if needed. if not vm_exists(): lima_yaml = HostPaths.LIMA_YAML if not lima_yaml.exists(): output(f"ERROR: VM config not found at {lima_yaml}") - output(" Run: brig init") + output(" Run: brig system init") return 1 - output(f"Creating VM '{VM_NAME}'...") + info(f"Creating VM '{VM_NAME}'...") result = subprocess.run( ["limactl", "create", f"--name={VM_NAME}", str(lima_yaml)], ) @@ -48,75 +65,149 @@ def cmd_up(args: Any) -> int: # Step 3: start VM if not running. if not check_vm(): - output(f"Starting VM '{VM_NAME}'...") + info(f"Starting VM '{VM_NAME}'...") result = subprocess.run(["limactl", "start", VM_NAME]) if result.returncode != 0: return 1 else: - output("VM is running") + info("VM is running") + + # Step 3.5: start the OTel collector BEFORE warden so warden has an emit + # target from cold start (warden is always pointed at the collector's OTLP + # endpoint). Best-effort: observability is non-critical, so a collector + # failure (e.g. unpinned image) must not block bringing the harness up. + from brig.observability import collector + if collector.is_running(): + info("OTel collector is running") + else: + info("Starting OTel collector...") + try: + if not collector.start(): + output(" (warn) OTel collector failed to start — " + "metrics / `brig cell network --otel` will be unavailable") + except BrigError as e: + output(f" (warn) OTel collector not started ({e}) — " + f"metrics / --otel unavailable") # Step 4: start warden if not running. Use warden.proxy.is_running() # (the same check warden.proxy.start() uses internally) so cmd_up and # start() can't disagree about state. - from warden.proxy import is_running, start + from warden.proxy import is_running, start, stop if is_running(): - output("Warden is running") + if addons_changed: + info("Reloading warden (addons changed)...") + stop() + if not start(): + output("ERROR: Failed to restart warden after addon sync") + return 1 + else: + info("Warden is running") else: - output("Starting warden...") + info("Starting warden...") if not start(): output("ERROR: Failed to start warden") return 1 - output("") - output("Brig is ready. Run a cell with:") - output(" brig run alpine -- echo hello") + # Re-launch any `restart: always` cells that went away (e.g. the VM + # restart that dropped every container). Best-effort; runs after warden so + # restored cells have an egress proxy. + from brig.cell.lifecycle import restore_persisted_cells + restore_persisted_cells() + + # Drift check: warn if a configured mount_root isn't actually mounted in the + # already-running VM. The instance config is synced above, so the fix is a + # lossless stop/start (vz applies the new mounts on restart; images survive). + # Fires whenever the VM lacks a root, not just on the run that edited config. + from brig.config import mount_root_slug, mount_roots + from brig.vm.shell import vm_run + missing = [ + r for r in mount_roots() + if vm_run(["test", "-d", f"/mnt/host/{mount_root_slug(r)}"]).returncode != 0 + ] + if missing: + info( + f"NOTE: mount_roots {missing} are configured but not yet mounted in " + f"the running VM — apply with a lossless restart: brig system down " + f"--vm && brig system up (your images are preserved)." + ) + + info("") + info("Brig is ready. Run a cell with:") + info(" brig run alpine -- echo hello") return 0 -def cmd_down(args: Any) -> int: - """Handle `brig down` — stop everything. +def cmd_down(args: argparse.Namespace) -> int: + """Handle `brig system down` — stop everything. Steps: 1. Stop all running cells (via stop_cell so ingress / host_socket bridges are torn down consistently per-cell). - 2. Sweep any orphan host_socket bridges whose cell already exited. + 2. Sweep orphan host_socket bridges and ingress routes whose cell + already exited (subnet reuse would otherwise let a future cell + inherit the prior cell's hashed auth token). 3. Stop warden. 4. Optionally stop VM (--vm flag). + + A stop failure on one cell is logged and counted but doesn't strand + the others; the command exits non-zero when any cell stop failed so + the operator notices. """ from brig.cell.lifecycle import list_cell_containers, stop_cell + from brig.network.ingress import sweep_orphan_routes + from brig.network.subnet import list_all, reclaim_orphan_subnets - # Stop each running cell through the full lifecycle path. Wrap in - # try/except so a single misbehaving cell doesn't strand the others - # (e.g., already mid-shutdown, deregistered externally, etc.). + failures = 0 for cell_name, _entry in list_cell_containers(include_stopped=False): - output(f"Stopping {cell_name}...") + info(f"Stopping {cell_name}...") try: stop_cell(cell_name) except BrigError as e: - debug(f"stop_cell({cell_name}) returned: {e}") + output(f" ERROR: {e}") + failures += 1 - # Sweep orphan host_socket bridges — launchd plists whose cells were - # already stopped (or removed) before this `brig down`. Idempotent; - # no-op when there's nothing to clean. _bootout_all_host_socket_bridges() + # Self-heal subnet allocations whose podman network is gone (raw-podman + # kills, crashes mid-rm), then sweep ingress routes against what's left — + # so orphans don't accumulate until a manual prune. + reclaimed = reclaim_orphan_subnets() + if reclaimed: + info(f"Reclaimed {len(reclaimed)} orphan subnet allocation(s)") + allocated = {info.cell_name for info in list_all()} + swept = sweep_orphan_routes(live_cells=allocated) + if swept: + info(f"Swept {swept} orphan ingress route(s)") + # Stop warden. from warden.proxy import stop - output("Stopping warden...") + info("Stopping warden...") stop() + # Stop the OTel collector (started by cmd_up). Best-effort. + from brig.observability import collector + if collector.is_running(): + info("Stopping OTel collector...") + try: + collector.stop() + except Exception as e: # noqa: BLE001 — teardown must not fail `down` + output(f" (warn) failed to stop OTel collector: {e}") + # Optionally stop VM. if getattr(args, "vm", False): - output(f"Stopping VM '{VM_NAME}'...") + info(f"Stopping VM '{VM_NAME}'...") subprocess.run(["limactl", "stop", VM_NAME], check=False) - output("Brig stopped") + if failures: + output(f"Brig stopped with {failures} cell shutdown error(s)") + return 1 + info("Brig stopped") return 0 def _bootout_all_host_socket_bridges() -> None: """Enumerate every loaded host_socket bridge plist and bootout it, - regardless of which cell it belongs to. Used by `brig down` so we + regardless of which cell it belongs to. Used by `brig system down` so we don't leak orphan bridges across system restarts. """ from brig.cell.host_sockets_bridge import ( @@ -135,15 +226,15 @@ def _bootout_all_host_socket_bridges() -> None: cell_name = rest.split(".", 1)[0] cells_with_bridges.add(cell_name) for cell_name in cells_with_bridges: - output(f"Tearing down host_socket bridges for {cell_name}...") + info(f"Tearing down host_socket bridges for {cell_name}...") try: stop_cell_bridges(cell_name) except Exception as e: output(f" (warn) failed to bootout bridges for {cell_name}: {e}") -def cmd_profiles(args: Any) -> int: - """Handle `brig profiles` — list available trust profiles.""" +def cmd_profiles(args: argparse.Namespace) -> int: + """Handle `brig system profiles` — list available trust profiles.""" from brig.cell.profiles import BUILTIN_PROFILES output(f"{'NAME':<15} {'MEMORY':<8} {'CPUS':<6} {'PIDS':<6} {'NETWORK':<10} {'NOTES'}") diff --git a/src/brig/commands/image_cmd.py b/src/brig/commands/image_cmd.py index dd1262c..64b1205 100644 --- a/src/brig/commands/image_cmd.py +++ b/src/brig/commands/image_cmd.py @@ -9,7 +9,7 @@ import subprocess import tarfile from pathlib import Path -from typing import Any +import argparse from brig.vm.shell import vm_run from brig.errors import BrigError @@ -50,12 +50,12 @@ def _resolve_warden_ip() -> str: # Prefer the proxy-external bridge; that's the one the build # container's host-networking namespace can reach. ext = nets.get(PROXY_EXTERNAL_NETWORK) or {} - ip = ext.get("IPAddress", "") + ip = str(ext.get("IPAddress", "")) if ip: return ip raise BrigError( f"Warden is not attached to network '{PROXY_EXTERNAL_NETWORK}'", - suggestion="brig system restart (force warden to rejoin)", + suggestion="brig system down && brig system up (force warden to rejoin)", ) @@ -228,7 +228,7 @@ def _safe_filter(info: tarfile.TarInfo) -> tarfile.TarInfo | None: return buf.getvalue() -def cmd_build(args: Any) -> int: +def cmd_build(args: argparse.Namespace) -> int: """Handle `brig image build ` — build a container image. Brig runs podman inside the Lima VM, and only ~/.brig/* is mounted @@ -278,12 +278,11 @@ def cmd_build(args: Any) -> int: build_args += ["--build-arg", ba] # --use-warden routes build-time HTTP(S) traffic through warden - # using the standard HTTPS_PROXY pattern. Closes the build/runtime - # asymmetry aitelier reported: today's build path is fast+unfiltered, - # runtime is slow+MITM'd, so operators pre-bake ~230 MB binaries to - # avoid timeouts. With --use-warden the build hits the same warden - # path as runtime; subsequent layer caches amortize the warden - # overhead. + # using the standard HTTPS_PROXY pattern. Without it the build path is + # fast+unfiltered while runtime is slow+MITM'd, tempting operators to + # pre-bake large binaries to avoid timeouts. With --use-warden the build + # hits the same warden path as runtime; subsequent layer caches amortize + # the warden overhead. extra_mounts: list[str] = [] if getattr(args, "use_warden", False): from brig.cell.ca_bundle import VM_WARDEN_CA_FILE @@ -291,7 +290,7 @@ def cmd_build(args: Any) -> int: if not proxy_running(): raise BrigError( "Warden proxy is not running", - suggestion="Start with: brig up (or omit --use-warden)", + suggestion="Start with: brig system up (or omit --use-warden)", ) # Pre-check the warden CA file exists. We mount it into the # build container below; if it's missing podman build fails @@ -302,7 +301,7 @@ def cmd_build(args: Any) -> int: raise BrigError( f"Warden CA cert is missing at {VM_WARDEN_CA_FILE}", suggestion=( - "Bring warden up first: brig up\n" + "Bring warden up first: brig system up\n" " (warden generates its CA at startup; the build " "would have no cert to mount otherwise)" ), @@ -313,7 +312,8 @@ def cmd_build(args: Any) -> int: # We pass the gateway IP literal — DNS for `warden.brig` isn't # plumbed into the build container's resolv.conf. warden_ip = _resolve_warden_ip() - proxy_url = f"http://{warden_ip}:8080" + from brig.config import PROXY_PORT + proxy_url = f"http://{warden_ip}:{PROXY_PORT}" # NO_PROXY excludes localhost so build-time sidecars (test # servers, etc.) don't try to proxy through warden. The list is # what apt/curl/npm/pip all recognize. @@ -365,17 +365,22 @@ def cmd_build(args: Any) -> int: return 0 -def cmd_pull(args: Any) -> int: - """Handle `brig pull` — pull and cache an image. +def cmd_pull(args: argparse.Namespace) -> int: + """Handle `brig image pull` — pull and cache an image. Streams podman's progress directly to the user's terminal (instead of capturing it) so large pulls don't look frozen. podman writes layer-by-layer progress to stderr; with capture=False the user sees it live. """ + if args.image.startswith("-"): + raise BrigError( + f"'{args.image}' must not start with '-' (would be parsed as a " + f"podman flag)" + ) info(f"Pulling {args.image}...") result = vm_run( - ["podman", "pull", args.image], + ["podman", "pull", "--", args.image], capture=False, ) if result.returncode != 0: @@ -384,32 +389,40 @@ def cmd_pull(args: Any) -> int: return 0 -def cmd_warmup(args: Any) -> int: - """Handle `brig warmup` — pre-pull images for a profile.""" - from brig.cell.profiles import load_profile +def cmd_warmup(args: argparse.Namespace) -> int: + """Handle `brig image warmup` — pre-pull the warden proxy base image. - profile_name = getattr(args, "profile", None) - if profile_name: - load_profile(profile_name) # Validate profile exists. - - # Warmup just ensures the proxy image is available. + Profiles don't carry an image (they set resource/network/policy + defaults), so there's nothing profile-specific to warm; this just ensures + the mitmproxy base image the warden image builds from is cached. + """ output("Warming up proxy image...") + from warden.proxy import BASE_IMAGE result = vm_run( - ["podman", "image", "exists", - "docker.io/mitmproxy/mitmproxy@sha256:39ef4ec493d10bf07c71189961c7797b24c445e640ee133efba87fea80d19268"], + ["podman", "image", "exists", BASE_IMAGE], ) if result.returncode != 0: output("Pulling proxy image...") - vm_run( - ["podman", "pull", - "docker.io/mitmproxy/mitmproxy@sha256:39ef4ec493d10bf07c71189961c7797b24c445e640ee133efba87fea80d19268"], + pull = vm_run( + ["podman", "pull", BASE_IMAGE], ) + if pull.returncode != 0: + raise BrigError( + f"Failed to warm up proxy image: {pull.stderr.strip()}", + suggestion="Check VM/network connectivity, then retry; " + "brig system doctor", + ) output("Warmup complete") return 0 -def cmd_verify_image(args: Any) -> int: - """Handle `brig image-verify` — verify image signature.""" +def cmd_verify_image(args: argparse.Namespace) -> int: + """Handle `brig image verify` — verify image signature.""" + if args.image.startswith("-"): + raise BrigError( + f"'{args.image}' must not start with '-' (would be parsed as a " + f"cosign/podman flag)" + ) ok, msg, details = verify_image_signature( args.image, key=getattr(args, "key", None), diff --git a/src/brig/commands/lifecycle_cmd.py b/src/brig/commands/lifecycle_cmd.py deleted file mode 100644 index 70baa2b..0000000 --- a/src/brig/commands/lifecycle_cmd.py +++ /dev/null @@ -1,1079 +0,0 @@ -""" -CLI handlers for cell lifecycle commands. - -Thin wrappers: parse args -> call domain modules -> format output. -All podman commands route through vm_run() to execute inside the Lima VM. -""" - -from __future__ import annotations - -import json -import sys -from typing import Any - -from brig.cell.lifecycle import kill_cell, rm_cell, run_cell, stop_cell -from brig.cell.profiles import apply_profile, load_profile -from brig.cell.spec import CellSpec, load_cell_definition, validate_cell_definition -from brig.config import CONTAINER_PREFIX, container_name -from brig.errors import BrigError -from brig.ops.logging import info, output -from brig.vm.shell import vm_run, vm_run_interactive - - -_DIGEST_RE = __import__("re").compile(r"@sha(?:256|512):[0-9a-f]{40,}$") - - -def _warn_unverified_image(image: str) -> None: - """Stderr-only warning if `image` is from a registry and lacks a - digest pin. Local builds (localhost/* and dotless single names) and - digest-pinned refs are silent. - - Brig doesn't refuse — verification is a publishing trust decision - that varies by user. We just make the absence visible so a careless - `brig run someorg/their-image:latest` doesn't slip through quietly. - - Honors the `suppress_unverified_image_warn` config flag (set via - `brig config set suppress_unverified_image_warn true`) for users - who have made an explicit trust decision and want quiet runs. - """ - if not image: - return - if image.startswith("localhost/"): - return - if _DIGEST_RE.search(image): - return - if _suppress_unverified_warn(): - return - # No registry component (e.g. "alpine") is an implicit Docker Hub pull, - # which is the most common trust-by-default footgun. Warn anyway. - info( - f"WARN: image {image!r} is unpinned and unverified. " - f"Pin a digest (image@sha256:...) or verify with: " - f"brig image verify {image}\n" - f" (silence with: brig config set suppress_unverified_image_warn true)" - ) - - -def _suppress_unverified_warn() -> bool: - """Read the suppress flag from CONFIG_FILE. Missing/unreadable - config is treated as 'warn' (the safe default).""" - import json as _json - from brig.config import CONFIG_FILE - try: - with open(CONFIG_FILE) as f: - return bool(_json.load(f).get("suppress_unverified_image_warn", False)) - except (FileNotFoundError, _json.JSONDecodeError, OSError): - return False - - -def cmd_run(args: Any) -> int: - """Handle `brig run`.""" - # Catch the common foot-gun: `brig run alpine -m 512m sh` puts -m and 512m - # into the container command instead of being parsed as a brig flag. - # nargs=REMAINDER swallows everything after the image, so flags must - # precede the image. If args.image looks like a flag, the user almost - # certainly meant to put it before the image. - if args.image and args.image.startswith("-"): - raise BrigError( - f"'{args.image}' looks like a flag but appears in image position", - suggestion="Brig flags must precede the image name. Did you forget '--' " - "before the container command? e.g. brig run --memory 512m alpine -- sh", - ) - - # `brig run cells/foo` is a common confusion — that's a build context, - # not an image ref. If the arg looks like a host directory (contains - # '/' AND exists on disk as a dir), surface the build vs run distinction - # before podman tries to pull and fails opaquely. - if args.image and "/" in args.image and not args.image.startswith("localhost/"): - from pathlib import Path as _P - if _P(args.image).is_dir(): - raise BrigError( - f"'{args.image}' is a directory, not an image reference", - suggestion=( - f"Did you mean to build it first?\n" - f" brig image build {args.image}\n" - f" brig run localhost/{_P(args.image).name}:latest ..." - ), - ) - - # Catch the inverse: flags AFTER a valid image. nargs=REMAINDER swallows - # everything so `brig run alpine --memory 256m sh` puts `--memory 256m sh` - # into container_cmd. The cell starts and tries to exec `--memory` as the - # command, fails with "executable not found", and the user can't tell - # why. Flag a brig-style flag in container_cmd[0] before that happens. - _BRIG_FLAG_TOKENS = { - "--memory", "-m", "--cpus", "--name", "-n", "--env", "-e", - "--secret", "-s", "--profile", "--file", "-f", "--network", - "--detach", "-d", "--rm", "--timeout", "--workspace-quota", - "--label", "-l", "--pids-limit", "--image-digest", "--workdir", - "--policy-allow", "--policy-deny", - } - cmd_tail = args.container_cmd or [] - # Strip the optional leading -- separator before checking. - first = cmd_tail[1] if cmd_tail and cmd_tail[0] == "--" else (cmd_tail[0] if cmd_tail else None) - if first in _BRIG_FLAG_TOKENS: - raise BrigError( - f"'{first}' looks like a brig flag but appears after the image", - suggestion=( - "Brig flags must precede the image. If you meant to pass it to " - f"the container, escape it: brig run ... {args.image} -- {first} ..." - ), - ) - - if args.image: - _warn_unverified_image(args.image) - - # Strip leading -- from REMAINDER args. - container_cmd = args.container_cmd or [] - if container_cmd and container_cmd[0] == "--": - container_cmd = container_cmd[1:] - - if not args.image and not args.file: - raise BrigError("Image is required unless --file is specified") - - # Name resolution: --name flag wins, then the yaml's name: field (if any), - # then auto-generate. The previous order auto-generated before loading the - # file, which made `--file foo.yaml` with `name: my-cell` get an auto-name. - spec_kwargs: dict[str, Any] = { - "name": args.name or "", # may be filled from yaml below - "image": args.image or "", - "command": container_cmd, - "env": args.env or [], - "secrets": args.secret or [], - "labels": args.label or [], - "detach": args.detach, - "rm": args.rm, - "image_digest": getattr(args, "image_digest", None), - "workdir": getattr(args, "workdir", None), - } - - # Precedence: CLI flag > yaml > profile > defaults. Build up from - # least-specific to most-specific: apply profile first, then merge - # yaml on top, then CLI flag overrides below. - - if args.profile: - profile = load_profile(args.profile) - merged = apply_profile(spec_kwargs, profile) - spec_kwargs.update(merged) - - if args.file: - cell_def = load_cell_definition(args.file) - errors = validate_cell_definition(cell_def, args.file) - if errors: - raise BrigError("Invalid cell definition:\n - " + "\n - ".join(errors)) - # Special-cased fields: - # - image / name: --flag wins over yaml (handled by the - # `not getattr(args, key, None)` check). - # - command: --container_cmd (positional) wins over yaml. - # - env: additive — yaml entries appended to --env entries. - # - policy: nested {allow, deny} flattens to policy_allow / - # policy_deny, extending whatever the profile contributed. - # All other CellSpec-valid fields fall through to the generic merge. - for key in ("image", "name"): - if key in cell_def and not getattr(args, key, None): - spec_kwargs[key] = cell_def[key] - if "command" in cell_def and not args.container_cmd: - cmd_val = cell_def["command"] - spec_kwargs["command"] = cmd_val if isinstance(cmd_val, list) else [cmd_val] - if "env" in cell_def: - env_list = cell_def["env"] - if isinstance(env_list, dict): - env_list = [f"{k}={v}" for k, v in env_list.items()] - spec_kwargs["env"] = (args.env or []) + env_list - if isinstance(cell_def.get("policy"), dict): - for src_key, dst_key in (("allow", "policy_allow"), - ("deny", "policy_deny"), - ("tls_passthrough", "policy_passthrough_tls")): - src = cell_def["policy"].get(src_key) or [] - if src: - spec_kwargs[dst_key] = list(spec_kwargs.get(dst_key) or []) + list(src) - # host_services from yaml EXTEND the profile baseline, same as - # policy.allow/deny. A plain generic-merge would replace, which - # silently drops profile-declared services when the yaml has any. - if isinstance(cell_def.get("host_services"), list): - spec_kwargs["host_services"] = ( - list(spec_kwargs.get("host_services") or []) - + list(cell_def["host_services"]) - ) - # Generic merge for everything else the validator accepts. - import dataclasses as _dc - _spec_field_names = {f.name for f in _dc.fields(CellSpec)} - _already_handled = { - "image", "name", "command", "env", "ingress", "policy", - "host_services", - } - for key, val in cell_def.items(): - if key in _spec_field_names and key not in _already_handled: - spec_kwargs[key] = val - - if args.memory: - spec_kwargs["memory"] = args.memory - if args.cpus: - spec_kwargs["cpus"] = args.cpus - if args.pids_limit: - spec_kwargs["pids_limit"] = args.pids_limit - if args.network: - spec_kwargs["network"] = args.network - if args.timeout: - spec_kwargs["timeout"] = args.timeout - if args.workspace_quota: - spec_kwargs["workspace_quota"] = args.workspace_quota - # --policy-allow / --policy-deny EXTEND profile + yaml entries - # (consistent with how yaml extends the profile baseline). To - # replace, edit the yaml. Single-tenant: there's no security gain - # from "CLI clobbers all" because the operator owns every layer. - if args.policy_allow: - spec_kwargs["policy_allow"] = ( - list(spec_kwargs.get("policy_allow") or []) + list(args.policy_allow) - ) - if args.policy_deny: - spec_kwargs["policy_deny"] = ( - list(spec_kwargs.get("policy_deny") or []) + list(args.policy_deny) - ) - - # Ingress from cell definition file (no CLI flag — file-only). - if args.file: - if "ingress" in cell_def: - spec_kwargs["ingress"] = cell_def["ingress"] - - # Yaml is canonical: a `--file ` invocation must spell its - # name. CLI shorthand (`brig run alpine echo hi`) still auto-names. - if not spec_kwargs.get("name"): - if args.file: - raise BrigError( - f"Cell yaml {args.file} is missing required field: name", - suggestion="Add `name: ` to the yaml", - ) - from brig.cell.names import generate_name - spec_kwargs["name"] = generate_name() - info(f"Auto-generated name: {spec_kwargs['name']}") - - # Filter to CellSpec fields only — profiles may add extra keys like 'runtime'. - import dataclasses - valid_fields = {f.name for f in dataclasses.fields(CellSpec)} - spec_kwargs = {k: v for k, v in spec_kwargs.items() if k in valid_fields} - - spec = CellSpec(**spec_kwargs) - _sync_cell_policy(spec) - # If this cell declares TCP host_services on ports warden hasn't - # bound yet, warden needs to restart (mitmproxy can't hot-add - # listeners). Prompts the operator unless --yes is set; refuses to - # continue silently because warden restart drops every other - # cell's open egress for ~5s. - _maybe_restart_warden_for_tcp(spec, yes=getattr(args, "yes", False)) - - from brig.ops.logging import Spinner - with Spinner(f"Starting cell '{spec.name}'...") as spinner: - result = run_cell(spec) - if result.success: - spinner.success(f"Cell '{spec.name}' started") - else: - spinner.fail(f"Cell '{spec.name}' failed") - - if result.container_id: - output(result.container_id[:12]) - - # Detect cells that exit immediately and turn the cryptic outcome into - # actionable feedback. The most common causes — image's CMD prints help - # and exits, or the image tried to write somewhere read-only — look like - # brig brokenness from the user's side. - if result.success and spec.detach: - _check_immediate_exit(spec.name) - return 0 - - -def _maybe_restart_warden_for_tcp(spec: CellSpec, yes: bool = False) -> None: - """Restart warden if this cell needs a TCP listener that isn't bound. - - mitmproxy can't hot-add `--mode reverse:tcp` listeners, so a new TCP - host_service in a cell yaml requires warden restart for the listener - to come up. Restart kills every live cell's open egress for ~5s, - so we prompt the operator unless `--yes`. Operators who would - rather defer the restart get a clean abort + suggestion. - - No-op for cells without TCP host_services, or when warden already - has all the cell's ports bound. - """ - spec_tcp_ports = sorted({ - e["port"] for e in (spec.host_services or []) - if isinstance(e, dict) and e.get("protocol") == "tcp" - and isinstance(e.get("port"), int) - }) - if not spec_tcp_ports: - return - from warden.proxy import get_bound_tcp_ports - bound = set(get_bound_tcp_ports()) - missing = [p for p in spec_tcp_ports if p not in bound] - if not missing: - return - info( - f"Cell '{spec.name}' declares TCP host_services on port(s) " - f"{missing} that warden hasn't bound yet." - ) - if not yes: - output( - "Restarting warden will drop every running cell's open " - "egress connections for ~5s while the new listener binds." - ) - try: - answer = input("Proceed with warden restart? [y/N] ").strip().lower() - except (EOFError, KeyboardInterrupt): - answer = "n" - if answer not in ("y", "yes"): - raise BrigError( - "Aborted: warden restart declined", - suggestion=( - "Re-run with --yes to auto-confirm, OR\n" - " brig system restart # bind the new listener " - "manually, then re-run this cell" - ), - ) - from warden.proxy import start as warden_start, stop as warden_stop - info("Restarting warden to bind new TCP listener(s)...") - warden_stop() - if not warden_start(): - raise BrigError( - "Warden restart failed", - suggestion="brig system doctor", - ) - info(f"Warden restarted; TCP listener(s) on {missing} now bound") - - -def _check_immediate_exit(cell_name: str) -> None: - """Sleep briefly, then probe whether the cell already exited. If it - did, scan its stderr/stdout for known error patterns and append a - suggestion. Otherwise just note that it stopped quickly. - - Called after detached `brig run` so users get a signal when the cell - they thought started is actually already gone. Best-effort: any - error here is swallowed (we don't want to fail the run on a - diagnostic). - """ - import time - time.sleep(1.5) - try: - cn = container_name(cell_name) - status = vm_run( - ["podman", "inspect", cn, "--format", "{{.State.Status}}"], - timeout=5, - ) - if status.returncode != 0: - return - if status.stdout.strip() == "running": - return # All good — cell is still alive. - - # Cell exited quickly. Pull recent logs and look for a known cause. - logs = vm_run( - ["podman", "logs", "--tail", "50", cn], timeout=5, - ) - log_text = (logs.stdout or "") + (logs.stderr or "") - hint = _diagnose_exit(log_text) - info( - f"NOTE: cell '{cell_name}' exited shortly after start.{hint} " - f"See: brig cell logs {cell_name}" - ) - except Exception: # noqa: BLE001 — pure diagnostic, never fail the run - pass - - -def _diagnose_exit(log_text: str) -> str: - """Heuristic — match common failure patterns and return an actionable - fragment that fits into the NOTE: line. Returns '' if no match.""" - s = log_text.lower() - if "read-only file system" in s or "errno 30" in s: - # Lead with the writable paths because relocating writes is the - # better fix; writable_rootfs:true is the escape hatch. - return ( - " Image tried to write to a read-only path. Writable paths " - "inside the cell: /work (workspace, persists), /tmp (tmpfs, " - "64m), /run (tmpfs, 16m). Common fix is to redirect writes " - "into one of these — e.g. `export HOME=/tmp/home` and stage " - "credentials there. If the image legitimately needs to write " - "outside those paths, set writable_rootfs: true in the cell spec." - ) - if "executable file not found" in s and "bash" in s: - return ( - " The image likely doesn't ship /bin/bash; try sh." - ) - return "" - - -def _sync_cell_policy(spec: CellSpec) -> None: - """Write the cell's allow / deny / host_services to its per-cell - policy file. Replace semantics: the yaml is the source of truth, - so anything in the file that's no longer in the spec is dropped. - - Skips the write if the on-disk policy already matches the spec — - idempotent re-runs don't churn mtime or noise the audit log. - """ - from brig.policy.policy import load_cell_policy, save_cell_policy - - desired = { - "allow": list(spec.policy_allow or []), - "deny": list(spec.policy_deny or []), - "host_services": list(spec.host_services or []), - "tls_passthrough": list(spec.policy_passthrough_tls or []), - } - existing = load_cell_policy(spec.name) or {} - current = { - "allow": existing.get("allow", []), - "deny": existing.get("deny", []), - "host_services": existing.get("host_services", []), - "tls_passthrough": existing.get("tls_passthrough", []), - } - if desired == current: - return - - # Preserve any other keys the on-disk policy may carry (e.g. - # rate_limits) so we don't accidentally drop unrelated config. - merged = dict(existing) - merged.update(desired) - save_cell_policy(spec.name, merged) - - # Log host_services diffs (the only field with operator-visible - # semantics worth flagging — allow/deny changes are too noisy to - # log line-by-line). - def _names(items): - return {e["name"] for e in items if isinstance(e, dict) and "name" in e} - added = _names(desired["host_services"]) - _names(current["host_services"]) - removed = _names(current["host_services"]) - _names(desired["host_services"]) - for n in sorted(added): - info(f"host_service granted: {spec.name} → {n} (from cell yaml)") - for n in sorted(removed): - info(f"host_service revoked: {spec.name} → {n} (no longer in cell yaml)") - - -def cmd_preflight(args: Any) -> int: - """Handle `brig cell preflight `. - - Dry-run check: parses the yaml and verifies every host-side - requirement it implies (declared secrets exist, declared - host_socket targets exist, declared ingress entries have a - token secret, image exists or is buildable). Prints a checklist - with a one-line fix for each gap. - - Replaces the iterative "brig run → error → fix one thing → - re-run" loop with a single diff. No mutations; safe to run - anytime. - """ - from brig.cell.spec import load_cell_definition, validate_cell_definition - from brig.config import HostPaths - - cell_def = load_cell_definition(args.file) - errors = validate_cell_definition(cell_def, args.file) - fail_count = 0 - - def _check(label: str, ok: bool, fix: str = "") -> None: - nonlocal fail_count - marker = "OK " if ok else "FAIL" - output(f" [{marker}] {label}") - if not ok: - fail_count += 1 - if fix: - output(f" fix: {fix}") - - cell_name = cell_def.get("name", "") - output(f"Preflight for cell '{cell_name}' ({args.file})") - output("=" * 60) - - # 1. Spec validity. - _check( - "cell yaml validates", - not errors, - fix=("Fix errors:\n " + "\n ".join(errors) - if errors else ""), - ) - - # 2. Secrets exist. - for secret in cell_def.get("secrets", []): - if not isinstance(secret, str): - continue - p = HostPaths.SECRETS_DIR / secret - _check( - f"secret: {secret}", p.exists(), - fix=f"brig secrets add {secret}", - ) - - # 3. Ingress token (if any ingress entry uses auth: token). - ingress_entries = cell_def.get("ingress") or [] - needs_token = any( - isinstance(e, dict) and e.get("auth") == "token" for e in ingress_entries - ) - if needs_token: - primary = HostPaths.SECRETS_DIR / f"{cell_name}-ingress-token" - fallback = HostPaths.SECRETS_DIR / "ingress-token" - ok = primary.exists() or fallback.exists() - _check( - f"ingress token: {primary.name}", ok, - fix=f"openssl rand -hex 32 | brig secrets add {primary.name} -", - ) - - # 4. host_sockets — targets exist and are sockets on the host. - import os as _os - import stat as _stat - for entry in cell_def.get("host_sockets") or []: - if not isinstance(entry, dict): - continue - host_path = entry.get("host_path", "") - name = entry.get("name", "?") - try: - st = _os.lstat(host_path) - is_sock = _stat.S_ISSOCK(st.st_mode) and not _stat.S_ISLNK(st.st_mode) - except (FileNotFoundError, OSError): - is_sock = False - _check( - f"host_socket target: {name} → {host_path}", is_sock, - fix="Start the service that provides this socket, or " - "correct host_path.", - ) - - # 5. socat installed if host_sockets are declared. - if cell_def.get("host_sockets"): - import shutil as _shutil - _check( - "socat installed (host_sockets bridge dependency)", - bool(_shutil.which("socat")), - fix="brew install socat", - ) - - output("=" * 60) - if fail_count: - output(f"FAILED: {fail_count} check(s) — fix above, then re-run") - return 1 - output("All preflight checks passed.") - return 0 - - -def cmd_stop(args: Any) -> int: - stop_cell(args.name) - return 0 - - -def cmd_kill(args: Any) -> int: - kill_cell(args.name) - return 0 - - -def cmd_rm(args: Any) -> int: - import sys - from brig.cell.lifecycle import _workspace_has_content - - keep = getattr(args, "keep_workspace", False) - # If the cell's workspace has files and the user didn't explicitly - # opt to keep or force, ask before deleting. Closes the silent-data-loss - # foot-gun where the user expects docker semantics (rm preserves - # volumes) and loses unexpected data. - if not keep and not args.force and _workspace_has_content(args.name): - if not sys.stdin.isatty(): - raise BrigError( - f"Cell '{args.name}' has files in its workspace; refusing to " - f"delete non-interactively.", - suggestion=( - f"brig cell rm {args.name} --keep-workspace # preserve files\n" - f" OR brig cell rm -f {args.name} # force delete" - ), - ) - prompt = ( - f"Cell '{args.name}' workspace contains files. " - f"Delete? [y/N/keep] " - ) - try: - answer = input(prompt).strip().lower() - except EOFError: - answer = "" - if answer in ("k", "keep"): - keep = True - elif answer not in ("y", "yes"): - output("Aborted.") - return 1 - - rm_cell(args.name, force=args.force, keep_workspace=keep) - return 0 - - -def cmd_list(args: Any) -> int: - from brig.cell.lifecycle import list_cell_containers - fmt = getattr(args, "format", "table") - cells = list_cell_containers(include_stopped=True) - - if fmt == "json": - output(json.dumps([entry for _, entry in cells], indent=2)) - return 0 - - if not cells: - output("No cells found") - return 0 - - if fmt == "wide": - output(f"{'NAME':<25} {'STATUS':<12} {'CREATED':<22} {'NETWORK':<25} {'IMAGE'}") - for cell, c in cells: - networks = c.get("Networks") or [] - network = ",".join(networks)[:25] if networks else "-" - created = c.get("CreatedAt", "")[:22] - output(f"{cell:<25} {c.get('State', ''):<12} {created:<22} {network:<25} {c.get('Image', '')}") - else: - output(f"{'NAME':<25} {'STATUS':<12} {'IMAGE':<30}") - for cell, c in cells: - output(f"{cell:<25} {c.get('State', ''):<12} {c.get('Image', ''):<30}") - return 0 - - -def cmd_inspect(args: Any) -> int: - cn = container_name(args.name) - result = vm_run(["podman", "inspect", cn, "--format", "json"]) - if result.returncode != 0: - raise BrigError( - f"Cell '{args.name}' does not exist", - suggestion="Use 'brig list' to see available cells", - ) - output(result.stdout) - return 0 - - -def cmd_files(args: Any) -> int: - """List workspace contents inside a cell.""" - cn = container_name(args.name) - path = getattr(args, "path", "/work") - return vm_run_interactive(["podman", "exec", cn, "ls", "-la", path]) - - -def cmd_read(args: Any) -> int: - """Handle `brig cell read ` — stream a workspace file - to stdout via the race-free safe_open primitive. - - This is the language-agnostic safe-read replacement for the (now - removed) workspace.host_path field in cell.json. Consumers in any - language can do: - - brig cell read mycell input.json > /tmp/local-copy - - instead of opening the host path directly. Each path component is - walked with O_NOFOLLOW so a cell-planted symlink raises - WorkspaceEscape rather than letting the host follow it. - """ - import sys - from brig.workspace.validation import safe_open, WorkspaceEscape - - try: - with safe_open(args.name, args.path, "rb") as f: - # Stream in chunks so a large file doesn't blow RAM. - while True: - chunk = f.read(64 * 1024) - if not chunk: - break - sys.stdout.buffer.write(chunk) - return 0 - except WorkspaceEscape as e: - raise BrigError( - f"Refused: {e}", - suggestion="A path component is a symlink or escapes the workspace root.", - ) - except FileNotFoundError: - raise BrigError(f"Not found: {args.path} in cell '{args.name}'") - - -def cmd_logs(args: Any) -> int: - """Tail a cell's container stdout/stderr (wraps `podman logs`). - - Empty output usually means the cell's app writes to a file inside - the container instead of stdout — common for daemons and agent - runtimes (SA, claude-acp, etc.). In that case, use - `brig cell exec -- cat /var/log/.log` or - `brig cell read ` to inspect the file directly. - - Hermes-feedback-driven: detect the empty case for the non-follow - path and surface the hint inline so operators don't waste time - wondering why `brig cell logs` is silent. - """ - cn = container_name(args.name) - cmd = ["podman", "logs"] - follow = getattr(args, "follow", False) - if follow: - cmd.append("-f") - if getattr(args, "tail", None): - cmd.extend(["--tail", str(args.tail)]) - cmd.append(cn) - # Follow mode wants TTY passthrough; can't capture. - if follow: - return vm_run_interactive(cmd) - # Snapshot mode: capture so we can detect empty output and hint. - result = vm_run(cmd, capture=True, timeout=30) - if result.returncode == 0: - sys.stdout.write(result.stdout) - sys.stderr.write(result.stderr) - # Empty stdout + stderr likely means file-based logging. - if not (result.stdout.strip() or result.stderr.strip()): - from brig.ops.logging import info - info( - f"(no stdout/stderr from '{args.name}' — if the app logs " - f"to a file, try: brig cell exec {args.name} -- " - f"cat /var/log/.log)" - ) - return 0 - sys.stderr.write(result.stderr) - return result.returncode - - -def _parse_cp_target(spec: str) -> tuple[str, str] | None: - """Parse 'cell:/path' into (cell, path), or return None if not a cell ref. - - Only treats the colon as a cell separator when the prefix matches the - canonical cell-name pattern. Avoids misreading paths containing colons - (e.g. './out:put.txt') as cell references. - """ - from brig.config import CELL_NAME_PATTERN - if ":" not in spec or spec.startswith("/") or spec.startswith("."): - return None - head, tail = spec.split(":", 1) - if not CELL_NAME_PATTERN.match(head): - return None - return head, tail - - -def cmd_cp(args: Any) -> int: - """Copy files to/from a cell with safety checks. - - Detects direction from the colon syntax (cell:path). The cell prefix - must match the canonical cell-name pattern, so a literal path like - './out:put.txt' is not misread as a cell reference. - Exports apply quarantine xattr and extension blocking by default. - """ - src, dst = args.src, args.dst - src_target = _parse_cp_target(src) - dst_target = _parse_cp_target(dst) - - if src_target and dst_target: - raise BrigError( - "Cannot copy between two cells", - suggestion="Copy from cell to host first, then from host to cell", - ) - if src_target: - from brig.workspace.workspace import copy_out - copy_out(src_target[0], src, dst, sanitize=True) - elif dst_target: - from brig.workspace.workspace import copy_in - copy_in(dst_target[0], src, dst) - else: - raise BrigError( - "Could not determine copy direction", - suggestion="Use cell:path syntax, e.g.: brig cp mycell:/work/out.txt ./", - ) - return 0 - - -def cmd_exec(args: Any) -> int: - cn = container_name(args.name) - cmd = ["podman", "exec"] - if getattr(args, "interactive", False): - cmd.append("-it") - cmd.append(cn) - cmd.extend(args.exec_cmd) - return vm_run_interactive(cmd) - - -def cmd_shell(args: Any) -> int: - cn = container_name(args.name) - return vm_run_interactive(["podman", "exec", "-it", cn, "/bin/sh"]) - - -def cmd_attach(args: Any) -> int: - cn = container_name(args.name) - return vm_run_interactive(["podman", "attach", cn]) - - -def cmd_start(args: Any) -> int: - # Invariant 9: proxy must be running before starting cells. - from brig.network.proxy import proxy_running - if not proxy_running(): - raise BrigError( - "Warden proxy is not running", - suggestion="Start with: brig up", - ) - # Refresh the cell metadata so /run/brig/cell.json's `started_at` - # reflects this start, not the original create. workspace_mount is - # preserved from whatever the cell was created with (bind mounts are - # fixed at container-create time; we can't change them on start). - _refresh_metadata_for_start(args.name) - cn = container_name(args.name) - result = vm_run(["podman", "start", cn]) - if result.returncode != 0: - raise BrigError( - f"Failed to start cell '{args.name}': {result.stderr.strip()}", - suggestion="Check if cell exists with: brig cell list", - ) - # Replay ingress registration with the freshly-inspected cell IP. - # podman may assign a different IP after stop/start, leaving the - # routes file pointing at a stale address; without this, external - # requests through warden's :8443 reverse proxy return 502 until - # the operator does `brig cell rm + brig run --file` to rebuild. - from brig.cell.lifecycle import register_ingress_for - from brig.cell.metadata import read_ingress - ingress_entries = read_ingress(args.name) - if ingress_entries: - register_ingress_for(args.name, ingress_entries) - info(f"Cell '{args.name}' started") - return 0 - - -def _refresh_metadata_for_start(cell_name: str) -> None: - """Rewrite /run/brig/cell.json on restart with a fresh `started_at`. - - Preserves the original workspace_mount, host_sockets, and ingress — - bind mounts and ingress configuration are fixed at create time, so - these come from the prior metadata write rather than being re-derived. - If the file is missing or unreadable (cell predates cell.json), write - a default-mount fallback. - """ - from brig.cell.metadata import refresh_metadata_if_present, write_metadata - if refresh_metadata_if_present(cell_name) is None: - write_metadata(cell_name, "/work") - - -def cmd_restart(args: Any) -> int: - """Handle `brig cell restart ` — stop (if running) then start. - - Composite of stop_cell + cmd_start. Refreshes the cell metadata's - started_at via cmd_start's existing path. - """ - from brig.cell.lifecycle import observe, stop_cell - actual = observe(args.name) - if not actual.exists: - raise BrigError( - f"Cell '{args.name}' does not exist", - suggestion="brig cell list # see what's there", - ) - if actual.running: - stop_cell(args.name) - return cmd_start(args) - - -def cmd_wait(args: Any) -> int: - cn = container_name(args.name) - result = vm_run(["podman", "wait", cn], timeout=None) - if result.returncode != 0: - raise BrigError( - f"Cell '{args.name}' does not exist", - suggestion="Use 'brig list' to see available cells", - ) - exit_code = result.stdout.strip() - output(exit_code) - return int(exit_code) if exit_code.isdigit() else 1 - - -def cmd_pause(args: Any) -> int: - cn = container_name(args.name) - result = vm_run(["podman", "pause", cn]) - if result.returncode != 0: - raise BrigError(f"Failed to pause cell '{args.name}': {result.stderr.strip()}") - info(f"Cell '{args.name}' paused") - return 0 - - -def cmd_unpause(args: Any) -> int: - cn = container_name(args.name) - result = vm_run(["podman", "unpause", cn]) - if result.returncode != 0: - raise BrigError(f"Failed to unpause cell '{args.name}': {result.stderr.strip()}") - info(f"Cell '{args.name}' unpaused") - return 0 - - -def cmd_rename(args: Any) -> int: - from brig.config import CELL_NAME_PATTERN - if not CELL_NAME_PATTERN.match(args.new_name): - raise BrigError( - f"Invalid cell name '{args.new_name}': must match {CELL_NAME_PATTERN.pattern}", - suggestion="Cell names: lowercase alphanumeric, hyphens, dots, max 63 chars", - ) - old_cn = container_name(args.old_name) - new_cn = container_name(args.new_name) - result = vm_run(["podman", "rename", old_cn, new_cn]) - if result.returncode != 0: - raise BrigError(f"Failed to rename: {result.stderr.strip()}") - info(f"Renamed '{args.old_name}' to '{args.new_name}'") - return 0 - - -def cmd_top(args: Any) -> int: - cn = container_name(args.name) - return vm_run_interactive(["podman", "top", cn]) - - -def cmd_diff(args: Any) -> int: - cn = container_name(args.name) - return vm_run_interactive(["podman", "diff", cn]) - - -def cmd_stats(args: Any) -> int: - cmd = ["podman", "stats", "--no-stream"] - if hasattr(args, "name") and args.name: - cmd.append(container_name(args.name)) - else: - cmd.extend(["--filter", f"name=^{CONTAINER_PREFIX}"]) - return vm_run_interactive(cmd) - - -def cmd_export(args: Any) -> int: - """Export a running cell's config as a reusable YAML cell definition.""" - cn = container_name(args.name) - result = vm_run(["podman", "inspect", cn, "--format", "json"]) - if result.returncode != 0: - raise BrigError( - f"Cell '{args.name}' does not exist", - suggestion="Use 'brig list' to see available cells", - ) - - try: - data = json.loads(result.stdout) - if isinstance(data, list): - data = data[0] - except json.JSONDecodeError: - raise BrigError("Could not parse container info") - - # Extract cell definition from container inspect. - host_config = data.get("HostConfig", {}) - config = data.get("Config", {}) - - cell_def = {"name": args.name} - - image = config.get("Image", "") - if image: - cell_def["image"] = image - - cmd = config.get("Cmd") - if cmd: - cell_def["command"] = cmd - - # Extract non-proxy env vars. - proxy_prefixes = ("http_proxy=", "https_proxy=", "HTTP_PROXY=", "HTTPS_PROXY=", "no_proxy=") - env_vars = [ - e for e in (config.get("Env") or []) - if not any(e.startswith(p) for p in proxy_prefixes) - and not e.endswith("_FILE=/run/secrets/" + e.split("=")[0].replace("_FILE", "").lower()) - ] - if env_vars: - cell_def["env"] = env_vars - - memory = host_config.get("Memory", 0) - if memory: - if memory >= 1024**3: - cell_def["memory"] = f"{memory // 1024**3}g" - elif memory >= 1024**2: - cell_def["memory"] = f"{memory // 1024**2}m" - - cpus = host_config.get("NanoCpus", 0) - if cpus: - cell_def["cpus"] = str(cpus / 1e9) - - pids = host_config.get("PidsLimit", 0) - if pids and pids > 0: - cell_def["pids_limit"] = pids - - # Per-cell policy (allow / deny / host_services). The yaml is - # canonical, the per-cell policy file mirrors it, and warden - # reads that file — so it's the authoritative running state. - from brig.policy.policy import load_cell_policy - cell_policy = load_cell_policy(args.name) or {} - allow = cell_policy.get("allow") or [] - deny = cell_policy.get("deny") or [] - if allow or deny: - policy_block: dict = {} - if allow: - policy_block["allow"] = list(allow) - if deny: - policy_block["deny"] = list(deny) - cell_def["policy"] = policy_block - host_services = cell_policy.get("host_services") or [] - if host_services: - cell_def["host_services"] = list(host_services) - - # Ingress routes (from the host-side ingress-routes.json, written - # at cell start by lifecycle._register_cell_ingress). - from brig.config import HostPaths - ingress = _ingress_for_cell(args.name, HostPaths.INGRESS_ROUTES_FILE) - if ingress: - cell_def["ingress"] = ingress - - # host_sockets — projected to {name, mount_point, mode} (host_path - # stays off the wire for the same reason it's omitted from the - # downward-API metadata). - sockets = _host_sockets_from_metadata(args.name) - if sockets: - cell_def["host_sockets"] = sockets - - output(f"# Cell definition exported from '{args.name}'") - output("# Self-contained: brig run --file will reproduce the cell.") - output("") - _emit_yaml(cell_def) - return 0 - - -def _ingress_for_cell(cell_name: str, routes_file) -> list: - try: - data = json.loads(routes_file.read_text()) - except (FileNotFoundError, json.JSONDecodeError, OSError): - return [] - routes = data.get("routes") or [] - return [ - {"name": r.get("name"), - "port": r.get("port"), - "path_prefix": r.get("path_prefix"), - "auth": "token"} - for r in routes - if isinstance(r, dict) and r.get("cell") == cell_name - ] - - -def _host_sockets_from_metadata(cell_name: str) -> list: - from brig.cell.metadata import _host_metadata_path - try: - meta = json.loads(_host_metadata_path(cell_name).read_text()) - except (FileNotFoundError, json.JSONDecodeError, OSError): - return [] - entries = meta.get("host_sockets") or [] - return [ - {"name": e["name"], "mount_point": e["mount_point"]} - for e in entries - if isinstance(e, dict) and "name" in e and "mount_point" in e - ] - - -def _emit_yaml(d: dict) -> None: - """Minimal yaml emitter that handles the shapes cell_def uses - (scalars, list of scalars, list of dicts, nested dicts). Avoids - pulling in pyyaml just for output. - - Strings are always JSON-quoted to dodge yaml metachars: bare - `*.example.com` parses as a yaml alias reference, bare `1.0` as - a float, leading-`!`/`&`/`>`/`|`/`#` as tag/anchor/block-scalar - markers. JSON quoting is yaml-valid for any string and keeps the - round-trip through load_cell_definition correct. - """ - def _scalar(v): - return json.dumps(v) - - for key, val in d.items(): - if isinstance(val, dict): - output(f"{key}:") - for k, v in val.items(): - if isinstance(v, list): - output(f" {k}:") - for item in v: - output(f" - {_scalar(item)}") - else: - output(f" {k}: {_scalar(v)}") - elif isinstance(val, list): - if val and all(isinstance(x, dict) for x in val): - output(f"{key}:") - for item in val: - first = True - for k, v in item.items(): - prefix = " - " if first else " " - output(f"{prefix}{k}: {_scalar(v)}") - first = False - else: - output(f"{key}:") - for item in val: - output(f" - {_scalar(item)}") - else: - output(f"{key}: {_scalar(val)}") diff --git a/src/brig/commands/lifecycle_control.py b/src/brig/commands/lifecycle_control.py new file mode 100644 index 0000000..f1a5c35 --- /dev/null +++ b/src/brig/commands/lifecycle_control.py @@ -0,0 +1,300 @@ +"""CLI handlers for cell state-changing commands. + +stop/kill/rm/start/restart/pause/unpause/wait/rename/exec/shell/attach. +Read-only commands live in lifecycle_inspect; `brig run` lives in +lifecycle_run. +""" + +from __future__ import annotations + +import argparse +import sys + +from brig.cell.lifecycle import kill_cell, rm_cell, stop_cell +from brig.config import container_name +from brig.errors import BrigError +from brig.ops.logging import info, output +from brig.vm.shell import vm_run, vm_run_interactive + + +def cmd_stop(args: argparse.Namespace) -> int: + stop_cell(args.name) + return 0 + + +def cmd_kill(args: argparse.Namespace) -> int: + kill_cell(args.name) + return 0 + + +def cmd_rm(args: argparse.Namespace) -> int: + from brig.cell.lifecycle import _workspace_has_content + + keep = getattr(args, "keep_workspace", False) + # Closes the silent-data-loss foot-gun where the user expects docker + # semantics (rm preserves volumes) and loses unexpected data. + if not keep and not args.force and _workspace_has_content(args.name): + if not sys.stdin.isatty(): + raise BrigError( + f"Cell '{args.name}' has files in its workspace; refusing to " + f"delete non-interactively.", + suggestion=( + f"brig cell rm {args.name} --keep-workspace # preserve files\n" + f" OR brig cell rm -f {args.name} # force delete" + ), + ) + prompt = ( + f"Cell '{args.name}' workspace contains files. " + f"Delete? [y/N/keep] " + ) + try: + answer = input(prompt).strip().lower() + except EOFError: + answer = "" + if answer in ("k", "keep"): + keep = True + elif answer not in ("y", "yes"): + output("Aborted.") + return 1 + + rm_cell(args.name, force=args.force, keep_workspace=keep) + return 0 + + +def cmd_exec(args: argparse.Namespace) -> int: + cn = container_name(args.name) + cmd = ["podman", "exec"] + if getattr(args, "interactive", False): + cmd.append("-it") + cmd.append(cn) + # argparse.REMAINDER keeps a leading '--' separator in the list; drop it + # so `brig cell exec mycell -- ls` runs `ls`, not `-- ls`. + exec_cmd = args.exec_cmd + if exec_cmd and exec_cmd[0] == "--": + exec_cmd = exec_cmd[1:] + cmd.extend(exec_cmd) + return vm_run_interactive(cmd) + + +def cmd_shell(args: argparse.Namespace) -> int: + cn = container_name(args.name) + return vm_run_interactive(["podman", "exec", "-it", cn, "/bin/sh"]) + + +def cmd_attach(args: argparse.Namespace) -> int: + cn = container_name(args.name) + return vm_run_interactive(["podman", "attach", cn]) + + +def cmd_start(args: argparse.Namespace) -> int: + # Invariant 9: proxy must be running before starting cells. + from brig.network.proxy import proxy_running + if not proxy_running(): + raise BrigError( + "Warden proxy is not running", + suggestion="Start with: brig system up", + ) + _verify_image_digest_on_start(args.name) + _refresh_metadata_for_start(args.name) + cn = container_name(args.name) + result = vm_run(["podman", "start", cn]) + if result.returncode != 0: + raise BrigError( + f"Failed to start cell '{args.name}': {result.stderr.strip()}", + suggestion="Check if cell exists with: brig cell list", + ) + # Post-start configuration must be all-or-nothing: a cell that comes up + # but can't be fully wired (ingress unrouteable, etc.) is worse than one + # that didn't start. Roll back (stop) on any failure so the operator isn't + # left with a misleading half-configured cell. + try: + _verify_proxy_connected_after_start(args.name) + + # Re-stage the Warden CA bundle. ca_bundle's design is per-start + # re-extraction; a `system down`/`up` cycle otherwise leaves a stale + # bundle and the cell's HTTPS fails "unknown ca" (and `brig verify` + # even suggests `restart` as the fix). Best-effort + idempotent — + # harmless for cells that don't mount it (airgapped / trust_warden_ca + # false). + try: + from brig.cell.ca_bundle import stage_bundle + stage_bundle(args.name) + except Exception as e: # noqa: BLE001 — never fail start on a re-stage hiccup + from brig.ops.logging import debug + debug(f"CA bundle re-stage on start failed: {e}") + + # host_sockets bridges are torn down on stop and are NOT recreated + # here: re-bridging needs the host_path, which is deliberately absent + # from the cell-readable metadata (it must not leak to the cell). Warn + # so the operator doesn't get a silently-dead mount. + from brig.cell.metadata import read_host_sockets + if read_host_sockets(args.name): + from brig.ops.logging import warn + warn( + f"Cell '{args.name}' declares host_sockets, but launchd bridges " + f"are not recreated on start. Re-run from yaml for working " + f"host_sockets: brig run --file " + ) + + from brig.cell.lifecycle import register_ingress_for + from brig.cell.metadata import read_ingress + ingress_entries = read_ingress(args.name) + if ingress_entries: + register_ingress_for(args.name, ingress_entries) + except BrigError: + stop_cell(args.name) + raise + info(f"Cell '{args.name}' started") + return 0 + + +def _verify_image_digest_on_start(cell_name: str) -> None: + """If the cell was created with a pinned image_digest, re-verify the + container's current image digest matches before letting it start. + + Closes the window where an operator could `podman commit` a new + image on top of the cell's pinned reference between stop and start. + No-op for cells that weren't pinned at create time. + """ + from brig.cell.metadata import read_image_digest + pinned = read_image_digest(cell_name) + if not pinned: + return + cn = container_name(cell_name) + inspect = vm_run(["podman", "inspect", "--format", "{{.ImageDigest}}", cn]) + if inspect.returncode != 0: + raise BrigError( + f"Cannot verify image digest for '{cell_name}': " + f"{inspect.stderr.strip()}" + ) + actual = inspect.stdout.strip() + # Fail closed: a pin was recorded at create time, so an empty/absent + # ImageDigest (e.g. a locally committed image with no registry digest) + # means we CANNOT prove the content matches — refuse rather than start on + # unverified content, which is the exact commit-swap window this closes. + if actual != pinned: + observed = actual or "" + raise BrigError( + f"Image digest drift on '{cell_name}': pinned {pinned}, " + f"container now references {observed}", + suggestion=( + f"Recreate the cell from yaml: " + f"brig cell rm -f {cell_name} && brig run --file " + ), + ) + + +def _verify_proxy_connected_after_start(cell_name: str) -> None: + """Refuse to leave a non-airgapped cell running when warden isn't + actually connected to its network. + + proxy_running() only proves warden is up — it doesn't prove warden + is attached to *this* cell's per-cell network. After `brig system + down`/`up` the proxy_connected attachment may be gone; without this + check the cell would start with no egress path and every outbound + request would silently fail. + """ + from brig.cell.lifecycle import observe + state = observe(cell_name) + if not (state.exists and state.running): + return + # Airgapped cells have no per-cell network — they were started with + # --network none and warden was never meant to be attached. + if not state.network_exists: + return + if state.proxy_connected: + return + from brig.cell.reconciler import _run_cmd + from brig.config import PROXY_NAME + cn = container_name(cell_name) + info(f"Reconnecting warden to cell network for '{cell_name}'") + _run_cmd(["podman", "network", "connect", cn, PROXY_NAME]) + state = observe(cell_name) + if not state.proxy_connected: + raise BrigError( + f"Cell '{cell_name}' started but warden isn't connected " + f"to its network. Egress will fail until reconnected.", + suggestion=f"brig cell rm -f {cell_name} && brig run --file ", + ) + + +def _refresh_metadata_for_start(cell_name: str) -> None: + """Rewrite /run/brig/cell.json on restart with a fresh `started_at`. + + Preserves the original workspace_mount, host_sockets, and ingress — + bind mounts and ingress configuration are fixed at create time, so + these come from the prior metadata write rather than being re-derived. + If the file is missing or unreadable (cell predates cell.json), write + a default-mount fallback. + """ + from brig.cell.metadata import refresh_metadata_if_present, write_metadata + if refresh_metadata_if_present(cell_name) is None: + write_metadata(cell_name, "/work") + + +def cmd_restart(args: argparse.Namespace) -> int: + """Handle `brig cell restart ` — stop (if running) then start. + + Composite of stop_cell + cmd_start. Refreshes the cell metadata's + started_at via cmd_start's existing path. + """ + from brig.cell.lifecycle import observe, stop_cell + actual = observe(args.name) + if not actual.exists: + raise BrigError( + f"Cell '{args.name}' does not exist", + suggestion="brig cell list # see what's there", + ) + if actual.running: + stop_cell(args.name) + return cmd_start(args) + + +def cmd_wait(args: argparse.Namespace) -> int: + cn = container_name(args.name) + result = vm_run(["podman", "wait", cn], timeout=None) + if result.returncode != 0: + raise BrigError( + f"Cell '{args.name}' does not exist", + suggestion="Use 'brig cell list' to see available cells", + ) + exit_code = result.stdout.strip() + output(exit_code) + return int(exit_code) if exit_code.isdigit() else 1 + + +def cmd_pause(args: argparse.Namespace) -> int: + cn = container_name(args.name) + result = vm_run(["podman", "pause", cn]) + if result.returncode != 0: + raise BrigError(f"Failed to pause cell '{args.name}': {result.stderr.strip()}") + info(f"Cell '{args.name}' paused") + return 0 + + +def cmd_unpause(args: argparse.Namespace) -> int: + cn = container_name(args.name) + result = vm_run(["podman", "unpause", cn]) + if result.returncode != 0: + raise BrigError(f"Failed to unpause cell '{args.name}': {result.stderr.strip()}") + info(f"Cell '{args.name}' unpaused") + return 0 + + +def cmd_rename(args: argparse.Namespace) -> int: + # Renaming only the podman container would orphan the cell's per-cell + # network, subnet allocation, state dir, per-cell policy file, and ingress + # routes — all keyed by the OLD name — leaving a half-renamed, + # inconsistent cell. A correct rename would have to migrate all of those + # atomically (network/subnet rename isn't atomic in podman), so it's not + # supported: recreate under the new name instead. + raise BrigError( + f"Renaming a cell is not supported — it can't safely migrate the " + f"cell's network, subnet, state, and policy (all keyed to " + f"'{args.old_name}').", + suggestion=( + f"Recreate it under the new name:\n" + f" brig cell rm {args.old_name}\n" + f" brig run --name {args.new_name} ... (or: brig run --file )" + ), + ) diff --git a/src/brig/commands/lifecycle_inspect.py b/src/brig/commands/lifecycle_inspect.py new file mode 100644 index 0000000..a284ef5 --- /dev/null +++ b/src/brig/commands/lifecycle_inspect.py @@ -0,0 +1,652 @@ +"""CLI handlers for read-only cell observation commands. + +These handlers inspect a cell's state without mutating it (list, inspect, +files, read, logs, cp, top, diff, stats, export, ingress, preflight). +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from typing import Any + +import yaml + +from brig.config import CONTAINER_PREFIX, container_name +from brig.errors import BrigError +from brig.ops.logging import info, output +from brig.vm.shell import vm_run, vm_run_interactive + + +def cmd_ingress(args: argparse.Namespace) -> int: + """Print reachable URLs for a cell's ingress endpoints. + + Surfaces three things the user otherwise has to dig for: the URL + `https://warden:8443///...` is reachable at on the + host (Lima forwards :8443), the token secret name to use for the + `Authorization: Bearer` header, and whether the cell currently has + routes registered (vs declared but not yet started). + """ + from brig.cell.metadata import read_ingress + from brig.config import HostPaths, INGRESS_PORT + from brig.network.ingress import _load_routes + + entries = read_ingress(args.name) + if not entries: + output(f"Cell '{args.name}' has no ingress declared.") + return 0 + + routes_file = HostPaths.INGRESS_ROUTES_FILE + registered = set() + if routes_file.exists(): + for r in _load_routes(routes_file).get("routes", []): + if r.get("cell") == args.name: + registered.add(r.get("name")) + + primary_token = f"{args.name}-ingress-token" + fallback_token = "ingress-token" + token = primary_token if (HostPaths.SECRETS_DIR / primary_token).exists() \ + else (fallback_token if (HostPaths.SECRETS_DIR / fallback_token).exists() else None) + + def _url(entry: dict) -> str: + return f"https://127.0.0.1:{INGRESS_PORT}/{args.name}{entry['path_prefix']}" + + output(f"Ingress for cell '{args.name}':") + for e in entries: + status = "REGISTERED" if e["name"] in registered else "DECLARED (not started)" + output(f" {e['name']}: {_url(e)} [{status}]") + output(f" cell port: {e['port']} auth: {e['auth']}") + if token: + output(f" Token secret: {token}") + output( + f" Example: curl -k -H \"Authorization: Bearer " + f"$(cat ~/.brig/secrets/{token})\" {_url(entries[0])}" + ) + else: + output( + f" WARNING: token secret '{primary_token}' missing — " + f"create with: openssl rand -hex 32 | brig secrets add {primary_token} -" + ) + return 0 + + +def cmd_preflight(args: argparse.Namespace) -> int: + """Handle `brig cell preflight `. + + Dry-run check: parses the yaml and verifies every host-side + requirement it implies (declared secrets exist, declared + host_socket targets exist, declared ingress entries have a + token secret, image exists or is buildable). Prints a checklist + with a one-line fix for each gap. + + Replaces the iterative "brig run → error → fix one thing → + re-run" loop with a single diff. No mutations; safe to run + anytime. + """ + from brig.cell.spec import load_cell_definition, validate_cell_definition + from brig.config import HostPaths + + cell_def = load_cell_definition(args.file) + errors = validate_cell_definition(cell_def, args.file) + fail_count = 0 + + def _check(label: str, ok: bool, fix: str = "") -> None: + nonlocal fail_count + marker = "OK " if ok else "FAIL" + output(f" [{marker}] {label}") + if not ok: + fail_count += 1 + if fix: + output(f" fix: {fix}") + + cell_name = cell_def.get("name", "") + output(f"Preflight for cell '{cell_name}' ({args.file})") + output("=" * 60) + + _check( + "cell yaml validates", + not errors, + fix=("Fix errors:\n " + "\n ".join(errors) + if errors else ""), + ) + + for secret in cell_def.get("secrets", []): + if not isinstance(secret, str): + continue + p = HostPaths.SECRETS_DIR / secret + _check( + f"secret: {secret}", p.exists(), + fix=f"brig secrets add {secret}", + ) + + ingress_entries = cell_def.get("ingress") or [] + needs_token = any( + isinstance(e, dict) and e.get("auth") == "token" for e in ingress_entries + ) + if needs_token: + primary = HostPaths.SECRETS_DIR / f"{cell_name}-ingress-token" + fallback = HostPaths.SECRETS_DIR / "ingress-token" + ok = primary.exists() or fallback.exists() + _check( + f"ingress token: {primary.name}", ok, + fix=f"openssl rand -hex 32 | brig secrets add {primary.name} -", + ) + + import os as _os + import stat as _stat + for entry in cell_def.get("host_sockets") or []: + if not isinstance(entry, dict): + continue + host_path = entry.get("host_path", "") + name = entry.get("name", "?") + try: + st = _os.lstat(host_path) + is_sock = _stat.S_ISSOCK(st.st_mode) and not _stat.S_ISLNK(st.st_mode) + except (FileNotFoundError, OSError): + is_sock = False + _check( + f"host_socket target: {name} → {host_path}", is_sock, + fix="Start the service that provides this socket, or " + "correct host_path.", + ) + + if cell_def.get("host_sockets"): + import shutil as _shutil + _check( + "socat installed (host_sockets bridge dependency)", + bool(_shutil.which("socat")), + fix="brew install socat", + ) + + output("=" * 60) + if fail_count: + output(f"FAILED: {fail_count} check(s) — fix above, then re-run") + return 1 + output("All preflight checks passed.") + return 0 + + +def cmd_list(args: argparse.Namespace) -> int: + from brig.cell.lifecycle import list_cell_containers + fmt = getattr(args, "format", "table") + cells = list_cell_containers(include_stopped=True) + + if fmt == "json": + output(json.dumps([entry for _, entry in cells], indent=2)) + return 0 + + if not cells: + output("No cells found") + return 0 + + if fmt == "wide": + output(f"{'NAME':<25} {'STATUS':<12} {'CREATED':<22} {'NETWORK':<25} {'IMAGE'}") + for cell, c in cells: + networks = c.get("Networks") or [] + network = ",".join(networks)[:25] if networks else "-" + created = c.get("CreatedAt", "")[:22] + output(f"{cell:<25} {c.get('State', ''):<12} {created:<22} {network:<25} {c.get('Image', '')}") + else: + output(f"{'NAME':<25} {'STATUS':<12} {'IMAGE':<30}") + for cell, c in cells: + output(f"{cell:<25} {c.get('State', ''):<12} {c.get('Image', ''):<30}") + return 0 + + +def cmd_inspect(args: argparse.Namespace) -> int: + cn = container_name(args.name) + result = vm_run(["podman", "inspect", cn, "--format", "json"]) + if result.returncode != 0: + raise BrigError( + f"Cell '{args.name}' does not exist", + suggestion="Use 'brig cell list' to see available cells", + ) + output(result.stdout) + return 0 + + +def cmd_files(args: argparse.Namespace) -> int: + """List workspace contents inside a cell.""" + cn = container_name(args.name) + path = getattr(args, "path", "/work") + return vm_run_interactive(["podman", "exec", cn, "ls", "-la", path]) + + +def cmd_read(args: argparse.Namespace) -> int: + """Handle `brig cell read ` — stream a workspace file + to stdout via the race-free safe_open primitive. + + This is the language-agnostic safe-read replacement for the (now + removed) workspace.host_path field in cell.json. Consumers in any + language can do: + + brig cell read mycell input.json > /tmp/local-copy + + instead of opening the host path directly. Each path component is + walked with O_NOFOLLOW so a cell-planted symlink raises + WorkspaceEscape rather than letting the host follow it. + """ + from brig.workspace.validation import safe_open, WorkspaceEscape + + try: + with safe_open(args.name, args.path, "rb") as f: + while True: + chunk = f.read(64 * 1024) + if not chunk: + break + sys.stdout.buffer.write(chunk) + return 0 + except WorkspaceEscape as e: + raise BrigError( + f"Refused: {e}", + suggestion="A path component is a symlink or escapes the workspace root.", + ) + except FileNotFoundError: + raise BrigError(f"Not found: {args.path} in cell '{args.name}'") + + +def cmd_logs(args: argparse.Namespace) -> int: + """Tail a cell's container stdout/stderr (wraps `podman logs`). + + Empty output usually means the cell's app writes to a file inside + the container instead of stdout — common for daemons and agent + runtimes. In that case, use + `brig cell exec -- cat /var/log/.log` or + `brig cell read ` to inspect the file directly. + """ + cn = container_name(args.name) + cmd = ["podman", "logs"] + follow = getattr(args, "follow", False) + if follow: + cmd.append("-f") + if getattr(args, "tail", None): + cmd.extend(["--tail", str(args.tail)]) + cmd.append(cn) + if follow: + return vm_run_interactive(cmd) + result = vm_run(cmd, capture=True, timeout=30) + if result.returncode == 0: + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + # Empty stdout + stderr likely means file-based logging. + if not (result.stdout.strip() or result.stderr.strip()): + info( + f"(no stdout/stderr from '{args.name}' — if the app logs " + f"to a file, try: brig cell exec {args.name} -- " + f"cat /var/log/.log)" + ) + return 0 + sys.stderr.write(result.stderr) + return result.returncode + + +def _parse_cp_target(spec: str) -> tuple[str, str] | None: + """Parse 'cell:/path' into (cell, path), or return None if not a cell ref. + + Only treats the colon as a cell separator when the prefix matches the + canonical cell-name pattern. Avoids misreading paths containing colons + (e.g. './out:put.txt') as cell references. + """ + from brig.config import CELL_NAME_PATTERN + if ":" not in spec or spec.startswith("/") or spec.startswith("."): + return None + head, tail = spec.split(":", 1) + if not CELL_NAME_PATTERN.match(head): + return None + return head, tail + + +def cmd_cp(args: argparse.Namespace) -> int: + """Copy files to/from a cell with safety checks. + + Detects direction from the colon syntax (cell:path). The cell prefix + must match the canonical cell-name pattern, so a literal path like + './out:put.txt' is not misread as a cell reference. + Exports apply quarantine xattr and extension blocking by default. + """ + src, dst = args.src, args.dst + src_target = _parse_cp_target(src) + dst_target = _parse_cp_target(dst) + + if src_target and dst_target: + raise BrigError( + "Cannot copy between two cells", + suggestion="Copy from cell to host first, then from host to cell", + ) + if src_target: + # The host path is operator argv; reject a leading dash so it can't be + # parsed as a `podman cp` (or downstream `xattr`) flag — matches the + # guard cmd_pull / build_run_command apply. + if dst.startswith("-"): + raise BrigError(f"Host path must not start with '-': {dst}") + from brig.workspace.workspace import copy_out + copy_out(src_target[0], src_target[1], dst, sanitize=True) + elif dst_target: + if src.startswith("-"): + raise BrigError(f"Host path must not start with '-': {src}") + from brig.workspace.workspace import copy_in + copy_in(dst_target[0], src, dst_target[1]) + else: + raise BrigError( + "Could not determine copy direction", + suggestion="Use cell:path syntax, e.g.: brig cell cp mycell:/work/out.txt ./", + ) + return 0 + + +def cmd_top(args: argparse.Namespace) -> int: + cn = container_name(args.name) + return vm_run_interactive(["podman", "top", cn]) + + +def cmd_diff(args: argparse.Namespace) -> int: + cn = container_name(args.name) + return vm_run_interactive(["podman", "diff", cn]) + + +def cmd_stats(args: argparse.Namespace) -> int: + cmd = ["podman", "stats", "--no-stream"] + if hasattr(args, "name") and args.name: + cmd.append(container_name(args.name)) + else: + cmd.extend(["--filter", f"name=^{CONTAINER_PREFIX}"]) + return vm_run_interactive(cmd) + + +def cmd_export(args: argparse.Namespace) -> int: + """Export a running cell's config as a reusable YAML cell definition.""" + cn = container_name(args.name) + result = vm_run(["podman", "inspect", cn, "--format", "json"]) + if result.returncode != 0: + raise BrigError( + f"Cell '{args.name}' does not exist", + suggestion="Use 'brig cell list' to see available cells", + ) + + try: + data = json.loads(result.stdout) + if isinstance(data, list): + data = data[0] + except json.JSONDecodeError: + raise BrigError( + "Could not parse container info", + suggestion="Check podman / VM health: brig system doctor", + ) + + host_config = data.get("HostConfig", {}) + config = data.get("Config", {}) + + cell_def: dict[str, Any] = {"name": args.name} + + image = config.get("Image", "") + if image: + cell_def["image"] = image + + cmd = config.get("Cmd") + if cmd: + cell_def["command"] = cmd + + proxy_prefixes = ("http_proxy=", "https_proxy=", "HTTP_PROXY=", "HTTPS_PROXY=", "no_proxy=") + # Drop brig-injected env: the HTTP(S)_PROXY vars and the secret-mount + # pointers (`_FILE=/run/secrets/`) — both are re-derived on + # the next run from the cell's `secrets:` block, not part of its definition. + env_vars = [ + e for e in (config.get("Env") or []) + if not any(e.startswith(p) for p in proxy_prefixes) + and "_FILE=/run/secrets/" not in e + ] + if env_vars: + cell_def["env"] = env_vars + + memory = host_config.get("Memory", 0) + if memory: + if memory >= 1024**3: + cell_def["memory"] = f"{memory // 1024**3}g" + elif memory >= 1024**2: + cell_def["memory"] = f"{memory // 1024**2}m" + + cpus = host_config.get("NanoCpus", 0) + if cpus: + cell_def["cpus"] = str(cpus / 1e9) + + pids = host_config.get("PidsLimit", 0) + if pids and pids > 0: + cell_def["pids_limit"] = pids + + # Round-trip the process user (podman --user → Config.User) when set. + user = config.get("User") + if user: + cell_def["user"] = user + + from brig.policy.policy import load_cell_policy + cell_policy = load_cell_policy(args.name) or {} + allow = cell_policy.get("allow") or [] + deny = cell_policy.get("deny") or [] + passthrough = cell_policy.get("tls_passthrough") or [] + if allow or deny or passthrough: + policy_block: dict = {} + if allow: + policy_block["allow"] = list(allow) + if deny: + policy_block["deny"] = list(deny) + if passthrough: + policy_block["tls_passthrough"] = list(passthrough) + cell_def["policy"] = policy_block + host_services = cell_policy.get("host_services") or [] + if host_services: + cell_def["host_services"] = list(host_services) + + # Prefer cell-metadata.json (works for stopped cells too) and fall + # back to ingress-routes.json for cells whose metadata predates the + # `ingress` field. + from brig.cell.metadata import read_ingress + from brig.config import HostPaths + ingress = read_ingress(args.name) + if not ingress: + ingress = _ingress_for_cell(args.name, HostPaths.INGRESS_ROUTES_FILE) + if ingress: + cell_def["ingress"] = ingress + + # Only {name, mount_point} are persisted (host_path stays off the + # downward-API surface). Reproducing the cell from the exported yaml + # requires re-supplying the host_path. + sockets = _host_sockets_from_metadata(args.name) + if sockets: + cell_def["host_sockets"] = sockets + + # Reconstruct `mounts:` from the live binds under /mnt/host/ (host_path + # mapped back via the configured mount_roots; mode from the bind's RW flag). + # The original `name` isn't persisted, so it's re-derived from the mount + # point and sanitized to MOUNT_NAME_PATTERN so the export round-trips. + mounts = [] + used_names: set[str] = set() + for m in data.get("Mounts", []) or []: + if not isinstance(m, dict): + continue + src = m.get("Source", "") + hp = _vm_mount_source_to_host(src) + if hp is None: + continue + dst = m.get("Destination", "") + mounts.append({ + "name": _safe_mount_name(dst, used_names), + "host_path": str(hp), + "mount_point": dst, + "mode": "rw" if m.get("RW") else "ro", + }) + if mounts: + cell_def["mounts"] = mounts + + # restart policy isn't recoverable from `podman inspect` (it's never passed + # to podman), so read it from the persisted spec — otherwise a + # restart:always cell would export as restart:no and not round-trip. + from brig.cell.metadata import read_cell_spec + persisted = read_cell_spec(args.name) + if persisted and persisted.get("restart") == "always": + cell_def["restart"] = "always" + + output(f"# Cell definition exported from '{args.name}'") + if sockets: + output("# NOTE: host_sockets entries lack `host_path` — re-add the host") + output("# socket paths before `brig run --file`; they are intentionally") + output("# not stored in cell metadata.") + if mounts: + output("# NOTE: `mounts:` requires the same mount_roots configured on the") + output("# target host (brig config set mount_roots ...) for host_path to resolve.") + if not sockets and not mounts: + output("# Self-contained: brig run --file reproduces the cell.") + output("") + output(yaml.safe_dump(cell_def, default_flow_style=False, sort_keys=False).rstrip()) + return 0 + + +def _ingress_for_cell(cell_name: str, routes_file) -> list: + try: + data = json.loads(routes_file.read_text()) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return [] + routes = data.get("routes") or [] + return [ + {"name": r.get("name"), + "port": r.get("port"), + "path_prefix": r.get("path_prefix"), + "auth": "token"} + for r in routes + if isinstance(r, dict) and r.get("cell") == cell_name + ] + + +def _host_sockets_from_metadata(cell_name: str) -> list: + from brig.cell.metadata import _host_metadata_path + try: + meta = json.loads(_host_metadata_path(cell_name).read_text()) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return [] + entries = meta.get("host_sockets") or [] + return [ + {"name": e["name"], "mount_point": e["mount_point"]} + for e in entries + if isinstance(e, dict) and "name" in e and "mount_point" in e + ] + + +def _vm_mount_source_to_host(source: str): + """Map a VM bind source under /mnt/host//... back to its macOS path, + or None if it isn't a mount_roots-backed bind.""" + from pathlib import Path + from brig.config import mount_root_slug, mount_roots + prefix = "/mnt/host/" + if not source.startswith(prefix): + return None + slug, _, rel = source[len(prefix):].partition("/") + for root in mount_roots(): + if mount_root_slug(root) == slug: + base = Path(root) + return base / rel if rel else base + return None + + +def _safe_mount_name(mount_point: str, used: set[str]) -> str: + """Derive a MOUNT_NAME_PATTERN-valid, unique name from a mount point. + + The original `mounts.name` isn't persisted, so it's re-derived for export; + sanitize to lowercase alphanumeric+hyphen (max 31) so `brig run --file` + accepts the result. + """ + base = re.sub(r"[^a-z0-9-]+", "-", mount_point.rsplit("/", 1)[-1].lower()).strip("-") + base = base[:28] or "mount" + name = base + suffix = 1 + while name in used: + name = f"{base}-{suffix}" + suffix += 1 + used.add(name) + return name + + +def cmd_mount_scan(args: argparse.Namespace) -> int: + """Scan a cell's host `mounts:` for symlinks that escape the mounted dir. + + A cell with a rw mount can plant a symlink pointing out of the shared + folder (e.g. -> ~/.ssh/id_rsa); a host process that follows it escapes the + folder. Reports such symlinks, or removes them with --quarantine. Run this + before a host process consumes files the cell wrote. + """ + from brig.workspace.workspace import ( + find_escaping_symlinks, + quarantine_escaping_symlinks, + ) + cn = container_name(args.name) + result = vm_run(["podman", "inspect", cn, "--format", "json"]) + if result.returncode != 0: + raise BrigError( + f"Cell '{args.name}' does not exist", + suggestion="Use 'brig cell list' to see available cells", + ) + try: + data = json.loads(result.stdout) + entry = data[0] if isinstance(data, list) else data + mounts = entry.get("Mounts", []) or [] + except (json.JSONDecodeError, IndexError, AttributeError): + raise BrigError( + f"Could not parse podman inspect for '{args.name}'", + suggestion="Check podman / VM health: brig system doctor", + ) + + host_dirs = [] + seen_dirs: set[str] = set() + unmapped = [] + for m in mounts: + if isinstance(m, dict): + src = m.get("Source", "") + hp = _vm_mount_source_to_host(src) + if hp is not None: + # One host_path can be bound at several mount points; scan it once. + if str(hp) not in seen_dirs: + seen_dirs.add(str(hp)) + host_dirs.append(hp) + elif src.startswith("/mnt/host/"): + # A live mounts: bind whose slug no longer maps (mount_roots + # edited after start). Don't claim a clean scan — fail loud. + unmapped.append(src) + + if unmapped: + raise BrigError( + f"Cell '{args.name}' has host mount(s) that no longer map to a " + f"configured mount_root: {', '.join(unmapped)}. Scan aborted — " + f"restore the mount_roots entry, or inspect them by hand.", + ) + if not host_dirs: + output(f"Cell '{args.name}' has no host mounts to scan.") + return 0 + + quarantine = getattr(args, "quarantine", False) + found = 0 # escaping symlinks seen + remaining = 0 # still escaping after a quarantine pass + for d in host_dirs: + if not d.exists(): + output(f" (skip) {d} — not present on host") + continue + escaping = find_escaping_symlinks(d) + found += len(escaping) + for link, target in escaping: + output(f" {'QUARANTINING' if quarantine else 'ESCAPES'} {link} -> {target}") + if quarantine: + quarantine_escaping_symlinks(d) + # Re-scan: unlink() can fail (perms, immutable flag, race), so count + # what's actually left rather than trusting the removal list. + remaining += len(find_escaping_symlinks(d)) + + if found == 0: + output(f"No escaping symlinks in '{args.name}' host mount(s).") + return 0 + if quarantine: + if remaining: + output(f"{found} found, but {remaining} could NOT be removed — investigate.") + return 1 + output(f"{found} escaping symlink(s) quarantined.") + return 0 + output(f"{found} escaping symlink(s) found.") + return 1 diff --git a/src/brig/commands/lifecycle_run.py b/src/brig/commands/lifecycle_run.py new file mode 100644 index 0000000..607f31e --- /dev/null +++ b/src/brig/commands/lifecycle_run.py @@ -0,0 +1,382 @@ +"""CLI handler for `brig run` and its parse/merge/diagnose helpers.""" + +from __future__ import annotations + +import argparse +import re +import sys +from typing import Any + +from brig.cell.lifecycle import run_cell +from brig.cell.profiles import apply_profile, load_profile +from brig.cell.spec import CellSpec, load_cell_definition, validate_cell_definition +from brig.config import container_name +from brig.errors import BrigError +from brig.ops.logging import info, output +from brig.vm.shell import vm_run + +_DIGEST_RE = re.compile(r"@sha(?:256|512):[0-9a-f]{40,}$") + + +def _warn_unverified_image(image: str) -> None: + """Stderr-only warning if `image` is from a registry and lacks a + digest pin. Local builds (localhost/* and dotless single names) and + digest-pinned refs are silent. + + Brig doesn't refuse — verification is a publishing trust decision + that varies by user. We just make the absence visible so a careless + `brig run someorg/their-image:latest` doesn't slip through quietly. + """ + if not image: + return + if image.startswith("localhost/"): + return + if _DIGEST_RE.search(image): + return + if _suppress_unverified_warn(): + return + info( + f"WARN: image {image!r} is unpinned and unverified. " + f"Pin a digest (image@sha256:...) or verify with: " + f"brig image verify {image}\n" + f" (silence with: brig config set suppress_unverified_image_warn true)" + ) + + +def _suppress_unverified_warn() -> bool: + """Read the suppress flag from CONFIG_FILE. Missing/unreadable + config is treated as 'warn' (the safe default).""" + import json as _json + from brig.config import CONFIG_FILE + try: + with open(CONFIG_FILE) as f: + return bool(_json.load(f).get("suppress_unverified_image_warn", False)) + except (FileNotFoundError, _json.JSONDecodeError, OSError): + return False + + +def cmd_run(args: argparse.Namespace) -> int: + """Handle `brig run`.""" + # Catch the common foot-gun: `brig run alpine -m 512m sh` puts -m and 512m + # into the container command instead of being parsed as a brig flag. + # nargs=REMAINDER swallows everything after the image, so flags must + # precede the image. If args.image looks like a flag, the user almost + # certainly meant to put it before the image. + if args.image and args.image.startswith("-"): + raise BrigError( + f"'{args.image}' looks like a flag but appears in image position", + suggestion="Brig flags must precede the image name. Did you forget '--' " + "before the container command? e.g. brig run --memory 512m alpine -- sh", + ) + + # `brig run cells/foo` is a common confusion — that's a build context, + # not an image ref. Detect host directories and steer the user toward + # `brig image build` before podman pulls and fails opaquely. + if args.image and "/" in args.image and not args.image.startswith("localhost/"): + from pathlib import Path as _P + if _P(args.image).is_dir(): + raise BrigError( + f"'{args.image}' is a directory, not an image reference", + suggestion=( + f"Did you mean to build it first?\n" + f" brig image build {args.image}\n" + f" brig run localhost/{_P(args.image).name}:latest ..." + ), + ) + + # Catch the inverse: flags AFTER a valid image. nargs=REMAINDER swallows + # everything so `brig run alpine --memory 256m sh` puts the flag tokens + # into container_cmd and the cell tries to exec `--memory`. Flag it before. + _BRIG_FLAG_TOKENS = { + "--memory", "-m", "--cpus", "--name", "-n", "--env", "-e", + "--secret", "-s", "--profile", "--file", "-f", "--network", + "--detach", "-d", "--rm", "--timeout", "--workspace-quota", + "--label", "-l", "--pids-limit", "--image-digest", "--workdir", + "--policy-allow", "--policy-deny", "-y", "--yes", + } + cmd_tail = args.container_cmd or [] + first = cmd_tail[1] if cmd_tail and cmd_tail[0] == "--" else (cmd_tail[0] if cmd_tail else None) + if first in _BRIG_FLAG_TOKENS: + raise BrigError( + f"'{first}' looks like a brig flag but appears after the image", + suggestion=( + "Brig flags must precede the image. If you meant to pass it to " + f"the container, escape it: brig run ... {args.image} -- {first} ..." + ), + ) + + if args.image: + _warn_unverified_image(args.image) + + container_cmd = args.container_cmd or [] + if container_cmd and container_cmd[0] == "--": + container_cmd = container_cmd[1:] + + if not args.image and not args.file: + raise BrigError("Image is required unless --file is specified") + + # Name resolution: --name flag wins, then the yaml's name: field (if any), + # then auto-generate. The auto-generate must happen LAST so `--file + # foo.yaml` with `name: my-cell` keeps `my-cell`. + spec_kwargs: dict[str, Any] = { + "name": args.name or "", + "image": args.image or "", + "command": container_cmd, + "env": args.env or [], + "secrets": args.secret or [], + "labels": args.label or [], + "detach": args.detach, + "rm": args.rm, + "image_digest": getattr(args, "image_digest", None), + "workdir": getattr(args, "workdir", None), + } + + # Precedence: CLI flag > yaml > profile > defaults. Build up from + # least-specific to most-specific. + if args.profile: + profile = load_profile(args.profile) + merged = apply_profile(spec_kwargs, profile) + spec_kwargs.update(merged) + # Record the profile name so the untrusted-profile guards (host_sockets, + # host_services, tls_passthrough) fire during validation below. Without + # this, --profile untrusted on the CLI silently voids those guards. + spec_kwargs["profile"] = args.profile + + cell_def: dict = {} + if args.file: + cell_def = load_cell_definition(args.file) + # The CLI --profile flag is authoritative over the yaml (CLI > yaml), + # so inject it for validation; otherwise an untrusted-profile run could + # declare host_sockets / tls_passthrough in the yaml unchecked. + if args.profile: + cell_def["profile"] = args.profile + errors = validate_cell_definition(cell_def, args.file) + if errors: + raise BrigError("Invalid cell definition:\n - " + "\n - ".join(errors)) + for key in ("image", "name"): + if key in cell_def and not getattr(args, key, None): + spec_kwargs[key] = cell_def[key] + if "command" in cell_def and not args.container_cmd: + cmd_val = cell_def["command"] + spec_kwargs["command"] = cmd_val if isinstance(cmd_val, list) else [cmd_val] + if "env" in cell_def: + env_list = cell_def["env"] + if isinstance(env_list, dict): + env_list = [f"{k}={v}" for k, v in env_list.items()] + spec_kwargs["env"] = (args.env or []) + env_list + if isinstance(cell_def.get("policy"), dict): + for src_key, dst_key in (("allow", "policy_allow"), + ("deny", "policy_deny"), + ("tls_passthrough", "policy_passthrough_tls")): + src = cell_def["policy"].get(src_key) or [] + if src: + spec_kwargs[dst_key] = list(spec_kwargs.get(dst_key) or []) + list(src) + # host_services and policy entries from yaml EXTEND the profile + # baseline. A generic-merge would replace, silently dropping + # profile-declared services when the yaml has any. + if isinstance(cell_def.get("host_services"), list): + spec_kwargs["host_services"] = ( + list(spec_kwargs.get("host_services") or []) + + list(cell_def["host_services"]) + ) + import dataclasses as _dc + _spec_field_names = {f.name for f in _dc.fields(CellSpec)} + _already_handled = { + "image", "name", "command", "env", "ingress", "policy", + "host_services", + } + for key, val in cell_def.items(): + if key in _spec_field_names and key not in _already_handled: + spec_kwargs[key] = val + + if args.memory: + spec_kwargs["memory"] = args.memory + if args.cpus: + spec_kwargs["cpus"] = args.cpus + if args.pids_limit: + spec_kwargs["pids_limit"] = args.pids_limit + if args.network: + spec_kwargs["network"] = args.network + if args.timeout: + spec_kwargs["timeout"] = args.timeout + if args.workspace_quota: + spec_kwargs["workspace_quota"] = args.workspace_quota + # --policy-allow / --policy-deny EXTEND profile + yaml entries, consistent + # with how yaml extends the profile baseline. Single-tenant — no security + # gain from "CLI clobbers all" because the operator owns every layer. + if args.policy_allow: + spec_kwargs["policy_allow"] = ( + list(spec_kwargs.get("policy_allow") or []) + list(args.policy_allow) + ) + if args.policy_deny: + spec_kwargs["policy_deny"] = ( + list(spec_kwargs.get("policy_deny") or []) + list(args.policy_deny) + ) + + if args.file and "ingress" in cell_def: + spec_kwargs["ingress"] = cell_def["ingress"] + + # Yaml is canonical: a `--file ` invocation must spell its name. + # CLI shorthand (`brig run alpine echo hi`) still auto-names. + if not spec_kwargs.get("name"): + if args.file: + raise BrigError( + f"Cell yaml {args.file} is missing required field: name", + suggestion="Add `name: ` to the yaml", + ) + from brig.cell.names import generate_name + spec_kwargs["name"] = generate_name() + info(f"Auto-generated name: {spec_kwargs['name']}") + + # Validate the FULLY-MERGED spec (CLI flags + yaml + profile), mirroring + # sdk.run_sync. The earlier per-file validation only saw the raw yaml; a + # flag-only `brig run` (no --file) would otherwise reach CellSpec with only + # name-validation, silently skipping domain/quota/format checks and the + # untrusted-profile guards. + merged_errors = validate_cell_definition(spec_kwargs) + if merged_errors: + raise BrigError("Invalid cell definition:\n - " + "\n - ".join(merged_errors)) + + import dataclasses + valid_fields = {f.name for f in dataclasses.fields(CellSpec)} + spec_kwargs = {k: v for k, v in spec_kwargs.items() if k in valid_fields} + + spec = CellSpec(**spec_kwargs) + # mitmproxy can't hot-add TCP listeners, so a new TCP host_service in a + # cell yaml requires warden restart. Prompts the operator unless --yes + # since restart drops every running cell's open egress for ~5s. (The + # per-cell policy itself is synced inside run_cell, for the SDK too.) + _maybe_restart_warden_for_tcp(spec, yes=getattr(args, "yes", False)) + + from brig.ops.logging import Spinner + with Spinner(f"Starting cell '{spec.name}'...") as spinner: + result = run_cell(spec) + if result.success: + spinner.success(f"Cell '{spec.name}' started") + else: + spinner.fail(f"Cell '{spec.name}' failed") + + if result.container_id: + output(result.container_id[:12]) + + # Detect cells that exit immediately and turn the cryptic outcome into + # actionable feedback. The most common causes — image's CMD prints help + # and exits, or the image tried to write somewhere read-only — look like + # brig brokenness from the user's side. + if result.success and spec.detach: + _check_immediate_exit(spec.name) + return 0 + + +def _maybe_restart_warden_for_tcp(spec: CellSpec, yes: bool = False) -> None: + """Restart warden if this cell needs a TCP listener that isn't bound. + + mitmproxy can't hot-add `--mode reverse:tcp` listeners, so a new TCP + host_service in a cell yaml requires warden restart for the listener + to come up. Restart kills every live cell's open egress for ~5s, so + we prompt the operator unless `--yes`. + """ + spec_tcp_ports = sorted({ + e["port"] for e in (spec.host_services or []) + if isinstance(e, dict) and e.get("protocol") == "tcp" + and isinstance(e.get("port"), int) + }) + if not spec_tcp_ports: + return + from warden.proxy import get_bound_tcp_ports + bound = set(get_bound_tcp_ports()) + missing = [p for p in spec_tcp_ports if p not in bound] + if not missing: + return + info( + f"Cell '{spec.name}' declares TCP host_services on port(s) " + f"{missing} that warden hasn't bound yet." + ) + if not yes: + if not sys.stdin.isatty(): + raise BrigError( + f"Cell '{spec.name}' needs a warden restart to bind its TCP " + f"host_service listener(s); declined in non-interactive use", + suggestion=( + "Re-run with --yes to auto-confirm, OR\n" + " brig system down && brig system up # bind the listener " + "manually, then re-run this cell" + ), + ) + output( + "Restarting warden will drop every running cell's open " + "egress connections for ~5s while the new listener binds." + ) + try: + answer = input("Proceed with warden restart? [y/N] ").strip().lower() + except (EOFError, KeyboardInterrupt): + answer = "n" + if answer not in ("y", "yes"): + raise BrigError( + "Aborted: warden restart declined", + suggestion=( + "Re-run with --yes to auto-confirm, OR\n" + " brig system down && brig system up # bind the new listener " + "manually, then re-run this cell" + ), + ) + from warden.proxy import start as warden_start, stop as warden_stop + info("Restarting warden to bind new TCP listener(s)...") + warden_stop() + if not warden_start(): + raise BrigError( + "Warden restart failed", + suggestion="brig system doctor", + ) + info(f"Warden restarted; TCP listener(s) on {missing} now bound") + + +def _check_immediate_exit(cell_name: str) -> None: + """Sleep briefly, then probe whether the cell already exited. If it + did, scan its stderr/stdout for known error patterns and append a + suggestion. Best-effort — any failure is swallowed.""" + import time + time.sleep(1.5) + try: + cn = container_name(cell_name) + status = vm_run( + ["podman", "inspect", cn, "--format", "{{.State.Status}}"], + timeout=5, + ) + if status.returncode != 0: + return + if status.stdout.strip() == "running": + return + + logs = vm_run(["podman", "logs", "--tail", "50", cn], timeout=5) + log_text = (logs.stdout or "") + (logs.stderr or "") + hint = _diagnose_exit(log_text) + info( + f"NOTE: cell '{cell_name}' exited shortly after start.{hint} " + f"See: brig cell logs {cell_name}" + ) + except Exception: # noqa: BLE001 — pure diagnostic, never fail the run + pass + + +def _diagnose_exit(log_text: str) -> str: + """Heuristic — match common failure patterns and return an actionable + fragment that fits into the NOTE: line.""" + s = log_text.lower() + if "read-only file system" in s or "errno 30" in s: + return ( + " Image tried to write to a read-only path. Writable paths " + "inside the cell: /work (workspace, persists), /tmp (tmpfs, " + "64m), /run (tmpfs, 16m). Common fix is to redirect writes " + "into one of these — e.g. `export HOME=/tmp/home` and stage " + "credentials there. If the image legitimately needs to write " + "outside those paths, set writable_rootfs: true in the cell spec." + ) + if "executable file not found" in s and "bash" in s: + return " The image likely doesn't ship /bin/bash; try sh." + return "" + + +# Backward-compatible re-export; the implementation lives in +# brig.cell.lifecycle and runs inside run_cell for both the CLI and the SDK. +from brig.cell.lifecycle import sync_cell_policy as _sync_cell_policy # noqa: E402,F401 diff --git a/src/brig/commands/network_cmd.py b/src/brig/commands/network_cmd.py index b64aae2..77250f6 100644 --- a/src/brig/commands/network_cmd.py +++ b/src/brig/commands/network_cmd.py @@ -4,17 +4,30 @@ from __future__ import annotations +import argparse import json -from typing import Any +import re -from brig.config import STATE_DIR, VMPaths +from brig.config import CELL_NAME_PATTERN, STATE_DIR, VMPaths from brig.errors import BrigError from brig.ops.logging import output from brig.vm.shell import vm_run -def cmd_network(args: Any) -> int: - """Handle `brig network` — view cell network activity from proxy logs. +def _require_valid_cell_name(name: str) -> None: + # `name` is a plain CLI positional; it flows into a VM log path read with + # `cat ... sudo`. Validate against CELL_NAME_PATTERN (which forbids '/') + # so a crafted name can't traverse to an arbitrary file (e.g. + # '../../etc/passwd' -> '/etc/passwd.jsonl'). + if not isinstance(name, str) or not CELL_NAME_PATTERN.match(name): + raise BrigError( + f"Invalid cell name '{name}': must start with a lowercase letter or " + f"digit, then up to 62 of [a-z0-9._-] — no uppercase, no '/'." + ) + + +def cmd_network(args: argparse.Namespace) -> int: + """Handle `brig cell network` — view cell network activity from proxy logs. Use --blocked to filter to only requests that warden blocked. This is the fastest way to answer "why was my request blocked?" — the block reason @@ -25,6 +38,7 @@ def cmd_network(args: Any) -> int: scripts continue to work. """ cell_name = args.name + _require_valid_cell_name(cell_name) if getattr(args, "otel", False): return _cmd_network_from_otel(args) @@ -33,7 +47,7 @@ def cmd_network(args: Any) -> int: # view of that path, and the Lima user can't read mitmproxy-owned files, # so read them with sudo via vm_run. log_path = VMPaths.LOG_DIR / f"{cell_name}.jsonl" - result = vm_run(["sudo", "cat", str(log_path)], timeout=10) + result = vm_run(["cat", str(log_path)], timeout=10, sudo=True) if result.returncode != 0: output(f"No network logs for cell '{cell_name}'") return 0 @@ -66,7 +80,7 @@ def cmd_network(args: Any) -> int: return 0 -def _cmd_network_from_otel(args: Any) -> int: +def _cmd_network_from_otel(args: argparse.Namespace) -> int: """Read the cell's request log from the OTel collector's logs file. The collector's file/logs exporter writes one OTLP ResourceLogs @@ -83,8 +97,11 @@ def _cmd_network_from_otel(args: Any) -> int: output("No collector logs available") return 0 + # Walk forward (oldest → newest) and collect every match, then take the + # last `tail`. A single batch line can hold more than `tail` records, so + # breaking early would render the oldest of that batch, not the newest. matches: list[dict] = [] - for line in reversed(result.stdout.splitlines()): + for line in result.stdout.splitlines(): if not line.strip(): continue try: @@ -114,25 +131,35 @@ def _cmd_network_from_otel(args: Any) -> int: "bytes_in": attrs.get("bytes_in", 0), "bytes_out": attrs.get("bytes_out", 0), }) - if len(matches) >= tail: - break - if len(matches) >= tail: - break - for entry in reversed(matches): + for entry in matches[-tail:]: _print_network_line(entry) return 0 +# C0 controls + DEL + C1 controls. Stripped from cell-controlled fields before +# they reach the operator's terminal (ANSI/CR log-line forgery defense). +_CONTROL_CHARS_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]") + + +def _clean(value: object) -> str: + return _CONTROL_CHARS_RE.sub("", str(value)) + + def _print_network_line(entry: dict) -> None: ts = entry.get("ts") or entry.get("timestamp", "") - method = entry.get("method", "") - host = entry.get("host", "") - path = entry.get("path", "") + # host/path/method/reason are cell-controlled and stored raw in the JSONL + # (and _redact_path runs unquote, which DECODES %1b→ESC / %0d→CR). Strip + # C0/C1 control bytes before printing so a cell can't inject ANSI escapes + # or CRs to forge/erase lines in the operator's terminal — mirrors the + # sanitize enforce.py applies before logging. + method = _clean(entry.get("method", "")) + host = _clean(entry.get("host", "")) + path = _clean(entry.get("path", "")) status = entry.get("status", entry.get("status_code", "")) blocked = entry.get("blocked") tag = " [BLOCKED]" if blocked else "" - reason = f" ({entry.get('block_reason', '')})" if blocked else "" + reason = f" ({_clean(entry.get('block_reason', ''))})" if blocked else "" # Invariant 11: passthrough flows have no method/path/status — only # SNI (host) + bytes. Render distinctly so operators can grep them # from MITM lines. @@ -146,10 +173,10 @@ def _print_network_line(entry: dict) -> None: return ingress_route = entry.get("ingress_route") if ingress_route: - ingress_src = entry.get("ingress_src_ip", entry.get("src_ip", "?")) + ingress_src = _clean(entry.get("ingress_src_ip", entry.get("src_ip", "?"))) output( f"{ts} INGRESS: {ingress_src} -> {method} {host}{path} " - f"-> {status} (route={ingress_route}){tag}{reason}" + f"-> {status} (route={_clean(ingress_route)}){tag}{reason}" ) else: output(f"{ts} OUT: {method} {host}{path} -> {status}{tag}{reason}") @@ -158,8 +185,14 @@ def _print_network_line(entry: dict) -> None: def _attrs_to_dict(attrs: list[dict]) -> dict: out: dict = {} for a in attrs: + if not isinstance(a, dict): + continue key = a.get("key") + if not isinstance(key, str): + continue val = a.get("value", {}) + if not isinstance(val, dict): + continue for typ in ("stringValue", "intValue", "doubleValue", "boolValue"): if typ in val: out[key] = val[typ] @@ -167,8 +200,8 @@ def _attrs_to_dict(attrs: list[dict]) -> dict: return out -def cmd_events(args: Any) -> int: - """Handle `brig events` — print lifecycle events. +def cmd_events(args: argparse.Namespace) -> int: + """Handle `brig cell events` — print lifecycle events. With --follow, blocks and prints new events as they appear. """ diff --git a/src/brig/commands/policy_cmd.py b/src/brig/commands/policy_cmd.py index 86f9e44..7ec9823 100644 --- a/src/brig/commands/policy_cmd.py +++ b/src/brig/commands/policy_cmd.py @@ -8,13 +8,13 @@ from __future__ import annotations +import argparse import json -from typing import Any from brig.errors import BrigError from brig.ops.history import log_policy_change from brig.ops.logging import info, output -from brig.policy.policy import delete_cell_policy, load_cell_policy, save_cell_policy +from brig.policy.policy import delete_cell_policy, load_cell_policy, mutate_cell_policy def register_parser(sub) -> None: @@ -41,7 +41,7 @@ def register_parser(sub) -> None: p_rm.add_argument("name", help="Cell name") -def cmd_policy_show(args: Any) -> int: +def cmd_policy_show(args: argparse.Namespace) -> int: policy = load_cell_policy(args.name) if policy is None: # No per-cell policy file = default deny. Don't error — show @@ -55,7 +55,7 @@ def cmd_policy_show(args: Any) -> int: return 0 -def cmd_policy_test(args: Any) -> int: +def cmd_policy_test(args: argparse.Namespace) -> int: """Simulate a request against a cell's policy.""" policy = load_cell_policy(args.name) if policy is None: @@ -98,13 +98,23 @@ def _name(rule): return 1 -def cmd_policy_set(args: Any) -> int: - policy = load_cell_policy(args.name) or {"allow": [], "deny": []} - old_policy = dict(policy) - changes = _apply_policy_changes(args, policy) +def cmd_policy_set(args: argparse.Namespace) -> int: + # Read-modify-write under one exclusive lock so a concurrent `brig run` + # re-sync or parallel `brig policy set` can't drop this update. + captured: dict = {} - save_cell_policy(args.name, policy) - log_policy_change(args.name, "update", changes, old_policy, policy) + def _mutate(policy: dict | None) -> dict: + import copy + policy = policy or {"allow": [], "deny": []} + # Deep copy: _apply_policy_changes mutates the inner allow/deny lists + # in place, so a shallow dict() copy would make the "old" snapshot + # alias the post-mutation lists and log old == new. + captured["old"] = copy.deepcopy(policy) + captured["changes"] = _apply_policy_changes(args, policy) + return policy + + new_policy = mutate_cell_policy(args.name, _mutate) + log_policy_change(args.name, "update", captured["changes"], captured["old"], new_policy) from brig.cell.metadata import refresh_metadata_if_present refresh_metadata_if_present(args.name) info(f"Updated policy for cell '{args.name}'") @@ -124,7 +134,7 @@ def _validate_domains(domains: list[str]) -> None: raise BrigError(f"Rejected: {suspicious}") -def _apply_policy_changes(args: Any, policy: dict) -> dict: +def _apply_policy_changes(args: argparse.Namespace, policy: dict) -> dict: changes: dict[str, list[str]] = {} if getattr(args, "allow", None): @@ -139,32 +149,31 @@ def _apply_policy_changes(args: Any, policy: dict) -> dict: if getattr(args, "remove_allow", None): allow_list = policy.get("allow", []) - for domain in args.remove_allow: - if domain in allow_list: - allow_list.remove(domain) - changes["remove_allow"] = args.remove_allow + removed = [d for d in args.remove_allow if d in allow_list] + for domain in removed: + allow_list.remove(domain) + # Record only what was actually removed, so the audit summary doesn't + # claim a removal that never happened for an absent domain. + changes["remove_allow"] = removed if getattr(args, "remove_deny", None): deny_list = policy.get("deny", []) - for domain in args.remove_deny: - if domain in deny_list: - deny_list.remove(domain) - changes["remove_deny"] = args.remove_deny + removed = [d for d in args.remove_deny if d in deny_list] + for domain in removed: + deny_list.remove(domain) + changes["remove_deny"] = removed return changes -def cmd_policy_rm(args: Any) -> int: +def cmd_policy_rm(args: argparse.Namespace) -> int: if delete_cell_policy(args.name): log_policy_change(args.name, "delete", changes={}) from brig.cell.metadata import refresh_metadata_if_present refresh_metadata_if_present(args.name) + # Warden auto-reloads per-cell policies on file-change (mtime poll), + # same as `brig policy set` — no explicit reload needed. info(f"Deleted policy for '{args.name}' (cell will block all egress)") - try: - from warden.proxy import reload_policy - reload_policy() - except Exception: - info("Note: run 'warden reload' to apply changes") return 0 raise BrigError(f"No policy for '{args.name}'") diff --git a/src/brig/commands/secrets_cmd.py b/src/brig/commands/secrets_cmd.py index bf7d994..0aec6eb 100644 --- a/src/brig/commands/secrets_cmd.py +++ b/src/brig/commands/secrets_cmd.py @@ -18,7 +18,7 @@ import os import sys -from typing import Any +import argparse from brig.config import HostPaths from brig.errors import BrigError @@ -44,11 +44,11 @@ def register_parser(sub) -> None: DISPATCH = {} # Populated below after handlers are defined. -def cmd_secrets_list(args: Any) -> int: +def cmd_secrets_list(args: argparse.Namespace) -> int: """Handle `brig secrets list`.""" secrets_dir = HostPaths.SECRETS_DIR if not secrets_dir.exists(): - output("No secrets directory. Run: brig init") + output("No secrets directory. Run: brig system init") return 0 secrets = sorted(f.name for f in secrets_dir.iterdir() if f.is_file()) @@ -72,7 +72,7 @@ def cmd_secrets_list(args: Any) -> int: return 0 -def cmd_secrets_add(args: Any) -> int: +def cmd_secrets_add(args: argparse.Namespace) -> int: """Handle `brig secrets add `. Reads value from: @@ -179,13 +179,27 @@ def cmd_secrets_add(args: Any) -> int: return 0 -def cmd_secrets_rm(args: Any) -> int: +def cmd_secrets_rm(args: argparse.Namespace) -> int: """Handle `brig secrets rm `. Requires --yes for non-interactive use, or an interactive y/N confirmation, because secret deletion is irreversible and a typo at the prompt permanently destroys credentials. """ + # Validate the name before building the path — `rm` must apply the same + # traversal/charset rules as `add`, otherwise `brig secrets rm ../../x` + # would unlink a file outside the secrets dir. + from brig.config import SECRET_NAME_PATTERN + name = args.name + if not name: + raise BrigError("Secret name must not be empty") + if "\x00" in name: + raise BrigError("Secret name must not contain null bytes") + if "/" in name or ".." in name: + raise BrigError("Secret name must not contain / or ..") + if not SECRET_NAME_PATTERN.match(name): + raise BrigError(f"Invalid secret name '{name}'") + path = HostPaths.SECRETS_DIR / args.name if not path.exists(): raise BrigError(f"Secret '{args.name}' not found") diff --git a/src/brig/commands/system_cmd.py b/src/brig/commands/system_cmd.py index 508dc19..bf25890 100644 --- a/src/brig/commands/system_cmd.py +++ b/src/brig/commands/system_cmd.py @@ -8,11 +8,12 @@ import time from brig.vm.shell import vm_run from pathlib import Path -from typing import Any +import argparse from brig.config import BRIG_HOME, CONTAINER_PREFIX, STATE_DIR, container_name from brig.errors import BrigError from brig.ops.atomic import atomic_write_json +from brig.ops.locking import locked_file from brig.ops.logging import debug, output from brig.security.verify import verify_all @@ -26,58 +27,67 @@ } -def cmd_init(args: Any) -> int: - """Handle `brig init`.""" - import shutil +def cmd_init(args: argparse.Namespace) -> int: + """Handle `brig system init`. - dirs = [ - BRIG_HOME / "cells" / "addons", - BRIG_HOME / "secrets", - BRIG_HOME / "state" / "system", - BRIG_HOME / "state" / "system" / "policies", - BRIG_HOME / "profiles", - ] - for d in dirs: - d.mkdir(parents=True, exist_ok=True) - - # Sensitive dirs must not be readable/writable by other users on the host. - # secrets: holds API keys, tokens. - # cells/addons: an attacker who can write here can replace enforce.py. - # state/system: holds subnet allocator state and operation history. - for sensitive in (BRIG_HOME / "secrets", - BRIG_HOME / "cells" / "addons", - BRIG_HOME / "state" / "system"): - sensitive.chmod(0o700) - - # Default network policy. - policy_file = BRIG_HOME / "cells" / "network-policy.json" - if not policy_file.exists(): - atomic_write_json(policy_file, _DEFAULT_POLICY) - output(f" Created default policy: {policy_file}") - - # Lima VM template. - lima_yaml = BRIG_HOME / "lima.yaml" - if not lima_yaml.exists(): - template = Path(__file__).parent.parent / "vm" / "lima.yaml.template" - if not template.exists(): - raise BrigError( - f"Lima VM template missing at {template}. " - f"This is a packaging bug — the wheel should ship " - f"src/brig/vm/lima.yaml.template; the editable install " - f"should expose the source tree directly." - ) - shutil.copy2(template, lima_yaml) - output(f" Created VM config: {lima_yaml}") + Serialized under fcntl on $BRIG_HOME/.init.lock so two concurrent + invocations can't race on the lima.yaml template copy (one would + win and overwrite a locally-edited file). + """ + import shutil - output(f"Initialized brig at {BRIG_HOME}") - output("") - output("Next steps:") - output(" brig up # create VM, start VM, start warden") - output(" brig run alpine echo hi # run your first cell") + BRIG_HOME.mkdir(parents=True, exist_ok=True) + with locked_file(BRIG_HOME / ".init.lock"): + dirs = [ + BRIG_HOME / "cells" / "addons", + BRIG_HOME / "secrets", + BRIG_HOME / "state" / "system", + BRIG_HOME / "state" / "system" / "policies", + BRIG_HOME / "profiles", + ] + for d in dirs: + d.mkdir(parents=True, exist_ok=True) + + # secrets: API keys / tokens. addons: an attacker who writes + # here replaces enforce.py. state/system: allocator + audit. + for sensitive in (BRIG_HOME / "secrets", + BRIG_HOME / "cells" / "addons", + BRIG_HOME / "state" / "system"): + sensitive.chmod(0o700) + + policy_file = BRIG_HOME / "cells" / "network-policy.json" + if not policy_file.exists(): + atomic_write_json(policy_file, _DEFAULT_POLICY) + output(f" Created default policy: {policy_file}") + + lima_yaml = BRIG_HOME / "lima.yaml" + if not lima_yaml.exists(): + template = Path(__file__).parent.parent / "vm" / "lima.yaml.template" + if not template.exists(): + raise BrigError( + f"Lima VM template missing at {template}. " + f"This is a packaging bug — the wheel should ship " + f"src/brig/vm/lima.yaml.template; the editable install " + f"should expose the source tree directly." + ) + shutil.copy2(template, lima_yaml) + output(f" Created VM config: {lima_yaml}") + + # Apply any configured mount_roots into the VM config's managed + # block (no-op when unset). Picked up on VM create. + from brig.vm.lima_mounts import sync_lima_mount_roots + if sync_lima_mount_roots(lima_yaml): + output(" Applied mount_roots to VM config") + + output(f"Initialized brig at {BRIG_HOME}") + output("") + output("Next steps:") + output(" brig system up # create VM, start VM, start warden") + output(" brig run alpine echo hi # run your first cell") return 0 -def cmd_verify(args: Any) -> int: +def cmd_verify(args: argparse.Namespace) -> int: """Handle `brig verify`.""" output("Verifying security invariants...") output("=" * 50) @@ -102,15 +112,15 @@ def cmd_verify(args: Any) -> int: return 0 -def cmd_doctor(args: Any) -> int: - """Handle `brig doctor` — deep environment + system check. +def cmd_doctor(args: argparse.Namespace) -> int: + """Handle `brig system doctor` — deep environment + system check. - Goes beyond `brig health`: verifies tooling on PATH, Lima version, gVisor - presence inside the VM, addons installed, port collisions, and disk - space. Prints a checklist with actionable suggestions on each failure. + Verifies tooling on PATH, Lima version, gVisor presence inside the VM, + addons installed, port collisions, and disk space. Prints a checklist + with actionable suggestions on each failure. - --quick: only the two essentials (proxy + VM). Equivalent to the - deprecated `brig health` and meant for scripting / readiness probes. + --quick: only the two essentials (proxy + VM) — a fast readiness probe + for scripting. """ if getattr(args, "quick", False): return _cmd_doctor_quick() @@ -119,7 +129,7 @@ def cmd_doctor(args: Any) -> int: from brig.network.proxy import proxy_running failures = [] - output("Running brig doctor...") + output("Running brig system doctor...") output("=" * 50) def _check(label: str, ok: bool, detail: str = "", suggestion: str = ""): @@ -173,7 +183,7 @@ def _check(label: str, ok: bool, detail: str = "", suggestion: str = ""): detail=f"mode: {oct(actual_mode)}", suggestion=f"chmod 0700 {dir_path}") else: - _check(f"{dir_path} exists", False, suggestion="brig init") + _check(f"{dir_path} exists", False, suggestion="brig system init") # 4. Required addons present. for addon in ["enforce.py", "logger.py", "_common.py"]: @@ -190,13 +200,19 @@ def _check(label: str, ok: bool, detail: str = "", suggestion: str = ""): except json.JSONDecodeError as e: _check(f"policy: {policy_path.name}", False, detail=str(e), - suggestion=f"Fix {policy_path} or re-seed: brig init") + suggestion=f"Fix {policy_path} or re-seed: brig system init") else: - _check(f"policy: {policy_path.name}", False, suggestion="brig init") + _check(f"policy: {policy_path.name}", False, suggestion="brig system init") # 6. Warden running. _check("warden proxy running", proxy_running(), - suggestion="brig up") + suggestion="brig system up") + + # 6b. OTel collector running — `brig system stats` and + # `brig cell network --otel` read from it; warden always emits to it. + from brig.observability import collector + _check("otel collector running", collector.is_running(), + suggestion="brig system down && brig system up") # 7. host_socket bridges — if any plists exist under LaunchAgents, # the corresponding bridge sockets must be present. Missing bridge @@ -204,7 +220,7 @@ def _check(label: str, ok: bool, detail: str = "", suggestion: str = ""): # also checked since the bridges require it. _check_host_socket_bridges(_check) - # 8. Warden CA staleness — aitelier hit a silent-TLS-hang foot-gun: + # 8. Warden CA staleness — a silent-TLS-hang foot-gun: # a cell entrypoint that ALSO sets SSL_CERT_FILE clobbers brig's # auto-mounted bundle. On the next warden restart, mitmproxy # regenerates its CA, brig re-stages bundles, but the cell's @@ -242,7 +258,10 @@ def _check_warden_ca_consistency(check) -> None: from brig.config import HostPaths ca_path = "/var/lib/warden/mitmproxy-state/mitmproxy-ca-cert.pem" - result = vm_run(["sudo", "cat", ca_path], timeout=5) + # Bind-mounted dir is uid-1000 owned (matches the lima user); no sudo + # needed. vm_run's auto-sudo whitelist intentionally doesn't cover + # `cat`, and a literal `sudo cat` here would diverge from that pattern. + result = vm_run(["cat", ca_path], timeout=5) if result.returncode != 0: # Warden hasn't run yet, or CA missing — separate check (#6 # "warden proxy running") handles that case loudly. @@ -276,7 +295,7 @@ def _check_warden_ca_consistency(check) -> None: suggestion=f"brig cell restart {entry.name}", ) - # Foot-gun catch (aitelier-flagged): a cell's image may set + # Foot-gun catch: a cell's image may set # SSL_CERT_FILE in its Config.Env, pointing at a path that # ISN'T brig's auto-mounted bundle. The TLS handshake against # warden's MITM cert then succeeds on the client side (whatever @@ -288,9 +307,8 @@ def _check_warden_ca_consistency(check) -> None: def _check_entrypoint_ssl_cert_override(check, cell_name: str) -> None: """Warn if a cell's effective env sets SSL_CERT_FILE differently - from brig's auto-mount target. Foot-gun aitelier diagnosed — - silent TLS hangs result when warden's CA rotates and the cell - trusts whatever the entrypoint pointed at. + from brig's auto-mount target. Silent TLS hangs result when warden's + CA rotates and the cell trusts whatever the entrypoint pointed at. Inspects the live container's env via `podman inspect`. No-op if the cell isn't running (the bundle mtime check above already @@ -377,9 +395,8 @@ def _check_host_socket_bridges(check) -> None: ) -def _cmd_doctor_quick(fmt: str = "table") -> int: - """The "quick" two-essentials check, shared by `brig doctor --quick` - and the deprecated `brig health`.""" +def _cmd_doctor_quick() -> int: + """The "quick" two-essentials check behind `brig system doctor --quick`.""" from brig.network.proxy import proxy_running checks = [("Proxy running", proxy_running())] @@ -392,10 +409,6 @@ def _cmd_doctor_quick(fmt: str = "table") -> int: ) checks.append(("VM reachable", vm_result.returncode == 0)) - if fmt == "json": - output(json.dumps([{"check": n, "passed": p} for n, p in checks], indent=2)) - return 0 if all(p for _, p in checks) else 1 - all_ok = True for name, passed in checks: status = "[OK]" if passed else "[FAIL]" @@ -407,7 +420,7 @@ def _cmd_doctor_quick(fmt: str = "table") -> int: -def cmd_diagnose(args: Any) -> int: +def cmd_diagnose(args: argparse.Namespace) -> int: """Handle `brig diagnose` — run diagnostic checks for a cell.""" cn = container_name(args.name) @@ -423,7 +436,10 @@ def cmd_diagnose(args: Any) -> int: if isinstance(data, list): data = data[0] except json.JSONDecodeError: - raise BrigError("Could not parse container info") + raise BrigError( + "Could not parse container info", + suggestion="Check podman / VM health: brig system doctor", + ) state = data.get("State", {}) output(f"Cell: {args.name}") @@ -438,7 +454,7 @@ def cmd_diagnose(args: Any) -> int: return 0 -def cmd_preflight(args: Any) -> int: +def cmd_preflight(args: argparse.Namespace) -> int: """Handle `brig preflight` — run pre-start checks.""" from warden.reconcile import reconcile_subnet_state @@ -454,7 +470,7 @@ def cmd_preflight(args: Any) -> int: return 0 -def cmd_metrics(args: Any) -> int: +def cmd_metrics(args: argparse.Namespace) -> int: """Handle `brig metrics` — output Prometheus-style metrics.""" from brig.network.subnet import list_all @@ -475,7 +491,7 @@ def cmd_metrics(args: Any) -> int: return 0 -def cmd_prune(args: Any) -> int: +def cmd_prune(args: argparse.Namespace) -> int: """Handle `brig prune` — clean up stopped cells, old logs, orphan subnets. With --dry-run, prints what would be removed without taking action. @@ -498,8 +514,8 @@ def cmd_prune(args: Any) -> int: freed_subnets = 0 # 1. Stopped cells AND orphan workspace dirs (state dirs with no - # container, left by `brig rm` versions before the workspace - # cleanup landed, or by externally-killed containers). + # container, left behind by externally-killed containers or an + # interrupted cell rm). if do_cells: from brig.cell.lifecycle import list_cell_containers live_cells: set[str] = set() @@ -566,6 +582,17 @@ def cmd_prune(args: Any) -> int: pass freed_subnets += 1 + # Drop ingress routes for cells whose subnet allocation is gone, so a + # reused /24 index can't inherit a stale route (old cell_ip + auth + # token). Keyed by the cells that still hold an allocation. + if not dry_run: + from brig.network.ingress import sweep_orphan_routes + swept = sweep_orphan_routes( + live_cells={info.cell_name for info in list_all()} + ) + if swept: + output(f" swept {swept} orphan ingress route(s)") + output("") output(f"Pruned: {removed_cells} cells, {removed_logs} log files, {freed_subnets} subnets") if dry_run: @@ -573,7 +600,7 @@ def cmd_prune(args: Any) -> int: return 0 -def cmd_history(args: Any) -> int: +def cmd_history(args: argparse.Namespace) -> int: """Handle `brig history` — show operation history.""" history_file = STATE_DIR / "system" / "operations.jsonl" if not history_file.exists(): diff --git a/src/brig/commands/watchdog_cmd.py b/src/brig/commands/watchdog_cmd.py index 90876f4..281f37b 100644 --- a/src/brig/commands/watchdog_cmd.py +++ b/src/brig/commands/watchdog_cmd.py @@ -9,12 +9,12 @@ import signal import time -from typing import Any +import argparse from brig.ops.logging import info, output, warn -def cmd_watchdog(args: Any) -> int: +def cmd_watchdog(args: argparse.Namespace) -> int: """Handle `brig watchdog` — monitor and restart warden.""" from brig.network.proxy import proxy_running from warden.proxy import start, stop diff --git a/src/brig/config.py b/src/brig/config.py index b1d9487..ac4f6bf 100644 --- a/src/brig/config.py +++ b/src/brig/config.py @@ -14,13 +14,13 @@ from pathlib import Path # Version. -VERSION = "0.3.1" +VERSION = "0.4.0" # Lima VM name. VM_NAME = "brig" # Canonical cell name pattern: lowercase alphanumeric start, max 63 chars. -CELL_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9._-]{0,62}$') +CELL_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9._-]{0,62}\Z') # Container naming prefix for cells. CONTAINER_PREFIX = "brig-" @@ -77,10 +77,10 @@ def container_name(cell_name: str) -> str: SENSITIVE_PATTERNS = {"password", "secret", "key", "token", "credential", "auth", "value"} # Valid memory suffixes. -MEMORY_PATTERN = r"^\d+[kmgKMG]?$" +MEMORY_PATTERN = r"^\d+[kmgKMG]?\Z" # Valid domain pattern for policy. -DOMAIN_PATTERN = r"^(\*\.)?[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)*$" +DOMAIN_PATTERN = r"^(\*\.)?[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)*\Z" # Suspicious domain patterns that could enable DNS rebinding attacks. SUSPICIOUS_DOMAIN_PATTERNS = [ @@ -90,31 +90,53 @@ def container_name(cell_name: str) -> str: # Host services (cell → host forwarding through Warden). # The .host.brig suffix is also defined as HOST_SERVICE_SUFFIX in -# src/addons/enforce.py — addons can't import from brig.* so the suffix +# src/brig/warden_addons/enforce.py — addons can't import from brig.* so the suffix # lives in both places. Keep them in sync. # # Single-tenant flattened model (see also host_sockets): cell yaml # declares both the name AND the host port. There is no separate # global registry — declaring in yaml IS the authorization. The # operator who wrote the yaml is the trust principal. -HOST_SERVICE_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9-]{0,30}$') +HOST_SERVICE_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9-]{0,30}\Z') MAX_HOST_SERVICES_PER_CELL = 16 +# Warden's HTTP(S) forward-proxy listen port — the egress choke point cells +# are pointed at via HTTP(S)_PROXY. Single source of truth for the proxy port. +PROXY_PORT = 8080 + # Ingress (authenticated reverse proxy through Warden). INGRESS_PORT = 8443 +# Minimum ingress bearer-token length. The salted hash lives in the +# untrusted routes file (invariant 4) and is offline-crackable at SHA-256 +# speed, so short tokens are rejected rather than warned. `openssl rand +# -hex 32` produces a 64-char token. +INGRESS_TOKEN_MIN_LEN = 32 MAX_INGRESS_PER_CELL = 8 -INGRESS_AUTH_METHODS = {"token"} -INGRESS_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9-]{0,30}$') -INGRESS_PATH_PREFIX_PATTERN = re.compile(r'^/[a-zA-Z0-9/_-]+$') +# "token": brig is the gate (Bearer-token perimeter auth). "none": transparent +# pass-through — brig does NOT authenticate; the cell's own app is the gate +# (opt-in, rejected on the untrusted profile, audited at run time). +INGRESS_AUTH_METHODS = {"token", "none"} +INGRESS_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9-]{0,30}\Z') +INGRESS_PATH_PREFIX_PATTERN = re.compile(r'^/[a-zA-Z0-9/_-]+\Z') # Host sockets — kernel-side channel between a cell and a macOS host # service via a bind-mounted unix socket. Bypasses Warden by design # (the bytes never reach the proxy), so the validators here are the # only thing standing between a cell yaml and a host file. -HOST_SOCKET_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9-]{0,30}$') +HOST_SOCKET_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9-]{0,30}\Z') MAX_HOST_SOCKETS_PER_CELL = 8 HOST_SOCKET_MOUNT_PREFIX = "/run/host/" HOST_SOCKET_MODES = {"ro", "rw"} + +# Scoped host-directory mounts — bind-mount an operator-chosen host directory +# into a non-untrusted cell (ro default / rw opt-in). Bytes bypass Warden, so +# the boundary is the validators + the mount_roots() allowlist + runtime +# realpath re-containment. A cell-side symlink can't escape the subtree +# (mount-namespace isolation); the residual host-side symlink risk is mitigated +# by `brig cell mount-scan`. See docs/design/host-mounts.md. +MOUNT_NAME_PATTERN = HOST_SOCKET_NAME_PATTERN +MAX_MOUNTS_PER_CELL = 8 +MOUNT_MODES = HOST_SOCKET_MODES # {"ro", "rw"} # Container-engine sockets — granting these to a cell is root-equivalent # on the host. Denied at parse time unless the operator passes the # (future) --allow-engine-socket override. @@ -127,7 +149,7 @@ def container_name(cell_name: str) -> str: # to a safe character set. Empty names, null bytes, leading dashes, and # shell metacharacters are excluded; an empty name would otherwise collapse # the per-secret bind mount to the whole secrets directory. -SECRET_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9._-]{0,62}$') +SECRET_NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9._-]{0,62}\Z') # Unsafe file extensions for --sanitize mode. UNSAFE_EXTENSIONS = { @@ -218,6 +240,146 @@ class VMPaths: # reached via the /state virtiofs mount. HOST_SOCKETS_DIR = SYSTEM_DIR / "host-sockets" + # Root under which each declared mount_root is Lima-mounted, at + # /mnt/host/. See mount_roots() / mount_root_slug(). + MOUNTS_DIR = Path("/mnt/host") + + +def mount_roots() -> list[str]: + """Operator-declared host trees that may be exposed to cells via `mounts:`. + + Read lazily from CONFIG_FILE (key `mount_roots`) — VM-level, not per-cell, + because Lima mounts are static and VM-wide (see docs/design/host-mounts.md). + Accepts a JSON list or a comma-separated string; returns absolute, + user-expanded, normalized paths. Empty (the default) means `mounts:` is + disabled. + """ + import json + try: + with open(HostPaths.CONFIG_FILE) as f: + raw = json.load(f).get("mount_roots", []) + except (OSError, ValueError): + return [] + if isinstance(raw, str): + raw = raw.split(",") + if not isinstance(raw, list): + return [] + out = [] + for p in raw: + if isinstance(p, str) and p.strip(): + out.append(os.path.normpath(os.path.expanduser(p.strip()))) + return out + + +def mount_root_slug(root: str) -> str: + """Stable VM-mount subdir name for a declared root (sanitized realpath + basename). + + Resolved via realpath so EVERY caller derives the same slug for a given root + — validation's collision check (config.py), the lima.yaml render + (vm/lima_mounts.py), and the reconciler's bind (cell/reconciler.py). If the + raw and realpath basenames differed, two symlinked roots could collide at one + `/mnt/host/` while validation, using a different basis, saw no + collision — silently shadowing one host tree with another. + """ + base = os.path.basename(os.path.realpath(os.path.expanduser(root))) or "root" + return re.sub(r"[^A-Za-z0-9._-]", "_", base) + + +def validate_mount_roots(roots: list[str]) -> list[str]: + """Validate operator-declared mount_roots; return a list of error strings. + + These trees are exposed to cells (and to the VM via lima.yaml), so a too-broad + or sensitive root, or one with a slug collision / YAML-hostile chars, is + refused — the floor docs/design/host-mounts.md promises. Each must be an + absolute, existing directory, not a catastrophe/secret tree (nor an ancestor + or descendant of one), with a unique VM-mount slug. + + Comparison is by real path (symlinks resolved) plus on-disk identity, so a + symlink, a realpath alias (/etc -> /private/etc), or a case variant on a + case-insensitive filesystem (~/.SSH == ~/.ssh) cannot dodge the floor — the + reconciler binds the realpath, so validation must reason about it too. + """ + home = Path.home() + + def _real(p: object) -> str: + return os.path.realpath(os.path.expanduser(str(p))) + + # "broad" trees: reject a root that EQUALS one or CONTAINS one (an ancestor + # would expose it). Descendants are fine — mounting ~/work (under $HOME) is + # the normal case; only mounting $HOME itself (or a parent) is too broad. + broad = [_real(p) for p in ("/", home, HostPaths.BRIG_HOME, "/etc")] + # secret trees: also reject a root that sits UNDER one (a slice of secrets). + secrets = [_real(p) for p in ( + home / ".ssh", home / ".aws", home / ".gnupg", + home / ".config" / "gcloud", home / ".kube", home / ".docker", + )] + errors: list[str] = [] + seen_slugs: dict[str, str] = {} + for root in roots: + if not isinstance(root, str) or not root.strip(): + errors.append("mount_roots entries must be non-empty strings") + continue + if "\n" in root or '"' in root: + errors.append(f"mount_roots entry {root!r} contains an illegal character") + continue + # Check absoluteness on the literal path: realpath() would turn a + # relative path into cwd-relative-but-absolute and mask the error. + if not os.path.isabs(os.path.expanduser(root.strip())): + errors.append(f"mount_root {root!r} must be an absolute path") + continue + rp_s = _real(root.strip()) + rp = Path(rp_s) + bad = False + for s in broad + secrets: + if _same_dir(rp_s, s) or _is_ancestor(rp, Path(s)): # root is s, or contains s + errors.append( + f"mount_root {root!r} is or contains a protected path ({s}); " + f"choose a narrower directory" + ) + bad = True + break + for s in secrets: + if not bad and _is_ancestor(Path(s), rp): # root is a slice of a secret tree + errors.append(f"mount_root {root!r} is inside a secret directory ({s})") + bad = True + break + if bad: + continue + if not rp.is_dir(): + errors.append(f"mount_root {root!r} is not an existing directory") + continue + slug = mount_root_slug(rp_s) + if slug in seen_slugs: + errors.append( + f"mount_roots {seen_slugs[slug]!r} and {root!r} collide at " + f"/mnt/host/{slug}; rename one so their basenames differ" + ) + continue + seen_slugs[slug] = root + return errors + + +def _is_ancestor(ancestor: Path, descendant: Path) -> bool: + """True if `descendant` is strictly under `ancestor`.""" + try: + descendant.relative_to(ancestor) + return descendant != ancestor + except ValueError: + return False + + +def _same_dir(a: str, b: str) -> bool: + """True if `a` and `b` are the same directory on disk — by path, or by inode + (catches case variants on case-insensitive filesystems where the realpath + strings differ but resolve to one directory).""" + if a == b: + return True + try: + return os.path.samefile(a, b) + except OSError: + return False + # Backward-compatible aliases used by modules that haven't been updated. BRIG_HOME = HostPaths.BRIG_HOME diff --git a/src/brig/network/ingress.py b/src/brig/network/ingress.py index 233e6e5..fc386f5 100644 --- a/src/brig/network/ingress.py +++ b/src/brig/network/ingress.py @@ -8,7 +8,6 @@ from __future__ import annotations -import fcntl import hashlib import json import os @@ -16,6 +15,7 @@ from brig.config import HostPaths from brig.ops.atomic import atomic_write_json +from brig.ops.locking import locked_file from brig.ops.logging import debug, info @@ -51,7 +51,7 @@ def register_ingress( cell_name: str, cell_ip: str, ingress_spec: list[dict], - auth_token: str, + auth_token: str | None = None, ) -> None: """Register ingress routes for a cell. @@ -59,7 +59,8 @@ def register_ingress( cell_name: Cell name. cell_ip: Cell's IP on its network (from podman inspect). ingress_spec: List of ingress entries from CellSpec. - auth_token: Raw bearer token for authentication. + auth_token: Raw bearer token. Required only for `auth: token` routes; + may be None when every route is `auth: none` (transparent). """ if not ingress_spec: return @@ -68,35 +69,34 @@ def register_ingress( lock_file = routes_file.with_suffix(".lock") routes_file.parent.mkdir(parents=True, exist_ok=True) - token_hash, token_salt = _hash_token(auth_token) - - with open(lock_file, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - try: - data = _load_routes(routes_file) - - # Remove any existing routes for this cell (idempotent). - data["routes"] = [ - r for r in data["routes"] if r.get("cell") != cell_name - ] - - # Add new routes. - for entry in ingress_spec: - data["routes"].append({ - "cell": cell_name, - "cell_ip": cell_ip, - "name": entry["name"], - "port": entry["port"], - "path_prefix": entry["path_prefix"], - "auth": entry["auth"], - "auth_secret_hash": token_hash, - "auth_salt": token_salt, - }) + # Hash once, attached only to `auth: token` routes. `auth: none` routes + # carry no secret — brig doesn't gate them. + token_hash, token_salt = _hash_token(auth_token) if auth_token else ("", "") - atomic_write_json(routes_file, data) - info(f"Registered {len(ingress_spec)} ingress routes for '{cell_name}'") - finally: - fcntl.flock(lock, fcntl.LOCK_UN) + with locked_file(lock_file): + data = _load_routes(routes_file) + + # Remove any existing routes for this cell (idempotent). + data["routes"] = [ + r for r in data["routes"] if r.get("cell") != cell_name + ] + + # Add new routes. + for entry in ingress_spec: + gated = entry["auth"] != "none" + data["routes"].append({ + "cell": cell_name, + "cell_ip": cell_ip, + "name": entry["name"], + "port": entry["port"], + "path_prefix": entry["path_prefix"], + "auth": entry["auth"], + "auth_secret_hash": token_hash if gated else "", + "auth_salt": token_salt if gated else "", + }) + + atomic_write_json(routes_file, data) + info(f"Registered {len(ingress_spec)} ingress routes for '{cell_name}'") def deregister_ingress(cell_name: str) -> None: @@ -107,18 +107,40 @@ def deregister_ingress(cell_name: str) -> None: if not routes_file.exists(): return - with open(lock_file, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - try: - data = _load_routes(routes_file) - before = len(data["routes"]) - data["routes"] = [ - r for r in data["routes"] if r.get("cell") != cell_name - ] - after = len(data["routes"]) - - if before != after: - atomic_write_json(routes_file, data) - debug(f"Deregistered {before - after} ingress routes for '{cell_name}'") - finally: - fcntl.flock(lock, fcntl.LOCK_UN) + with locked_file(lock_file): + data = _load_routes(routes_file) + before = len(data["routes"]) + data["routes"] = [ + r for r in data["routes"] if r.get("cell") != cell_name + ] + after = len(data["routes"]) + + if before != after: + atomic_write_json(routes_file, data) + debug(f"Deregistered {before - after} ingress routes for '{cell_name}'") + + +def sweep_orphan_routes(live_cells: set[str]) -> int: + """Drop routes for cells not in `live_cells`. Returns count removed. + + When the subnet allocator reuses a freed index, a new cell at the + same private IP would otherwise inherit the prior cell's hashed + auth token. Sweeping at known-safe moments (after `brig system + down`, on `brig system prune`) keeps the routes file in sync with + allocated cells. + """ + routes_file = HostPaths.INGRESS_ROUTES_FILE + if not routes_file.exists(): + return 0 + lock_file = routes_file.with_suffix(".lock") + with locked_file(lock_file): + data = _load_routes(routes_file) + before = len(data["routes"]) + data["routes"] = [ + r for r in data["routes"] if r.get("cell") in live_cells + ] + removed = before - len(data["routes"]) + if removed: + atomic_write_json(routes_file, data) + debug(f"Swept {removed} orphan ingress routes") + return removed diff --git a/src/brig/network/proxy.py b/src/brig/network/proxy.py index 3586ef0..00bf5f9 100644 --- a/src/brig/network/proxy.py +++ b/src/brig/network/proxy.py @@ -6,11 +6,8 @@ from __future__ import annotations -import json - -from brig.config import PROXY_NAME, container_name +from brig.config import PROXY_NAME from brig.ops.cache import cached, set_cache -from brig.ops.logging import debug from brig.vm.shell import vm_run @@ -18,7 +15,7 @@ def proxy_running() -> bool: """Check if the Warden proxy container is running. Cached for 2 seconds.""" hit, val = cached("proxy_running") if hit: - return val # type: ignore[return-value] + return bool(val) result = vm_run( ["podman", "inspect", PROXY_NAME, "--format", "{{.State.Status}}"], @@ -27,44 +24,3 @@ def proxy_running() -> bool: running = result.returncode == 0 and result.stdout.strip() == "running" set_cache("proxy_running", running) return running - - -def get_proxy_ip(cell_name: str) -> str | None: - """Get the proxy's IP address on a cell's network.""" - network_name = container_name(cell_name) - result = vm_run( - ["podman", "inspect", PROXY_NAME, "--format", "json"], - timeout=5, - ) - if result.returncode != 0: - return None - - try: - info = json.loads(result.stdout) - if isinstance(info, list): - info = info[0] - networks = info.get("NetworkSettings", {}).get("Networks", {}) - ip = networks.get(network_name, {}).get("IPAddress", "") - return ip if ip else None - except json.JSONDecodeError: - return None - - -def connect_proxy_to_network(cell_name: str) -> bool: - """Connect the proxy container to a cell's network.""" - network_name = container_name(cell_name) - result = vm_run(["podman", "network", "connect", network_name, PROXY_NAME]) - if result.returncode != 0: - debug(f"Failed to connect proxy to {network_name}: {result.stderr}") - return False - return True - - -def disconnect_proxy_from_network(cell_name: str) -> bool: - """Disconnect the proxy container from a cell's network.""" - network_name = container_name(cell_name) - result = vm_run(["podman", "network", "disconnect", network_name, PROXY_NAME]) - if result.returncode != 0: - debug(f"Failed to disconnect proxy from {network_name}: {result.stderr}") - return False - return True diff --git a/src/brig/network/subnet.py b/src/brig/network/subnet.py index 2645b32..9b053e2 100644 --- a/src/brig/network/subnet.py +++ b/src/brig/network/subnet.py @@ -9,7 +9,6 @@ from __future__ import annotations -import fcntl import json import os from dataclasses import dataclass @@ -18,6 +17,7 @@ from brig.config import ALLOCATOR_LOCK_FILE, CELL_NAME_PATTERN, SUBNET_STATE_FILE from brig.ops.atomic import atomic_write_json +from brig.ops.locking import locked_file SUBNET_PREFIX = "10.60" MIN_INDEX = 1 @@ -49,7 +49,7 @@ def _load_state(state_file: Path = SUBNET_STATE_FILE) -> dict: UNLOCKED — caller must hold the allocator file lock (fcntl.LOCK_SH or LOCK_EX on ALLOCATOR_LOCK_FILE). Direct reads without the lock can observe a torn file in the rename window. All callers in this module - (allocate/free/get/list_all/get_subnet_map) wrap _load_state in a lock; + (allocate/free/get/list_all) wrap _load_state in a lock; if you add a new caller, do the same. """ default: dict = {"next_index": 1, "allocated": {}, "freed": []} @@ -78,6 +78,48 @@ def _load_state(state_file: Path = SUBNET_STATE_FILE) -> dict: # Filter out invalid freed indices. state["freed"] = [i for i in state["freed"] if isinstance(i, int) and validate_index(i)] + + # Validate allocated indices too (subnets.json is untrusted, invariant 4): + # drop entries whose index is non-int / out-of-range / colliding with an + # already-seen index. An unchecked index flows into index_to_subnet and + # would yield a malformed CIDR or two cells sharing a /24. + seen_indices: set[int] = set() + clean_allocated: dict = {} + for cell_name, info in state["allocated"].items(): + if not isinstance(info, dict): + continue + idx = info.get("index") + if not isinstance(idx, int) or not validate_index(idx) or idx in seen_indices: + continue + # get()/list_all() read info["allocated_at"]; a tampered entry missing + # it (or with a non-str value) would KeyError out of unwrapped callers + # (brig system metrics/prune/sweep). Normalize to "" so reads degrade + # gracefully rather than crash (invariant 4: state dir is untrusted). + if not isinstance(info.get("allocated_at"), str): + info = {**info, "allocated_at": ""} + seen_indices.add(idx) + clean_allocated[cell_name] = info + state["allocated"] = clean_allocated + + # Enforce disjointness across allocated / freed / next_index — a tampered + # file could otherwise hand out an in-use /24 (two cells, one subnet): + # - drop freed indices that collide with an allocated one (allocate() + # pops freed first and would re-issue the live index), and dedup freed + # so the same index can't be popped for two cells; + # - clamp next_index above every allocated/freed index so the sequential + # path can't issue an index already in use. + deduped_freed: list[int] = [] + freed_seen: set[int] = set() + for i in state["freed"]: + if i in seen_indices or i in freed_seen: + continue + freed_seen.add(i) + deduped_freed.append(i) + state["freed"] = deduped_freed + + used = seen_indices | freed_seen + if used: + state["next_index"] = max(state["next_index"], max(used) + 1) return state @@ -121,47 +163,55 @@ def allocate( """Allocate a /24 subnet for a cell. Thread/process-safe via file locking. Raises ValueError on failure. + + Idempotent per cell name: a subnet is keyed by cell name, so re-allocating + for a name that already holds one returns the existing allocation rather + than failing. This lets `brig run` reclaim a same-named orphan (e.g. after + a VM restart leaves the host-side allocation but drops the podman network) + instead of erroring with "already has subnet allocated". """ if not CELL_NAME_PATTERN.match(cell_name): raise ValueError(f"Invalid cell name '{cell_name}'") - lock_file.parent.mkdir(parents=True, exist_ok=True) - with open(lock_file, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - try: - state = _load_state(state_file) - - if cell_name in state["allocated"]: - raise ValueError(f"Cell '{cell_name}' already has subnet allocated") - - # Prefer freed indices, then next_index. - if state["freed"]: - index = state["freed"].pop(0) - if not validate_index(index): - raise ValueError(f"Corrupted state: freed index {index} out of range") - else: - index = state["next_index"] - if not validate_index(index): - raise ValueError(f"No more subnets available (max {MAX_INDEX} cells)") - state["next_index"] = index + 1 - - allocated_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - state["allocated"][cell_name] = { - "index": index, - "allocated_at": allocated_at, - } - - _save_state(state, state_file) - _write_subnet_map(state, map_file=state_file.parent / "subnet-map.json") + with locked_file(lock_file): + state = _load_state(state_file) + existing = state["allocated"].get(cell_name) + if existing is not None: + index = existing["index"] return SubnetInfo( cell_name=cell_name, index=index, subnet=index_to_subnet(index), - allocated_at=allocated_at, + allocated_at=existing["allocated_at"], ) - finally: - fcntl.flock(lock, fcntl.LOCK_UN) + + # Prefer freed indices, then next_index. + if state["freed"]: + index = state["freed"].pop(0) + if not validate_index(index): + raise ValueError(f"Corrupted state: freed index {index} out of range") + else: + index = state["next_index"] + if not validate_index(index): + raise ValueError(f"No more subnets available (max {MAX_INDEX} cells)") + state["next_index"] = index + 1 + + allocated_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + state["allocated"][cell_name] = { + "index": index, + "allocated_at": allocated_at, + } + + _save_state(state, state_file) + _write_subnet_map(state, map_file=state_file.parent / "subnet-map.json") + + return SubnetInfo( + cell_name=cell_name, + index=index, + subnet=index_to_subnet(index), + allocated_at=allocated_at, + ) def free( @@ -173,26 +223,21 @@ def free( if not CELL_NAME_PATTERN.match(cell_name): raise ValueError(f"Invalid cell name '{cell_name}'") - lock_file.parent.mkdir(parents=True, exist_ok=True) - with open(lock_file, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - try: - state = _load_state(state_file) + with locked_file(lock_file): + state = _load_state(state_file) - if cell_name not in state["allocated"]: - raise ValueError(f"Cell '{cell_name}' has no subnet allocated") + if cell_name not in state["allocated"]: + raise ValueError(f"Cell '{cell_name}' has no subnet allocated") - index = state["allocated"][cell_name]["index"] - del state["allocated"][cell_name] + index = state["allocated"][cell_name]["index"] + del state["allocated"][cell_name] - if index not in state["freed"]: - state["freed"].append(index) - state["freed"].sort() + if index not in state["freed"]: + state["freed"].append(index) + state["freed"].sort() - _save_state(state, state_file) - _write_subnet_map(state, map_file=state_file.parent / "subnet-map.json") - finally: - fcntl.flock(lock, fcntl.LOCK_UN) + _save_state(state, state_file) + _write_subnet_map(state, map_file=state_file.parent / "subnet-map.json") def get( @@ -201,22 +246,17 @@ def get( lock_file: Path = ALLOCATOR_LOCK_FILE, ) -> SubnetInfo | None: """Get subnet info for a cell. Returns None if not allocated.""" - lock_file.parent.mkdir(parents=True, exist_ok=True) - with open(lock_file, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_SH) - try: - state = _load_state(state_file) - if cell_name not in state["allocated"]: - return None - info = state["allocated"][cell_name] - return SubnetInfo( - cell_name=cell_name, - index=info["index"], - subnet=index_to_subnet(info["index"]), - allocated_at=info["allocated_at"], - ) - finally: - fcntl.flock(lock, fcntl.LOCK_UN) + with locked_file(lock_file, exclusive=False): + state = _load_state(state_file) + if cell_name not in state["allocated"]: + return None + info = state["allocated"][cell_name] + return SubnetInfo( + cell_name=cell_name, + index=info["index"], + subnet=index_to_subnet(info["index"]), + allocated_at=info["allocated_at"], + ) def list_all( @@ -224,36 +264,43 @@ def list_all( lock_file: Path = ALLOCATOR_LOCK_FILE, ) -> list[SubnetInfo]: """List all allocated subnets, sorted by index.""" - lock_file.parent.mkdir(parents=True, exist_ok=True) - with open(lock_file, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_SH) - try: - state = _load_state(state_file) - result = [] - for cell_name, info in sorted( - state["allocated"].items(), key=lambda x: x[1]["index"] - ): - result.append(SubnetInfo( - cell_name=cell_name, - index=info["index"], - subnet=index_to_subnet(info["index"]), - allocated_at=info["allocated_at"], - )) - return result - finally: - fcntl.flock(lock, fcntl.LOCK_UN) - - -def get_subnet_map( - state_file: Path = SUBNET_STATE_FILE, - lock_file: Path = ALLOCATOR_LOCK_FILE, -) -> dict[str, str]: - """Get the subnet -> cell_name mapping (for enforce.py consumption).""" - lock_file.parent.mkdir(parents=True, exist_ok=True) - with open(lock_file, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_SH) - try: - state = _load_state(state_file) - return _build_subnet_map(state) - finally: - fcntl.flock(lock, fcntl.LOCK_UN) + with locked_file(lock_file, exclusive=False): + state = _load_state(state_file) + result = [] + for cell_name, info in sorted( + state["allocated"].items(), key=lambda x: x[1]["index"] + ): + result.append(SubnetInfo( + cell_name=cell_name, + index=info["index"], + subnet=index_to_subnet(info["index"]), + allocated_at=info["allocated_at"], + )) + return result + + +def reclaim_orphan_subnets() -> list[str]: + """Free subnet allocations whose podman network no longer exists. + + Returns the freed cell names. Self-heals leaks from a raw `podman` kill or a + crash mid-`rm` that left the /24 allocated after the network/container were + gone — otherwise the bounded 254-subnet space (and stale ingress routes) + accumulate until a manual `brig system prune`. Fails safe: if the network + list can't be read, nothing is freed (a live subnet must never be reclaimed + just because enumeration failed). + """ + from brig.config import CONTAINER_PREFIX + from brig.vm.shell import vm_run + result = vm_run(["podman", "network", "ls", "--format", "{{.Name}}"]) + if result.returncode != 0: + return [] + existing = set(result.stdout.strip().split("\n")) + freed: list[str] = [] + for info in list_all(): + if f"{CONTAINER_PREFIX}{info.cell_name}" not in existing: + try: + free(info.cell_name) + freed.append(info.cell_name) + except ValueError: + pass + return freed diff --git a/src/brig/network/validation.py b/src/brig/network/validation.py index ca61dc6..1c80ee8 100644 --- a/src/brig/network/validation.py +++ b/src/brig/network/validation.py @@ -7,9 +7,32 @@ from __future__ import annotations +import ipaddress +import socket + from brig.config import SUSPICIOUS_DOMAIN_PATTERNS +def _is_ip_literal(host: str) -> bool: + """True if host denotes an IP address in any encoding (canonical or not). + + Egress targets must be domain names; literal IPs are blocked at the proxy. + Catches the alternate IPv4 encodings (integer/hex/octal/short-dotted) that + a plain ipaddress parse misses, so they can't slip into an allow or + tls_passthrough list and reach an internal address. + """ + try: + ipaddress.ip_address(host) + return True + except ValueError: + pass + try: + socket.inet_aton(host) + return True + except (OSError, UnicodeError): + return False + + def is_suspicious_domain(domain: str) -> str: """Check if domain pattern is suspicious for DNS rebinding. @@ -17,6 +40,10 @@ def is_suspicious_domain(domain: str) -> str: """ domain_lower = domain.lower() + base = domain_lower[2:] if domain_lower.startswith("*.") else domain_lower + if _is_ip_literal(base): + return f"'{domain}' is an IP literal; egress targets must be domain names" + for pattern in SUSPICIOUS_DOMAIN_PATTERNS: if domain_lower == pattern: return f"'{domain}' is too broad and could allow DNS rebinding" diff --git a/src/brig/observability/collector.py b/src/brig/observability/collector.py index c4cf764..d6d75aa 100644 --- a/src/brig/observability/collector.py +++ b/src/brig/observability/collector.py @@ -2,8 +2,8 @@ Runs as a sibling container to warden inside the Lima VM. Receives OTLP from warden, exposes a Prometheus endpoint for the brig CLI to -read, and writes traces/logs to rotated files. Started before warden -on `brig system up` so warden has an emit target from cold start. +read, and writes log records to a rotated file. Started before warden +on `brig system up` (by cmd_up) so warden has an emit target from cold start. Failure-closed: refuses to start if the image digest is not pinned in brig.config.COLLECTOR_IMAGE_DIGEST (run scripts/pin-collector-image.sh diff --git a/src/brig/observability/collector_config.yaml b/src/brig/observability/collector_config.yaml index c802fe2..7a5d4c7 100644 --- a/src/brig/observability/collector_config.yaml +++ b/src/brig/observability/collector_config.yaml @@ -21,8 +21,8 @@ processors: limit_mib: 256 spike_limit_mib: 64 batch: - # 200ms is small enough that `brig system stats --watch` feels live - # but large enough that gRPC overhead is amortized. + # 200ms is small enough that live metric reads feel current but large + # enough that gRPC overhead is amortized. timeout: 200ms send_batch_size: 1024 @@ -34,16 +34,8 @@ exporters: # Convert OTel histograms to prometheus histograms. send_timestamps: true metric_expiration: 24h - # Default in-memory trace store; the CLI reads recent spans via - # the debug exporter's exported file. For real trace storage, - # operators add an OTLP exporter pointing at Tempo / Jaeger here. - file/traces: - path: /var/lib/otel/traces.jsonl - rotation: - max_megabytes: 100 - max_days: 7 - max_backups: 3 - # Same shape for logs — brig cell network reads from here. + # File store for warden's per-request log records — brig cell network reads + # from here. file/logs: path: /var/lib/otel/logs.jsonl rotation: @@ -60,10 +52,6 @@ service: receivers: [otlp] processors: [memory_limiter, batch] exporters: [prometheus] - traces: - receivers: [otlp] - processors: [memory_limiter, batch] - exporters: [file/traces] logs: receivers: [otlp] processors: [memory_limiter, batch] diff --git a/src/brig/observability/traces.py b/src/brig/observability/traces.py deleted file mode 100644 index d5b7b40..0000000 --- a/src/brig/observability/traces.py +++ /dev/null @@ -1,147 +0,0 @@ -"""brig cell trace — read OTel spans from the collector's trace file. - -The collector writes JSONL traces to /var/lib/otel/traces.jsonl -inside the VM. Each line is one ResourceSpans batch (the OTLP -default). We grep by trace_id, then pretty-print the span tree -sorted by start time. -""" - -from __future__ import annotations - -import json -from dataclasses import dataclass, field -from typing import Any - -from brig.errors import BrigError -from brig.ops.logging import output -from brig.vm.shell import vm_run - -TRACES_PATH = "/var/lib/otel/traces.jsonl" - - -@dataclass -class Span: - trace_id: str - span_id: str - parent_span_id: str - name: str - start_ns: int - end_ns: int - attributes: dict[str, Any] = field(default_factory=dict) - status_code: int = 0 - cell: str = "" - - @property - def duration_ms(self) -> float: - return (self.end_ns - self.start_ns) / 1_000_000 - - -def fetch_trace_file() -> str: - """Read the collector's trace file via vm_run.""" - result = vm_run(["cat", TRACES_PATH], timeout=10) - if result.returncode != 0: - raise BrigError( - f"could not read {TRACES_PATH}", - suggestion="Is the collector running and has it seen traffic yet?", - ) - return result.stdout - - -def parse_spans(jsonl: str) -> list[Span]: - """Flatten the nested OTLP ResourceSpans format into plain Spans.""" - spans: list[Span] = [] - for line in jsonl.splitlines(): - if not line.strip(): - continue - try: - batch = json.loads(line) - except json.JSONDecodeError: - continue - for rs in batch.get("resourceSpans", []): - resource_attrs = _attrs_to_dict( - rs.get("resource", {}).get("attributes", []), - ) - for ss in rs.get("scopeSpans", []): - for raw in ss.get("spans", []): - sp_attrs = _attrs_to_dict(raw.get("attributes", [])) - spans.append(Span( - trace_id=raw.get("traceId", ""), - span_id=raw.get("spanId", ""), - parent_span_id=raw.get("parentSpanId", ""), - name=raw.get("name", ""), - start_ns=int(raw.get("startTimeUnixNano", 0)), - end_ns=int(raw.get("endTimeUnixNano", 0)), - attributes=sp_attrs, - status_code=raw.get("status", {}).get("code", 0), - cell=sp_attrs.get("cell") or resource_attrs.get("cell", ""), - )) - return spans - - -def _attrs_to_dict(attrs: list[dict]) -> dict[str, Any]: - out: dict[str, Any] = {} - for a in attrs: - key = a.get("key") - if not isinstance(key, str): - continue - val = a.get("value", {}) - # OTel attribute values are tagged: stringValue, intValue, etc. - for typ in ("stringValue", "intValue", "doubleValue", "boolValue"): - if typ in val: - out[key] = val[typ] - break - return out - - -def render_trace(spans: list[Span]) -> str: - """Print a tree view of one trace's spans, sorted by start_ns.""" - if not spans: - return "(no spans for that trace_id)" - - by_id = {s.span_id: s for s in spans} - children: dict[str, list[Span]] = {} - roots: list[Span] = [] - for s in spans: - parent = by_id.get(s.parent_span_id) - if parent is None: - roots.append(s) - else: - children.setdefault(s.parent_span_id, []).append(s) - for kids in children.values(): - kids.sort(key=lambda s: s.start_ns) - roots.sort(key=lambda s: s.start_ns) - - lines: list[str] = [] - lines.append(f"TRACE {spans[0].trace_id}") - - def _walk(s: Span, depth: int) -> None: - bar = " " * depth + ("└─ " if depth else "") - attrs = "" - if s.cell: - attrs += f" cell={s.cell}" - for k in ("http.method", "http.host", "http.target", "http.status_code"): - if k in s.attributes: - attrs += f" {k.split('.')[-1]}={s.attributes[k]}" - status_marker = " [error]" if s.status_code == 2 else "" - lines.append( - f"{bar}{s.name} ({s.duration_ms:.1f}ms){attrs}{status_marker}" - ) - for kid in children.get(s.span_id, []): - _walk(kid, depth + 1) - - for r in roots: - _walk(r, 0) - return "\n".join(lines) - - -def cmd_trace(args) -> int: - """Handle `brig cell trace `.""" - body = fetch_trace_file() - all_spans = parse_spans(body) - matching = [s for s in all_spans if s.trace_id == args.trace_id] - if not matching: - # Allow prefix match for ergonomics. - prefix = args.trace_id - matching = [s for s in all_spans if s.trace_id.startswith(prefix)] - output(render_trace(matching)) - return 0 if matching else 1 diff --git a/src/brig/ops/addon_deploy.py b/src/brig/ops/addon_deploy.py new file mode 100644 index 0000000..4ae23a9 --- /dev/null +++ b/src/brig/ops/addon_deploy.py @@ -0,0 +1,59 @@ +"""Keep the deployed warden addons in lockstep with the brig package. + +Warden loads its addons from HostPaths.ADDONS_DIR (mounted into the container at +/addons). They ship with brig as package-data (`brig/warden_addons/`); the +deployed copy used to be refreshed only by a build step, so editing an addon left +warden running stale code until a manual re-copy. `brig system up` now syncs +them, so the deployed data plane can't silently drift from the installed package. +""" + +from __future__ import annotations + +import hashlib +import shutil +from pathlib import Path + +from brig.config import HostPaths +from brig.ops.logging import debug + + +def addon_source_dir() -> Path | None: + """The brig-shipped addon source directory, or None if not locatable. + + Resolved as brig package-data, so it works for both editable and wheel + installs. Returns None only if the package somehow ships without the addons + — the sync then no-ops and warden uses whatever was previously staged. + """ + try: + from importlib.resources import files + src = Path(str(files("brig").joinpath("warden_addons"))) + except (ModuleNotFoundError, TypeError, FileNotFoundError, NotADirectoryError): + return None + return src if src.is_dir() else None + + +def _digest(p: Path) -> str: + return hashlib.sha256(p.read_bytes()).hexdigest() + + +def sync_addons() -> bool: + """Copy changed addon `*.py` from the package into HostPaths.ADDONS_DIR. + + Returns True if any file was written — the caller (cmd_up) bounces warden + so it reloads, since mitmproxy's script-watcher reliably hot-reloads only + the entry scripts, not their sibling helper modules. Copy-only (never + deletes) so an operator-placed file is left alone. + """ + src = addon_source_dir() + if src is None: + debug("addon source dir not found; skipping addon sync") + return False + dst = HostPaths.ADDONS_DIR + dst.mkdir(parents=True, exist_ok=True) + changed = False + for f in sorted(src.glob("*.py")): + target = dst / f.name + if not target.exists() or _digest(target) != _digest(f): + shutil.copy2(f, target) + changed = True + return changed diff --git a/src/brig/ops/atomic.py b/src/brig/ops/atomic.py index 97d477b..05a11dc 100644 --- a/src/brig/ops/atomic.py +++ b/src/brig/ops/atomic.py @@ -3,7 +3,8 @@ POSIX rename within a filesystem is atomic: readers always see either the old contents or the new contents, never a partial write. We follow the standard -pattern: tempfile in the same directory, write + flush + fsync, then rename. +pattern: tempfile in the same directory, write + flush + fsync, rename, then +fsync the parent directory so the rename itself survives a crash. """ from __future__ import annotations @@ -15,30 +16,45 @@ from typing import Any, Optional -def atomic_write_json( - path: Path, - data: Any, - indent: Optional[int] = 2, - mode: Optional[int] = None, -) -> None: - """Write JSON atomically. Creates parent dirs if missing. +def _fsync_dir(directory: Path) -> None: + """fsync a directory so a rename into it is durable across a crash. - If `mode` is given, the file's permission bits are set BEFORE the - rename (via fchmod on the open fd). This ensures readers that race - with the rename see the final mode, not the mkstemp default of 0600. + Best-effort: some platforms/filesystems reject O_RDONLY fsync on a + directory — the rename's atomicity (torn-read protection) is unaffected + either way; this only hardens crash durability. + """ + try: + dir_fd = os.open(str(directory), os.O_RDONLY) + except OSError: + return + try: + os.fsync(dir_fd) + except OSError: + pass + finally: + os.close(dir_fd) + + +def _atomic_write(path: Path, write_fn: Any, mode: Optional[int] = None) -> None: + """Atomic-write core: temp file in the same dir, content via `write_fn(file)`, + optional fchmod-before-rename, fsync, POSIX-atomic rename, parent fsync, and + unlink-on-error — the skeleton both public helpers share. + + If `mode` is given, permission bits are set BEFORE the rename (fchmod on the + open fd) so readers racing the rename see the final mode, not mkstemp's 0600; + a failing chmod raises rather than silently falling through. """ path.parent.mkdir(parents=True, exist_ok=True) fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp") try: with os.fdopen(fd, "w") as f: - json.dump(data, f, indent=indent) + write_fn(f) f.flush() if mode is not None: - # fchmod on the fd before fsync+rename — if chmod fails, - # raise (no silent fallthrough to 0600). os.fchmod(f.fileno(), mode) os.fsync(f.fileno()) os.rename(tmp_path, str(path)) + _fsync_dir(path.parent) except Exception: try: os.unlink(tmp_path) @@ -47,19 +63,17 @@ def atomic_write_json( raise +def atomic_write_json( + path: Path, + data: Any, + indent: Optional[int] = 2, + mode: Optional[int] = None, +) -> None: + """Write JSON atomically. Creates parent dirs if missing. See `_atomic_write` + for the `mode` semantics.""" + _atomic_write(path, lambda f: json.dump(data, f, indent=indent), mode=mode) + + def atomic_write_text(path: Path, text: str) -> None: """Write text atomically. Creates parent dirs if missing.""" - path.parent.mkdir(parents=True, exist_ok=True) - fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp") - try: - with os.fdopen(fd, "w") as f: - f.write(text) - f.flush() - os.fsync(f.fileno()) - os.rename(tmp_path, str(path)) - except Exception: - try: - os.unlink(tmp_path) - except OSError: - pass - raise + _atomic_write(path, lambda f: f.write(text)) diff --git a/src/brig/ops/history.py b/src/brig/ops/history.py index afe41aa..930796d 100644 --- a/src/brig/ops/history.py +++ b/src/brig/ops/history.py @@ -6,7 +6,6 @@ from __future__ import annotations -import fcntl import json import re import time @@ -22,6 +21,7 @@ POLICY_AUDIT_FILE, SENSITIVE_PATTERNS, ) +from brig.ops.locking import locked_file from brig.ops.logging import debug @@ -62,8 +62,7 @@ def _append_jsonl(path: Path, entry: dict[str, Any]) -> None: """ path.parent.mkdir(parents=True, exist_ok=True) lock_path = path.with_suffix(path.suffix + ".lock") - with open(lock_path, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) + with locked_file(lock_path): _maybe_rotate(path) with open(path, "a") as f: f.write(json.dumps(entry) + "\n") diff --git a/src/brig/ops/locking.py b/src/brig/ops/locking.py new file mode 100644 index 0000000..7a06a24 --- /dev/null +++ b/src/brig/ops/locking.py @@ -0,0 +1,33 @@ +"""Shared advisory file-lock context manager. + +Collapses the `open(lock_file) + fcntl.flock + guaranteed-unlock` idiom that the +subnet allocator and ingress route store both use into one place, so the +boilerplate (and the explicit-unlock detail) lives once. POSIX advisory lock on +a dedicated lock file; released on context exit. +""" + +from __future__ import annotations + +import fcntl +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import IO + + +@contextmanager +def locked_file(lock_file: Path, *, exclusive: bool = True) -> Iterator[IO[str]]: + """Hold an advisory lock on `lock_file` for the duration of the block. + + `exclusive=True` takes `LOCK_EX` (writers); `exclusive=False` takes `LOCK_SH` + (concurrent readers). Creates the lock file's parent dir if missing. The + explicit `LOCK_UN` on exit is belt-and-suspenders — closing the fd releases + the lock too — but mirrors the prior call sites' behavior exactly. + """ + lock_file.parent.mkdir(parents=True, exist_ok=True) + with open(lock_file, "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + try: + yield lock + finally: + fcntl.flock(lock, fcntl.LOCK_UN) diff --git a/src/brig/ops/logging.py b/src/brig/ops/logging.py index 7784afc..468c6db 100644 --- a/src/brig/ops/logging.py +++ b/src/brig/ops/logging.py @@ -75,21 +75,6 @@ def colorize(text: str, color: str) -> str: return text -def status_color(status: str) -> str: - """Get colorized status string.""" - status_lower = status.lower() - if status_lower == "running": - return colorize(status, "green") - elif status_lower == "paused": - return colorize(status, "yellow") - elif status_lower in ("exited", "stopped", "dead"): - return colorize(status, "red") - elif status_lower == "created": - return colorize(status, "blue") - else: - return status - - def log(level: int, msg: str, level_name: str | None = None) -> None: """Log a message at the specified level.""" if level < _state["log_level"]: @@ -129,9 +114,14 @@ def info(msg: str) -> None: def output(msg: str) -> None: - """Print output message (respects quiet mode).""" - if not _state["quiet"]: - print(msg) + """Print primary command output to stdout. + + Always prints — `--quiet` suppresses advisory chatter (the [INFO]/[DEBUG] + stderr lines via log()), NOT the command's actual results. Suppressing + results would break scripting (`brig cell list --quiet`, `--format json`, + inspect/export/read all produce their data through this path). + """ + print(msg) def warn(msg: str) -> None: diff --git a/src/brig/ops/ratelimit.py b/src/brig/ops/ratelimit.py index 38e460c..0c9150c 100644 --- a/src/brig/ops/ratelimit.py +++ b/src/brig/ops/ratelimit.py @@ -1,72 +1,88 @@ """ Rate limiting for cell creation. -Uses file locking to prevent TOCTOU races between concurrent brig invocations. -Fails closed: denies operation when rate limit state is unreadable. +check_rate_limit is a read-only gate; record_rate_limit reserves a slot. +Splitting them means a run that fails or no-ops (already running, bad image, +rolled-back reconcile) does not burn quota — only an actual creation does. +Fails closed: denies when rate limit state is unreadable. """ from __future__ import annotations -import fcntl import json -import os import time from pathlib import Path from brig.config import RATE_LIMIT_FILE, RATE_LIMIT_MAX, RATE_LIMIT_WINDOW +from brig.ops.atomic import atomic_write_json +from brig.ops.locking import locked_file from brig.ops.logging import debug +def _live_timestamps(rate_limit_file: Path, now: float, window: int) -> list[float]: + """Read in-window timestamps. Corrupt/missing state reads as empty. + + Lock-free: record_rate_limit replaces the file via atomic rename, so a + reader always sees a complete old-or-new file, never a partial write. + """ + if not rate_limit_file.exists(): + return [] + try: + with open(rate_limit_file) as f: + data = json.loads(f.read()) + except (json.JSONDecodeError, IOError): + return [] + # Tolerate any corrupt shape (top-level non-dict, non-numeric timestamps): + # a tampered/garbage file reads as empty rather than crashing the caller. + if not isinstance(data, dict): + return [] + cutoff = now - window + return [ + ts for ts in (data.get("timestamps") or []) + if isinstance(ts, (int, float)) and not isinstance(ts, bool) and ts > cutoff + ] + + def check_rate_limit( rate_limit_file: Path = RATE_LIMIT_FILE, max_cells: int = RATE_LIMIT_MAX, window: int = RATE_LIMIT_WINDOW, ) -> bool: - """Check if cell creation is rate limited. Returns True if allowed. + """Return True if a new cell creation is within the rate limit. + + Read-only — it does NOT reserve a slot. Call record_rate_limit() once the + cell is actually created. Fails closed (False) if state is unreadable. - Args: - rate_limit_file: Path to rate limit state file. - max_cells: Maximum cells created per window. - window: Window in seconds. + The cap is best-effort under concurrency: because the check and the record + are separate steps with no reservation between them, N simultaneous runs at + max-1 can all pass and then all record, briefly overshooting the cap. This + is an intentional trade-off — the rate limit is a soft abuse guard, not a + hard boundary (the Lima VM is the only hard boundary). """ try: - rate_limit_file.parent.mkdir(parents=True, exist_ok=True) - - now = time.time() - - # Use separate lock file so the data file can be atomically replaced. - lock_path = rate_limit_file.with_suffix(".lock") - with open(lock_path, "w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - - # Read current data. - timestamps: list[float] = [] - if rate_limit_file.exists(): - try: - with open(rate_limit_file) as f: - data = json.loads(f.read()) - timestamps = data.get("timestamps", []) - except (json.JSONDecodeError, IOError): - timestamps = [] + return len(_live_timestamps(rate_limit_file, time.time(), window)) < max_cells + except (IOError, OSError) as e: + debug(f"Rate limit check failed: {e}") + return False # Fail closed on error. - # Filter to only timestamps within the window. - cutoff = now - window - timestamps = [ts for ts in timestamps if ts > cutoff] - # Check if limit exceeded. - if len(timestamps) >= max_cells: - return False +def record_rate_limit( + rate_limit_file: Path = RATE_LIMIT_FILE, + window: int = RATE_LIMIT_WINDOW, +) -> None: + """Record one cell creation timestamp atomically under an exclusive lock. - # Add current timestamp and save atomically. + Best-effort: a write failure must not fail an already-successful run — the + rate limit is a soft abuse guard, not a hard boundary. + """ + try: + rate_limit_file.parent.mkdir(parents=True, exist_ok=True) + now = time.time() + # Separate lock file so the data file can be atomically replaced. + lock_path = rate_limit_file.with_suffix(".lock") + with locked_file(lock_path): + timestamps = _live_timestamps(rate_limit_file, now, window) timestamps.append(now) - tmp_path = rate_limit_file.with_suffix(".tmp") - with open(tmp_path, "w") as tmp: - json.dump({"timestamps": timestamps}, tmp) - tmp.flush() - os.fsync(tmp.fileno()) - tmp_path.rename(rate_limit_file) - - return True + atomic_write_json(rate_limit_file, {"timestamps": timestamps}) except (IOError, OSError) as e: - debug(f"Rate limit check failed: {e}") - return False # Fail closed on error. + debug(f"Rate limit record failed: {e}") diff --git a/src/brig/policy/policy.py b/src/brig/policy/policy.py index bb23f53..4f3ce17 100644 --- a/src/brig/policy/policy.py +++ b/src/brig/policy/policy.py @@ -1,28 +1,22 @@ """ -Cell policy management — load, save, delete, validate. +Cell policy management — load, save, delete. -Supports both JSON and YAML policy files. +Per-cell policies are stored as JSON, one file per cell. """ from __future__ import annotations -import fcntl import json -import re +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path -from typing import Any +from typing import Any, Callable -from brig.config import DOMAIN_PATTERN, POLICY_DIR -from brig.network.validation import is_suspicious_domain +from brig.config import POLICY_DIR from brig.ops.atomic import atomic_write_json +from brig.ops.locking import locked_file from brig.ops.logging import debug -try: - import yaml - YAML_AVAILABLE = True -except ImportError: - YAML_AVAILABLE = False - def _policy_lock_path(policy_dir: Path) -> Path: """Path of the policy-directory write lock. @@ -38,7 +32,7 @@ def _policy_lock_path(policy_dir: Path) -> Path: def _encode_idn(domain: str) -> str: """Return the punycode form of `domain` or fall back to lowercase. - Mirrors src/addons/_policy.py:PolicyRule._normalize_domain so the + Mirrors src/brig/warden_addons/_policy.py:PolicyRule._normalize_domain so the host-side `policy.test` and the warden-side runtime match decisions agree on unicode/IDN inputs — without punycode encoding here, a unicode rule pair can pass cross-field validation on the host (UTF-8 @@ -55,7 +49,7 @@ def _encode_idn(domain: str) -> str: def domain_matches_rule(rule: str, host: str) -> bool: """Brig-CLI-side wildcard suffix match for "is host allowed by this rule". - Mirrors src/addons/_policy.py:PolicyRule.matches_domain. The host-side + Mirrors src/brig/warden_addons/_policy.py:PolicyRule.matches_domain. The host-side `brig policy test` path and the in-warden `enforce.py` evaluator must agree on IDN encoding — see _encode_idn. @@ -85,7 +79,7 @@ def load_cell_policy( against an in-flight `save_cell_policy`. Atomic-rename writes mean readers without the lock already see a consistent file, but the lock also serializes against the read-modify-write callers in - lifecycle_cmd._sync_cell_policy. + brig.cell.lifecycle.sync_cell_policy. """ path = get_cell_policy_path(cell_name, policy_dir) if not path.exists(): @@ -110,7 +104,7 @@ def save_cell_policy( Atomic-write alone protects against torn reads, but concurrent read-modify-write sequences (two `brig run` invocations on the same - cell, or `brig policy add` in parallel with `brig run`) can each + cell, or `brig policy set` in parallel with `brig run`) can each load the previous state, merge, and write — silently dropping the other's update. The lock mirrors the subnet allocator's pattern (src/brig/network/subnet.py). @@ -119,6 +113,38 @@ def save_cell_policy( atomic_write_json(get_cell_policy_path(cell_name, policy_dir), policy) +def mutate_cell_policy( + cell_name: str, + mutator: Callable[[dict[str, Any] | None], dict[str, Any] | None], + policy_dir: Path = POLICY_DIR, +) -> dict[str, Any] | None: + """Atomically read-modify-write a cell's policy under one exclusive lock. + + `mutator` receives the current policy dict (or None if no policy file + exists yet) and returns the policy dict to persist — or None to signal + "no change", in which case nothing is written (avoids churning mtime and + triggering a needless warden reload). Holding a single exclusive lock + across the load and the store closes the race where two concurrent + writers each load the same baseline and last-write-wins, silently + dropping one update. Returns the persisted (or unchanged current) policy. + """ + path = get_cell_policy_path(cell_name, policy_dir) + with _locked_policy_dir(policy_dir, exclusive=True): + current: dict[str, Any] | None = None + if path.exists(): + try: + with open(path) as f: + current = json.load(f) + except (json.JSONDecodeError, IOError) as e: + debug(f"Failed to load policy for {cell_name}: {e}") + current = None + result = mutator(current) + if result is None: + return current + atomic_write_json(path, result) + return result + + def delete_cell_policy( cell_name: str, policy_dir: Path = POLICY_DIR, @@ -133,100 +159,12 @@ def delete_cell_policy( return False -class _locked_policy_dir: - """Context manager: fcntl.flock on the policy-directory lock file. - - Creates policy_dir (and the lockfile) if missing. Use shared lock for - reads and exclusive lock for writes. Operates as a no-op silently if - the platform lacks fcntl (Windows isn't supported, but tests may run - in stripped environments). +@contextmanager +def _locked_policy_dir(policy_dir: Path, exclusive: bool) -> Iterator[None]: + """Advisory lock on the policy-directory lock file — shared for reads, + exclusive for writes — so the per-cell load-modify-write sequence is + serialized. Creates policy_dir if missing. """ - - def __init__(self, policy_dir: Path, exclusive: bool): - self.policy_dir = policy_dir - self.exclusive = exclusive - self._lock_fh: Any = None - - def __enter__(self) -> "_locked_policy_dir": - self.policy_dir.mkdir(parents=True, exist_ok=True) - lock_path = _policy_lock_path(self.policy_dir) - # Open with O_CREAT semantics via "a" — read mode would fail if - # the lockfile didn't exist yet, write mode would truncate - # contents we don't care about anyway. "a" is the simplest no-op. - self._lock_fh = open(lock_path, "a") - try: - fcntl.flock( - self._lock_fh, - fcntl.LOCK_EX if self.exclusive else fcntl.LOCK_SH, - ) - except OSError: - self._lock_fh.close() - self._lock_fh = None - raise - return self - - def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None: - if self._lock_fh is not None: - try: - fcntl.flock(self._lock_fh, fcntl.LOCK_UN) - finally: - self._lock_fh.close() - self._lock_fh = None - - -def load_policy_file(path: Path) -> dict[str, Any]: - """Load a policy from a JSON or YAML file. - - Detects format by extension. Raises ValueError on parse errors. - """ - content = path.read_text() - suffix = path.suffix.lower() - - if suffix in (".yaml", ".yml"): - if not YAML_AVAILABLE: - raise ValueError("YAML support requires pyyaml: pip install pyyaml") - try: - result = yaml.safe_load(content) - if not isinstance(result, dict): - raise ValueError("Policy file must contain a YAML mapping") - return result - except yaml.YAMLError as e: - raise ValueError(f"Failed to parse YAML: {e}") - else: - try: - result = json.loads(content) - if not isinstance(result, dict): - raise ValueError("Policy file must contain a JSON object") - return result - except json.JSONDecodeError as e: - raise ValueError(f"Failed to parse JSON: {e}") - - -def validate_policy(policy: dict[str, Any]) -> list[str]: - """Validate a policy dict and return list of errors.""" - errors: list[str] = [] - - if not isinstance(policy, dict): - return ["Policy must be a dict"] - - for key in ("allow", "deny"): - if key in policy: - rules = policy[key] - if not isinstance(rules, list): - errors.append(f"'{key}' must be a list") - continue - for domain in rules: - if isinstance(domain, str): - if not re.match(DOMAIN_PATTERN, domain): - errors.append(f"Invalid domain '{domain}' in {key}") - elif key == "allow": - suspicious = is_suspicious_domain(domain) - if suspicious: - errors.append(f"Security: {suspicious}") - elif isinstance(domain, dict): - if "domain" not in domain: - errors.append(f"Rule in '{key}' missing 'domain' field") - else: - errors.append(f"'{key}' items must be strings or dicts") - - return errors + policy_dir.mkdir(parents=True, exist_ok=True) + with locked_file(_policy_lock_path(policy_dir), exclusive=exclusive): + yield diff --git a/src/brig/sdk.py b/src/brig/sdk.py index 03a662c..63c4732 100644 --- a/src/brig/sdk.py +++ b/src/brig/sdk.py @@ -169,6 +169,33 @@ def stop(self) -> bool: return stop() +def _require_tcp_listeners_bound(spec: CellSpec) -> None: + """Fail loudly if the cell declares a TCP host_service whose warden + `reverse:tcp` listener isn't bound yet. + + mitmproxy can't hot-add reverse:tcp listeners, so a new TCP port needs a + warden restart. The CLI prompts the operator for that; the SDK has no + operator, so it raises rather than return a Cell whose `.host.brig` + TCP connection would silently have no listener. + """ + tcp_ports = sorted({ + e["port"] for e in (spec.host_services or []) + if isinstance(e, dict) and e.get("protocol") == "tcp" + and isinstance(e.get("port"), int) + }) + if not tcp_ports: + return + from warden.proxy import get_bound_tcp_ports + missing = [p for p in tcp_ports if p not in set(get_bound_tcp_ports())] + if missing: + raise BrigError( + f"Cell '{spec.name}' declares TCP host_services on port(s) {missing} " + f"that warden hasn't bound (mitmproxy can't hot-add reverse:tcp " + f"listeners).", + suggestion="brig system down && brig system up # rebind, then re-run", + ) + + class Brig: """Main SDK entry point for cell management.""" @@ -191,6 +218,21 @@ async def run( timeout: str | None = None, labels: list[str] | None = None, host_sockets: list[dict[str, Any]] | None = None, + host_services: list[dict[str, Any]] | None = None, + mounts: list[dict[str, Any]] | None = None, + ingress: list[dict[str, Any]] | None = None, + policy_allow: list[str] | None = None, + policy_deny: list[str] | None = None, + policy_passthrough_tls: list[str] | None = None, + image_digest: str | None = None, + trust_warden_ca: bool = True, + workdir: str | None = None, + workspace_quota: str | None = None, + workspace_mount: str = "/work", + writable_rootfs: bool = False, + seccomp_profile: str | None = None, + restart: str = "no", + user: str | None = None, ) -> Cell: """Run a new cell. Returns a Cell handle.""" return await asyncio.to_thread( @@ -200,6 +242,13 @@ async def run( pids_limit=pids_limit, network=network, profile=profile, detach=detach, timeout=timeout, labels=labels, host_sockets=host_sockets, + host_services=host_services, mounts=mounts, ingress=ingress, + policy_allow=policy_allow, policy_deny=policy_deny, + policy_passthrough_tls=policy_passthrough_tls, + image_digest=image_digest, trust_warden_ca=trust_warden_ca, + workdir=workdir, workspace_quota=workspace_quota, + workspace_mount=workspace_mount, writable_rootfs=writable_rootfs, + seccomp_profile=seccomp_profile, restart=restart, user=user, ) def run_sync( @@ -218,12 +267,35 @@ def run_sync( timeout: str | None = None, labels: list[str] | None = None, host_sockets: list[dict[str, Any]] | None = None, + host_services: list[dict[str, Any]] | None = None, + mounts: list[dict[str, Any]] | None = None, + ingress: list[dict[str, Any]] | None = None, + policy_allow: list[str] | None = None, + policy_deny: list[str] | None = None, + policy_passthrough_tls: list[str] | None = None, + image_digest: str | None = None, + trust_warden_ca: bool = True, + workdir: str | None = None, + workspace_quota: str | None = None, + workspace_mount: str = "/work", + writable_rootfs: bool = False, + seccomp_profile: str | None = None, + restart: str = "no", + user: str | None = None, ) -> Cell: """Synchronous version of run(). host_sockets: list of dicts with keys {name, host_path, mount_point, mode?}. Same validation rules as the cell yaml (see docs/design/cell-definition.md). Bypasses Warden by design. + + mounts: list of dicts {name, host_path, mount_point, mode?} — + bind-mount a host dir (under a configured mount_roots entry) into the + cell, ro default / rw opt-in. Rejected on the untrusted profile. + + host_services / ingress / policy / image_digest etc accept the + same shapes as the cell yaml fields of the same name; see + docs/design/cell-definition.md for the full schema. """ if not CELL_NAME_PATTERN.match(name): raise BrigError(f"Invalid cell name: {name}") @@ -241,23 +313,44 @@ def run_sync( "detach": detach, "labels": labels or [], "host_sockets": host_sockets or [], + "host_services": host_services or [], + "mounts": mounts or [], + "ingress": ingress or [], + "policy_allow": policy_allow or [], + "policy_deny": policy_deny or [], + "policy_passthrough_tls": policy_passthrough_tls or [], + "trust_warden_ca": trust_warden_ca, + "workspace_mount": workspace_mount, + "writable_rootfs": writable_rootfs, } - if timeout: spec_kwargs["timeout"] = timeout + if image_digest: + spec_kwargs["image_digest"] = image_digest + if workdir: + spec_kwargs["workdir"] = workdir + if workspace_quota: + spec_kwargs["workspace_quota"] = workspace_quota + if seccomp_profile: + spec_kwargs["seccomp_profile"] = seccomp_profile + if restart and restart != "no": + spec_kwargs["restart"] = restart + if user: + spec_kwargs["user"] = user if profile: try: prof = load_profile(profile) spec_kwargs = apply_profile(spec_kwargs, prof) + spec_kwargs["profile"] = profile except ValueError as e: raise ProfileError(str(e)) - # CellSpec.__post_init__ only validates name + coerces numeric - # strings. The security boundary for host_sockets / ingress / - # policy lives in validate_cell_definition — without this call, - # an SDK caller bypasses the engine-socket denylist, path - # traversal checks, and the untrusted-profile rejection. + # validate_cell_definition accepts the SDK's flat policy_* form + # and the yaml's nested `policy: {...}` form via the same entry + # point, so the untrusted-profile guards and SSRF wildcard checks + # fire on SDK calls without the SDK needing to know the nested + # shape. from brig.cell.spec import validate_cell_definition validation_errors = validate_cell_definition(spec_kwargs) if validation_errors: @@ -265,12 +358,12 @@ def run_sync( "Invalid cell spec:\n " + "\n ".join(validation_errors), ) - # Filter to only CellSpec fields (profiles may add extra keys like 'runtime'). import dataclasses valid_fields = {f.name for f in dataclasses.fields(CellSpec)} spec_kwargs = {k: v for k, v in spec_kwargs.items() if k in valid_fields} spec = CellSpec(**spec_kwargs) + _require_tcp_listeners_bound(spec) run_cell(spec) return Cell(name) @@ -355,7 +448,7 @@ async def list_cells(self) -> list[CellInfo]: return await asyncio.to_thread(self.list_sync) def list_sync(self) -> list[CellInfo]: - """Synchronous version of list().""" + """Synchronous version of list_cells().""" from brig.cell.lifecycle import list_cell_containers return [ CellInfo(name=cell, status=c.get("State", ""), image=c.get("Image", "")) diff --git a/src/brig/security/image.py b/src/brig/security/image.py index 97cb381..b9e3c34 100644 --- a/src/brig/security/image.py +++ b/src/brig/security/image.py @@ -18,7 +18,8 @@ def _parse_cosign_output(stdout: str) -> dict: try: data = json.loads(stdout) if isinstance(data, list) and data: - return data[0] + first = data[0] + return first if isinstance(first, dict) else {} return {} except (json.JSONDecodeError, IndexError): return {} @@ -35,15 +36,20 @@ def verify_image_signature( Returns (success, message, details) tuple. Note: cosign runs on macOS host, not in the VM. + + Keyless verification is ADVISORY unless both certificate_identity and + certificate_oidc_issuer are supplied: with neither, cosign accepts a + signature from any Fulcio identity, so a success means "signed by + someone", not "signed by who you trust". Image integrity at run time is + enforced by digest pinning, not by this command. """ result = subprocess.run( ["which", "cosign"], check=False, capture_output=True, text=True, ) if result.returncode != 0: - # The previous fallback ran `podman image trust show` and accepted - # the image whenever ANY policy line said "accept" — even if the - # specific image wasn't covered by that policy's scope. That's - # vacuous trust. cosign is now a hard prerequisite for verification. + # cosign is a hard prerequisite: `podman image trust show` accepts an + # image whenever ANY policy line says "accept", even when the specific + # image isn't in that policy's scope — vacuous trust, so no fallback. return ( False, "cosign is not installed. Install from https://docs.sigstore.dev/cosign/. " diff --git a/src/brig/security/verify.py b/src/brig/security/verify.py index 61d57f9..868b774 100644 --- a/src/brig/security/verify.py +++ b/src/brig/security/verify.py @@ -2,7 +2,10 @@ Security invariant verification checks. Each check is a pure function returning CheckResult(passed, message, details). -All 9 invariants from docs/design/security.md are covered. +Covers the 6 of the 12 invariants that are verifiable at runtime via podman +inspect: 1 (network isolation), 5 (gVisor), 6 (proxy-external membership), +7 (cell-network members), 8 (single-homed), 9 (proxy running). The others are +enforced at cell-definition parse / build time, not re-checked here. """ from __future__ import annotations @@ -29,19 +32,19 @@ def _run(cmd: list[str], timeout: int = 10) -> subprocess.CompletedProcess[str]: return vm_run(cmd, timeout=timeout) -def _get_cell_containers() -> tuple[list[str], list[dict]] | None: - """Shared query: list cell containers and their inspect data. +def _get_cell_containers() -> list[dict] | None: + """Shared query: inspect data for all cell containers. - Returns (cell_names, container_infos) or None on failure. Uses the - module-local `_run` alias for both calls so tests can mock the - verify-side subprocess seam independently of list_cell_containers. + Returns the container info list ([] if none) or None on failure. Uses the + module-local `_run` alias for both calls so tests can mock the verify-side + subprocess seam independently of list_cell_containers. """ result = _run([ "podman", "ps", "-a", "--format", "json", "--filter", f"name=^{CONTAINER_PREFIX}", ]) if not result.stdout.strip(): - return [], [] + return [] try: containers = json.loads(result.stdout) @@ -50,24 +53,26 @@ def _get_cell_containers() -> tuple[list[str], list[dict]] | None: def _name(c: dict) -> str: n = c.get("Names", "") - return n[0] if isinstance(n, list) else n + if isinstance(n, list): + return str(n[0]) if n else "" + return str(n) from brig.config import INFRA_CONTAINER_NAMES cell_names = [_name(c) for c in containers if _name(c) not in INFRA_CONTAINER_NAMES] if not cell_names: - return [], [] + return [] inspect = _run(["podman", "inspect", "--format", "json"] + cell_names) try: - return cell_names, json.loads(inspect.stdout) + return json.loads(inspect.stdout) # type: ignore[no-any-return] except json.JSONDecodeError: return None -def _get_cell_networks() -> tuple[list[str], list[dict]] | None: - """Shared query: list cell networks and their inspect data. +def _get_cell_networks() -> list[dict] | None: + """Shared query: inspect data for all cell networks. - Returns (network_names, network_infos) or None on failure. + Returns the network info list ([] if none) or None on failure. """ result = _run(["podman", "network", "ls", "--format", "{{.Name}}"]) cell_networks = [ @@ -75,11 +80,11 @@ def _get_cell_networks() -> tuple[list[str], list[dict]] | None: if net.startswith(CONTAINER_PREFIX) and net != PROXY_EXTERNAL_NETWORK ] if not cell_networks: - return [], [] + return [] inspect = _run(["podman", "network", "inspect"] + cell_networks) try: - return cell_networks, json.loads(inspect.stdout) + return json.loads(inspect.stdout) # type: ignore[no-any-return] except json.JSONDecodeError: return None @@ -99,9 +104,36 @@ def verify_proxy_network() -> CheckResult: "{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}", ]) networks = result.stdout.strip().split() - if PROXY_EXTERNAL_NETWORK in networks: - return CheckResult(True, "Proxy attached to proxy-external") - return CheckResult(False, "Proxy not on proxy-external network") + if PROXY_EXTERNAL_NETWORK not in networks: + return CheckResult(False, "Proxy not on proxy-external network") + + # Confirming warden is attached is not enough: a second container on + # proxy-external would share warden's egress-capable (non-internal) + # network and defeat the choke point. Enumerate the members and assert + # only infrastructure containers are attached. + from brig.config import INFRA_CONTAINER_NAMES + inspect = _run(["podman", "network", "inspect", PROXY_EXTERNAL_NETWORK]) + try: + net_infos = json.loads(inspect.stdout) + except json.JSONDecodeError: + return CheckResult(False, "Could not inspect proxy-external network") + + members: set[str] = set() + for net_info in net_infos: + containers = net_info.get("containers") or {} + for cid, m in containers.items(): + if not isinstance(m, dict): + continue + # A member with no name still occupies the network and must be + # accounted for — fall back to the container ID so it can't slip + # past the enumeration (fail closed, not open). + members.add(m.get("name") or f"") + unexpected = sorted(n for n in members if n not in INFRA_CONTAINER_NAMES) + if unexpected: + return CheckResult( + False, "Non-infrastructure containers on proxy-external", unexpected + ) + return CheckResult(True, "Proxy-external has only infrastructure containers") def verify_gvisor_runtime(container_infos: list[dict] | None = None) -> CheckResult: @@ -110,7 +142,7 @@ def verify_gvisor_runtime(container_infos: list[dict] | None = None) -> CheckRes data = _get_cell_containers() if data is None: return CheckResult(False, "Could not query containers") - _, container_infos = data + container_infos = data if not container_infos: return CheckResult(True, "No cells to check") @@ -118,9 +150,15 @@ def verify_gvisor_runtime(container_infos: list[dict] | None = None) -> CheckRes for c_info in container_infos: name = c_info.get("Name", "").lstrip("/") cell = name[len(CONTAINER_PREFIX):] if name.startswith(CONTAINER_PREFIX) else name - oci_runtime = c_info.get("HostConfig", {}).get("Runtime", "") - if oci_runtime and oci_runtime != RUNTIME: - issues.append(f"{cell} uses {oci_runtime} instead of {RUNTIME}") + # podman reports the OCI *category* ("oci") in HostConfig.Runtime; + # the named runtime (runsc/crun) lives in the top-level OCIRuntime + # field. Reading HostConfig.Runtime cannot tell runsc from a crun + # downgrade. + oci_runtime = c_info.get("OCIRuntime", "") + if oci_runtime != RUNTIME: + issues.append( + f"{cell} uses {oci_runtime or ''} instead of {RUNTIME}" + ) if issues: return CheckResult(False, "gVisor runtime violations", issues) @@ -133,7 +171,7 @@ def verify_network_isolation(network_infos: list[dict] | None = None) -> CheckRe data = _get_cell_networks() if data is None: return CheckResult(False, "Could not query networks") - _, network_infos = data + network_infos = data if not network_infos: return CheckResult(True, "No cell networks to check") @@ -154,7 +192,7 @@ def verify_single_homed(container_infos: list[dict] | None = None) -> CheckResul data = _get_cell_containers() if data is None: return CheckResult(False, "Could not query containers") - _, container_infos = data + container_infos = data if not container_infos: return CheckResult(True, "No cells to check") @@ -162,7 +200,9 @@ def verify_single_homed(container_infos: list[dict] | None = None) -> CheckResul for c_info in container_infos: name = c_info.get("Name", "").lstrip("/") networks = list(c_info.get("NetworkSettings", {}).get("Networks", {}).keys()) - if len(networks) != 1: + # Airgapped cells (--network none) legitimately have 0 networks and + # are strictly more isolated than single-homed; only >1 is a violation. + if len(networks) > 1: issues.append(f"{name} has {len(networks)} networks") if issues: @@ -176,7 +216,7 @@ def verify_cell_network_members(network_infos: list[dict] | None = None) -> Chec data = _get_cell_networks() if data is None: return CheckResult(False, "Could not query networks") - _, network_infos = data + network_infos = data if not network_infos: return CheckResult(True, "No cell networks to check") @@ -185,11 +225,12 @@ def verify_cell_network_members(network_infos: list[dict] | None = None) -> Chec net_name = net_info.get("name", "") containers = net_info.get("containers") or {} member_names = { - m.get("name", "") for m in containers.values() + (m.get("name") or f"") + for cid, m in containers.items() if isinstance(m, dict) } expected = {PROXY_NAME, net_name} - unexpected = {n for n in member_names if n and n not in expected} + unexpected = {n for n in member_names if n not in expected} if unexpected: issues.append( f"Network {net_name} has non-warden/non-cell containers: " @@ -213,7 +254,7 @@ def verify_all() -> list[CheckResult]: if container_data is None: results.append(CheckResult(False, "Could not query containers")) else: - _, container_infos = container_data + container_infos = container_data results.append(verify_gvisor_runtime(container_infos)) results.append(verify_single_homed(container_infos)) @@ -222,7 +263,7 @@ def verify_all() -> list[CheckResult]: if network_data is None: results.append(CheckResult(False, "Could not query networks")) else: - _, network_infos = network_data + network_infos = network_data results.append(verify_network_isolation(network_infos)) results.append(verify_cell_network_members(network_infos)) diff --git a/src/brig/vm/lima.yaml.template b/src/brig/vm/lima.yaml.template index b25c5b2..8776868 100644 --- a/src/brig/vm/lima.yaml.template +++ b/src/brig/vm/lima.yaml.template @@ -1,5 +1,5 @@ # Brig Lima VM Configuration -# Generated by: brig init +# Generated by: brig system init # Reference: https://lima-vm.io/docs/reference/ images: @@ -8,6 +8,13 @@ images: - location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img" arch: "aarch64" +# Pin the backend explicitly rather than relying on Lima's version-dependent +# default: Virtualization.framework + virtiofs is the validated configuration +# (see docs/design/architecture.md). Either backend is a full HW-virtualized +# VM, so this is reproducibility, not a boundary change. +vmType: "vz" +mountType: "virtiofs" + cpus: 4 memory: "8GiB" disk: "50GiB" @@ -22,6 +29,8 @@ mounts: - location: "~/.brig/state" mountPoint: "/state" writable: true + # brig:mount_roots:begin (managed — set via `brig config set mount_roots`; VM restart to apply) + # brig:mount_roots:end provision: - mode: system @@ -76,7 +85,13 @@ provision: crun = ["/usr/bin/crun"] CONF - # Disable IPv6 (simplifies network policy enforcement) + # Disable IPv6 (simplifies network policy enforcement). Persist to + # /etc/sysctl.d so a guest reboot doesn't silently restore defaults; + # `sysctl -w` alone is runtime-only. + cat > /etc/sysctl.d/99-brig-no-ipv6.conf <<'SYSCTL' + net.ipv6.conf.all.disable_ipv6 = 1 + net.ipv6.conf.default.disable_ipv6 = 1 + SYSCTL sysctl -w net.ipv6.conf.all.disable_ipv6=1 sysctl -w net.ipv6.conf.default.disable_ipv6=1 diff --git a/src/brig/vm/lima_mounts.py b/src/brig/vm/lima_mounts.py new file mode 100644 index 0000000..3a3dd44 --- /dev/null +++ b/src/brig/vm/lima_mounts.py @@ -0,0 +1,126 @@ +"""Render the operator-declared `mount_roots` into the Lima VM config. + +`mount_roots` (config.json) lists host trees that may be exposed to cells via +the per-cell `mounts:` block. Lima mounts are static + VM-wide, so each root is +mounted into the VM at /mnt/host/ and the reconciler binds sub-paths from +there into cells. See docs/design/host-mounts.md. + +The mounts live between managed markers in lima.yaml so we can rewrite just that +block (idempotent) without disturbing the rest of the operator's config. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from brig.config import ( + VM_NAME, + HostPaths, + mount_root_slug, + mount_roots, + validate_mount_roots, +) +from brig.errors import BrigError +from brig.ops.atomic import atomic_write_text + +_BEGIN = "# brig:mount_roots:begin" +_END = "# brig:mount_roots:end" + + +def render_mount_roots_block(roots: list[str]) -> str: + """The lima.yaml lines (between the markers) for the given roots. + + Raises BrigError if the roots don't validate (catastrophe/sensitive root, + non-dir, slug collision, illegal chars). `location` is JSON-quoted so a path + with YAML-hostile characters can't break out of the string. + """ + errs = validate_mount_roots(roots) + if errs: + raise BrigError("Invalid mount_roots: " + "; ".join(errs)) + lines: list[str] = [] + for root in roots: + slug = mount_root_slug(root) + # json.dumps gives a safely-quoted YAML scalar (YAML is a JSON superset). + # Writable at the VM layer; per-cell ro/rw is enforced by podman's + # `-v ...:` (the reconciler), not here. + lines.append(f' - location: {json.dumps(root)}') + lines.append(f' mountPoint: "/mnt/host/{slug}"') + lines.append(' writable: true') + return "\n".join(lines) + + +def sync_lima_mount_roots(lima_yaml: Path = HostPaths.LIMA_YAML) -> bool: + """Rewrite the managed mount_roots block in lima.yaml from config. + + Returns True if the file content changed (caller may then advise a VM + restart). No-op (returns False) if lima.yaml is missing or predates the + markers — we never inject mounts into an unmarked file (would risk + corrupting hand-edited YAML). Raises BrigError on malformed markers or + invalid mount_roots rather than producing broken YAML. + """ + if not lima_yaml.exists(): + return False + text = lima_yaml.read_text() + if _BEGIN not in text or _END not in text: + return False + # Exactly one well-ordered marker pair, else refuse (don't corrupt the file). + if text.count(_BEGIN) != 1 or text.count(_END) != 1 or text.index(_BEGIN) > text.index(_END): + raise BrigError( + f"Malformed brig:mount_roots markers in {lima_yaml} — expected one " + f"begin before one end. Fix or regenerate with: brig system init" + ) + + pre, rest = text.split(_BEGIN, 1) + mid_and_end, post = rest.split(_END, 1) + + block = render_mount_roots_block(mount_roots()) + middle = f"\n{block}\n" if block else "\n" + # Preserve the begin marker's trailing descriptor (up to the newline) and + # the END marker's leading indentation so the managed block stays tidy. + begin_line_tail = mid_and_end[: mid_and_end.index("\n")] if "\n" in mid_and_end else "" + last_nl = mid_and_end.rfind("\n") + end_indent = mid_and_end[last_nl + 1:] if last_nl != -1 else "" # ws before _END + + new_text = f"{pre}{_BEGIN}{begin_line_tail}{middle}{end_indent}{_END}{post}" + if new_text != text: + atomic_write_text(lima_yaml, new_text) + return True + return False + + +def lima_instance_config() -> Path: + """The live Lima instance config Lima reads on `limactl start`. Distinct + from HostPaths.LIMA_YAML (brig's template, used only on VM create).""" + return Path.home() / ".lima" / VM_NAME / "lima.yaml" + + +def sync_all_lima_mount_roots() -> bool: + """Sync the managed mount block into BOTH the template (used on VM create) + and the live instance config (read on `limactl start`). + + Returns True if the INSTANCE config changed — i.e. the running VM needs a + `brig system down --vm && brig system up` (a lossless stop/start; vz applies + the new virtiofs mounts on restart and the container/image store survives — + only `limactl delete` would wipe it). + + The two files are synced independently so a fault in one (e.g. operator- + corrupted markers in `~/.lima`) doesn't strand the other; any errors are + aggregated into one BrigError that names the offending file(s). + """ + instance = lima_instance_config() + errors: list[str] = [] + instance_changed = False + for target in (HostPaths.LIMA_YAML, instance): + if not target.exists(): + continue + try: + changed = sync_lima_mount_roots(target) + except BrigError as e: + errors.append(str(e)) + continue + if target == instance: + instance_changed = changed + if errors: + raise BrigError("; ".join(dict.fromkeys(errors))) + return instance_changed diff --git a/src/brig/vm/shell.py b/src/brig/vm/shell.py index 3a13d22..ecce8a6 100644 --- a/src/brig/vm/shell.py +++ b/src/brig/vm/shell.py @@ -60,11 +60,49 @@ def _is_sensitive_env(name: str) -> bool: return any(s in upper for s in _SENSITIVE_ENV_SUBSTRINGS) +# Single source of truth for which command basenames need sudo inside +# the Lima VM. Rootful podman owns the container store; the mkdir/du/chown/cp/rm +# helpers run against root-owned paths under /state/. Anything not in this +# set runs as the unprivileged lima user. +_AUTOSUDO_COMMANDS = frozenset({"podman", "mkdir", "du", "chown", "cp", "rm"}) + + +def _prepend_sudo(cmd: list[str], sudo: bool | None) -> list[str]: + """Decide whether to prepend `sudo` based on caller intent. + + `sudo=None` (default) → auto: infer from cmd[0]. Use this for + podman / standard helpers — keeps the call sites uncluttered. + `sudo=True` → force sudo. Use when running a command whose basename + isn't in `_AUTOSUDO_COMMANDS` but still needs root (e.g. `sh -c