diff --git a/.dockerignore b/.dockerignore index e79720e87..6b9a54382 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,7 @@ -# The Hermes image builds from the repo root (images/hermes/Dockerfile): it -# carries the fragment CLI, of the root workspace. Its context is only what -# its build reads; everything else (targets, the cell's builds, node_modules, -# git) stays out. The stub image builds from images/ (images/.dockerignore). +# The computer images build from the repo root (images/hermes/Dockerfile, +# images/stub/Dockerfile): each carries the fragment CLI, of the root +# workspace. Their context is only what their builds read; everything else +# (targets, the cell's builds, node_modules, git) stays out. * !Cargo.toml !Cargo.lock diff --git a/.github/actions/e2e-build/action.yml b/.github/actions/e2e-build/action.yml index 8caa49d60..9c2606625 100644 --- a/.github/actions/e2e-build/action.yml +++ b/.github/actions/e2e-build/action.yml @@ -110,6 +110,10 @@ runs: # commit in eight changes one) is its layers fetched in seconds, not # its bridge compiled (about 100 s alone, 250 beside the Rust); a # changed one compiles here, before the Rust rather than beside it. + # Its CLI stage (the fragment CLI its agents ask with) is the same: + # cached until the CLI or a crate it builds from changes, then built + # beside the bridge's stage. The context is the repo's root, cut to + # what the images build from (.dockerignore). # Saved when asked, every stage's layers (mode=max: the build stage's # base too, 345 MB in all, each layer kept once by its digest). - id: buildx @@ -120,7 +124,7 @@ runs: DOCKER_BUILD_SUMMARY: "false" DOCKER_BUILD_RECORD_UPLOAD: "false" with: - context: images + context: . file: images/stub/Dockerfile platforms: linux/amd64 provenance: false diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index cd1da7193..db062c3ca 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -155,10 +155,11 @@ jobs: - run: cargo test -p fragment-bridge --test docker --locked --no-run - name: the runner run: nproc; free -g; df -h /; docker version --format '{{.Server.Version}}' - # as the tests' own `build` does (docker.rs), the stub from images/, the - # Hermes image from the repo's root + # as the tests' own `build` does (docker.rs), both from the repo's root + # (each carries the fragment CLI, of the root workspace) - name: build the stub image - run: docker build --platform linux/amd64 -f stub/Dockerfile -t fragment-stub:test . + working-directory: . + run: docker build --platform linux/amd64 -f images/stub/Dockerfile -t fragment-stub:test . - name: build the Hermes image working-directory: . run: docker build --platform linux/amd64 -f images/hermes/Dockerfile -t fragment-hermes:test . diff --git a/cell/wrangler.jsonc b/cell/wrangler.jsonc index 85195ae28..9247c40f3 100644 --- a/cell/wrangler.jsonc +++ b/cell/wrangler.jsonc @@ -69,9 +69,9 @@ "class_name": "Computer", "scheduling_policy": "durable_object", "images": { - "stub": { "dockerfile": "../images/stub/Dockerfile", "build_context": "../images" }, + "stub": { "dockerfile": "../images/stub/Dockerfile", "build_context": ".." }, // the next build of the same image: the e2e's upgrade and rollback - "stub-next": { "dockerfile": "../images/stub/Dockerfile", "build_context": "../images", "build_vars": { "STUB_VERSION": "2" } } + "stub-next": { "dockerfile": "../images/stub/Dockerfile", "build_context": "..", "build_vars": { "STUB_VERSION": "2" } } } } ], diff --git a/crates/e2e/src/lanes/computers.rs b/crates/e2e/src/lanes/computers.rs index 9cd8d8192..5fdd65495 100644 --- a/crates/e2e/src/lanes/computers.rs +++ b/crates/e2e/src/lanes/computers.rs @@ -1727,6 +1727,28 @@ pub fn computers(s: &mut Suite, api: &Api) -> Result<()> { let led = s.eventually(wake, || agent_replies(&records(api, &owner, &chat_name, "chat"), &identity).len() == replies_so_far); s.ok("unmentioned, the lead answers", led && agent_replies(&records(api, &owner, &chat_name, "chat"), &maple_id).len() == 1, ""); + // an ask (docs/chat-records.md, "An ask") with the CLI the stub carries, + // as our Hermes image's agents ask: the lead asks the second agent; the + // question is in the two agents' chat with their owner, answered there, + // and the answer comes back here as a turn of the lead's own + let label = "an agent asks another with the CLI its image carries: the question in the two agents' chat with their owner (its return_to this chat), answered there, and the answer back here as the asker's own turn"; + if scripted { + let said = fetched(s, 22, &format!("ask {maple_label} what is the time"))?; + replies_so_far += 1; + let answer = |r: &Value| r["body"]["text"].as_str().is_some_and(|t| t.contains(" answered your question in ")); + let answered = s.eventually(wake, || agent_replies(&records(api, &owner, &chat_name, "chat"), &identity).iter().any(answer)); + replies_so_far += 1; + let listed = api.signed(&owner, "GET", "/api/fragments", None)?.body; + let both = |f: &Value| f["agents"].as_array().is_some_and(|a| a.len() == 2 && a.iter().any(|x| x == identity.as_str()) && a.iter().any(|x| x == maple_id.as_str())); + let pair = listed["fragments"].as_array().into_iter().flatten().find(|f| f["kind"] == "chat" && f["name"] != chat_name.as_str() && both(f)).and_then(|f| f["name"].as_str()).unwrap_or("").to_string(); + let asked = if pair.is_empty() { vec![] } else { records(api, &owner, &pair, "chat") }; + let question = asked.iter().any(|r| r["principal"] == identity.as_str() && r["body"]["to"][0] == maple_id.as_str() && r["body"]["return_to"] == chat_name.as_str()); + let echoed = agent_replies(&asked, &maple_id).len() == 1; + s.ok(label, said.starts_with("asked ") && question && echoed && answered, json!({ "said": said, "pair": pair, "asked": asked.iter().map(|r| &r["body"]).collect::>() })); + } else { + s.skip(label, "it needs the stub's scripted runtime"); + } + // a placeholder names its agent, not whoever sends it: the second // agent's, sent by the lead, is the second's (a person's agents are not // fenced: decision 44); removed from the computer, its tags are refused diff --git a/docs/computers.md b/docs/computers.md index 1861957dc..486b6e060 100644 --- a/docs/computers.md +++ b/docs/computers.md @@ -1798,9 +1798,12 @@ settings and state): and the fragment API, with the chat records of docs/chat-records.md. Its core names no runtime; `relay` (Hermes' Relay) and `script` (a deterministic agent) are its two runtimes. -- `images/stub`: the bridge with `script`, and a screen page. 9.6 MB; a - local start follows its chats in 0.3 s. The platform's own lanes run - against it. +- `images/stub`: the bridge with `script`, the fragment CLI (its agents + ask each other with it, `ask `, as our Hermes image's + do; built from the CLI's inputs alone, its dependencies a layer of their + own, unoptimized), and a screen page. 9.6 MB and a 17 MB CLI; a local + start follows its chats in 0.3 s. The platform's own lanes run against + it. - `images/hermes`: Hermes v0.21.7's desktop image, `hermes-boot`, the bridge as Hermes' Relay connector, the screen. The base is pinned by digest to v0.21.7's release build (its versioned tag diff --git a/images/.dockerignore b/images/.dockerignore deleted file mode 100644 index 473d46cc4..000000000 --- a/images/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -target -**/target -*.md diff --git a/images/bridge/src/runtime/script.rs b/images/bridge/src/runtime/script.rs index 14f0d9c5b..d7ae0d524 100644 --- a/images/bridge/src/runtime/script.rs +++ b/images/bridge/src/runtime/script.rs @@ -84,6 +84,14 @@ //! (only the agent's own key's, as every request of its is); one signed //! by a key that is no door's. The reply is `door linked`, `door said //! `, or `door refused: `; +//! - `ask `: another agent of its computer asked `` +//! with the fragment CLI, `fragment ask --return-to `, as the +//! agent acting for its owner, as our Hermes image's `fragment_ask` runs +//! it: the question posted in the two agents' chat with their owner, and +//! its answer back here once that agent's turn is over, as a turn of this +//! agent's own (docs/chat-records.md, "An ask"). Its post id is the +//! turn's, so a turn read again asks once. The reply: `asked `, +//! or `ask refused: `; //! - a message with attachments: the reply names them; //! - a turn told something first (`TurnStart::note`: its agent's turn before //! it in the chat was cut by a restart): the reply ends with @@ -415,6 +423,37 @@ async fn schedule(agent: &crate::runtime::Agent, chat: &str, rest: &str) -> Stri asked.unwrap_or_else(|e| format!("schedule refused: {e}")) } +/// The fragment CLI the stub image carries (images/stub/Dockerfile). +const CLI: &str = "/usr/local/bin/fragment"; +/// An ask is a few platform calls (who may be asked, their chat, made the +/// first time, its post): at most this long. +const ASK_WAIT: Duration = Duration::from_secs(60); + +/// `ask `: the module's list. +async fn ask_another(agent: &crate::runtime::Agent, chat: &str, turn: &str, rest: &str) -> String { + let Some((who, words)) = rest.trim().split_once(char::is_whitespace) else { + return "say `ask `".into(); + }; + let run = tokio::process::Command::new(CLI) + .args(["--json", "ask", "--return-to", chat, "--id", &format!("ask-{turn}"), "--", who, words.trim()]) + .env("FRAGMENT_AS_AGENT", &agent.fragment) + .env("FRAGMENT_FOR", &agent.owner) + .kill_on_drop(true) + .output(); + let out = match tokio::time::timeout(ASK_WAIT, run).await { + Ok(Ok(out)) => out, + Ok(Err(e)) => return format!("ask refused: the CLI did not run ({e})"), + Err(_) => return format!("ask refused: no answer in {} s", ASK_WAIT.as_secs()), + }; + let said: serde_json::Value = String::from_utf8_lossy(&out.stdout).lines().last().and_then(|l| serde_json::from_str(l).ok()).unwrap_or_default(); + if said["ok"] != true { + let why = said["error"]["message"].as_str().map(str::to_string).unwrap_or_else(|| String::from_utf8_lossy(&out.stderr).chars().take(300).collect()); + return format!("ask refused: {why}"); + } + let asked = &said["data"]["asked"]; + format!("asked {}", asked["title"].as_str().or(asked["name"].as_str()).unwrap_or(who)) +} + /// The door keys `door link` made, by agent, for the process's life. static DOORS: std::sync::Mutex> = std::sync::Mutex::new(BTreeMap::new()); @@ -741,6 +780,9 @@ async fn turn(cfg: ScriptConfig, ts: TurnStart, mut rx: mpsc::Receiver, e if let Some(rest) = ts.text.strip_prefix("schedule ") { reply = schedule(&ts.agent, &ts.fragment, rest).await; } + if let Some(rest) = ts.text.strip_prefix("ask ") { + reply = ask_another(&ts.agent, &ts.fragment, &id, rest).await; + } if let Some(rest) = ts.text.strip_prefix("relay ") { reply = relay(&ts.agent.fragment, rest).await.unwrap_or_else(|e| format!("relay failed: {e}")); } diff --git a/images/bridge/tests/docker.rs b/images/bridge/tests/docker.rs index 5ebdf4a03..e9a9c00c3 100644 --- a/images/bridge/tests/docker.rs +++ b/images/bridge/tests/docker.rs @@ -39,8 +39,8 @@ fn images_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().expect("images/").to_path_buf() } -/// The repo's root: the Hermes image's build context (it carries the -/// fragment CLI, of the root workspace). +/// The repo's root: both images' build context (each carries the fragment +/// CLI, of the root workspace). fn repo_dir() -> PathBuf { images_dir().parent().expect("the repo").to_path_buf() } @@ -194,7 +194,7 @@ impl Container { #[tokio::test] #[ignore = "needs Docker: cargo test -p fragment-bridge --test docker -- --ignored"] async fn the_stub_image() { - let built = build(&images_dir(), "stub/Dockerfile", "fragment-stub:test"); + let built = build(&repo_dir(), "images/stub/Dockerfile", "fragment-stub:test"); eprintln!("stub: built in {:.1} s, {} MB", built.as_secs_f64(), size("fragment-stub:test") / 1_000_000); let fake = Fake::start("0.0.0.0:0", &["juniper"]).await; let model = Model::start("0.0.0.0:0").await; diff --git a/images/stub/Dockerfile b/images/stub/Dockerfile index 5226faf6a..7eca39526 100644 --- a/images/stub/Dockerfile +++ b/images/stub/Dockerfile @@ -1,10 +1,12 @@ # syntax=docker/dockerfile:1.7 # The stub computer image (docs/computers.md): the bridge with its `script` -# runtime and a screen page on 6080, nothing else. The platform's own lanes -# run against it, which is the proof that the platform needs nothing -# Hermes-specific (docs/cloudflare-v1.md, the rule). +# runtime, the fragment CLI its agents ask each other with (`ask`, as our +# Hermes image's do), and a screen page on 6080, nothing else. The +# platform's own lanes run against it, which is the proof that the platform +# needs nothing Hermes-specific (docs/cloudflare-v1.md, the rule). # -# Build from images/: docker build --platform linux/amd64 -f stub/Dockerfile . +# Build from the repo's root (it carries the CLI, of the root workspace): +# docker build --platform linux/amd64 -f images/stub/Dockerfile . # The bridge, static (Alpine's Rust targets musl), built where it runs. Its # sources are touched first: the target cache is every build's on this @@ -12,22 +14,67 @@ # than its last build (images/hermes/Dockerfile). FROM rust:1.99-alpine AS build WORKDIR /src -COPY Cargo.toml Cargo.lock ./ -COPY bridge bridge -COPY hermes/boot hermes/boot +COPY images/Cargo.toml images/Cargo.lock ./ +COPY images/bridge bridge +COPY images/hermes/boot hermes/boot RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/src/target \ find bridge -type f -exec touch {} + \ && cargo build --release --locked -p fragment-bridge \ && cp target/release/fragment-bridge /fragment-bridge +# The fragment CLI (cli/), static, its agents ask each other with. Its +# dependencies first, built from the manifests and the lock alone (each of +# our crates an empty source, the templates' build script none), so the +# layer that holds them is reused until a dependency changes, and a change +# to our own crates (the CLI, core, proto) rebuilds only them; then our +# crates, whole. Its target is the layers' own (no cache mount), so CI's +# BuildKit cache keeps the dependencies' build. Unoptimized: the stub's +# agents make a few calls with it, and it builds in a third of the time. +# Its inputs are the CLI's alone, as our Hermes image's CLI stage is (that +# one says why each), and xtask's host test +# `the_cli_stage_copies_what_the_cli_builds_from` keeps them so. +FROM rust:1.99-alpine AS cli +WORKDIR /src +ENV CARGO_PROFILE_RELEASE_OPT_LEVEL=0 +COPY Cargo.toml Cargo.lock ./ +COPY cli/Cargo.toml cli/Cargo.toml +COPY crates/core/Cargo.toml crates/core/Cargo.toml +COPY crates/nip98/Cargo.toml crates/nip98/Cargo.toml +COPY crates/proto/Cargo.toml crates/proto/Cargo.toml +COPY crates/templates/Cargo.toml crates/templates/Cargo.toml +COPY crates/devstack/Cargo.toml crates/devstack/Cargo.toml +COPY crates/e2e/Cargo.toml crates/e2e/Cargo.toml +COPY crates/fakes/Cargo.toml crates/fakes/Cargo.toml +COPY xtask/Cargo.toml xtask/Cargo.toml +RUN --mount=type=cache,id=stub-cli-registry,target=/usr/local/cargo/registry \ + for m in crates/core crates/nip98 crates/proto crates/templates crates/devstack crates/e2e crates/fakes xtask; do mkdir -p "$m/src" && : > "$m/src/lib.rs"; done \ + && mkdir -p cli/src && echo 'fn main() {}' > cli/src/main.rs && echo 'fn main() {}' > crates/templates/build.rs \ + && cargo build --release --locked -p fragment-cli +COPY cli cli +COPY crates/core crates/core +COPY crates/nip98 crates/nip98 +COPY crates/proto crates/proto +COPY crates/templates crates/templates +COPY templates/blank templates/blank +COPY templates/calories templates/calories +COPY templates/doc templates/doc +COPY templates/inbox templates/inbox +COPY templates/notes templates/notes +COPY templates/todo templates/todo +RUN --mount=type=cache,id=stub-cli-registry,target=/usr/local/cargo/registry \ + find cli crates templates -type f -exec touch {} + \ + && cargo build --release --locked -p fragment-cli \ + && mkdir -p /out && cp target/release/fragment /out/ + # busybox for the few commands the Computer DO runs in a container (`true` # while it polls, `touch` for the restore marker); sandbox-shim for its # DirectoryBackup. FROM busybox:1.37-musl COPY --from=docker.io/cloudflare/sandbox:1.0.0 /usr/local/bin/sandbox-shim /usr/local/bin/sandbox-shim COPY --from=build /fragment-bridge /usr/local/bin/fragment-bridge -COPY stub/screen /opt/fragment/screen +COPY --from=cli /out/fragment /usr/local/bin/fragment +COPY images/stub/screen /opt/fragment/screen # which build this is, served beside the screen (the platform's upgrade and # rollback lane reads it through the computer's port) ARG STUB_VERSION=1 diff --git a/xtask/src/build.rs b/xtask/src/build.rs index d27d41521..a19ae4d97 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -411,8 +411,8 @@ mod tests { /// "Stable and canary": what changes a version), so a change to the /// cell, the shell, a blessed template, the e2e, xtask or the docs /// builds the same image. Method: its CLI stage - /// (images/hermes/Dockerfile) against cargo's own account of the - /// workspace. It copies, whole, the CLI and each workspace crate it + /// (images/hermes/Dockerfile), and the stub's, the same, against + /// cargo's own account of the workspace. It copies, whole, the CLI and each workspace crate it /// builds from (its normal and build dependencies by path, and theirs); /// each other member's manifest alone, and stubs exactly those; the /// root manifest and lock; and each template but the platform's own, @@ -455,23 +455,6 @@ mod tests { let stubbed: BTreeSet = dirs.iter().filter(|(n, _)| !closure.contains(*n)).map(|(_, d)| d.clone()).collect(); assert!(whole.contains("cli") && whole.contains("crates/templates") && stubbed.contains("crates/e2e"), "{whole:?} {stubbed:?}"); - let dockerfile = std::fs::read_to_string(root.join("images/hermes/Dockerfile")).expect("the Dockerfile"); - let stage: Vec<&str> = dockerfile.lines().skip_while(|l| !l.starts_with("FROM rust:") || !l.ends_with(" AS cli")).skip(1).take_while(|l| !l.starts_with("FROM ")).collect(); - assert!(!stage.is_empty(), "the Dockerfile has its CLI stage (`FROM rust:… AS cli`)"); - let copies: Vec> = stage.iter().filter_map(|l| l.strip_prefix("COPY ")).map(|l| l.split_whitespace().collect()).collect(); - let lock = ["Cargo.toml", "Cargo.lock", "./"]; - assert!(copies.iter().any(|c| c.as_slice() == lock), "the root manifest and lock: {copies:?}"); - let paths: Vec<&str> = copies.iter().filter(|c| c.as_slice() != lock).map(|c| { - assert!(c.len() == 2 && c[0] == c[1] && !c[0].starts_with("--"), "each copy keeps its path: {c:?}"); - c[0] - }).collect(); - let copied_whole: BTreeSet = paths.iter().filter(|p| !p.ends_with("/Cargo.toml") && !p.starts_with("templates/")).map(|p| p.to_string()).collect(); - let copied_manifests: BTreeSet = paths.iter().filter_map(|p| p.strip_suffix("/Cargo.toml")).map(str::to_string).collect(); - assert_eq!(copied_whole, whole, "the stage copies the CLI and each crate it builds from, whole"); - assert_eq!(copied_manifests, stubbed, "and each other member's manifest alone"); - let stubs = stage.iter().find_map(|l| l.trim().strip_prefix("for m in ").and_then(|l| l.split_once(';'))).map(|(list, _)| list.split_whitespace().map(str::to_string).collect::>()); - assert_eq!(stubs, Some(stubbed), "and stubs exactly those members' sources (`for m in …;`)"); - let mut templates: BTreeSet = BTreeSet::new(); for entry in std::fs::read_dir(root.join("templates")).expect("templates/").flatten() { let name = entry.file_name().to_string_lossy().into_owned(); @@ -479,7 +462,32 @@ mod tests { templates.insert(format!("templates/{name}")); } } - let copied_templates: BTreeSet = paths.iter().filter(|p| p.starts_with("templates/")).map(|p| p.to_string()).collect(); - assert_eq!(copied_templates, templates, "and each template the CLI scaffolds: every one but the platform's own"); + // both images carry the CLI. Hermes' stage builds it whole, its other + // members' manifests and empty sources beside it; the stub's builds + // its dependencies first, from every member's manifest and empty + // sources (the CLI's a `main`), then the CLI's crates whole + let every: BTreeSet = whole.union(&stubbed).cloned().collect(); + let emptied: BTreeSet = every.iter().filter(|d| *d != "cli").cloned().collect(); + for (file, manifests, empties) in [("images/hermes/Dockerfile", &stubbed, &stubbed), ("images/stub/Dockerfile", &every, &emptied)] { + let dockerfile = std::fs::read_to_string(root.join(file)).expect("the Dockerfile"); + let stage: Vec<&str> = dockerfile.lines().skip_while(|l| !l.starts_with("FROM rust:") || !l.ends_with(" AS cli")).skip(1).take_while(|l| !l.starts_with("FROM ")).collect(); + assert!(!stage.is_empty(), "{file} has its CLI stage (`FROM rust:… AS cli`)"); + let copies: Vec> = stage.iter().filter_map(|l| l.strip_prefix("COPY ")).map(|l| l.split_whitespace().collect()).collect(); + let lock = ["Cargo.toml", "Cargo.lock", "./"]; + assert!(copies.iter().any(|c| c.as_slice() == lock), "{file}: the root manifest and lock: {copies:?}"); + let paths: Vec<&str> = copies.iter().filter(|c| c.as_slice() != lock).map(|c| { + assert!(c.len() == 2 && c[0] == c[1] && !c[0].starts_with("--"), "each copy keeps its path: {c:?}"); + c[0] + }).collect(); + let copied_whole: BTreeSet = paths.iter().filter(|p| !p.ends_with("/Cargo.toml") && !p.starts_with("templates/")).map(|p| p.to_string()).collect(); + let copied_manifests: BTreeSet = paths.iter().filter_map(|p| p.strip_suffix("/Cargo.toml")).map(str::to_string).collect(); + assert_eq!(copied_whole, whole, "{file}'s stage copies the CLI and each crate it builds from, whole"); + assert_eq!(&copied_manifests, manifests, "{file}: and the members' manifests its stage copies alone"); + let stubs = stage.iter().find_map(|l| l.trim().strip_prefix("for m in ").and_then(|l| l.split_once(';'))).map(|(list, _)| list.split_whitespace().map(str::to_string).collect::>()); + assert_eq!(stubs.as_ref(), Some(empties), "{file}: and empties exactly those members' sources (`for m in …;`)"); + + let copied_templates: BTreeSet = paths.iter().filter(|p| p.starts_with("templates/")).map(|p| p.to_string()).collect(); + assert_eq!(copied_templates, templates, "{file}: and each template the CLI scaffolds: every one but the platform's own"); + } } }