From 50a90ccb374b56ae149e4f8b8e01ef3e9b23f402 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 09:07:25 +0800 Subject: [PATCH 1/3] feat(harness): add capability diagnostics and loading samples --- .github/workflows/ci.yml | 3 + README.md | 3 + README.zh-CN.md | 3 + bin/claude-everything-workflow.js | 12 ++ docs/capability-diagnostics.md | 69 ++++++ docs/loading-baseline-2026-11-02.md | 49 +++++ harness/loading/cases.json | 41 ++++ package.json | 5 +- scripts/diagnostics/capabilities.js | 237 +++++++++++++++++++++ scripts/diagnostics/capabilities.test.js | 140 ++++++++++++ scripts/diagnostics/loading.js | 129 +++++++++++ scripts/diagnostics/loading.test.js | 76 +++++++ scripts/diagnostics/sample-loading.js | 88 ++++++++ scripts/diagnostics/sample-loading.test.js | 26 +++ scripts/install.ps1 | 8 +- scripts/install.sh | 8 +- scripts/verify/install-rules.test.js | 4 + scripts/verify/metadata-checks.js | 6 + 18 files changed, 904 insertions(+), 3 deletions(-) create mode 100644 docs/capability-diagnostics.md create mode 100644 docs/loading-baseline-2026-11-02.md create mode 100644 harness/loading/cases.json create mode 100644 scripts/diagnostics/capabilities.js create mode 100644 scripts/diagnostics/capabilities.test.js create mode 100644 scripts/diagnostics/loading.js create mode 100644 scripts/diagnostics/loading.test.js create mode 100644 scripts/diagnostics/sample-loading.js create mode 100644 scripts/diagnostics/sample-loading.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4de7dc0..46bd6e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,9 @@ jobs: - name: Test plugin distribution run: npm run test:plugins + - name: Test capability diagnosis and loading reports + run: npm run test:diagnostics + - name: Validate npm package run: npm run pack:dry-run diff --git a/README.md b/README.md index a748c30..6b2b039 100644 --- a/README.md +++ b/README.md @@ -141,12 +141,15 @@ npm run test:manifest npm run test:invocation npm run test:continuation npm run test:install-rules +npm run test:diagnostics npm run verify:upstream npm run pack:dry-run ``` `verify:upstream` checks the recorded capability mapping; it does not establish parity with the latest upstream version. `npx claude-everything-workflow verify` validates the published distribution rather than your local project. Neither command proves an agent's effectiveness on real tasks. +[Installed capability diagnosis and five-case loading samples](docs/capability-diagnostics.md): `cew doctor --host codex` / `cew loading --trace FILE --case small-fix`. + For contributions, describe the problem and expected behavior, keep the change focused, and include the checks you actually ran. See the [Git rules](rules/05-git-workflow.md). ## Acknowledgments diff --git a/README.zh-CN.md b/README.zh-CN.md index 5632bd9..c6cffbf 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -141,12 +141,15 @@ npm run test:manifest npm run test:invocation npm run test:continuation npm run test:install-rules +npm run test:diagnostics npm run verify:upstream npm run pack:dry-run ``` `verify:upstream` 检查已登记的上游能力映射，不代表与最新上游完全一致。`npx claude-everything-workflow verify` 检查已发布的分发包，而不是当前业务项目。两者都不能证明 Agent 的真实任务效果。 +[实际能力诊断与五类加载采样](docs/capability-diagnostics.md)：`cew doctor --host codex` / `cew loading --trace FILE --case small-fix`。 + 提交贡献时说明问题与预期行为，保持改动范围集中，并列出实际运行的检查。具体要求见 [Git 规则](rules/05-git-workflow.md)。 ## 致谢 diff --git a/bin/claude-everything-workflow.js b/bin/claude-everything-workflow.js index 09545c0..a71a583 100755 --- a/bin/claude-everything-workflow.js +++ b/bin/claude-everything-workflow.js @@ -14,6 +14,16 @@ function main() { process.exit(0); } + if (command === "loading") { + require("../scripts/diagnostics/loading").main(args); + return; + } + + if (command === "doctor") { + require("../scripts/diagnostics/capabilities").main(args); + return; + } + if (command === "install") { runInstall(args); return; @@ -95,6 +105,8 @@ function printHelp() { Usage: cew install [--claude-only|--codex-only] [--dry-run] cew verify + cew doctor [--host all|codex|claude] [--json] + cew loading --trace FILE --case ID [--root DIR] [--json] Examples: npx claude-everything-workflow install diff --git a/docs/capability-diagnostics.md b/docs/capability-diagnostics.md new file mode 100644 index 0000000..b426aed --- /dev/null +++ b/docs/capability-diagnostics.md @@ -0,0 +1,69 @@ +# 实际能力诊断与加载采样 + +诊断沿用 `cew`，不修改个人文件，不修复或升级安装。它补充源码校验 `cew verify`，不替代真实任务验收。没有新增常驻规则、Hook 或技能路由。 + +## 只读检查安装 + +在当前源码目录执行，或把下面的 `node bin/claude-everything-workflow.js` 换成已安装的 `cew`： + +```sh +node bin/claude-everything-workflow.js doctor --host codex +node bin/claude-everything-workflow.js doctor --host claude --json +node bin/claude-everything-workflow.js doctor --host codex --project /path/to/project --plugin-root /path/to/plugin --json +``` + +对照版本是运行命令所在的分发包或 checkout；不会联网推测“最新版”。报告记录该位置、包版本、正文与引用文件的 SHA-256 指纹。比较前仅统一 CRLF/LF 换行，不忽略其他正文差异。登记表相同也会独立比较每个技能正文。 + +默认检查两个宿主。`--host codex|claude` 限定宿主；`--home DIR` 使用 DIR 下的 `.codex` / `.claude`，便于隔离验证。未传 `--home` 时尊重 `CODEX_HOME`、`CLAUDE_CONFIG_DIR`。`--project` 默认当前调用目录，不是 CLI 包的目录。 + +检查项目和用户级技能目录、Codex 的 `.agents/skills`，以及宿主 `plugins/cache` 中能确认包名的 CEW 插件。插件缓存扫描最多六层、4000 个目录，不递归跟随目录软链接；自定义目录、marketplace checkout 或其他宿主位置用可重复的 `--plugin-root` 显式指定。扫描错误会作为 finding 报告。扫描不是宿主完整配置解析；禁用和旧缓存也可能出现。 + +| 字段 | 含义 | +| --- | --- | +| `DISCOVERED` / `NOT_FOUND` | 磁盘上发现或未发现入口 | +| `MATCH` / `DIFFERENT` / `NO_REFERENCE` | 内容与对照版本一致、有差异，或个人引用在分发包中没有对照；不判断旧安装还是个人修改 | +| `STATIC_READY` | 入口及直接、具体的本地引用可读，仅为静态前置条件 | +| `BLOCKED` / `UNVERIFIED` | 引用缺失、不可读，或引用超出可确认范围 | +| `selection: UNVERIFIED` | 未证明宿主实际选择了该副本 | +| `DUPLICATE_CANDIDATES` | 同一宿主有多份物理副本；不等于确认重复加载 | + +相同物理文件的软链接别名会去重。规则路径沿用 `AGENTS.md`：项目优先，再用户目录；Claude 专项规则先保留个人 `rules/common`，再回退 `references/rules/common`。插件包里的规则不自动充当用户规则回退。 + +引用检查覆盖正文中具体的 Markdown 链接、代码格式文件路径和路由代码块内的文件路径；合并对照正文与实际正文里的引用，避免旧安装漏掉新依赖。它包含条件分支的潜在直接引用，不递归推断整个执行图，不解析脚本依赖、自然语言里的动态路径、系统工具、账号、权限或生产环境。因此 `STATIC_READY` 不是“已执行通过”，也不能保证每个条件分支可运行。差异只报告，不覆盖。 + +退出码：`0` 无发现项，`1` 有差异、缺失或重复候选，`2` 参数或诊断本身出错。缺少一个宿主的安装也会报告；只使用一个宿主时传 `--host`。 + +## 五类加载基线 + +[harness/loading/cases.json](../harness/loading/cases.json) 保存普通问答、小修复、正式设计、执行验收、建立验收能力五类自然语言请求。它是采样用例，不是第二套路由表。没有改变自然语言触发或安全、审查、验证规则。 + +```sh +codex exec --json '你的自然语言任务' > /private/local/trace.jsonl +node bin/claude-everything-workflow.js loading --trace /private/local/trace.jsonl --case small-fix --json +``` + +实际操作应在相应的隔离项目运行第一条命令。额外的 `--root DIR` 将实际安装或项目里的规则、技能及参考文件加入比对，可重复使用；默认仅对照当前包。读取有个人修改的安装时传入其宿主根目录，才能匹配那份正文。 + +报告只从 Codex CLI 的 `item.completed / command_execution / aggregated_output` 中匹配完整正文，输出逻辑路径、候选物理路径、内容指纹、读取次数、累计及重复字符量。它不从命令字符串或文件名出现推测“已读”。CRLF/LF 归一后匹配；一个完整正文在输出里出现两次，计为两次。相同内容的候选副本不重复计费，也不能由此判断宿主选择。 + +**观测边界：** 宿主自动注入、部分读取、加行号或其他变形输出、其他工具类型及上下文压缩不在统计内。这是完整文件工具输出的下界，不是完整会话上下文账单。`actualFileTokens` 留空，字符数除以四只是粗略估算；宿主 usage 单独记录整个 turn 的用量，不归因到某份文件，也不据此声称节省量。 + +`NO_VIOLATION_OBSERVED` 只表示没有观察到该用例禁止的完整正文；不是任务或安全验收通过。缺少 `turn.completed` 是 `INSUFFICIENT_EVIDENCE`。仍要按用例的 `review` 核对最终行为、必要规则和验证证据。实际总 Token 节省须后续在相同任务、宿主和模型条件下做对照实测。 + +退出码：`0` 未观察到禁止读取，`1` 观察到禁止读取或会话不完整，`2` 输入/格式错误。输入目前仅支持 Codex CLI JSONL，Claude 轨迹未适配。 + +## 可选的真实模型采样 + +```sh +node scripts/diagnostics/sample-loading.js --output docs/superpowers/evidence/loading-run-1 +# 只运行一种 +node scripts/diagnostics/sample-loading.js --output docs/superpowers/evidence/loading-run-2 --case small-fix +``` + +这会消耗当前 Codex CLI 账号的模型额度，默认顺序启动五个独立会话，沿用当前宿主配置和技能；不会被 `verify`、诊断、测试或 CI 自动调用。它创建本次所属的临时 Git 项目，保留轨迹、错误、结果和工作成果用于复核。每次调用最多三分钟，可用 `--timeout-ms 1000..180000` 下调；模型调用失败即停止，不自动循环重试。输出目录必须尚不存在，避免覆盖历史证据。 + +采样项目只包含 Node 标准库、一个公开函数及测试；执行验收用例有现成入口，建立验收用例没有。为使完整读取可观测，项目说明要求用不截断的 `cat` 读取材料，每次最多三份，但不要求加载任何额外文件。这是受控采样，不能代表所有真实项目或桌面 App。自动报告的行为审核初始为 `NOT_RUN`，复核后另行记录，不把模型自报完成当作通过。 + +原始轨迹可能包含用户上下文和路径，只保存在本地忽略目录；长期报告只保留必要统计和证据指纹，不提交原始会话。复核后只清理输出中列明的本次临时项目，按项目约定保留证据；不删除用户的安装或其他任务工作区。 + +首轮实测结果见[五类加载基线](loading-baseline-2026-11-02.md)，其中三类完整、两类超时；没有 Token 节省结论。 diff --git a/docs/loading-baseline-2026-11-02.md b/docs/loading-baseline-2026-11-02.md new file mode 100644 index 0000000..a002f10 --- /dev/null +++ b/docs/loading-baseline-2026-11-02.md @@ -0,0 +1,49 @@ +# 五类任务的首轮加载采样（2026-11-02） + +本轮取得五类实际 Codex CLI 轨迹：普通问答、小修复、执行验收完整结束；正式设计、建立验收能力在三分钟上限超时，属于不完整样本。没有观察到普通任务提前加载 Spec、ticket 或验收建设参考；这个结论只覆盖匹配到的完整正文，不代表所有实际加载行为。 + +## 环境与口径 + +- 对照：本地 CEW `0.2.5`，基线 `42dda80522ba96f0c02c2c3913eadf3d1f4d92a9`，本轮新增诊断与采样工具；没有修改技能、规则或路由正文。 +- 宿主：macOS、`codex-cli 0.160.1`，沿用当前用户配置和技能；有效模型未在本次 CLI JSONL 中记录，不能据此建立跨模型比较。 +- 每类一个独立会话和隔离的 Node 标准库项目；没有外部服务、账号操作或生产数据。 +- 读入量来自完整正文在工具输出中的匹配，包含项目验收入口；同一输出位置的分发正文/个人较长变体不重复累计。自动注入、局部或变形读取不在统计内。 +- 文件指纹和轨迹指纹按 LF 换行归一后计算 SHA-256。表内轨迹指纹仅展示前 12 位；本地原始报告保留全值。 + +## 实测结果 + +| 类别 | 行为审核 | 文件数 / 读取次数 | 观察到的字符量 | 重复字符量 | 轨迹指纹前缀 | +| --- | --- | --- | --- | --- | --- | +| 普通问答 | PASS | 0 / 0 | 0 | 0 | `b1f2a19a1721` | +| 小修复 | PASS | 8 / 8 | 14,450 | 0 | `eaa745e5f31d` | +| 正式设计 | BLOCKED | 8 / 8 | 20,749 | 0 | `7b2e17fe2a64` | +| 执行验收 | PASS | 7 / 7 | 6,990 | 0 | `366b8250dffa` | +| 建立验收能力 | BLOCKED | 9 / 9 | 17,467 | 0 | `095abeef3463` | + +这里的 `PASS` 仅指本次小型采样任务，不能外推到实际产品、桌面 App 或所有项目。两项 `BLOCKED` 的字符量是中断前的下界，不能与完整任务作为等价成本比较。没有发布实测 Token 节省量；字符除以四的粗估值未作为性能结论。 + +## 逐类核查 + +- **普通问答：PASS。** 回答了 map/forEach 的区别与示例，没有工具调用，也没有观察到技能正文读取。宿主自动注入内容仍未统计。 +- **小修复：PASS。** 读取路由、调试、TDD、基础/实现/Git/禁止规则与测试规则；先观察空数组回归失败，再增加初值，公开函数的两个测试通过。只修改实现和测试，差异检查通过；没有读取 Spec、ticket 或验收建设参考。 +- **正式设计：BLOCKED。** 读取了 Spec 及其模板、自审参考，并保存草稿；采样超时，缺少 `turn.completed` 和最终待审交付。不得视为 Spec 审核流程通过，也没有用户批准。未观察到实施、拆票或验收建设材料。 +- **执行验收：PASS。** 复用项目已有 Skill 和功能地图，`npm run verify` 两项公开函数用例通过，证据日志已回读，代码及设施没有改动；没有读取验收建设参考。 +- **建立验收能力：BLOCKED。** 正确读取了验收建设参考，已有入口、地图和证据草稿；采样超时且缺少完整最终交付，不能把草稿称为已交付能力。未观察到 Spec 或 ticket 材料。 + +完整样本均没有重复完整正文读取。小修复与执行验收观察到禁止事项及测试规则，且真实测试与改动边界符合该小项目请求；这不证明所有安全约束在其他项目中均有效。因此本轮不据此缩减安全或验证要求，也不调整路由。 + +## 安装核查及工具验证 + +在 CEW 仓库作为目标项目时，Codex 和 Claude 各发现 20 个 CEW 技能，未发现正文、登记表、有效直接引用差异或重复候选。换成隔离采样项目后，Codex 用户级 `rules/02-implementation.md` 与分发包有差异，诊断如实报告；在 CEW 仓库中它由项目规则优先覆盖。Claude 的隔离项目检查没有发现项。所有差异只报告、未覆盖。此前报告的两个技能正文差异本次未复现，原因未追溯。此结果仍不证明宿主选择顺序、条件分支或外部工具可执行。 + +回归覆盖“登记表相同但正文不同”、缺失与个人引用、Claude 规则回退、重复候选/软链接、路由代码块引用、重复/重叠输出、格式错误与不完整轨迹。真实采样发现超时子进程可能返回退出码 0，工具现结合错误、终止信号和退出码判断失败，已有外部进程回归；不会把这种情况报告为成功。 + +这次两项超时发生于修正失败停止判定之前的同一轮采样，因此保留了后续类别的轨迹；修正后的采样器遇到这类失败会停止，不自动重试。这里不以重跑替换失败记录。 + +本轮新增 16 项诊断/采样测试、现有流程与安装校验通过；独立审查通过。打包后在独立目录安装生产依赖，CLI 的 `verify`、`doctor` 与 `loading` 冒烟通过。安装执行回归限于本机 Shell；PowerShell 清单已核对，未做 Windows 实机验收。 + +## 证据与限制 + +完整本地证据保存在当前任务工作区的 `docs/superpowers/evidence/loading-20261102/`，每类包含 `trace.jsonl`、`stderr.log`、`report.json`，总表为 `summary.json`；宿主诊断为相邻的 `doctor-codex-20261102.json`、`doctor-claude-20261102.json`，隔离项目检查为对应的 `doctor-*-isolated-20261102.json`。目录已被现有忽略规则排除，不提交原始会话或个人路径。临时项目位置见本地报告，保留供复核。 + +未做安装升级实机验收、Windows 实机验收、Claude 会话采样、桌面 App 验收或 Token 成本对照。下一轮若要评价长任务加载成本，需要让两项中断样本完整结束，再在相同模型、宿主和任务条件下比较；本轮不把这些未测项计为通过。 diff --git a/harness/loading/cases.json b/harness/loading/cases.json new file mode 100644 index 0000000..90edca5 --- /dev/null +++ b/harness/loading/cases.json @@ -0,0 +1,41 @@ +{ + "schemaVersion": 1, + "description": "Natural-language sampling cases, not a second routing policy. Assertions observe reads only; successful routing and behavior still need transcript review.", + "cases": [ + { + "id": "question", + "label": "普通问答", + "prompt": "解释一下 JavaScript 的 Array.map 与 Array.forEach 的区别，给一个简短例子。", + "forbidden": ["skills/"], + "review": "直接回答；不因普通问答进入工程交付。" + }, + { + "id": "small-fix", + "label": "小修复", + "prompt": "修复 src/total.js：total([]) 应返回 0，现在会抛错。补相称回归验证，只完成这个本地修复，不提交。", + "forbidden": ["skills/spec-gate/", "skills/to-tickets/", "skills/implement/references/ticket-delivery.md", "skills/feature-acceptance/references/project-verification.md"], + "review": "修复和回归验证完成；审查、测试与范围要求仍生效。" + }, + { + "id": "formal-design", + "label": "正式设计", + "prompt": "为 total(values) 增加可选的 round 参数，写一份正式 Spec，先不实现。已决定默认不舍入，传 0 到 6 的整数时对最终结果舍入到对应小数位，其他值抛 RangeError；保留现有调用方式，不增加依赖。", + "forbidden": ["skills/to-tickets/", "skills/implement/references/ticket-delivery.md", "skills/feature-acceptance/references/project-verification.md"], + "review": "形成可审核 Spec，停在用户批准关口；不实施、不拆票。" + }, + { + "id": "acceptance-run", + "label": "执行验收", + "prompt": "按项目现有验收入口验收 total 功能，沿用功能地图，仅报告真实命令和结果，不建立或维护验收基础设施。", + "forbidden": ["skills/spec-gate/", "skills/to-tickets/", "skills/implement/references/ticket-delivery.md", "skills/feature-acceptance/references/project-verification.md"], + "review": "复用现有入口；PASS/FAIL/BLOCKED/NOT RUN 与真实结果一致。" + }, + { + "id": "acceptance-setup", + "label": "建立验收能力", + "prompt": "为这个小型 Node 项目建立最小的项目验收入口与功能地图，覆盖 total 的正常输入和空数组；沿用 node --test，不增加依赖或服务，不提交。", + "forbidden": ["skills/spec-gate/", "skills/to-tickets/", "skills/implement/references/ticket-delivery.md"], + "review": "建立并执行可重复验收，功能地图引用真实入口和证据。" + } + ] +} diff --git a/package.json b/package.json index a8ed191..9ba29ff 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "test:install-rules": "node scripts/verify/install-rules.test.js", "test:plugins": "node --test scripts/verify/plugin-distribution.test.js", "verify:upstream": "node scripts/verify-upstream-capability-map.js", - "pack:dry-run": "npm pack --dry-run" + "pack:dry-run": "npm pack --dry-run", + "test:diagnostics": "node --test scripts/diagnostics/*.test.js" }, "dependencies": { "commonmark": "0.31.2", @@ -39,6 +40,8 @@ "README.md", "README.zh-CN.md", "docs/plugins.md", + "docs/capability-diagnostics.md", + "docs/loading-baseline-2026-11-02.md", "settings.json", "agents/", "bin/", diff --git a/scripts/diagnostics/capabilities.js b/scripts/diagnostics/capabilities.js new file mode 100644 index 0000000..63f4f8b --- /dev/null +++ b/scripts/diagnostics/capabilities.js @@ -0,0 +1,237 @@ +"use strict"; + +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const crypto = require("node:crypto"); +const { Parser } = require("commonmark"); +const { parseFrontmatter } = require("../verify/skill-manifest-checks"); + +const source = path.resolve(__dirname, "../.."); +const normalize = (value) => value.replace(/\r\n/g, "\n"); +const hash = (value) => crypto.createHash("sha256").update(normalize(value)).digest("hex"); +const isFile = (file) => { try { return fs.statSync(file).isFile(); } catch { return false; } }; +const within = (root, file) => file === root || file.startsWith(root + path.sep); + +function read(file) { + try { + if (!fs.statSync(file).isFile()) return { status: "NOT_FILE" }; + return { status: "READABLE", body: fs.readFileSync(file, "utf8") }; + } catch (error) { + return { status: error.code === "ENOENT" ? "MISSING" : "UNREADABLE", error: error.code }; + } +} + +function comparison(expected, actual) { + const current = read(actual); + const original = read(expected); + return { + status: current.status !== "READABLE" ? current.status : original.status !== "READABLE" ? "NO_REFERENCE" + : hash(original.body) === hash(current.body) ? "MATCH" : "DIFFERENT", + expectedHash: original.body === undefined ? null : hash(original.body), + actualHash: current.body === undefined ? null : hash(current.body), + }; +} + +function options(args, allowed, repeat = []) { + const result = {}; + for (let i = 0; i < args.length; i++) { + const key = args[i]; + if (!allowed.includes(key)) throw new Error(`Unknown option: ${key}`); + if (key === "--json" || key === "--help") { result[key] = true; continue; } + const value = args[++i]; + if (!value || value.startsWith("--")) throw new Error(`Missing value: ${key}`); + if (repeat.includes(key)) (result[key] ||= []).push(value); + else if (key in result) throw new Error(`Repeated option: ${key}`); + else result[key] = value; + } + return result; +} + +function pluginRoots(searchRoot, host, issues) { + const found = []; + const seen = new Set(); + let count = 0; + function visit(dir, depth) { + if (depth > 6) return; + let entries; + try { + const real = fs.realpathSync(dir); + if (seen.has(real)) return; + seen.add(real); + if (++count > 4000) throw new Error("scan limit"); + const manifest = read(path.join(dir, host === "codex" ? ".codex-plugin/plugin.json" : ".claude-plugin/plugin.json")); + if (manifest.status === "READABLE") { + const metadata = JSON.parse(manifest.body); + if (metadata.name === "claude-everything-workflow") { + found.push({ path: dir, kind: "plugin-cache", activation: "UNVERIFIED" }); + return; + } + } + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch (error) { + if (error.code !== "ENOENT") issues.push({ code: "SCAN_INCOMPLETE", path: dir, reason: error.code || error.message }); + return; + } + for (const entry of entries) if (entry.isDirectory() && entry.name !== "node_modules" && !entry.name.startsWith(".")) { + visit(path.join(dir, entry.name), depth + 1); + } + } + visit(searchRoot, 0); + return found; +} + +// Only concrete, distributed paths are dependencies. Project artifacts and external URLs are not. +// A missing new relative Markdown link in a personal skill is still reported, never silently accepted. +function references(body, relative) { + const found = new Set(); + const local = new Set(); + const add = (target, link = false) => { + if (/^(?:[a-z]+:|#|\/)/i.test(target) || /[<>*{}$]/.test(target)) return; + target = target.split("#")[0]; + let resolved; + const sibling = path.posix.normalize(path.posix.join(path.posix.dirname(relative), target)); + if (isFile(path.join(source, sibling))) resolved = sibling; + else if (isFile(path.join(source, target))) resolved = target; + else if (/^(?:skills|rules|agents|commands|scripts|hooks|harness)\//.test(target)) resolved = path.posix.normalize(target); + else if (link || target.startsWith("references/")) resolved = sibling; + if (!resolved) return; + if (resolved.startsWith("../")) { local.add(target); return; } + found.add(resolved); + }; + for (const match of body.matchAll(/\[[^\]\n]*\]\(([^\s)]+)(?:\s+[^)]*)?\)/g)) add(match[1], true); + for (const match of body.matchAll(/`([^`\n]+)`/g)) { + if (/^(?:skills|rules|references|agents|commands|scripts|hooks|harness)\/[\w./-]+\.[\w]+$/.test(match[1])) add(match[1]); + } + const walker = new Parser().parse(body).walker(); + let event; + while ((event = walker.next())) { + if (!event.entering || event.node.type !== "code_block") continue; + for (const match of event.node.literal.matchAll(/(?:^|[\s`(])((?:skills|rules|references|agents|commands|scripts|hooks|harness)\/[\w./-]+\.[\w]+)/g)) add(match[1]); + } + return { paths: [...found].sort(), unresolved: [...local] }; +} + +function dependencyPath(relative, root, hostRoot, project, host) { + if (!relative.startsWith("rules/")) return path.join(root, relative); + const candidates = [path.join(project, relative), path.join(hostRoot, relative)]; + if (host === "claude" && relative.startsWith("rules/common/")) candidates.push(path.join(hostRoot, "references", relative)); + // Plugin-bundled rules are not an automatic fallback: AGENTS.md does not define one. + return candidates.find((file) => fs.existsSync(file)) || candidates.at(-1); +} + +function skillMetadata(body, name) { + try { + const parsed = parseFrontmatter(body || ""); + if (!parsed || parsed.name !== name || typeof parsed.description !== "string" || !parsed.description.trim()) { + return { status: "INVALID", reason: "Missing or mismatched name/description" }; + } + return { status: "VALID" }; + } catch { return { status: "INVALID", reason: "Invalid frontmatter YAML" }; } +} + +function candidate(skill, root, hostRoot, project, host) { + const file = path.join(root.path, skill.path); + const content = comparison(path.join(source, skill.path), file); + const actual = read(file); + const metadata = skillMetadata(actual.body, skill.name); + const expected = read(path.join(source, skill.path)); + const refs = references((expected.body || "") + "\n" + (actual.body || ""), skill.path); + const dependencies = refs.paths.map((relative) => { + const target = dependencyPath(relative, root.path, hostRoot, project, host); + return { reference: relative, path: target, ...comparison(path.join(source, relative), target) }; + }); + const manifest = comparison(path.join(source, "harness/manifest.json"), path.join(root.path, "harness/manifest.json")); + const unavailable = dependencies.some((item) => !["MATCH", "DIFFERENT", "NO_REFERENCE"].includes(item.status)); + return { + path: file, realPath: fs.realpathSync(file), kind: root.kind, selection: "UNVERIFIED", + manifestMatch: manifest.status === "MATCH", manifestStatus: manifest.status, content, metadata, + readiness: actual.status !== "READABLE" || metadata.status === "INVALID" || unavailable ? "BLOCKED" + : refs.unresolved.length || dependencies.some((item) => item.status === "NO_REFERENCE") ? "UNVERIFIED" : "STATIC_READY", + dependencies, unresolvedReferences: refs.unresolved, + }; +} + +function diagnose({ host = "all", home, project = process.cwd(), plugins = [] } = {}) { + if (!["all", "codex", "claude"].includes(host)) throw new Error("--host must be all, codex or claude"); + const manifest = JSON.parse(fs.readFileSync(path.join(source, "harness/manifest.json"), "utf8")); + const issues = []; + const hosts = (host === "all" ? ["codex", "claude"] : [host]).map((name) => { + const userHome = path.resolve(home || os.homedir()); + const hostRoot = path.resolve(home ? path.join(userHome, `.${name}`) + : (name === "codex" ? process.env.CODEX_HOME : process.env.CLAUDE_CONFIG_DIR) || path.join(userHome, `.${name}`)); + const roots = [ + { path: hostRoot, kind: "user-copy" }, + { path: path.resolve(project, `.${name}`), kind: "project" }, + ...(name === "codex" ? [ + { path: path.join(userHome, ".agents"), kind: "shared-user" }, + { path: path.resolve(project, ".agents"), kind: "shared-project" }, + ] : []), + ...pluginRoots(path.join(hostRoot, "plugins/cache"), name, issues), + ...plugins.map((dir) => ({ path: path.resolve(dir), kind: "explicit-plugin", activation: "UNVERIFIED" })), + ]; + const skills = manifest.skills.map((skill) => { + const seen = new Set(); + const candidates = []; + for (const root of roots) { + const file = path.join(root.path, skill.path); + if (!fs.existsSync(file)) continue; + const real = fs.realpathSync(file); + if (seen.has(real)) continue; + seen.add(real); + const item = candidate(skill, root, hostRoot, project, name); + candidates.push(item); + const manifestPath = path.join(root.path, "harness/manifest.json"); + if (!item.manifestMatch && !issues.some((issue) => issue.host === name && issue.path === manifestPath)) { + issues.push({ code: "MANIFEST_" + item.manifestStatus, host: name, path: manifestPath }); + } + if (item.content.status !== "MATCH") issues.push({ code: "BODY_" + item.content.status, host: name, skill: skill.name, path: file }); + if (item.readiness !== "STATIC_READY") issues.push({ code: "DEPENDENCIES_" + item.readiness, host: name, skill: skill.name, path: file }); + for (const dep of item.dependencies.filter((dep) => dep.status === "DIFFERENT")) { + issues.push({ code: "DEPENDENCY_DIFFERENT", host: name, skill: skill.name, path: dep.path }); + } + } + if (candidates.length > 1) issues.push({ code: "DUPLICATE_CANDIDATES", host: name, skill: skill.name }); + return { name: skill.name, discovery: candidates.length ? "DISCOVERED" : "NOT_FOUND", candidates }; + }); + const discovered = skills.filter((skill) => skill.candidates.length).length; + if (!discovered) issues.push({ code: "NO_INSTALLATION_FOUND", host: name }); + else for (const skill of skills.filter((skill) => !skill.candidates.length)) issues.push({ code: "SKILL_NOT_FOUND", host: name, skill: skill.name }); + return { host: name, hostRoot, discovered, roots, skills }; + }); + return { + schemaVersion: 1, reference: { root: source, version: require("../../package.json").version, manifestHash: hash(JSON.stringify(manifest)) }, + scope: "Read-only disk inspection; host selection, conditional execution and external tools are unverified. STATIC_READY covers direct concrete local references only.", + hosts, issues, + }; +} + +function main(args) { + try { + const opts = options(args, ["--host", "--home", "--project", "--plugin-root", "--json", "--help"], ["--plugin-root"]); + if (opts["--help"]) { + console.log("cew doctor [--host all|codex|claude] [--home DIR] [--project DIR] [--plugin-root DIR] [--json]\nRead-only; exit 0: no findings, 1: findings, 2: invalid input. Candidate discovery does not prove host selection."); + return; + } + const report = diagnose({ host: opts["--host"], home: opts["--home"], project: opts["--project"], plugins: opts["--plugin-root"] }); + if (opts["--json"]) console.log(JSON.stringify(report, null, 2)); + else { + console.log(`CEW ${report.reference.version} — read-only capability diagnosis`); + for (const host of report.hosts) { + console.log(`${host.host}: ${host.discovered}/${host.skills.length} skills discovered; host selection UNVERIFIED`); + for (const skill of host.skills) for (const item of skill.candidates) { + console.log(` ${skill.name}: ${item.content.status}, ${item.readiness} — ${item.path}`); + for (const dep of item.dependencies.filter((dep) => dep.status !== "MATCH")) console.log(` ${dep.status}: ${dep.path}`); + } + } + for (const issue of report.issues) console.log(`${issue.code}: ${issue.host || ""} ${issue.skill || ""} ${issue.path || ""}`); + console.log(report.scope); + } + process.exitCode = report.issues.length ? 1 : 0; + } catch (error) { + console.error(`doctor: ${error.message}`); + process.exitCode = 2; + } +} + +module.exports = { diagnose, main, options, hash, normalize }; diff --git a/scripts/diagnostics/capabilities.test.js b/scripts/diagnostics/capabilities.test.js new file mode 100644 index 0000000..d81d314 --- /dev/null +++ b/scripts/diagnostics/capabilities.test.js @@ -0,0 +1,140 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const { test } = require("node:test"); + +const source = path.resolve(__dirname, "../.."); +const cli = path.join(source, "bin/claude-everything-workflow.js"); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "cew-doctor-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const home = path.join(root, "home"); + const project = path.join(root, "project"); + fs.mkdirSync(project, { recursive: true }); + const installed = path.join(home, ".codex"); + for (const dir of ["skills", "rules", "harness", "references", "agents", "commands"]) { + fs.cpSync(path.join(source, dir), path.join(installed, dir), { recursive: true }); + } + const run = (extra = []) => spawnSync(process.execPath, [cli, "doctor", "--host", "codex", "--home", home, + "--project", project, "--json", ...extra], { encoding: "utf8" }); + return { root, home, project, installed, run }; +} + +test("doctor detects changed bodies even when the installed manifest is identical, without writes", (t) => { + const { installed, run } = fixture(t); + const file = path.join(installed, "skills/using-superpowers/SKILL.md"); + const personal = fs.readFileSync(file, "utf8") + "\nPersonal instruction.\n"; + fs.writeFileSync(file, personal); + const result = run(); + assert.equal(result.status, 1); + const report = JSON.parse(result.stdout); + const candidate = report.hosts[0].skills.find((s) => s.name === "using-superpowers").candidates[0]; + assert.equal(candidate.manifestMatch, true); + assert.equal(candidate.content.status, "DIFFERENT"); + assert.equal(candidate.selection, "UNVERIFIED"); + assert.equal(fs.readFileSync(file, "utf8"), personal); + assert.deepEqual(fs.readdirSync(path.dirname(file)).filter((name) => name.includes("bak")), []); +}); + +test("a discovered skill is blocked when its required reference is missing", (t) => { + const { installed, run } = fixture(t); + fs.unlinkSync(path.join(installed, "skills/feature-acceptance/references/project-verification.md")); + const result = run(); + const skill = JSON.parse(result.stdout).hosts[0].skills.find((s) => s.name === "feature-acceptance"); + assert.equal(skill.discovery, "DISCOVERED"); + assert.equal(skill.candidates[0].readiness, "BLOCKED"); + assert.ok(skill.candidates[0].dependencies.some((dep) => dep.reference.endsWith("project-verification.md") && dep.status === "MISSING")); +}); + +test("Claude rule resolution preserves personal common rules ahead of migrated references", (t) => { + const { home, project, installed } = fixture(t); + const dest = path.join(home, ".claude"); + fs.renameSync(installed, dest); + fs.mkdirSync(path.join(dest, "references/rules"), { recursive: true }); + fs.renameSync(path.join(dest, "rules/common"), path.join(dest, "references/rules/common")); + const invoke = () => spawnSync(process.execPath, [cli, "doctor", "--host", "claude", "--home", home, "--project", project, "--json"], { encoding: "utf8" }); + let report = JSON.parse(invoke().stdout); + let dep = report.hosts[0].skills.find((s) => s.name === "using-superpowers").candidates[0].dependencies.find((d) => d.reference === "rules/common/testing.md"); + assert.equal(dep.path, path.join(dest, "references/rules/common/testing.md")); + fs.mkdirSync(path.join(dest, "rules/common")); + fs.writeFileSync(path.join(dest, "rules/common/testing.md"), "personal rule"); + report = JSON.parse(invoke().stdout); + dep = report.hosts[0].skills.find((s) => s.name === "using-superpowers").candidates[0].dependencies.find((d) => d.reference === "rules/common/testing.md"); + assert.equal(dep.path, path.join(dest, "rules/common/testing.md")); + assert.equal(dep.status, "DIFFERENT"); +}); + +test("npm and cached plugin copies are duplicate candidates, never claimed as active", (t) => { + const { installed, run } = fixture(t); + const plugin = path.join(installed, "plugins/cache/cew/claude-everything-workflow/0.2.5"); + fs.mkdirSync(path.join(plugin, ".codex-plugin"), { recursive: true }); + fs.writeFileSync(path.join(plugin, ".codex-plugin/plugin.json"), '{"name":"claude-everything-workflow"}'); + fs.cpSync(path.join(installed, "skills"), path.join(plugin, "skills"), { recursive: true }); + const report = JSON.parse(run().stdout); + assert.ok(report.issues.some((issue) => issue.code === "DUPLICATE_CANDIDATES")); + const candidates = report.hosts[0].skills.find((s) => s.name === "using-superpowers").candidates; + assert.equal(candidates.length, 2); + assert.equal(candidates[1].selection, "UNVERIFIED"); + fs.rmSync(path.join(plugin, "skills"), { recursive: true }); + fs.symlinkSync(path.join(installed, "skills"), path.join(plugin, "skills"), "dir"); + assert.equal(JSON.parse(run().stdout).hosts[0].skills[0].candidates.length, 1); +}); + +test("new missing local references in an installed body block readiness", (t) => { + const { installed, run } = fixture(t); + fs.appendFileSync(path.join(installed, "skills/implement/SKILL.md"), '\nRead `rules/personal-required.md` first.\n'); + const candidate = JSON.parse(run().stdout).hosts[0].skills.find((s) => s.name === "implement").candidates[0]; + assert.equal(candidate.readiness, "BLOCKED"); + assert.ok(candidate.dependencies.some((dep) => dep.reference === "rules/personal-required.md" && dep.status === "MISSING")); +}); + +test("doctor rejects malformed options", (t) => { + const { run } = fixture(t); + assert.equal(run(["--host"]).status, 2); + assert.equal(run(["--unknown"]).status, 2); +}); + +test("a malformed discovered skill is not statically ready", (t) => { + const { installed, run } = fixture(t); + fs.writeFileSync(path.join(installed, "skills/implement/SKILL.md"), '---\nname: [unterminated\n---\n'); + const candidate = JSON.parse(run().stdout).hosts[0].skills.find((s) => s.name === "implement").candidates[0]; + assert.equal(candidate.readiness, "BLOCKED"); + assert.equal(candidate.metadata.status, "INVALID"); +}); + +test("doctor reports manifest drift separately and resolves project rules from the caller's directory", (t) => { + const { installed, project, run } = fixture(t); + const file = path.join(installed, "harness/manifest.json"); + fs.appendFileSync(file, "\n"); + fs.mkdirSync(path.join(project, "rules/common"), { recursive: true }); + fs.writeFileSync(path.join(project, "rules/common/testing.md"), "project-specific test policy"); + const report = JSON.parse(run().stdout); + const candidate = report.hosts[0].skills.find((s) => s.name === "using-superpowers").candidates[0]; + assert.equal(candidate.manifestMatch, false); + assert.ok(report.issues.some((item) => item.code === "MANIFEST_DIFFERENT")); + assert.equal(candidate.dependencies.find((d) => d.reference === "rules/common/testing.md").path, path.join(project, "rules/common/testing.md")); +}); + +test("readable personal references without a distribution counterpart are unverified, not missing", (t) => { + const { installed, run } = fixture(t); + const file = path.join(installed, "skills/implement/SKILL.md"); + fs.appendFileSync(file, '\nRead [guide](references/personal.md).\n'); + fs.writeFileSync(path.join(installed, "skills/implement/references/personal.md"), "Personal guide."); + const candidate = JSON.parse(run().stdout).hosts[0].skills.find((s) => s.name === "implement").candidates[0]; + assert.equal(candidate.readiness, "UNVERIFIED"); + assert.equal(candidate.dependencies.find((d) => d.reference.endsWith("personal.md")).status, "NO_REFERENCE"); +}); + +test("router references in fenced code participate in readiness checks", (t) => { + const { installed, run } = fixture(t); + fs.unlinkSync(path.join(installed, "agents/harness-optimizer.md")); + const report = JSON.parse(run().stdout); + const candidate = report.hosts[0].skills.find((s) => s.name === "using-superpowers").candidates[0]; + assert.equal(candidate.readiness, "BLOCKED"); + assert.ok(candidate.dependencies.some((dep) => dep.reference === "agents/harness-optimizer.md" && dep.status === "MISSING")); +}); diff --git a/scripts/diagnostics/loading.js b/scripts/diagnostics/loading.js new file mode 100644 index 0000000..0f7b062 --- /dev/null +++ b/scripts/diagnostics/loading.js @@ -0,0 +1,129 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { options, hash, normalize } = require("./capabilities"); +const source = path.resolve(__dirname, "../.."); +const cases = require("../../harness/loading/cases.json").cases; + +function inventory(roots) { + const files = new Map(); + function visit(root, relative) { + const file = path.join(root, relative); + let stat; + try { stat = fs.statSync(file); } catch (error) { if (error.code === "ENOENT") return; throw error; } + if (stat.isDirectory()) { + // Source references may be symlinks; don't recursively follow directory links. + if (fs.lstatSync(file).isSymbolicLink()) return; + for (const name of fs.readdirSync(file).sort()) visit(root, path.join(relative, name)); + } else if (stat.isFile() && relative.endsWith(".md")) { + const body = normalize(fs.readFileSync(file, "utf8")); + if (!body.trim()) return; + const logical = relative.split(path.sep).join("/"); + const digest = hash(body); + const key = digest; + const item = files.get(key) || { path: logical, sha256: digest, body, aliases: [], candidatePaths: [], reads: 0 }; + if (!item.aliases.includes(logical)) item.aliases.push(logical); + item.candidatePaths.push(file); + files.set(key, item); + } + } + for (const root of roots) for (const dir of ["AGENTS.md", "CLAUDE.md", "rules", "skills", "references", "agents", "commands", ".agents/skills", ".claude/skills"]) visit(root, dir); + return [...files.values()]; +} + +function analyze({ trace, caseId, roots = [] }) { + const scenario = cases.find((item) => item.id === caseId); + if (!scenario) throw new Error(`Unknown case: ${caseId}; choose ${cases.map((c) => c.id).join(", ")}`); + const content = fs.readFileSync(trace, "utf8"); + const events = content.split(/\r?\n/).filter((line) => line.trim()).map((line, i) => { + try { return JSON.parse(line); } catch { throw new Error(`Invalid JSONL at line ${i + 1}`); } + }); + if (!events.length) throw new Error("Empty trace"); + const files = inventory([...new Set([source, ...roots.map((root) => path.resolve(root))])]); + let commands = 0; + let completed = false; + let usage = null; + for (const event of events) { + if (event.type === "turn.completed") { + completed = true; + if (event.usage) { + usage ||= {}; + for (const [key, value] of Object.entries(event.usage)) { + if (typeof value === "number") usage[key] = (usage[key] || 0) + value; + } + } + } + if (["turn.started", "turn.failed", "error"].includes(event.type)) completed = false; + if (event.type !== "item.completed" || event.item?.type !== "command_execution") continue; + commands++; + const body = normalize(event.item.aggregated_output || ""); + const matches = []; + for (const file of files) { + let offset = 0; + let position; + while ((position = body.indexOf(file.body, offset)) !== -1) { + matches.push({ file, start: position, end: position + file.body.length }); + offset = position + file.body.length; + } + } + // Prefer the longer installed variant over a contained reference body. One output span + // represents one read even if the same text exists at several candidate paths. + const selected = []; + matches.sort((a, b) => b.file.body.length - a.file.body.length); + for (const match of matches) { + if (selected.some((other) => match.start < other.end && other.start < match.end)) continue; + selected.push(match); + match.file.reads++; + } + } + const observed = files.filter((file) => file.reads).map(({ body, ...file }) => ({ + ...file, charactersPerRead: body.length, + observedCharacters: body.length * file.reads, repeatedCharacters: body.length * (file.reads - 1), + estimatedTokens: Math.ceil(body.length * file.reads / 4), + })); + const violations = observed.filter((file) => scenario.forbidden.some((prefix) => file.aliases.some((alias) => alias.startsWith(prefix)))) + .map((file) => ({ path: file.path, reads: file.reads })); + return { + schemaVersion: 1, caseId, label: scenario.label, traceHash: hash(content), + reference: { version: require("../../package.json").version, root: source }, + coverage: "full-file-tool-output-only", sampleStatus: completed ? "COMPLETE" : "INCOMPLETE", + assessment: !completed ? "INSUFFICIENT_EVIDENCE" : violations.length ? "VIOLATION_OBSERVED" : "NO_VIOLATION_OBSERVED", + limitations: ["Host-injected context, partial or transformed reads and other tool types are not counted.", + "candidatePaths locate matching content, not proof of which installed copy the host selected.", + "Characters/4 is an estimate, not measured model tokens or proven savings. Host usage covers the entire turn.", + "Read assertions do not prove successful behavior, safety or verification; review the transcript against the case."], + summary: { + commands, uniqueFiles: observed.length, reads: observed.reduce((n, file) => n + file.reads, 0), + observedCharacters: observed.reduce((n, file) => n + file.observedCharacters, 0), + repeatedCharacters: observed.reduce((n, file) => n + file.repeatedCharacters, 0), + estimatedTokens: observed.reduce((n, file) => n + file.estimatedTokens, 0), actualFileTokens: null, + }, + hostUsage: usage, violations, files: observed, review: scenario.review, + }; +} + +function main(args) { + try { + const opts = options(args, ["--trace", "--case", "--root", "--json", "--help"], ["--root"]); + if (opts["--help"]) { + console.log("cew loading --trace FILE --case ID [--root DIR] [--json]\nCases: " + cases.map((c) => c.id).join(", ") + "\nCodex exec --json input. Exit 0: no forbidden read observed, 1: forbidden read/incomplete trace, 2: invalid input."); + return; + } + if (!opts["--trace"] || !opts["--case"]) throw new Error("--trace and --case are required"); + const report = analyze({ trace: opts["--trace"], caseId: opts["--case"], roots: opts["--root"] }); + if (opts["--json"]) console.log(JSON.stringify(report, null, 2)); + else { + console.log(`${report.label}: ${report.assessment} (${report.coverage})`); + console.log(`${report.summary.uniqueFiles} files, ${report.summary.reads} reads, ${report.summary.observedCharacters} characters; repeated ${report.summary.repeatedCharacters}`); + for (const file of report.files) console.log(` ${file.reads}x ${file.path} (${file.sha256})`); + for (const violation of report.violations) console.log(`Unexpected read: ${violation.path}`); + report.limitations.forEach((item) => console.log(item)); + } + process.exitCode = report.assessment === "NO_VIOLATION_OBSERVED" ? 0 : 1; + } catch (error) { + console.error(`loading: ${error.message}`); + process.exitCode = 2; + } +} +module.exports = { main, analyze }; diff --git a/scripts/diagnostics/loading.test.js b/scripts/diagnostics/loading.test.js new file mode 100644 index 0000000..ebde140 --- /dev/null +++ b/scripts/diagnostics/loading.test.js @@ -0,0 +1,76 @@ +"use strict"; +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const { test } = require("node:test"); +const root = path.resolve(__dirname, "../.."); +const cli = path.join(root, "bin/claude-everything-workflow.js"); + +function run(t, events, caseId = "small-fix") { + const temp = fs.mkdtempSync(path.join(os.tmpdir(), "cew-loading-")); + t.after(() => fs.rmSync(temp, { recursive: true, force: true })); + const trace = path.join(temp, "trace.jsonl"); + fs.writeFileSync(trace, [...events, { type: "turn.completed" }].map((event) => JSON.stringify(event)).join("\n") + "\n"); + const result = spawnSync(process.execPath, [cli, "loading", "--trace", trace, "--case", caseId, "--json"], { encoding: "utf8" }); + return { result, trace }; +} + +const output = (body, code = 0) => ({ type: "item.completed", item: { type: "command_execution", command: "cat SKILL.md", aggregated_output: body, exit_code: code } }); +const read = (relative) => fs.readFileSync(path.join(root, relative), "utf8"); + +test("loading counts observed output and repeated reads, not file mentions or failed empty reads", (t) => { + const file = "skills/implement/SKILL.md"; + const { result } = run(t, [output(read(file)), output("skills/spec-gate/SKILL.md\n"), output("", 1), output(read(file))]); + assert.equal(result.status, 0, result.stderr); + const report = JSON.parse(result.stdout); + assert.equal(report.files.length, 1); + assert.equal(report.files[0].path, file); + assert.equal(report.files[0].reads, 2); + assert.equal(report.summary.observedCharacters, read(file).length * 2); + assert.equal(report.summary.repeatedCharacters, read(file).length); + assert.equal(report.summary.actualFileTokens, null); + assert.equal(report.coverage, "full-file-tool-output-only"); +}); + +test("loading flags construction references in ordinary acceptance without exposing raw output", (t) => { + const file = "skills/feature-acceptance/references/project-verification.md"; + const { result } = run(t, [output(read(file) + "\nprivate customer text")], "acceptance-run"); + assert.equal(result.status, 1); + const report = JSON.parse(result.stdout); + assert.equal(report.assessment, "VIOLATION_OBSERVED"); + assert.deepEqual(report.violations.map((item) => item.path), [file]); + assert.ok(!result.stdout.includes("private customer text")); +}); + +test("loading supports CRLF outputs and natural-language question samples with zero skill reads", (t) => { + const { result } = run(t, [output(read("rules/01-base.md").replace(/\n/g, "\r\n"))], "question"); + assert.equal(result.status, 0); + assert.equal(JSON.parse(result.stdout).summary.uniqueFiles, 1); + assert.equal(run(t, [{ type: "item.completed", item: { type: "agent_message", text: "map returns an array" } }], "question").result.status, 0); +}); + +test("loading rejects an incomplete or malformed trace instead of claiming a passing sample", (t) => { + const { result, trace } = run(t, [], "question"); + fs.writeFileSync(trace, JSON.stringify(output(read("skills/implement/SKILL.md"))) + "\n"); + const invoke = () => spawnSync(process.execPath, [cli, "loading", "--trace", trace, "--case", "question", "--json"], { encoding: "utf8" }); + assert.equal(JSON.parse(invoke().stdout).assessment, "INSUFFICIENT_EVIDENCE"); + fs.appendFileSync(trace, "{broken\n"); + assert.equal(invoke().status, 2); + assert.equal(result.status, 0); +}); + +test("a longer installed variant does not count its reference-body substring a second time", (t) => { + const { trace } = run(t, []); + const host = path.join(path.dirname(trace), "host"); + const relative = "skills/implement/SKILL.md"; + fs.mkdirSync(path.dirname(path.join(host, relative)), { recursive: true }); + const personal = read(relative) + "\nPersonal local instruction.\n"; + fs.writeFileSync(path.join(host, relative), personal); + fs.writeFileSync(trace, [output(personal), { type: "turn.completed" }].map(JSON.stringify).join("\n")); + const result = spawnSync(process.execPath, [cli, "loading", "--trace", trace, "--case", "small-fix", "--root", host, "--json"], { encoding: "utf8" }); + const report = JSON.parse(result.stdout); + assert.equal(report.summary.reads, 1); + assert.equal(report.summary.observedCharacters, personal.length); +}); diff --git a/scripts/diagnostics/sample-loading.js b/scripts/diagnostics/sample-loading.js new file mode 100644 index 0000000..822f9b5 --- /dev/null +++ b/scripts/diagnostics/sample-loading.js @@ -0,0 +1,88 @@ +#!/usr/bin/env node +"use strict"; + +// Opt-in local model experiment. Never invoked by doctor, loading, verify or CI. +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const { options } = require("./capabilities"); +const { analyze } = require("./loading"); +const source = path.resolve(__dirname, "../.."); + +function write(root, relative, body) { + const file = path.join(root, relative); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, body); +} + +function prepare(scenario, root) { + const broken = scenario.id === "small-fix"; + write(root, "src/total.js", `"use strict";\nexports.total = (values) => values.reduce((a, b) => a + b${broken ? "" : ", 0"});\n`); + write(root, "test/total.test.js", 'const { test } = require("node:test");\nconst assert = require("node:assert/strict");\nconst { total } = require("../src/total");\ntest("adds values", () => assert.equal(total([1, 2, 3]), 6));\n' + + (broken ? "" : 'test("empty array", () => assert.equal(total([]), 0));\n')); + write(root, "package.json", JSON.stringify({ name: "cew-loading-fixture", private: true, scripts: { test: "node --test", verify: "node --test" } }, null, 2) + "\n"); + write(root, ".gitignore", "docs/superpowers/\n.evidence/\n"); + const acceptance = scenario.id === "acceptance-run"; + write(root, "AGENTS.md", fs.readFileSync(path.join(source, "AGENTS.md"), "utf8") + + '\n## 采样项目\n测试入口 `npm test`。仅处理当前用户请求，不提交、推送或访问外部服务。\n' + + '为准确采样，读取规则、技能和参考文件时使用不截断的 cat，单次最多读取三份文件；不因此多读额外文件。\n' + + (acceptance ? '验收入口 `.agents/skills/verify-loading-fixture/SKILL.md`，功能地图见该目录的 `references/features.md`。\n' : "")); + if (acceptance) { + write(root, ".agents/skills/verify-loading-fixture/SKILL.md", '---\nname: verify-loading-fixture\ndescription: Verify the total library through its public API.\n---\n\nRun `npm run verify` from the project root; require exit 0 and both scenarios passing.\nFeature map: [features](references/features.md). Record code revision, command, exit code and output in `.evidence/`; read the evidence again before reporting. No service or account is required.\n'); + write(root, ".agents/skills/verify-loading-fixture/references/features.md", '# total 功能地图\n\n- 正常输入：公开函数 total([1,2,3]) 返回 6，test/total.test.js 覆盖。\n- 空数组：total([]) 返回 0，同一测试文件覆盖。\n- 范围：纯本地 Node 库，不涉及 UI、账号或网络。\n'); + } + for (const args of [["init", "--initial-branch=sample-task"], ["add", "."], ["-c", "user.name=CEW Sample", "-c", "user.email=sample@example.invalid", "-c", "core.hooksPath=/dev/null", "commit", "-m", "fixture baseline"]]) { + const result = spawnSync("git", args, { cwd: root, encoding: "utf8" }); + if (result.status !== 0) throw new Error(result.stderr); + } +} + +function main() { + const opts = options(process.argv.slice(2), ["--output", "--case", "--timeout-ms", "--help"]); + if (opts["--help"]) { + console.log("node scripts/diagnostics/sample-loading.js --output NEW_DIR [--case ID] [--timeout-ms 1000..180000]\nRuns five fresh Codex CLI model sessions by default; consumes quota. Uses current host config and user skills. Stores local traces, fixtures and reports. Never edits the CEW installation."); + return; + } + if (!opts["--output"]) throw new Error("--output is required"); + const timeout = opts["--timeout-ms"] === undefined ? 180000 : Number(opts["--timeout-ms"]); + if (!Number.isInteger(timeout) || timeout < 1000 || timeout > 180000) throw new Error("--timeout-ms must be an integer from 1000 to 180000"); + const output = path.resolve(opts["--output"]); + if (fs.existsSync(output)) throw new Error("Output directory already exists; choose a new one to preserve evidence"); + const cases = require("../../harness/loading/cases.json").cases.filter((item) => !opts["--case"] || item.id === opts["--case"]); + if (!cases.length) throw new Error("Unknown case"); + fs.mkdirSync(output, { recursive: true }); + const cliVersion = spawnSync("codex", ["--version"], { encoding: "utf8" }); + if (cliVersion.status !== 0) throw new Error("Codex CLI is unavailable"); + const summary = []; + for (const scenario of cases) { + const dir = path.join(output, scenario.id); + fs.mkdirSync(dir); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "cew-loading-fixture-")); + prepare(scenario, root); + const trace = path.join(dir, "trace.jsonl"); + const stderr = path.join(dir, "stderr.log"); + const out = fs.openSync(trace, "w"); + const err = fs.openSync(stderr, "w"); + console.log(`Sampling ${scenario.id} in ${root}`); + let result; + try { + result = spawnSync("codex", ["exec", "--json", "--sandbox", "workspace-write", "-C", root, scenario.prompt], { + stdio: ["ignore", out, err], timeout, + }); + } finally { fs.closeSync(out); fs.closeSync(err); } + const report = { caseId: scenario.id, commandExitCode: result.status, signal: result.signal, error: result.error?.message || null, + cliVersion: cliVersion.stdout.trim(), fixture: root, prompt: scenario.prompt, behavioralReview: "NOT_RUN" }; + try { + const hostRoot = process.env.CODEX_HOME || path.join(os.homedir(), ".codex"); + report.loading = analyze({ trace, caseId: scenario.id, roots: [hostRoot, root] }); + } catch (error) { report.analysisError = error.message; } + write(dir, "report.json", JSON.stringify(report, null, 2) + "\n"); + summary.push(report); + write(output, "summary.json", JSON.stringify(summary, null, 2) + "\n"); + console.log(`${scenario.id}: exit=${result.status} ${report.loading?.assessment || report.analysisError}`); + // Preserve incomplete evidence and stop on a failed model invocation; no quota-burning retry loop. + if (result.error || result.signal || result.status !== 0) { process.exitCode = 1; break; } + } +} +try { main(); } catch (error) { console.error(error.message); process.exitCode = 2; } diff --git a/scripts/diagnostics/sample-loading.test.js b/scripts/diagnostics/sample-loading.test.js new file mode 100644 index 0000000..4b3599e --- /dev/null +++ b/scripts/diagnostics/sample-loading.test.js @@ -0,0 +1,26 @@ +"use strict"; +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const { test } = require("node:test"); + +// A real external process boundary: emulate Codex handling SIGTERM with exit 0. +test("sampling stops after a timed-out process even when it exits zero", { skip: process.platform === "win32" }, (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "cew-sampler-timeout-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const fakeBin = path.join(root, "bin"); + const output = path.join(root, "results"); + fs.mkdirSync(fakeBin); + fs.writeFileSync(path.join(fakeBin, "codex"), '#!/usr/bin/env node\nif (process.argv.includes("--version")) { console.log("fake Codex"); process.exit(0); }\nprocess.on("SIGTERM", () => process.exit(0));\nsetInterval(() => {}, 100);\n', { mode: 0o755 }); + const result = spawnSync(process.execPath, [path.join(__dirname, "sample-loading.js"), "--output", output, "--timeout-ms", "1000"], { + encoding: "utf8", env: { ...process.env, PATH: fakeBin + path.delimiter + process.env.PATH }, timeout: 10000, + }); + assert.equal(result.status, 1, result.stderr); + const summary = JSON.parse(fs.readFileSync(path.join(output, "summary.json"), "utf8")); + assert.equal(summary.length, 1); + assert.match(summary[0].error, /ETIMEDOUT/); + assert.equal(summary[0].commandExitCode, 0); + t.after(() => fs.rmSync(summary[0].fixture, { recursive: true, force: true })); +}); diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 73d9ac7..2736ba2 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -201,7 +201,13 @@ function Remove-PackageOnlyPaths { "scripts\verify\skill-invocation.test.js", "scripts\verify\continuation-checks.test.js", "scripts\verify\install-rules.test.js", - "scripts\verify\plugin-distribution.test.js" + "scripts\verify\plugin-distribution.test.js", + "scripts\diagnostics\capabilities.js", + "scripts\diagnostics\loading.js", + "scripts\diagnostics\sample-loading.js", + "scripts\diagnostics\capabilities.test.js", + "scripts\diagnostics\loading.test.js", + "scripts\diagnostics\sample-loading.test.js" ) foreach ($relative in $packageOnlyFiles) { diff --git a/scripts/install.sh b/scripts/install.sh index 3988088..c1cc579 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -170,7 +170,13 @@ remove_package_only_paths() { "scripts/verify/skill-invocation.test.js" \ "scripts/verify/continuation-checks.test.js" \ "scripts/verify/install-rules.test.js" \ - "scripts/verify/plugin-distribution.test.js" + "scripts/verify/plugin-distribution.test.js" \ + "scripts/diagnostics/capabilities.js" \ + "scripts/diagnostics/loading.js" \ + "scripts/diagnostics/sample-loading.js" \ + "scripts/diagnostics/capabilities.test.js" \ + "scripts/diagnostics/loading.test.js" \ + "scripts/diagnostics/sample-loading.test.js" do if [ -f "$dest/$file" ]; then run rm -f "$dest/$file" diff --git a/scripts/verify/install-rules.test.js b/scripts/verify/install-rules.test.js index 74f7755..448f51b 100644 --- a/scripts/verify/install-rules.test.js +++ b/scripts/verify/install-rules.test.js @@ -251,6 +251,10 @@ function checkInstallerEntrypoint() { "安装不应保留仅供包内验证的续接测试"); assert.equal(fs.existsSync(path.join(profile, host, "scripts/verify/plugin-distribution.test.js")), false, "安装不应保留仅供包内验证的插件分发测试"); + for (const file of ["capabilities.js", "loading.js", "sample-loading.js", "capabilities.test.js", "loading.test.js", "sample-loading.test.js"]) { + assert.equal(fs.existsSync(path.join(profile, host, "scripts/diagnostics", file)), false, + "诊断 CLI 及采样属于分发包，不应复制到用户运行目录"); + } assert.match(read(path.join(profile, host, "skills/handoff/agents/openai.yaml")), /allow_implicit_invocation: true/); } diff --git a/scripts/verify/metadata-checks.js b/scripts/verify/metadata-checks.js index f8e29f8..bcff7ce 100644 --- a/scripts/verify/metadata-checks.js +++ b/scripts/verify/metadata-checks.js @@ -32,6 +32,12 @@ const PACKAGE_ONLY_PATHS = [ "scripts/verify/continuation-checks.test.js", "scripts/verify/install-rules.test.js", "scripts/verify/plugin-distribution.test.js", + "scripts/diagnostics/capabilities.js", + "scripts/diagnostics/loading.js", + "scripts/diagnostics/sample-loading.js", + "scripts/diagnostics/capabilities.test.js", + "scripts/diagnostics/loading.test.js", + "scripts/diagnostics/sample-loading.test.js", ]; function bindContext(context) { From 5ee38f1d8cc8be18dfd2a52c5cd379db6ece6664 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 09:40:24 +0800 Subject: [PATCH 2/3] fix(diagnostics): resolve project skills and personal references --- docs/capability-diagnostics.md | 4 ++-- scripts/diagnostics/capabilities.js | 15 ++++++++---- scripts/diagnostics/capabilities.test.js | 30 ++++++++++++++++++++++++ scripts/diagnostics/loading.js | 2 +- scripts/diagnostics/loading.test.js | 20 ++++++++++++++++ 5 files changed, 63 insertions(+), 8 deletions(-) diff --git a/docs/capability-diagnostics.md b/docs/capability-diagnostics.md index b426aed..3d8fa1d 100644 --- a/docs/capability-diagnostics.md +++ b/docs/capability-diagnostics.md @@ -29,7 +29,7 @@ node bin/claude-everything-workflow.js doctor --host codex --project /path/to/pr 相同物理文件的软链接别名会去重。规则路径沿用 `AGENTS.md`：项目优先，再用户目录；Claude 专项规则先保留个人 `rules/common`，再回退 `references/rules/common`。插件包里的规则不自动充当用户规则回退。 -引用检查覆盖正文中具体的 Markdown 链接、代码格式文件路径和路由代码块内的文件路径；合并对照正文与实际正文里的引用，避免旧安装漏掉新依赖。它包含条件分支的潜在直接引用，不递归推断整个执行图，不解析脚本依赖、自然语言里的动态路径、系统工具、账号、权限或生产环境。因此 `STATIC_READY` 不是“已执行通过”，也不能保证每个条件分支可运行。差异只报告，不覆盖。 +引用检查覆盖正文中具体的 Markdown 链接、代码格式文件路径和路由代码块内的文件路径；合并对照正文与实际正文里的引用，避免旧安装漏掉新依赖。它包含条件分支的潜在直接引用，不递归推断整个执行图，不解析脚本依赖、自然语言里的动态路径、系统工具、账号、权限或生产环境。因此 `STATIC_READY` 不是“已执行通过”，也不能保证每个条件分支可运行。实际正文的相对引用会结合候选技能目录解析；存在但没有分发对照的个人文件报告为 `NO_REFERENCE` / `UNVERIFIED`。分发正文的引用仍独立核对，避免同名根目录文件掩盖技能内的缺失依赖。差异只报告，不覆盖。 退出码：`0` 无发现项，`1` 有差异、缺失或重复候选，`2` 参数或诊断本身出错。缺少一个宿主的安装也会报告；只使用一个宿主时传 `--host`。 @@ -44,7 +44,7 @@ node bin/claude-everything-workflow.js loading --trace /private/local/trace.json 实际操作应在相应的隔离项目运行第一条命令。额外的 `--root DIR` 将实际安装或项目里的规则、技能及参考文件加入比对，可重复使用；默认仅对照当前包。读取有个人修改的安装时传入其宿主根目录，才能匹配那份正文。 -报告只从 Codex CLI 的 `item.completed / command_execution / aggregated_output` 中匹配完整正文，输出逻辑路径、候选物理路径、内容指纹、读取次数、累计及重复字符量。它不从命令字符串或文件名出现推测“已读”。CRLF/LF 归一后匹配；一个完整正文在输出里出现两次，计为两次。相同内容的候选副本不重复计费，也不能由此判断宿主选择。 +报告只从 Codex CLI 的 `item.completed / command_execution / aggregated_output` 中匹配完整正文，输出逻辑路径、候选物理路径、内容指纹、读取次数、累计及重复字符量。项目 `.agents/skills/`、`.claude/skills/` 统一为 `skills/` 逻辑路径，再应用用例的禁止读取前缀；`candidatePaths` 保留实际位置。它不从命令字符串或文件名出现推测“已读”。CRLF/LF 归一后匹配；一个完整正文在输出里出现两次，计为两次。相同内容的候选副本不重复计费，也不能由此判断宿主选择。 **观测边界：** 宿主自动注入、部分读取、加行号或其他变形输出、其他工具类型及上下文压缩不在统计内。这是完整文件工具输出的下界，不是完整会话上下文账单。`actualFileTokens` 留空，字符数除以四只是粗略估算；宿主 usage 单独记录整个 turn 的用量，不归因到某份文件，也不据此声称节省量。 diff --git a/scripts/diagnostics/capabilities.js b/scripts/diagnostics/capabilities.js index 63f4f8b..ee36bb6 100644 --- a/scripts/diagnostics/capabilities.js +++ b/scripts/diagnostics/capabilities.js @@ -81,9 +81,9 @@ function pluginRoots(searchRoot, host, issues) { return found; } -// Only concrete, distributed paths are dependencies. Project artifacts and external URLs are not. +// Resolve concrete local references in the distribution and actual candidate without executing them. // A missing new relative Markdown link in a personal skill is still reported, never silently accepted. -function references(body, relative) { +function references(body, relative, root = source) { const found = new Set(); const local = new Set(); const add = (target, link = false) => { @@ -91,8 +91,8 @@ function references(body, relative) { target = target.split("#")[0]; let resolved; const sibling = path.posix.normalize(path.posix.join(path.posix.dirname(relative), target)); - if (isFile(path.join(source, sibling))) resolved = sibling; - else if (isFile(path.join(source, target))) resolved = target; + if (isFile(path.join(root, sibling)) || isFile(path.join(source, sibling))) resolved = sibling; + else if (isFile(path.join(root, target)) || isFile(path.join(source, target))) resolved = target; else if (/^(?:skills|rules|agents|commands|scripts|hooks|harness)\//.test(target)) resolved = path.posix.normalize(target); else if (link || target.startsWith("references/")) resolved = sibling; if (!resolved) return; @@ -136,7 +136,12 @@ function candidate(skill, root, hostRoot, project, host) { const actual = read(file); const metadata = skillMetadata(actual.body, skill.name); const expected = read(path.join(source, skill.path)); - const refs = references((expected.body || "") + "\n" + (actual.body || ""), skill.path); + const expectedRefs = references(expected.body || "", skill.path); + const actualRefs = references(actual.body || "", skill.path, root.path); + const refs = { + paths: [...new Set([...expectedRefs.paths, ...actualRefs.paths])].sort(), + unresolved: [...new Set([...expectedRefs.unresolved, ...actualRefs.unresolved])], + }; const dependencies = refs.paths.map((relative) => { const target = dependencyPath(relative, root.path, hostRoot, project, host); return { reference: relative, path: target, ...comparison(path.join(source, relative), target) }; diff --git a/scripts/diagnostics/capabilities.test.js b/scripts/diagnostics/capabilities.test.js index d81d314..9c6a9be 100644 --- a/scripts/diagnostics/capabilities.test.js +++ b/scripts/diagnostics/capabilities.test.js @@ -138,3 +138,33 @@ test("router references in fenced code participate in readiness checks", (t) => assert.equal(candidate.readiness, "BLOCKED"); assert.ok(candidate.dependencies.some((dep) => dep.reference === "agents/harness-optimizer.md" && dep.status === "MISSING")); }); + +test("personal scripts resolve from the installed skill directory without a distribution counterpart", (t) => { + const { installed, run } = fixture(t); + const skill = path.join(installed, "skills/visual-companion"); + const entry = path.join(skill, "SKILL.md"); + const script = path.join(skill, "scripts/personal-check.js"); + const body = '"use strict";\n// Private read-only helper.\n'; + fs.appendFileSync(entry, '\nRead `scripts/personal-check.js` before use.\n'); + fs.writeFileSync(script, body); + const candidate = JSON.parse(run().stdout).hosts[0].skills.find((s) => s.name === "visual-companion").candidates[0]; + assert.equal(candidate.readiness, "UNVERIFIED"); + const dependency = candidate.dependencies.find((d) => d.reference.endsWith("personal-check.js")); + assert.equal(dependency.reference, "skills/visual-companion/scripts/personal-check.js"); + assert.equal(dependency.path, script); + assert.equal(dependency.status, "NO_REFERENCE"); + assert.equal(dependency.expectedHash, null); + assert.match(dependency.actualHash, /^[a-f0-9]{64}$/); + assert.equal(fs.readFileSync(script, "utf8"), body); +}); + +test("missing distributed sibling references stay blocked even if a root file has the same name", (t) => { + const { installed, run } = fixture(t); + const sibling = path.join(installed, "skills/visual-companion/scripts/start-server.sh"); + fs.unlinkSync(sibling); + fs.mkdirSync(path.join(installed, "scripts")); + fs.writeFileSync(path.join(installed, "scripts/start-server.sh"), "root helper, not the skill's server\n"); + const candidate = JSON.parse(run().stdout).hosts[0].skills.find((s) => s.name === "visual-companion").candidates[0]; + assert.equal(candidate.readiness, "BLOCKED"); + assert.ok(candidate.dependencies.some((d) => d.path === sibling && d.status === "MISSING")); +}); diff --git a/scripts/diagnostics/loading.js b/scripts/diagnostics/loading.js index 0f7b062..ae000c6 100644 --- a/scripts/diagnostics/loading.js +++ b/scripts/diagnostics/loading.js @@ -19,7 +19,7 @@ function inventory(roots) { } else if (stat.isFile() && relative.endsWith(".md")) { const body = normalize(fs.readFileSync(file, "utf8")); if (!body.trim()) return; - const logical = relative.split(path.sep).join("/"); + const logical = relative.split(path.sep).join("/").replace(/^\.(?:agents|claude)\/skills\//, "skills/"); const digest = hash(body); const key = digest; const item = files.get(key) || { path: logical, sha256: digest, body, aliases: [], candidatePaths: [], reads: 0 }; diff --git a/scripts/diagnostics/loading.test.js b/scripts/diagnostics/loading.test.js index ebde140..5cdf6d7 100644 --- a/scripts/diagnostics/loading.test.js +++ b/scripts/diagnostics/loading.test.js @@ -74,3 +74,23 @@ test("a longer installed variant does not count its reference-body substring a s assert.equal(report.summary.reads, 1); assert.equal(report.summary.observedCharacters, personal.length); }); + +for (const directory of [".agents/skills", ".claude/skills"]) { + test(`project skills in ${directory} use logical paths for forbidden reads`, (t) => { + const { trace } = run(t, [], "question"); + const project = path.join(path.dirname(trace), "project"); + const file = path.join(project, directory, "personal/SKILL.md"); + const body = `# Project-only skill\nUnique personal instructions from ${directory}.\n`; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, body); + fs.writeFileSync(trace, [output(body), { type: "turn.completed" }].map(JSON.stringify).join("\n")); + const result = spawnSync(process.execPath, [cli, "loading", "--trace", trace, "--case", "question", "--root", project, "--json"], { encoding: "utf8" }); + const report = JSON.parse(result.stdout); + assert.equal(result.status, 1); + assert.equal(report.assessment, "VIOLATION_OBSERVED"); + assert.deepEqual(report.violations, [{ path: "skills/personal/SKILL.md", reads: 1 }]); + assert.equal(report.files[0].path, "skills/personal/SKILL.md"); + assert.deepEqual(report.files[0].candidatePaths, [file]); + assert.equal(report.summary.reads, 1); + }); +} From 24d1c73c28a7d81fc4b3aff77a07b9141c4345ae Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 2 Nov 2026 09:47:16 +0800 Subject: [PATCH 3/3] fix(diagnostics): include shared user skills in sampling --- docs/capability-diagnostics.md | 2 +- scripts/diagnostics/sample-loading.js | 2 +- scripts/diagnostics/sample-loading.test.js | 30 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/docs/capability-diagnostics.md b/docs/capability-diagnostics.md index 3d8fa1d..5dc76d4 100644 --- a/docs/capability-diagnostics.md +++ b/docs/capability-diagnostics.md @@ -60,7 +60,7 @@ node scripts/diagnostics/sample-loading.js --output docs/superpowers/evidence/lo node scripts/diagnostics/sample-loading.js --output docs/superpowers/evidence/loading-run-2 --case small-fix ``` -这会消耗当前 Codex CLI 账号的模型额度，默认顺序启动五个独立会话，沿用当前宿主配置和技能；不会被 `verify`、诊断、测试或 CI 自动调用。它创建本次所属的临时 Git 项目，保留轨迹、错误、结果和工作成果用于复核。每次调用最多三分钟，可用 `--timeout-ms 1000..180000` 下调；模型调用失败即停止，不自动循环重试。输出目录必须尚不存在，避免覆盖历史证据。 +这会消耗当前 Codex CLI 账号的模型额度，默认顺序启动五个独立会话，沿用当前宿主配置和技能，比对清单包含宿主根目录、用户共享的 `~/.agents/skills` 及临时项目；不会被 `verify`、诊断、测试或 CI 自动调用。它创建本次所属的临时 Git 项目，保留轨迹、错误、结果和工作成果用于复核。每次调用最多三分钟，可用 `--timeout-ms 1000..180000` 下调；模型调用失败即停止，不自动循环重试。输出目录必须尚不存在，避免覆盖历史证据。 采样项目只包含 Node 标准库、一个公开函数及测试；执行验收用例有现成入口，建立验收用例没有。为使完整读取可观测，项目说明要求用不截断的 `cat` 读取材料，每次最多三份，但不要求加载任何额外文件。这是受控采样，不能代表所有真实项目或桌面 App。自动报告的行为审核初始为 `NOT_RUN`，复核后另行记录，不把模型自报完成当作通过。 diff --git a/scripts/diagnostics/sample-loading.js b/scripts/diagnostics/sample-loading.js index 822f9b5..f3c99bd 100644 --- a/scripts/diagnostics/sample-loading.js +++ b/scripts/diagnostics/sample-loading.js @@ -75,7 +75,7 @@ function main() { cliVersion: cliVersion.stdout.trim(), fixture: root, prompt: scenario.prompt, behavioralReview: "NOT_RUN" }; try { const hostRoot = process.env.CODEX_HOME || path.join(os.homedir(), ".codex"); - report.loading = analyze({ trace, caseId: scenario.id, roots: [hostRoot, root] }); + report.loading = analyze({ trace, caseId: scenario.id, roots: [hostRoot, path.join(os.homedir(), ".agents"), root] }); } catch (error) { report.analysisError = error.message; } write(dir, "report.json", JSON.stringify(report, null, 2) + "\n"); summary.push(report); diff --git a/scripts/diagnostics/sample-loading.test.js b/scripts/diagnostics/sample-loading.test.js index 4b3599e..b09c17e 100644 --- a/scripts/diagnostics/sample-loading.test.js +++ b/scripts/diagnostics/sample-loading.test.js @@ -24,3 +24,33 @@ test("sampling stops after a timed-out process even when it exits zero", { skip: assert.equal(summary[0].commandExitCode, 0); t.after(() => fs.rmSync(summary[0].fixture, { recursive: true, force: true })); }); + +test("sampling includes shared user skills alongside a custom Codex home", { skip: process.platform === "win32" }, (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "cew-sampler-shared-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const fakeBin = path.join(root, "bin"); + const output = path.join(root, "results"); + const fixtureHome = path.join(root, "user-home"); + const file = path.join(fixtureHome, ".agents/skills/personal/SKILL.md"); + const body = "# Shared personal skill\nUnique shared user instructions observed in a complete tool output.\n"; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, body); + fs.mkdirSync(fakeBin); + const events = [ + { type: "item.completed", item: { type: "command_execution", command: `cat ${file}`, aggregated_output: body, exit_code: 0 } }, + { type: "turn.completed" }, + ]; + fs.writeFileSync(path.join(fakeBin, "codex"), '#!/usr/bin/env node\nif (process.argv.includes("--version")) { console.log("fake Codex"); process.exit(0); }\n' + + 'const events = ' + JSON.stringify(events) + ';\nfor (const event of events) console.log(JSON.stringify(event));\n', { mode: 0o755 }); + const result = spawnSync(process.execPath, [path.join(__dirname, "sample-loading.js"), "--output", output, "--case", "question"], { + encoding: "utf8", timeout: 10000, + env: { ...process.env, HOME: fixtureHome, CODEX_HOME: path.join(root, "custom-codex"), PATH: fakeBin + path.delimiter + process.env.PATH }, + }); + assert.equal(result.status, 0, result.stderr); + const [report] = JSON.parse(fs.readFileSync(path.join(output, "summary.json"), "utf8")); + t.after(() => fs.rmSync(report.fixture, { recursive: true, force: true })); + assert.equal(report.loading.summary.reads, 1); + assert.equal(report.loading.assessment, "VIOLATION_OBSERVED"); + assert.equal(report.loading.files[0].path, "skills/personal/SKILL.md"); + assert.deepEqual(report.loading.files[0].candidatePaths, [file]); +});