diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1268b8a63c7..50096ea84dc 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -50,6 +50,8 @@ jobs: run: bash scripts/tests/test_telegram_route.sh - name: capability mode tests run: bash scripts/tests/test_skill_mode.sh + - name: vuln-scanner status contract tests + run: python3 scripts/tests/test_vuln_scanner_status.py - name: per-skill requires parse tests run: bash scripts/tests/test_skill_requires.sh - name: run actions summary tests diff --git a/eyebrowlock.json b/eyebrowlock.json index 1af2c9295c1..e39842e5af5 100644 --- a/eyebrowlock.json +++ b/eyebrowlock.json @@ -117,7 +117,7 @@ "files": [ { "path": "SKILL.md", - "hash": "b17491de609e1fa3b241a55012e60f21d63eaaac6ad9251c43509f40ec635872" + "hash": "1328de6f3288fa062a50af59e80ffd23e9013e1c498bbfb920b9541e413bb02d" } ], "findings": [ @@ -144,7 +144,7 @@ "severity": "medium", "owasp": "ASK-03", "file": "SKILL.md", - "line": 304, + "line": 328, "snippet": "- `exec` / `spawn` / `system` / `eval` sinks + subprocess with string interpolation (RCE)", "explanation": "uses a process-execution primitive" }, @@ -153,12 +153,12 @@ "severity": "critical", "owasp": "ASK-01", "file": "SKILL.md", - "line": 1013, + "line": 1057, "snippet": "1. **Install** — the binaries (`semgrep`, `trufflehog`, `osv-scanner`, `slither`) are **not pre-installed**. Stage the…", "explanation": "downloads and executes remote code via a pipe to a shell" } ], - "contentHash": "sha256-a81f0757d7b3bafe3baaa0c3d2c0e735eb284a7ebc9768264bdfe4e747b4aa9a", + "contentHash": "sha256-fd1fda98b06f22aa90c7434393c3cc605dd1c3c7527c5925dfdf5a454e24512a", "discoveredFrom": "skills/vuln-scanner/SKILL.md" }, { diff --git a/scripts/skill_mode.sh b/scripts/skill_mode.sh index 66daaff027a..dc304e8d874 100755 --- a/scripts/skill_mode.sh +++ b/scripts/skill_mode.sh @@ -75,6 +75,9 @@ WRITE_TOOLS="Write,Edit,Bash(gh:*),Bash(git:*),Bash(python3:*),Bash(python:*)" # a live-test showed the run logging that denial as "Blocked by sandbox". These are # read-only static-analysis tools (no repo/network mutation of their own). WRITE_TOOLS="$WRITE_TOOLS,Bash(semgrep:*),Bash(osv-scanner:*),Bash(trufflehog:*),Bash(slither:*)" +# Bounded scanner calls start with the wrapper, not the scanner's bare name. +# These can execute arbitrary commands, so grant them only in the write tier. +WRITE_TOOLS="$WRITE_TOOLS,Bash(timeout:*),Bash(gtimeout:*)" # cargo (vuln-scanner Arm A, step A3.5 — dynamic testing). Staged by # scripts/stage-vuln-scanner.sh (nightly toolchain + cargo-fuzz, workflow step, # same reason as Foundry below — the sandbox denies toolchain installs in-run). diff --git a/scripts/tests/test_skill_mode.sh b/scripts/tests/test_skill_mode.sh index 9b21a4db85c..18c3afc86fb 100755 --- a/scripts/tests/test_skill_mode.sh +++ b/scripts/tests/test_skill_mode.sh @@ -34,6 +34,17 @@ echo "$WT" | grep -q "Write" && echo "$WT" | grep -q "Edit" \ # allowed-tools: read-only tier drops mutation tools but keeps read+notify+curl RT=$(bash "$M" allowed-tools read-only) +# Timeout wrappers are command heads, not covered by Bash(trufflehog:*). +# Keep these general command runners out of the read-only tier. +for timer in timeout gtimeout; do + echo "$WT" | tr ',' '\n' | grep -qxF "Bash($timer:*)" \ + && pass "write tier includes $timer wrapper" || bad "write tier missing $timer wrapper" + if echo "$RT" | tr ',' '\n' | grep -qxF "Bash($timer:*)"; then + bad "read-only tier exposes $timer wrapper" + else + pass "read-only tier excludes $timer wrapper" + fi +done if echo "$RT" | grep -q "Write" || echo "$RT" | grep -q "Edit" \ || echo "$RT" | grep -q "Bash(git:\*)" || echo "$RT" | grep -q "Bash(gh:\*)"; then bad "read-only tier drops Write/Edit/git/gh" diff --git a/scripts/tests/test_vuln_scanner_status.py b/scripts/tests/test_vuln_scanner_status.py new file mode 100644 index 00000000000..36ad4774a64 --- /dev/null +++ b/scripts/tests/test_vuln_scanner_status.py @@ -0,0 +1,51 @@ +"""Contract checks, not a claim that a model followed the reporting instructions.""" +from pathlib import Path +import unittest +import subprocess +import tempfile + +SKILL = (Path(__file__).resolve().parents[2] / "skills/vuln-scanner/SKILL.md").read_text() + + +class ScannerStatusContract(unittest.TestCase): + def test_history_status_uses_exit_code_not_finding_count(self): + block = SKILL.split('if [ "${TRUFFLEHOG_GIT_RC:-1}"', 1)[1].split('\necho "osv=', 1)[0] + block = 'if [ "${TRUFFLEHOG_GIT_RC:-1}"' + block + with tempfile.TemporaryDirectory() as directory: + # Partial findings must not turn a later process failure into ok. + (Path(directory) / "trufflehog-git.json").write_text('{}\n') + for rc, expected in [(0, "ok"), (1, "fail"), (124, "timeout"), (137, "fail")]: + with self.subTest(rc=rc): + # Execute the actual status block with partial findings. + script = block.replace("/tmp/vuln-scan", directory) + result = subprocess.run(["bash", "-c", f"TRUFFLEHOG_GIT_RC={rc}\n{script}"], check=True) + self.assertEqual(result.returncode, 0) + rows = (Path(directory) / "sources.txt").read_text().splitlines() + self.assertEqual(rows[-1], f"trufflehog-git={expected}") + + def test_filesystem_clean_empty_stream_is_ok(self): + row = next(line for line in SKILL.splitlines() if line.startswith('echo "trufflehog=')) + with tempfile.TemporaryDirectory() as directory: + for rc, expected in [(0, "ok"), (1, "fail")]: + subprocess.run(["bash", "-c", f"TRUFFLEHOG_RC={rc}\n" + row.replace("/tmp/vuln-scan", directory)], check=True) + self.assertEqual((Path(directory) / "sources.txt").read_text().splitlines()[-1], f"trufflehog={expected}") + + def test_report_preserves_history_status(self): + report = SKILL.split("### A7. Write local report", 1)[1].split("### A8.", 1)[0] + self.assertIn("trufflehog-git", report) + self.assertIn("sources.txt", report) + + def test_both_notification_templates_preserve_history_status(self): + notify = SKILL.split("### A8. Notify", 1)[1].split("## Arm D", 1)[0] + rows = [line for line in notify.splitlines() if "Scanners:" in line] + self.assertEqual(len(rows), 2) + for row in rows: + self.assertIn("trufflehog-git=", row) + + def test_log_preserves_history_status(self): + log = SKILL.split("## Log", 1)[1].split("## Network note", 1)[0] + self.assertIn("trufflehog-git=ok|fail|timeout", log) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/vuln-scanner/SKILL.md b/skills/vuln-scanner/SKILL.md index c70ae6a52fb..3d19e50a813 100644 --- a/skills/vuln-scanner/SKILL.md +++ b/skills/vuln-scanner/SKILL.md @@ -140,11 +140,23 @@ fi # --- Secrets: TruffleHog (only-verified = actually authenticates) --- if command -v trufflehog >/dev/null 2>&1; then + TRUFFLEHOG_RC=0 trufflehog filesystem . --only-verified --json \ - > /tmp/vuln-scan/trufflehog.json 2>/dev/null || true - # Also scan full git history for secrets - trufflehog git file://. --only-verified --json \ - > /tmp/vuln-scan/trufflehog-git.json 2>/dev/null || true + > /tmp/vuln-scan/trufflehog.json 2>/dev/null || TRUFFLEHOG_RC=$? + # Also scan full git history for secrets — BOUNDED. An unbounded `trufflehog git` + # walks every commit's every tree, and a large packed history (measured: 200 + # commits / ~369MB on one real run) can eat the whole turn budget by itself, + # with nothing to show it happened until the run reports "success" anyway + # having produced no report at all. `timeout` turns that silent budget-burn + # into an ordinary, honestly-recorded `fail` — same as an install failure, + # never a reason to write a "still running, will resume" placeholder as the + # final output. There is no resume: a workflow_dispatch run is one shot, and + # a note promising to pick back up later is not truthful about what a single + # run can actually do. + TRUFFLEHOG_GIT_RC=0 + timeout 300 trufflehog git file://. --only-verified --json \ + > /tmp/vuln-scan/trufflehog-git.json 2>/dev/null || TRUFFLEHOG_GIT_RC=$? + [ "$TRUFFLEHOG_GIT_RC" = 124 ] && echo "VULN_SCANNER_TIMEOUT: trufflehog git history scan exceeded 300s on a large packed history — recorded as fail, not retried, not left unfinished" else echo "VULN_SCANNER_SKIPPED: trufflehog not available" fi @@ -180,7 +192,19 @@ fi # Record what succeeded (empty output ≠ clean, could be tool failure) echo "semgrep=$([ -s /tmp/vuln-scan/semgrep.json ] && echo ok || echo fail)" > /tmp/vuln-scan/sources.txt -echo "trufflehog=$([ -s /tmp/vuln-scan/trufflehog.json ] && echo ok || echo fail)" >> /tmp/vuln-scan/sources.txt +# TruffleHog JSON is finding-only: an exit-0 empty stream is a clean scan. +echo "trufflehog=$([ "${TRUFFLEHOG_RC:-1}" = 0 ] && echo ok || echo fail)" >> /tmp/vuln-scan/sources.txt +# Recorded separately from the filesystem pass above: they can genuinely diverge +# (filesystem scan clean and fast, git-history scan timed out on a large packed +# repo, or vice versa) and collapsing both into one trufflehog= line hides +# whichever one actually failed. +if [ "${TRUFFLEHOG_GIT_RC:-1}" = 124 ]; then + echo "trufflehog-git=timeout" >> /tmp/vuln-scan/sources.txt +elif [ "${TRUFFLEHOG_GIT_RC:-1}" = 0 ]; then + echo "trufflehog-git=ok" >> /tmp/vuln-scan/sources.txt +else + echo "trufflehog-git=fail" >> /tmp/vuln-scan/sources.txt +fi echo "osv=${OSV_STATUS:-fail}" >> /tmp/vuln-scan/sources.txt ``` @@ -605,7 +629,25 @@ Append to `memory/vuln-scanned.json` (create if missing) so future runs skip thi ### A7. Write local report -Save to `output/articles/vuln-scan-${today}.md` with sections for: repo metadata, scanner sources (ok/fail per tool), candidate count, confirmed findings with severity and channel, PoC gate status (`verified` with verifier/chain/block, `not-required` with reason, or `needs-verification`), and dedup note. Do **not** include exploit details for findings disclosed via PVR — redact file/line and link to the advisory ID instead. +**There is no resume.** The git-history pass has a timeout; this does not bound +every other scanner or installation step. If you are running low on turns, finish +the report with whatever scanners actually completed, record the rest `fail` in +`sources.txt` (§A3's rule: unfinished is `fail`, not a pending state), and write +A7/A8 now. A single `workflow_dispatch` run is one shot with no continuation — +writing "still running, will pick this up automatically" as the final output is +not true of this run path (it was live-observed: a run reported workflow +`success` having written that sentence instead of a report, with no ledger entry +at all — the operator had to notice and re-dispatch by hand). A shorter, honest +report with some scanners marked `fail` is a completed task; a promise to +resume is not. + +Save to `output/articles/vuln-scan-${today}.md` with sections for: repo metadata, scanner sources (ok/fail per tool — `trufflehog` and `trufflehog-git` are two separate rows, not one; folding a timed-out history scan into a clean filesystem-scan's `ok` is exactly the silent-masking this split exists to prevent), candidate count, confirmed findings with severity and channel, PoC gate status (`verified` with verifier/chain/block, `not-required` with reason, or `needs-verification`), and dedup note. Do **not** include exploit details for findings disclosed via PVR — redact file/line and link to the advisory ID instead. + +Copy each scanner status from `sources.txt` into the report, notification and log. +Keep `trufflehog` (filesystem) and `trufflehog-git` (history) separate, preserving +`timeout` exactly. Missing status is `fail`, never inferred `ok`. If any pass failed +or timed out, say "limited audit" and name the incomplete coverage, even when zero +findings were confirmed. Do not describe incomplete coverage as a clean audit. ### A8. Notify @@ -615,13 +657,15 @@ Use `./notify`. One paragraph. Lead with the verdict. *Vuln Scanner — * confirmed findings (). Disclosed via: -Scanners: semgrep=, trufflehog=, osv=, fuzz=. PoC gate: . +Scanners: semgrep=, trufflehog=, trufflehog-git=, osv=, fuzz=. PoC gate: . ``` -If the audit was clean: +`trufflehog-git=timeout` must always be spelled out here, never folded into a plain `trufflehog=ok` — a clean filesystem pass and a timed-out history pass are different facts, and this is the durable line an operator actually reads. Silently dropping the git-history state here reproduces the exact masking this field exists to prevent. + +If no findings were confirmed (choose clean or limited according to actual coverage): ``` *Vuln Scanner — * -Clean audit. candidates reviewed, 0 confirmed. Scanners: semgrep=ok, trufflehog=ok, osv=ok, fuzz=skip, agentic=ok. +. candidates reviewed, 0 confirmed. Scanners: semgrep=, trufflehog=, trufflehog-git=, osv=, fuzz=, agentic=. ``` Then log per the **Log** section below with `Mode: scan`. @@ -983,7 +1027,7 @@ specific bullets. - Candidates: N | Confirmed: M - Channels used: PVR (x), public PR (y), skipped (z) - Prior-art check: N candidates checked, 0 matches | matched #123 → skipped/commented -- Scanner status: semgrep=ok trufflehog=ok osv=ok fuzz=ok|fail|skip agentic=ok|skip poc=verified|not-required|needs-verification +- Scanner status: semgrep=ok|fail trufflehog=ok|fail trufflehog-git=ok|fail|timeout osv=ok|fail|none|skipped fuzz=ok|fail|skip agentic=ok|skip poc=verified|not-required|needs-verification - Advisory/PR links: [...] ```