From 043353d7d2a076663ffba92fb6fe9b509b0698fa Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Sat, 29 Aug 2026 16:35:15 +0900 Subject: [PATCH 1/2] fix(sandbox): use the verifier-setup budget for hardening execs --- src/benchflow/sandbox/lockdown.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/benchflow/sandbox/lockdown.py b/src/benchflow/sandbox/lockdown.py index 9fbfde900..d42800f45 100644 --- a/src/benchflow/sandbox/lockdown.py +++ b/src/benchflow/sandbox/lockdown.py @@ -709,14 +709,16 @@ async def _trusted_verifier_path( checks prove they are root-owned directories and not group/world writable. Runtime locations and sandbox-user writable locations stay excluded. """ - path_result = await env.exec("printenv PATH", user="root", timeout_sec=10) + path_result = await env.exec( + "printenv PATH", user="root", timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC + ) raw_path = path_result.stdout or "" if not raw_path.strip(): return _SAFE_VERIFIER_PATH cmd = _trusted_path_extras_cmd( raw_path, _blocked_verifier_path_prefixes(sandbox_user, workspace) ) - result = await env.exec(cmd, user="root", timeout_sec=10) + result = await env.exec(cmd, user="root", timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC) if _exec_return_code(result) != 0: logger.debug( "Trusted verifier PATH extras unavailable; using safe PATH.%s", @@ -748,14 +750,16 @@ async def _trusted_verifier_pythonpath( is chowned to root before verification. """ pp_result = await env.exec( - "printenv PYTHONPATH 2>/dev/null || true", user="root", timeout_sec=10 + "printenv PYTHONPATH 2>/dev/null || true", + user="root", + timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC, ) raw_pp = (pp_result.stdout or "").strip() if not raw_pp: return "" blocked = _blocked_verifier_pythonpath_prefixes(sandbox_user) cmd = _trusted_path_extras_cmd(raw_pp, blocked) - result = await env.exec(cmd, user="root", timeout_sec=10) + result = await env.exec(cmd, user="root", timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC) try: extras = _json.loads(result.stdout or "[]") except _json.JSONDecodeError: @@ -1052,7 +1056,7 @@ async def _kill_sandbox_user_procs(env, sandbox_user: str) -> None: await env.exec( f"pkill -u {sandbox_user} 2>/dev/null; " f"sleep 1; pkill -9 -u {sandbox_user} 2>/dev/null || true", - timeout_sec=10, + timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC, ) # Second pass: catch any processes that slipped through (e.g. cron/at jobs). await env.exec( From c56ded51c0b40c1f632fd3bcb3e8361ce8108122 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Tue, 1 Sep 2026 16:59:25 -0700 Subject: [PATCH 2/2] fix(sandbox): cover every verifier setup exec budget --- src/benchflow/sandbox/lockdown.py | 7 ++++-- tests/test_sandbox_hardening.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/benchflow/sandbox/lockdown.py b/src/benchflow/sandbox/lockdown.py index d42800f45..74eff863e 100644 --- a/src/benchflow/sandbox/lockdown.py +++ b/src/benchflow/sandbox/lockdown.py @@ -609,7 +609,7 @@ def add_plugin(name: str) -> None: result = await env.exec( f"python3 -c {shlex.quote(_DISCOVER_PYTEST_PLUGINS_SCRIPT)}", user="root", - timeout_sec=15, + timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC, ) if result.stderr: logger.debug(f"Plugin discovery stderr: {result.stderr.strip()}") @@ -684,7 +684,9 @@ async def _distro_pip_env(env) -> dict[str, str]: """ try: result = await env.exec( - "cat /etc/os-release 2>/dev/null || true", user="root", timeout_sec=5 + "cat /etc/os-release 2>/dev/null || true", + user="root", + timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC, ) except Exception as e: logger.warning("distro detection failed (%s); skipping pip env tweaks", e) @@ -1063,6 +1065,7 @@ async def _kill_sandbox_user_procs(env, sandbox_user: str) -> None: f"! pgrep -u {sandbox_user} > /dev/null 2>&1 || " f"(sleep 1 && pkill -9 -u {sandbox_user}; sleep 1)", user="root", + timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC, ) diff --git a/tests/test_sandbox_hardening.py b/tests/test_sandbox_hardening.py index cbabbc600..584800567 100644 --- a/tests/test_sandbox_hardening.py +++ b/tests/test_sandbox_hardening.py @@ -268,6 +268,43 @@ async def test_pr_942_tree_hardening_uses_shared_setup_budget(self): == VERIFIER_SETUP_TIMEOUT_SEC ) + @pytest.mark.asyncio + async def test_pr_1062_all_verifier_setup_probes_share_budget(self): + """Guards PR #1062 against short or unbounded Daytona setup execs.""" + from benchflow.sandbox.lockdown import ( + VERIFIER_SETUP_TIMEOUT_SEC, + build_reclaim_caches_cmd, + harden_before_verify, + ) + + def side_effect(command, **kwargs): + if command == "printenv PATH": + stdout = "/usr/local/bin:/usr/bin" + elif command.startswith("printenv PYTHONPATH"): + stdout = "/usr/local/lib/python3.12/site-packages" + elif command.startswith("python3 -c"): + stdout = "[]" + elif command.startswith("cat /etc/os-release"): + stdout = "ID=ubuntu\n" + else: + stdout = "" + return MagicMock(stdout=stdout, stderr="", exit_code=0) + + env = _make_env(side_effect) + await harden_before_verify( + env, _make_task(), sandbox_user="agent", workspace="/app" + ) + + reclaim_command = build_reclaim_caches_cmd("/app") + setup_calls = [ + call for call in env.exec.call_args_list if call.args[0] != reclaim_command + ] + assert len(setup_calls) >= 12 + assert all( + call.kwargs.get("timeout_sec") == VERIFIER_SETUP_TIMEOUT_SEC + for call in setup_calls + ) + @pytest.mark.asyncio async def test_pr_942_workspace_chown_failure_is_fatal(self): """Guards PR #942: failed ownership freezing cannot reach verification."""