Skip to content

fix(smoke-tests): don't let a log-fetch timeout mask the real failure #129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
spal1 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(smoke-tests): don't let a log-fetch timeout mask the real failure #129

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 32 additions & 9 deletions bin/test_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@
import requests

POLL_INTERVAL = 15 # seconds between status checks
# Log dumps are best-effort diagnostics, so they get a tighter budget than the
# operations the test depends on.
LOG_FETCH_TIMEOUT = 120 # seconds
TIMEOUT_RETURNCODE = 124 # conventional exit code for a timed-out command
TERMINAL_STATUSES = {
"TRAINING_JOB_COMPLETED",
"TRAINING_JOB_DEPLOY_FAILED",
Expand Down Expand Up @@ -97,6 +101,15 @@ def request_with_retry(method, url, max_retries=3, **kwargs):
# ---------------------------------------------------------------------------


def _echo_cli_output(stdout: str | None, stderr: str | None) -> None:
if stdout:
for line in stdout.rstrip().split("\n"):
print(f" {line}")
if stderr:
for line in stderr.rstrip().split("\n"):
print(f" [stderr] {line}", file=sys.stderr)


def run_truss_cli(
args: list[str], check: bool = True, timeout: int = 300
) -> subprocess.CompletedProcess:
Expand All @@ -106,15 +119,23 @@ def run_truss_cli(
"""
cmd = ["uv", "run", "truss"] + args + ["--non-interactive"]
print(f" $ {' '.join(cmd)}")
result = subprocess.run(
cmd, capture_output=True, text=True, timeout=timeout
)
if result.stdout:
for line in result.stdout.rstrip().split("\n"):
print(f" {line}")
if result.stderr:
for line in result.stderr.rstrip().split("\n"):
print(f" [stderr] {line}", file=sys.stderr)
try:
result = subprocess.run(
cmd, capture_output=True, text=True, timeout=timeout
)
except subprocess.TimeoutExpired as exc:
# A hung CLI call must not become the reported failure. Echo whatever it
# emitted and hand a synthetic result back so a check=False caller (a
# best-effort log dump) can carry on and report the real outcome.
_echo_cli_output(exc.stdout, exc.stderr)
message = f"CLI command timed out after {timeout}s: {' '.join(cmd)}"
if check:
raise RuntimeError(message) from exc
print(f" {message}")
return subprocess.CompletedProcess(
cmd, TIMEOUT_RETURNCODE, exc.stdout or "", exc.stderr or ""
)
_echo_cli_output(result.stdout, result.stderr)
if check and result.returncode != 0:
raise RuntimeError(
f"CLI command failed (exit {result.returncode}): {' '.join(cmd)}"
Expand Down Expand Up @@ -294,6 +315,7 @@ def poll_job_status(
run_truss_cli(
["train", "logs", "--job-id", job_id, "--remote", remote],
check=False,
timeout=LOG_FETCH_TIMEOUT,
)
# Stop the timed-out job via CLI
run_truss_cli(
Expand Down Expand Up @@ -322,6 +344,7 @@ def poll_job_status(
run_truss_cli(
["train", "logs", "--job-id", job_id, "--remote", remote],
check=False,
timeout=LOG_FETCH_TIMEOUT,
)
return status

Expand Down