diff --git a/miles/utils/audit_utils/checksum_utils.py b/miles/utils/audit_utils/checksum_utils.py index 64a5f1e1f0..6b6fe8d8d1 100644 --- a/miles/utils/audit_utils/checksum_utils.py +++ b/miles/utils/audit_utils/checksum_utils.py @@ -14,7 +14,7 @@ def flatten_inference_engine_checksums(check_weights_result: Any) -> list[Infere def _flatten_to_inference_engine_bodies(check_weights_result: Any) -> list[Any]: - return [engine_body for server in check_weights_result for server_group in server for engine_body in server_group] + return [engine_body for server_group in check_weights_result for engine_body in server_group] def _merge_inference_engine_ranks(engine_body: dict[str, Any]) -> InferenceEngineChecksums: @@ -24,11 +24,18 @@ def _merge_inference_engine_ranks(engine_body: dict[str, Any]) -> InferenceEngin ranks: list[dict[str, Any]] = engine_body.get("ranks", []) or [] assert ranks, f"check_weights engine body has no ranks: {engine_body!r}" - ranks_sorted = sorted(ranks, key=lambda r: r["parallelism_info"]["rank"]) + ranks_sorted = sorted(ranks, key=_gpu_rank) merged: InferenceEngineChecksums = {} for rank_info in ranks_sorted: - rank = rank_info["parallelism_info"]["rank"] + rank = _gpu_rank(rank_info) for name, value in rank_info["checksums"].items(): merged[f"rank{rank}/{name}"] = value return merged + + +def _gpu_rank(rank_info: dict[str, Any]) -> int: + parallelism_info = rank_info["parallelism_info"] + gpu_ranks = {role_info["rank"] for role_info in parallelism_info} + assert len(gpu_ranks) == 1, f"expected one GPU rank across roles, got {gpu_ranks}: {rank_info!r}" + return next(iter(gpu_ranks)) diff --git a/tests/e2e/ft/README.md b/tests/e2e/ft/README.md index 20d637284d..9d0413a5d0 100644 --- a/tests/e2e/ft/README.md +++ b/tests/e2e/ft/README.md @@ -3,7 +3,7 @@ ## Layout - Scenario logic lives in `conftest_ft/scenario_.py`. -- CI runs it via thin per-mode entry files `test_trainer_ft__.py`, each registered with `register_cuda_ci(est_time=..., suite="stage-c-8-gpu-h100", labels=["ft-short"])` (comparison scenarios) or `labels=["ft-long"]` (soak scenarios). +- CI runs it via thin per-mode entry files `test_trainer_ft__.py`, each registered with `register_cuda_ci(est_time=..., suite="stage-c-8-gpu-h200", labels=["ft-short"])` (comparison scenarios) or `labels=["ft-long"]` (soak scenarios). - The CUDA CI runner executes each entry as bare `python3 ` (exit code = pass/fail); the entry just calls the scenario's `run_ci(mode)`. | Scenario (`conftest_ft/scenario_*.py`) | Type | What it verifies | @@ -36,7 +36,7 @@ ### In CI -- Gated on the `run-ci-ft-short` / `run-ci-ft-long` PR labels (FT is expensive — not run on every PR). `ft-short` covers the comparison scenarios (no_failure / deterministic / with_failure, minutes each); `ft-long` covers the soak scenarios (random-crash survival, realistic-gsm8k convergence — tens of minutes to hours). With a label set, the matching entries run on `stage-c-8-gpu-h100`. +- Gated on the `run-ci-ft-short` / `run-ci-ft-long` PR labels (FT is expensive — not run on every PR). `ft-short` covers the comparison scenarios (no_failure / deterministic / with_failure, minutes each); `ft-long` covers the soak scenarios (random-crash survival, realistic-gsm8k convergence — tens of minutes to hours). With a label set, the matching entries run on `stage-c-8-gpu-h200`. - Add a `(scenario, mode)` to CI: copy an entry file, change `run_ci(...)`'s mode. - Add a new label: edit `tests/ci/labels.py` and create the matching `run-ci-