From 4cb0877d93f1f0c8646e098212cf3750be6b34e9 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Fri, 10 Jul 2026 11:59:18 +0800 Subject: [PATCH 01/13] Re-enable ft-short FT e2e tests in CI Remove the disabled= markers from all 11 ft-short FT e2e tests (deterministic / no_failure / with_failure x pp2 / tp2_ep2 / dp4_cp2 + the three real_rollout variants). They were opted out on the 2026-07-08 dev image rebuild pending stabilization; this follow-up PR re-enables them and drives them green via the run-ci-ft-short label. --- tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py | 1 - .../e2e/ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py | 1 - tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py | 1 - tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py | 1 - tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py | 1 - tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py | 1 - tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py | 1 - tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py | 1 - tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py | 1 - .../test_trainer_ft_with_failure_dp2_cp2_real_rollout_dense.py | 1 - tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py | 1 - 11 files changed, 11 deletions(-) diff --git a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py index 198eac6daa..35b173de94 100644 --- a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py +++ b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py @@ -10,7 +10,6 @@ est_time=900, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="FT e2e suite disabled in CI pending stabilization on current image (follow-up PR)", ) _MODE: str = "dp2_cp2_pp2" diff --git a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py index ac8bf1c520..28bd8fab0e 100644 --- a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py +++ b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py @@ -10,7 +10,6 @@ est_time=1500, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="real_rollout deterministic-inference sglang hang on current image", ) _MODE: str = "dp2_cp2_real_rollout" diff --git a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py index e86fc26025..7d5e70327f 100644 --- a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py +++ b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py @@ -10,7 +10,6 @@ est_time=900, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="FT e2e suite disabled in CI pending stabilization on current image (follow-up PR)", ) _MODE: str = "dp2_cp2_tp2_ep2" diff --git a/tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py b/tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py index 339b83a9c7..06ca410553 100644 --- a/tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py +++ b/tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py @@ -10,7 +10,6 @@ est_time=900, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="FT e2e suite disabled in CI pending stabilization on current image (follow-up PR)", ) _MODE: str = "dp4_cp2" diff --git a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py index 9d3ccfcdb1..75d05b9794 100644 --- a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py +++ b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py @@ -10,7 +10,6 @@ est_time=600, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="FT e2e suite disabled in CI pending stabilization on current image (follow-up PR)", ) _MODE: str = "dp2_cp2_pp2" diff --git a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py index bf9787ed64..6a342cdc2d 100644 --- a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py +++ b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py @@ -10,7 +10,6 @@ est_time=1200, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="real_rollout deterministic-inference sglang hang on current image", ) _MODE: str = "dp2_cp2_real_rollout" diff --git a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py index 30a9b78f05..1e08985e17 100644 --- a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py +++ b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py @@ -10,7 +10,6 @@ est_time=600, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="FT e2e suite disabled in CI pending stabilization on current image (follow-up PR)", ) _MODE: str = "dp2_cp2_tp2_ep2" diff --git a/tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py b/tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py index 16222bf9d9..5d6ec03f73 100644 --- a/tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py +++ b/tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py @@ -10,7 +10,6 @@ est_time=600, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="FT e2e suite disabled in CI pending stabilization on current image (follow-up PR)", ) _MODE: str = "dp4_cp2" diff --git a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py index f3a0789aa6..c8463a3b0c 100644 --- a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py +++ b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py @@ -10,7 +10,6 @@ est_time=900, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="FT e2e suite disabled in CI pending stabilization on current image (follow-up PR)", ) _MODE: str = "dp2_cp2_pp2" diff --git a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_real_rollout_dense.py b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_real_rollout_dense.py index 68dc1fa2c6..3f340e5759 100644 --- a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_real_rollout_dense.py +++ b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_real_rollout_dense.py @@ -10,7 +10,6 @@ est_time=1500, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="real_rollout deterministic-inference sglang hang on current image", ) _MODE: str = "dp2_cp2_real_rollout_dense" diff --git a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py index b1f4dc4b27..5eb6755bd9 100644 --- a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py +++ b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py @@ -10,7 +10,6 @@ est_time=900, suite="stage-c-8-gpu-h100", labels=["ft-short"], - disabled="FT e2e suite disabled in CI pending stabilization on current image (follow-up PR)", ) _MODE: str = "dp2_cp2_tp2_ep2" From 6d5a122106f6da1ae0c97ea6671a711271972c86 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Fri, 10 Jul 2026 14:38:54 +0800 Subject: [PATCH 02/13] Fix inference-engine weight-checksum consumer Two mismatches on the real_rollout per-rollout weight-checksum path, both masked while the FT e2e tests were disabled in CI: 1. Nesting. RolloutManager.check_weights returns the single updatable server's result (server_group -> engine_body); #1329 established this (and test_check_weights_targets_only_updatable_model pins it). But flatten_inference_engine_checksums iterated one extra level, treating dict keys as engine bodies -> AttributeError: 'str' object has no attribute 'get'. Flatten the actual server_group -> engine_body shape. 2. Per-role parallelism_info. sglang's /weights_checker now returns each rank's parallelism_info as a per-role list ([{role: target, ...}, {role: draft, ...}]) for target + draft models, but the merge indexed it as a bare {rank: int} dict -> TypeError: list indices must be integers or slices, not str. Collapse the per-role list to the single shared GPU rank (asserting the roles agree). Update the checksum unit tests and the test_group mock to the real check_weights output and sglang schema, with regression coverage for the multi-role and rank-disagreement cases. --- miles/utils/audit_utils/checksum_utils.py | 13 +++++-- tests/fast/ray/train/test_group.py | 10 +++-- tests/fast/utils/test_checksum_utils.py | 47 ++++++++++++++++------- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/miles/utils/audit_utils/checksum_utils.py b/miles/utils/audit_utils/checksum_utils.py index 64a5f1e1f0..6b6fe8d8d1 100644 --- a/miles/utils/audit_utils/checksum_utils.py +++ b/miles/utils/audit_utils/checksum_utils.py @@ -14,7 +14,7 @@ def flatten_inference_engine_checksums(check_weights_result: Any) -> list[Infere def _flatten_to_inference_engine_bodies(check_weights_result: Any) -> list[Any]: - return [engine_body for server in check_weights_result for server_group in server for engine_body in server_group] + return [engine_body for server_group in check_weights_result for engine_body in server_group] def _merge_inference_engine_ranks(engine_body: dict[str, Any]) -> InferenceEngineChecksums: @@ -24,11 +24,18 @@ def _merge_inference_engine_ranks(engine_body: dict[str, Any]) -> InferenceEngin ranks: list[dict[str, Any]] = engine_body.get("ranks", []) or [] assert ranks, f"check_weights engine body has no ranks: {engine_body!r}" - ranks_sorted = sorted(ranks, key=lambda r: r["parallelism_info"]["rank"]) + ranks_sorted = sorted(ranks, key=_gpu_rank) merged: InferenceEngineChecksums = {} for rank_info in ranks_sorted: - rank = rank_info["parallelism_info"]["rank"] + rank = _gpu_rank(rank_info) for name, value in rank_info["checksums"].items(): merged[f"rank{rank}/{name}"] = value return merged + + +def _gpu_rank(rank_info: dict[str, Any]) -> int: + parallelism_info = rank_info["parallelism_info"] + gpu_ranks = {role_info["rank"] for role_info in parallelism_info} + assert len(gpu_ranks) == 1, f"expected one GPU rank across roles, got {gpu_ranks}: {rank_info!r}" + return next(iter(gpu_ranks)) diff --git a/tests/fast/ray/train/test_group.py b/tests/fast/ray/train/test_group.py index f03a8d3243..3c80eff00f 100644 --- a/tests/fast/ray/train/test_group.py +++ b/tests/fast/ray/train/test_group.py @@ -984,12 +984,16 @@ def test_with_normal_and_error_cells(self): def _checksum_response(engine_checksums: list[dict[str, str]]) -> list: - """Build a nested servers->groups->engines check_weights('checksum') response.""" + """Build a nested groups->engines check_weights('checksum') response.""" engines = [ - {"success": True, "message": "ok", "ranks": [{"checksums": cs, "parallelism_info": {"rank": 0}}]} + { + "success": True, + "message": "ok", + "ranks": [{"checksums": cs, "parallelism_info": [{"role": "target", "rank": 0}]}], + } for cs in engine_checksums ] - return [[engines]] + return [engines] class TestMaybeLogInferenceEngineWeightChecksums: diff --git a/tests/fast/utils/test_checksum_utils.py b/tests/fast/utils/test_checksum_utils.py index 46d047c843..69f1e31931 100644 --- a/tests/fast/utils/test_checksum_utils.py +++ b/tests/fast/utils/test_checksum_utils.py @@ -15,53 +15,72 @@ def _engine_body(*, success: bool, ranks: list[dict[str, Any]] | None) -> dict[s def _rank(rank: int, checksums: dict[str, str]) -> dict[str, Any]: - return {"checksums": checksums, "parallelism_info": {"rank": rank}} + return {"checksums": checksums, "parallelism_info": [{"role": "target", "rank": rank}]} class TestFlattenInferenceEngineChecksums: - def test_single_server_group_engine_single_rank(self) -> None: - """One server/group/engine with one rank yields one prefixed checksum dict.""" - result = [[[_engine_body(success=True, ranks=[_rank(0, {"w": "aaa"})])]]] + def test_single_group_engine_single_rank(self) -> None: + """One server-group/engine with one rank yields one prefixed checksum dict.""" + result = [[_engine_body(success=True, ranks=[_rank(0, {"w": "aaa"})])]] assert flatten_inference_engine_checksums(result) == [{"rank0/w": "aaa"}] def test_multiple_engines_flattened_in_order(self) -> None: - """Engines across servers/groups are flattened into a single ordered list.""" + """Engines across server-groups are flattened into a single ordered list.""" result = [ - [[_engine_body(success=True, ranks=[_rank(0, {"w": "e0"})])]], - [[_engine_body(success=True, ranks=[_rank(0, {"w": "e1"})])]], + [_engine_body(success=True, ranks=[_rank(0, {"w": "e0"})])], + [_engine_body(success=True, ranks=[_rank(0, {"w": "e1"})])], ] assert flatten_inference_engine_checksums(result) == [{"rank0/w": "e0"}, {"rank0/w": "e1"}] def test_none_node_rank_payloads_filtered(self) -> None: """None engine bodies (non-zero node ranks) are dropped before indexing.""" - result = [[[_engine_body(success=True, ranks=[_rank(0, {"w": "e0"})]), None]]] + result = [[_engine_body(success=True, ranks=[_rank(0, {"w": "e0"})]), None]] assert flatten_inference_engine_checksums(result) == [{"rank0/w": "e0"}] def test_all_none_fails_loud(self) -> None: """A None-only result means the checksum action did nothing, so fail loud.""" - result = [[[None, None]]] + result = [[None, None]] with pytest.raises(AssertionError, match="no non-None engine bodies"): flatten_inference_engine_checksums(result) def test_multi_rank_merged_with_rank_prefix(self) -> None: """Multiple ranks of one engine merge into one dict, prefixed by rank to avoid clobber.""" - result = [[[_engine_body(success=True, ranks=[_rank(0, {"w": "r0"}), _rank(1, {"w": "r1"})])]]] + result = [[_engine_body(success=True, ranks=[_rank(0, {"w": "r0"}), _rank(1, {"w": "r1"})])]] assert flatten_inference_engine_checksums(result) == [{"rank0/w": "r0", "rank1/w": "r1"}] def test_ranks_out_of_order_sorted_by_parallelism_rank(self) -> None: """Ranks arriving out of order (zmq) are sorted by parallelism rank deterministically.""" - out_of_order = [[[_engine_body(success=True, ranks=[_rank(1, {"w": "r1"}), _rank(0, {"w": "r0"})])]]] - in_order = [[[_engine_body(success=True, ranks=[_rank(0, {"w": "r0"}), _rank(1, {"w": "r1"})])]]] + out_of_order = [[_engine_body(success=True, ranks=[_rank(1, {"w": "r1"}), _rank(0, {"w": "r0"})])]] + in_order = [[_engine_body(success=True, ranks=[_rank(0, {"w": "r0"}), _rank(1, {"w": "r1"})])]] assert flatten_inference_engine_checksums(out_of_order) == flatten_inference_engine_checksums(in_order) def test_engine_failure_fails_loud(self) -> None: """An engine reporting success=False fails loud rather than silently dropping it.""" - result = [[[_engine_body(success=False, ranks=[_rank(0, {"w": "aaa"})])]]] + result = [[_engine_body(success=False, ranks=[_rank(0, {"w": "aaa"})])]] with pytest.raises(AssertionError, match="reported failure"): flatten_inference_engine_checksums(result) def test_engine_without_ranks_fails_loud(self) -> None: """A success body with no ranks fails loud (nothing to compare).""" - result = [[[_engine_body(success=True, ranks=None)]]] + result = [[_engine_body(success=True, ranks=None)]] with pytest.raises(AssertionError, match="no ranks"): flatten_inference_engine_checksums(result) + + def test_multi_role_parallelism_info_collapses_to_one_gpu_rank(self) -> None: + """Per-role parallelism_info (target + draft) sharing a GPU rank uses that single rank.""" + rank_info = { + "checksums": {"w": "aaa"}, + "parallelism_info": [{"role": "target", "rank": 0}, {"role": "draft", "rank": 0}], + } + result = [[_engine_body(success=True, ranks=[rank_info])]] + assert flatten_inference_engine_checksums(result) == [{"rank0/w": "aaa"}] + + def test_parallelism_info_rank_disagreement_fails_loud(self) -> None: + """Roles reporting different GPU ranks for one engine rank is a contract violation.""" + rank_info = { + "checksums": {"w": "aaa"}, + "parallelism_info": [{"role": "target", "rank": 0}, {"role": "draft", "rank": 1}], + } + result = [[_engine_body(success=True, ranks=[rank_info])]] + with pytest.raises(AssertionError, match="one GPU rank"): + flatten_inference_engine_checksums(result) From 4349c4759c6e0798a3a2cac0455dd1ab230249ae Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Fri, 10 Jul 2026 18:17:42 +0800 Subject: [PATCH 03/13] Raise ft-short non-real-rollout est_time to 2400s The 06-13 phase_a/phase_b unification made phase_a run the full phase_b workload, roughly doubling these tests' cost; their est_time (600-900s, from the 06-12 measurements) now under-budgets the run so run_suite's per-file timeout (max(1800, est*1.25)) kills them mid dump-comparison. Raising est_time only lifts the timeout ceiling (the test still exits when done). --- tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py | 2 +- tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py | 2 +- tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py | 2 +- tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py | 2 +- tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py | 2 +- tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py | 2 +- tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py | 2 +- tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py index 35b173de94..2eda018fa0 100644 --- a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py +++ b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_pp2.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_deterministic import run_ci register_cuda_ci( - est_time=900, + est_time=2400, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) diff --git a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py index 7d5e70327f..d47f4da1b5 100644 --- a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py +++ b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_tp2_ep2.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_deterministic import run_ci register_cuda_ci( - est_time=900, + est_time=2400, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) diff --git a/tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py b/tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py index 06ca410553..4d316ba163 100644 --- a/tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py +++ b/tests/e2e/ft/test_trainer_ft_deterministic_dp4_cp2.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_deterministic import run_ci register_cuda_ci( - est_time=900, + est_time=2400, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) diff --git a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py index 75d05b9794..80c87e0803 100644 --- a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py +++ b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_pp2.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_no_failure import run_ci register_cuda_ci( - est_time=600, + est_time=2400, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) diff --git a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py index 1e08985e17..f4d6807858 100644 --- a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py +++ b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_no_failure import run_ci register_cuda_ci( - est_time=600, + est_time=2400, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) diff --git a/tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py b/tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py index 5d6ec03f73..344693e3bb 100644 --- a/tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py +++ b/tests/e2e/ft/test_trainer_ft_no_failure_dp4_cp2.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_no_failure import run_ci register_cuda_ci( - est_time=600, + est_time=2400, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) diff --git a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py index c8463a3b0c..24399b0f0d 100644 --- a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py +++ b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_pp2.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_with_failure import run_ci register_cuda_ci( - est_time=900, + est_time=2400, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) diff --git a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py index 5eb6755bd9..c6579be1d7 100644 --- a/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py +++ b/tests/e2e/ft/test_trainer_ft_with_failure_dp2_cp2_tp2_ep2.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_with_failure import run_ci register_cuda_ci( - est_time=900, + est_time=2400, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) From 068bc698b083d0d262bdebb14bc9fdab416a52ab Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Fri, 10 Jul 2026 18:25:58 +0800 Subject: [PATCH 04/13] Start the FT-test ray cluster with actor + rollout GPUs When MILES_SCRIPT_EXTERNAL_RAY is unset (CI), execute_train boots the ray head with --num-gpus , and run_training passed only mode.train_gpus_per_node (the actor GPUs). For the disaggregated real_rollout modes the sglang engines live on separate GPUs (actor 4 + rollout 4 = 8), so ray came up with just 4 GPUs and the 8-GPU placement group could never be scheduled (scheduling_state NO_RESOURCES) -> train.py hung ~30min at ray.get(pg.ready()) until the CI timeout. Non-real-rollout modes were fine because train_gpus_per_node already equals the node total. On rcli MILES_SCRIPT_EXTERNAL_RAY=1 uses an external 8-GPU ray cluster, which masked this. Pass mode.total_node_gpus (train + rollout), matching what scenario_realistic_gsm8k already does for its real-rollout phase. --- tests/e2e/ft/conftest_ft/execution.py | 2 +- tests/e2e/ft/conftest_ft/modes.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/e2e/ft/conftest_ft/execution.py b/tests/e2e/ft/conftest_ft/execution.py index c8fdb6df69..8b2cedf4d4 100644 --- a/tests/e2e/ft/conftest_ft/execution.py +++ b/tests/e2e/ft/conftest_ft/execution.py @@ -203,7 +203,7 @@ def run_training( } U.execute_train( train_args=train_args, - num_gpus_per_node=mode.train_gpus_per_node, + num_gpus_per_node=mode.total_node_gpus, megatron_model_type=mode.megatron_model_type, extra_env_vars=merged_env_vars, megatron_path=_MEGATRON_PATH, diff --git a/tests/e2e/ft/conftest_ft/modes.py b/tests/e2e/ft/conftest_ft/modes.py index 34baf9e761..07d7e0c2e8 100644 --- a/tests/e2e/ft/conftest_ft/modes.py +++ b/tests/e2e/ft/conftest_ft/modes.py @@ -40,6 +40,10 @@ def has_real_rollout(self) -> bool: def total_rollout_gpus(self) -> int: return self.rollout_num_engines * self.rollout_gpus_per_engine + @property + def total_node_gpus(self) -> int: + return self.train_gpus_per_node + self.total_rollout_gpus + MODES: dict[str, FTTestMode] = { # --- 1-node (8 GPUs) variants --- From 297efdecb4d7e2165f3980893ebc556ddc9e9f58 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Sat, 11 Jul 2026 00:17:08 +0800 Subject: [PATCH 05/13] Clean up FT e2e dumps after each test to avoid filling CI disk The comparison scenarios write dense per-param value/grad dumps (+ ckpt, rollout data, events) under /node_public/dumps/, which on CI lives on the runner's container-writable layer (not a mounted volume). run_pipeline never removed a test's dump root, so across a shard's tests they accumulated and eventually filled the runner disk -> 'No space left on device' mid-run. Remove the per-test dump root in a finally once its comparison is done (dumps are only needed within the test), so disk is reclaimed between tests. --- tests/e2e/ft/conftest_ft/app.py | 38 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/e2e/ft/conftest_ft/app.py b/tests/e2e/ft/conftest_ft/app.py index a25406c6e9..ef7aa358d0 100644 --- a/tests/e2e/ft/conftest_ft/app.py +++ b/tests/e2e/ft/conftest_ft/app.py @@ -1,6 +1,7 @@ # NOTE: You MUST read tests/e2e/ft/README.md as source-of-truth and documentations import os +import shutil from collections.abc import Callable from pathlib import Path from typing import Annotated @@ -44,23 +45,26 @@ def run_pipeline( prepare(ft_mode) - for phase in effective_phases: - baseline_dump = f"{dump_dir}/{_dump_subdir('baseline', phase)}" - run_training( - train_args=build_baseline_args(ft_mode, baseline_dump, enable_dumper), - mode=ft_mode, - dump_dir=baseline_dump, - ) - - target_dump = f"{dump_dir}/{_dump_subdir('target', phase)}" - run_training( - train_args=build_target_args(ft_mode, target_dump, enable_dumper), - mode=ft_mode, - dump_dir=target_dump, - ) - - if enable_dumper: - compare_fn(dump_dir, ft_mode) + try: + for phase in effective_phases: + baseline_dump = f"{dump_dir}/{_dump_subdir('baseline', phase)}" + run_training( + train_args=build_baseline_args(ft_mode, baseline_dump, enable_dumper), + mode=ft_mode, + dump_dir=baseline_dump, + ) + + target_dump = f"{dump_dir}/{_dump_subdir('target', phase)}" + run_training( + train_args=build_target_args(ft_mode, target_dump, enable_dumper), + mode=ft_mode, + dump_dir=target_dump, + ) + + if enable_dumper: + compare_fn(dump_dir, ft_mode) + finally: + shutil.rmtree(dump_dir, ignore_errors=True) def create_comparison_app_and_run_ci( From 10cfce39d038c2805e7dc5cf9d51a48c47c0558c Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Sat, 11 Jul 2026 07:09:31 +0800 Subject: [PATCH 06/13] Use expandable_segments for FT real_rollout actor to avoid H100 OOM The disaggregated real_rollout modes put the actor on only train_gpus_per_node GPUs (4) with no tensor/pipeline/expert sharding, so each actor GPU holds the full unsharded 30B model plus the deterministic dumper's value/grad tensors. On an 80GB H100 the true peak fits, but the caching allocator fragments (~11GiB reserved-but-unallocated against a 10.5GiB request) and spuriously OOMs midway through deterministic_dp2_cp2_real_rollout. Set PYTORCH_CUDA_ALLOC_CONF= expandable_segments:True for has_real_rollout modes so those reserved segments can be reused. Safe: no --offload-train and sglang enable_memory_saver is off, so it cannot collide with torch_memory_saver; determinism is unaffected. --- tests/e2e/ft/conftest_ft/execution.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/e2e/ft/conftest_ft/execution.py b/tests/e2e/ft/conftest_ft/execution.py index 8b2cedf4d4..2be71a6166 100644 --- a/tests/e2e/ft/conftest_ft/execution.py +++ b/tests/e2e/ft/conftest_ft/execution.py @@ -201,6 +201,15 @@ def run_training( "SGLANG_LOG_MS": "1", **(extra_env_vars or {}), } + if mode.has_real_rollout: + # Disaggregated real_rollout puts the actor on only train_gpus_per_node GPUs (4) with no + # tensor/pipeline/expert sharding, so each actor GPU holds the full unsharded model plus the + # deterministic dumper's value/grad tensors. On 80GB H100 the peak fits but the allocator + # fragments (observed ~11GiB reserved-but-unallocated vs a 10.5GiB request -> spurious OOM); + # expandable_segments coalesces that. Safe here: no --offload-train and enable_memory_saver + # is off, so it cannot collide with torch_memory_saver. Determinism is unaffected (both + # baseline and target use it, and it only changes virtual-address arena management). + merged_env_vars["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" U.execute_train( train_args=train_args, num_gpus_per_node=mode.total_node_gpus, From f4cd575eca29b3f2fd7f4f780c55dcb4c3d9f979 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Sat, 11 Jul 2026 07:09:31 +0800 Subject: [PATCH 07/13] Raise est_time for the 30B real_rollout FT tests deterministic_dp2_cp2_real_rollout runs the same 2-phase deterministic compare as deterministic_dp2_cp2_pp2 (which took ~2008s on H100) plus real sglang generation, but its est_time=1500 capped the per-file timeout at 1875s. Raise it to 3600 (timeout 4500s) and no_failure_dp2_cp2_real_rollout 1200->2400 so real rollout generation on H100 has headroom. est_time only lifts the timeout ceiling and rebalances sharding; it does not extend actual runtime. --- .../ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py | 2 +- tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py index 28bd8fab0e..7224a53291 100644 --- a/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py +++ b/tests/e2e/ft/test_trainer_ft_deterministic_dp2_cp2_real_rollout.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_deterministic import run_ci register_cuda_ci( - est_time=1500, + est_time=3600, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) diff --git a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py index 6a342cdc2d..09546ed8ef 100644 --- a/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py +++ b/tests/e2e/ft/test_trainer_ft_no_failure_dp2_cp2_real_rollout.py @@ -7,7 +7,7 @@ from tests.e2e.ft.conftest_ft.scenario_no_failure import run_ci register_cuda_ci( - est_time=1200, + est_time=2400, suite="stage-c-8-gpu-h100", labels=["ft-short"], ) From 731dc5a6916435f04f9826fbae566b83d626adb6 Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Sat, 11 Jul 2026 07:15:07 +0800 Subject: [PATCH 08/13] Do not disable sglang cuda graph in FT real_rollout tests Drop --sglang-disable-cuda-graph so the real_rollout tests exercise sglang with cuda graphs enabled, matching real usage. --- tests/e2e/ft/conftest_ft/execution.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/e2e/ft/conftest_ft/execution.py b/tests/e2e/ft/conftest_ft/execution.py index 2be71a6166..f8ecd4d888 100644 --- a/tests/e2e/ft/conftest_ft/execution.py +++ b/tests/e2e/ft/conftest_ft/execution.py @@ -104,7 +104,6 @@ def get_common_train_args( "--rollout-temperature 0.8 " "--rollout-batch-size 32 " "--n-samples-per-prompt 8 " - "--sglang-disable-cuda-graph " # Required for reproducibility (ref: https://github.com/THUDM/slime/pull/370) "--sglang-enable-deterministic-inference " "--sglang-attention-backend flashinfer " From d901760fb98b765d4f8ad9692edbae576434c51c Mon Sep 17 00:00:00 2001 From: fzyzcjy Date: Sat, 11 Jul 2026 07:24:26 +0800 Subject: [PATCH 09/13] Scope real_rollout expandable_segments to the train actor via --train-env-vars The H100 actor OOM fix previously set PYTORCH_CUDA_ALLOC_CONF job-wide via the ray runtime env, which also reached the sglang engine. The OOM is purely on the train actor (disaggregated real_rollout puts sglang on separate GPUs), so scope it to the train workers via --train-env-vars instead, matching the existing precedent in tests/e2e/fsdp/test_qwen3_0.6B_megatron_fsdp_align.py. This leaves the sglang engine's allocator untouched. A shared get_train_env_vars_arg helper builds a single --train-env-vars per scenario (merging the deterministic env vars where needed) so there is never a duplicate argument. --- tests/e2e/ft/conftest_ft/execution.py | 27 ++++++++++++------- .../ft/conftest_ft/scenario_deterministic.py | 10 ++----- .../e2e/ft/conftest_ft/scenario_no_failure.py | 14 ++++++---- .../ft/conftest_ft/scenario_with_failure.py | 3 ++- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/tests/e2e/ft/conftest_ft/execution.py b/tests/e2e/ft/conftest_ft/execution.py index f8ecd4d888..1637ffe0a0 100644 --- a/tests/e2e/ft/conftest_ft/execution.py +++ b/tests/e2e/ft/conftest_ft/execution.py @@ -173,6 +173,24 @@ def get_ft_args(mode: FTTestMode) -> str: } +def get_train_env_vars_arg(mode: FTTestMode, *, deterministic: bool) -> str: + env_vars: dict[str, str] = {} + if deterministic: + env_vars.update(_DETERMINISTIC_ENV_VARS) + if mode.has_real_rollout: + # Disaggregated real_rollout puts the actor on only train_gpus_per_node GPUs (4) with no + # tensor/pipeline/expert sharding, so each actor GPU holds the full unsharded model plus the + # deterministic dumper's value/grad tensors. On 80GB H100 the peak fits but the allocator + # fragments (observed ~11GiB reserved-but-unallocated vs a 10.5GiB request -> spurious OOM); + # expandable_segments coalesces that. Scoped to the train actor (not the sglang engine), + # which is safe: no --offload-train, so it cannot collide with torch_memory_saver, and + # determinism is unaffected (it only changes virtual-address arena management). + env_vars["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" + if not env_vars: + return "" + return f"--train-env-vars '{json.dumps(env_vars)}' " + + def run_training( train_args: str, mode: FTTestMode, @@ -200,15 +218,6 @@ def run_training( "SGLANG_LOG_MS": "1", **(extra_env_vars or {}), } - if mode.has_real_rollout: - # Disaggregated real_rollout puts the actor on only train_gpus_per_node GPUs (4) with no - # tensor/pipeline/expert sharding, so each actor GPU holds the full unsharded model plus the - # deterministic dumper's value/grad tensors. On 80GB H100 the peak fits but the allocator - # fragments (observed ~11GiB reserved-but-unallocated vs a 10.5GiB request -> spurious OOM); - # expandable_segments coalesces that. Safe here: no --offload-train and enable_memory_saver - # is off, so it cannot collide with torch_memory_saver. Determinism is unaffected (both - # baseline and target use it, and it only changes virtual-address arena management). - merged_env_vars["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" U.execute_train( train_args=train_args, num_gpus_per_node=mode.total_node_gpus, diff --git a/tests/e2e/ft/conftest_ft/scenario_deterministic.py b/tests/e2e/ft/conftest_ft/scenario_deterministic.py index d63f80ade1..7a785b4886 100644 --- a/tests/e2e/ft/conftest_ft/scenario_deterministic.py +++ b/tests/e2e/ft/conftest_ft/scenario_deterministic.py @@ -5,7 +5,7 @@ from pathlib import Path from tests.e2e.ft.conftest_ft.app import create_comparison_app_and_run_ci -from tests.e2e.ft.conftest_ft.execution import get_common_train_args, get_ft_args +from tests.e2e.ft.conftest_ft.execution import get_common_train_args, get_ft_args, get_train_env_vars_arg from tests.e2e.ft.conftest_ft.modes import FTTestMode from miles.utils.test_utils.comparisons.dumps import ( @@ -22,12 +22,6 @@ TOTAL_NUM_ROLLOUTS: int = 2 * NUM_ROLLOUTS_PER_PHASE PHASE_START_ROLLOUT_IDS: dict[str, int] = {"phase_a": 0, "phase_b": NUM_ROLLOUTS_PER_PHASE} -_DETERMINISTIC_ENV_VARS: str = ( - '--train-env-vars \'{"NCCL_ALGO": "Ring", ' - '"NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0", ' - '"CUBLAS_WORKSPACE_CONFIG": ":4096:8"}\' ' -) - def _build_actions(phase_start_rollout_id: int) -> list[dict]: heal_trigger_rollout_id: int = phase_start_rollout_id + 1 @@ -64,7 +58,7 @@ def _build_phase_args(mode: FTTestMode, dump_dir: str, *, is_target: bool, enabl phase_start_rollout_id: int = PHASE_START_ROLLOUT_IDS[phase_name] base = get_common_train_args(mode, dump_dir=dump_dir, num_steps=TOTAL_NUM_ROLLOUTS, enable_dumper=enable_dumper) - base += "--deterministic-mode " + _DETERMINISTIC_ENV_VARS + base += "--deterministic-mode " + get_train_env_vars_arg(mode, deterministic=True) base += "--debug-deterministic-collective " if is_target: diff --git a/tests/e2e/ft/conftest_ft/scenario_no_failure.py b/tests/e2e/ft/conftest_ft/scenario_no_failure.py index 3e0c86163b..7ee2c022f6 100644 --- a/tests/e2e/ft/conftest_ft/scenario_no_failure.py +++ b/tests/e2e/ft/conftest_ft/scenario_no_failure.py @@ -3,7 +3,7 @@ from tests.e2e.ft.conftest_ft.app import create_comparison_app_and_run_ci -from tests.e2e.ft.conftest_ft.execution import get_common_train_args, get_ft_args +from tests.e2e.ft.conftest_ft.execution import get_common_train_args, get_ft_args, get_train_env_vars_arg from tests.e2e.ft.conftest_ft.modes import FTTestMode from miles.utils.test_utils.comparisons.dumps import ( @@ -17,13 +17,17 @@ def _build_baseline_args(mode: FTTestMode, dump_dir: str, enable_dumper: bool = True) -> str: - return get_common_train_args(mode, dump_dir=dump_dir, num_steps=NUM_STEPS, enable_dumper=enable_dumper) + return get_common_train_args( + mode, dump_dir=dump_dir, num_steps=NUM_STEPS, enable_dumper=enable_dumper + ) + get_train_env_vars_arg(mode, deterministic=False) def _build_target_args(mode: FTTestMode, dump_dir: str, enable_dumper: bool = True) -> str: - return get_common_train_args( - mode, dump_dir=dump_dir, num_steps=NUM_STEPS, enable_dumper=enable_dumper - ) + get_ft_args(mode) + return ( + get_common_train_args(mode, dump_dir=dump_dir, num_steps=NUM_STEPS, enable_dumper=enable_dumper) + + get_ft_args(mode) + + get_train_env_vars_arg(mode, deterministic=False) + ) def _compare(dump_dir: str, mode: FTTestMode) -> None: diff --git a/tests/e2e/ft/conftest_ft/scenario_with_failure.py b/tests/e2e/ft/conftest_ft/scenario_with_failure.py index cc1214cc9d..c48cf620b3 100644 --- a/tests/e2e/ft/conftest_ft/scenario_with_failure.py +++ b/tests/e2e/ft/conftest_ft/scenario_with_failure.py @@ -5,7 +5,7 @@ from pathlib import Path from tests.e2e.ft.conftest_ft.app import create_comparison_app_and_run_ci -from tests.e2e.ft.conftest_ft.execution import get_common_train_args, get_ft_args +from tests.e2e.ft.conftest_ft.execution import get_common_train_args, get_ft_args, get_train_env_vars_arg from tests.e2e.ft.conftest_ft.modes import FTTestMode from miles.utils.test_utils.comparisons.dumps import ( @@ -88,6 +88,7 @@ def _expected_reconfigures(*, is_target: bool, phase: str, num_cells: int) -> li def _build_phase_args(mode: FTTestMode, dump_dir: str, *, is_target: bool, enable_dumper: bool = True) -> str: is_phase_a: bool = dump_dir.endswith("phase_a") base = get_common_train_args(mode, dump_dir=dump_dir, num_steps=NUM_PHASE_B_STEPS, enable_dumper=enable_dumper) + base += get_train_env_vars_arg(mode, deterministic=False) if is_target: base += get_ft_args(mode) From 0930f59fa4612c753164d6dc579635a44aed6feb Mon Sep 17 00:00:00 2001 From: Jiajun Li Date: Fri, 10 Jul 2026 16:14:53 -0700 Subject: [PATCH 10/13] Run fault-tolerance suites on full-node H200 CI Add the stage-c-8-gpu-h200 workflow job and register it as a per-commit CUDA suite. Move every ft-short and ft-long entry to that stage so PR #1622 runs the re-enabled fault-tolerance tests on the H200 fleet, while keeping CI docs and roster tests aligned. --- .github/workflows/pr-test.yml | 19 +++++++++++++++++++ docs/ci/00-stage.md | 1 + tests/ci/run_suite.py | 9 +++++---- tests/ci/test/test_run_suite.py | 1 + tests/e2e/ft/README.md | 4 ++-- ...st_trainer_ft_deterministic_dp2_cp2_pp2.py | 2 +- ...r_ft_deterministic_dp2_cp2_real_rollout.py | 2 +- ...rainer_ft_deterministic_dp2_cp2_tp2_ep2.py | 2 +- .../test_trainer_ft_deterministic_dp4_cp2.py | 2 +- .../test_trainer_ft_no_failure_dp2_cp2_pp2.py | 2 +- ...iner_ft_no_failure_dp2_cp2_real_rollout.py | 2 +- ...t_trainer_ft_no_failure_dp2_cp2_tp2_ep2.py | 2 +- .../ft/test_trainer_ft_no_failure_dp4_cp2.py | 2 +- ..._trainer_ft_random_dp2_cp2_real_rollout.py | 2 +- .../test_trainer_ft_random_dp2_cp2_tp2_ep2.py | 2 +- .../e2e/ft/test_trainer_ft_realistic_gsm8k.py | 2 +- ...est_trainer_ft_with_failure_dp2_cp2_pp2.py | 2 +- ...with_failure_dp2_cp2_real_rollout_dense.py | 2 +- ...trainer_ft_with_failure_dp2_cp2_tp2_ep2.py | 2 +- 19 files changed, 42 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 659a2ab385..2487e2d72d 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -157,6 +157,25 @@ jobs: ${{ (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--continue-on-error' || '' }} secrets: inherit + stage-c-8-gpu-h200: + needs: [resolve-ci-image, stage-a-cpu] + if: | + always() && !cancelled() && + needs.resolve-ci-image.result == 'success' && + (needs.stage-a-cpu.result == 'success' || + (needs.stage-a-cpu.result == 'failure' && (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))))) && + ((github.event_name == 'workflow_dispatch') || ((github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) || (github.event.pull_request)) + uses: ./.github/workflows/_run-ci.yml + with: + runs_on: '["h200", "8gpu"]' + container_image: ${{ needs.resolve-ci-image.outputs.container_image }} + execute_command: >- + python tests/ci/run_suite.py --hw cuda --suite stage-c-8-gpu-h200 + --labels ${{ join(github.event.pull_request.labels.*.name, ' ') }} + ${{ (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-ci-image') || contains(github.event.pull_request.labels.*.name, 'run-ci-all') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--match-all-labels' || '' }} + ${{ (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--continue-on-error' || '' }} + secrets: inherit + stage-c-4-gpu-h200: needs: [resolve-ci-image, stage-a-cpu] if: | diff --git a/docs/ci/00-stage.md b/docs/ci/00-stage.md index 758ad1a6d6..405d800351 100644 --- a/docs/ci/00-stage.md +++ b/docs/ci/00-stage.md @@ -27,6 +27,7 @@ Stage names follow `stage---` (or `stage--` for CPU, e | `stage-c-2-gpu-h200` | 2× H200 | `["h200","2gpu"]` | 2 | `resolve-ci-image`, `stage-a-cpu` | | `stage-c-4-gpu-h200` | 4× H200 | `["h200","4gpu"]` | 3 | `resolve-ci-image`, `stage-a-cpu` | | `stage-c-8-gpu-h100` | 8× H100 | `["h100","8gpu"]` | 2 | `resolve-ci-image`, `stage-a-cpu` | +| `stage-c-8-gpu-h200` | 8× H200 | `["h200","8gpu"]` | 1 | `resolve-ci-image`, `stage-a-cpu` | `tier a` (CPU fast) gates the GPU fleet after `resolve-ci-image`; the GPU stages (`b` / `c`) all depend on `resolve-ci-image` and `stage-a-cpu`, and run concurrently with each other — the `b` / `c` letters classify role, they are not a sequential pipeline. diff --git a/tests/ci/run_suite.py b/tests/ci/run_suite.py index 621897fc38..a6ca287051 100644 --- a/tests/ci/run_suite.py +++ b/tests/ci/run_suite.py @@ -22,10 +22,10 @@ # runtime by `filter_tests` via the `--labels` arg, not via per-suite jobs). # # CUDA suites: each is served by a matching workflow job in -# .github/workflows/pr-test.yml. `stage-c-8-gpu-h100` runs on the full-node -# 8-GPU H100 host; the H200 fleet is one 8-GPU node split into 2+2+4 workers -# via per-runner CUDA_VISIBLE_DEVICES (see pr-test.yml stage-c-4-gpu-h200 / -# stage-b-2-gpu-h200 / stage-c-2-gpu-h200 job comments). +# .github/workflows/pr-test.yml. `stage-c-8-gpu-h100` and `stage-c-8-gpu-h200` +# run on full-node 8-GPU hosts; the split H200 fleet is one 8-GPU node divided +# into 2+2+4 workers via per-runner CUDA_VISIBLE_DEVICES (see pr-test.yml +# stage-c-4-gpu-h200 / stage-b-2-gpu-h200 / stage-c-2-gpu-h200 job comments). PER_COMMIT_SUITES = { HWBackend.CPU: [ "stage-a-cpu", @@ -34,6 +34,7 @@ HWBackend.CUDA: [ "stage-b-2-gpu-h200", "stage-c-8-gpu-h100", + "stage-c-8-gpu-h200", "stage-c-4-gpu-h200", "stage-c-2-gpu-h200", ], diff --git a/tests/ci/test/test_run_suite.py b/tests/ci/test/test_run_suite.py index 29a8061e42..b08afb5bf7 100644 --- a/tests/ci/test/test_run_suite.py +++ b/tests/ci/test/test_run_suite.py @@ -79,6 +79,7 @@ def test_cuda_suites_exact(self): assert PER_COMMIT_SUITES[HWBackend.CUDA] == [ "stage-b-2-gpu-h200", "stage-c-8-gpu-h100", + "stage-c-8-gpu-h200", "stage-c-4-gpu-h200", "stage-c-2-gpu-h200", ] diff --git a/tests/e2e/ft/README.md b/tests/e2e/ft/README.md index 20d637284d..9d0413a5d0 100644 --- a/tests/e2e/ft/README.md +++ b/tests/e2e/ft/README.md @@ -3,7 +3,7 @@ ## Layout - Scenario logic lives in `conftest_ft/scenario_.py`. -- CI runs it via thin per-mode entry files `test_trainer_ft__.py`, each registered with `register_cuda_ci(est_time=..., suite="stage-c-8-gpu-h100", labels=["ft-short"])` (comparison scenarios) or `labels=["ft-long"]` (soak scenarios). +- CI runs it via thin per-mode entry files `test_trainer_ft__.py`, each registered with `register_cuda_ci(est_time=..., suite="stage-c-8-gpu-h200", labels=["ft-short"])` (comparison scenarios) or `labels=["ft-long"]` (soak scenarios). - The CUDA CI runner executes each entry as bare `python3 ` (exit code = pass/fail); the entry just calls the scenario's `run_ci(mode)`. | Scenario (`conftest_ft/scenario_*.py`) | Type | What it verifies | @@ -36,7 +36,7 @@ ### In CI -- Gated on the `run-ci-ft-short` / `run-ci-ft-long` PR labels (FT is expensive — not run on every PR). `ft-short` covers the comparison scenarios (no_failure / deterministic / with_failure, minutes each); `ft-long` covers the soak scenarios (random-crash survival, realistic-gsm8k convergence — tens of minutes to hours). With a label set, the matching entries run on `stage-c-8-gpu-h100`. +- Gated on the `run-ci-ft-short` / `run-ci-ft-long` PR labels (FT is expensive — not run on every PR). `ft-short` covers the comparison scenarios (no_failure / deterministic / with_failure, minutes each); `ft-long` covers the soak scenarios (random-crash survival, realistic-gsm8k convergence — tens of minutes to hours). With a label set, the matching entries run on `stage-c-8-gpu-h200`. - Add a `(scenario, mode)` to CI: copy an entry file, change `run_ci(...)`'s mode. - Add a new label: edit `tests/ci/labels.py` and create the matching `run-ci-