diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index b15f31286b..0e39931d35 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -6,8 +6,8 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review, labeled] schedule: - # Nightly full CI on main: a scheduled run is treated like workflow_dispatch - # (every stage, --match-all-labels). 15:00 UTC = 08:00 PDT / 07:00 PST (cron is UTC, no DST). + # Nightly CI on main runs every enabled tag except ft-long, with fast-fail disabled. + # 15:00 UTC = 08:00 PDT / 07:00 PST (cron is UTC, no DST). - cron: '0 15 * * *' workflow_dispatch: inputs: @@ -43,9 +43,12 @@ concurrency: # `--labels`. run_suite.py strips the `run-ci-` prefix internally and ignores # any label that does not start with `run-ci-` (see tests/ci/run_suite.py # `strip_run_ci_prefix`). For `workflow_dispatch`, no PR labels exist, so the -# `--labels` list collapses to empty; `--match-all-labels` is then added -# unconditionally to bypass the labels predicate inside run_suite.py and run -# every enabled test in the suite. +# `--labels` list collapses to empty; `--match-all-labels` is then added to +# bypass the labels predicate inside run_suite.py. Broad scopes can add a final +# `--exclude-labels`: `run-ci-image` excludes ft-short and ft-long, while +# nightly excludes only ft-long. `run-ci-all` and workflow_dispatch pass no +# exclusions and therefore include every enabled tag. When scope labels +# overlap, all > nightly > image. # # The `bypass-fastfail` PR label opts a PR out of fast-fail on two levels: GPU # stages run even when stage-a-cpu fails (cross-stage), and every stage passes @@ -97,6 +100,7 @@ jobs: --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 4 --labels ${{ join(github.event.pull_request.labels.*.name, ' ') }} ${{ (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-ci-image') || contains(github.event.pull_request.labels.*.name, 'run-ci-all') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--match-all-labels' || '' }} + ${{ ((github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly')) && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && '--exclude-labels ft-long') || (contains(github.event.pull_request.labels.*.name, 'run-ci-image') && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'nightly') && '--exclude-labels ft-short ft-long') || '' }} ${{ (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--continue-on-error' || '' }} secrets: inherit @@ -111,6 +115,7 @@ jobs: python tests/ci/run_suite.py --hw cpu --suite stage-b-cpu --labels ${{ join(github.event.pull_request.labels.*.name, ' ') }} ${{ (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-ci-image') || contains(github.event.pull_request.labels.*.name, 'run-ci-all') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--match-all-labels' || '' }} + ${{ ((github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly')) && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && '--exclude-labels ft-long') || (contains(github.event.pull_request.labels.*.name, 'run-ci-image') && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'nightly') && '--exclude-labels ft-short ft-long') || '' }} ${{ (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--continue-on-error' || '' }} secrets: inherit @@ -130,6 +135,7 @@ jobs: python tests/ci/run_suite.py --hw cuda --suite stage-b-2-gpu-h200 --labels ${{ join(github.event.pull_request.labels.*.name, ' ') }} ${{ (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-ci-image') || contains(github.event.pull_request.labels.*.name, 'run-ci-all') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--match-all-labels' || '' }} + ${{ ((github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly')) && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && '--exclude-labels ft-long') || (contains(github.event.pull_request.labels.*.name, 'run-ci-image') && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'nightly') && '--exclude-labels ft-short ft-long') || '' }} ${{ (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--continue-on-error' || '' }} secrets: inherit @@ -154,6 +160,7 @@ jobs: --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2 --labels ${{ join(github.event.pull_request.labels.*.name, ' ') }} ${{ (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-ci-image') || contains(github.event.pull_request.labels.*.name, 'run-ci-all') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--match-all-labels' || '' }} + ${{ ((github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly')) && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && '--exclude-labels ft-long') || (contains(github.event.pull_request.labels.*.name, 'run-ci-image') && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'nightly') && '--exclude-labels ft-short ft-long') || '' }} ${{ (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--continue-on-error' || '' }} secrets: inherit @@ -178,6 +185,7 @@ jobs: --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2 --labels ${{ join(github.event.pull_request.labels.*.name, ' ') }} ${{ (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-ci-image') || contains(github.event.pull_request.labels.*.name, 'run-ci-all') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--match-all-labels' || '' }} + ${{ ((github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly')) && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && '--exclude-labels ft-long') || (contains(github.event.pull_request.labels.*.name, 'run-ci-image') && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'nightly') && '--exclude-labels ft-short ft-long') || '' }} ${{ (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--continue-on-error' || '' }} secrets: inherit @@ -202,6 +210,7 @@ jobs: --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 3 --labels ${{ join(github.event.pull_request.labels.*.name, ' ') }} ${{ (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-ci-image') || contains(github.event.pull_request.labels.*.name, 'run-ci-all') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--match-all-labels' || '' }} + ${{ ((github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly')) && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && '--exclude-labels ft-long') || (contains(github.event.pull_request.labels.*.name, 'run-ci-image') && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'nightly') && '--exclude-labels ft-short ft-long') || '' }} ${{ (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--continue-on-error' || '' }} secrets: inherit @@ -226,5 +235,6 @@ jobs: --auto-partition-id ${{ matrix.partition_id }} --auto-partition-size 2 --labels ${{ join(github.event.pull_request.labels.*.name, ' ') }} ${{ (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-ci-image') || contains(github.event.pull_request.labels.*.name, 'run-ci-all') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--match-all-labels' || '' }} + ${{ ((github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly')) && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && '--exclude-labels ft-long') || (contains(github.event.pull_request.labels.*.name, 'run-ci-image') && !contains(github.event.pull_request.labels.*.name, 'run-ci-all') && github.event_name != 'schedule' && !contains(github.event.pull_request.labels.*.name, 'nightly') && '--exclude-labels ft-short ft-long') || '' }} ${{ (contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') || (github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'nightly'))) && '--continue-on-error' || '' }} secrets: inherit diff --git a/docs/ci/00-stage.md b/docs/ci/00-stage.md index 7576852070..0c0447c396 100644 --- a/docs/ci/00-stage.md +++ b/docs/ci/00-stage.md @@ -33,9 +33,11 @@ Stage names follow `stage---` (or `stage--` for CPU, e ## What each stage does -**Image resolution (`resolve-ci-image`).** Before the GPU stages, a small `ubuntu-latest` job resolves the container image: it reads `ci-image-tag:` from the PR description (or the `ci_image_tag` dispatch input), defaults to `dev`, validates it is a bare tag, and outputs `radixark/miles:`. Every GPU stage uses this as its `container_image`. Distinct from this, the **`run-ci-image` label** (alongside `run-ci-all`, any `workflow_dispatch`, and the nightly `schedule` run on `main`) makes each stage add `--match-all-labels`, running the full suite regardless of per-test labels — this is how you validate a PR that bumps the image. +**Image resolution (`resolve-ci-image`).** Before the GPU stages, a small `ubuntu-latest` job resolves the container image: it reads `ci-image-tag:` from the PR description (or the `ci_image_tag` dispatch input), defaults to `dev`, validates it is a bare tag, and outputs `radixark/miles:`. Every GPU stage uses this as its `container_image`. Distinct from this, the **`run-ci-image` label** makes each stage add `--match-all-labels --exclude-labels ft-short ft-long`, running every enabled non-FT tag — this is how you validate a PR that bumps the image without scheduling FT tests. -A **nightly** run is the full suite with fast-fail off, triggered either by the nightly `schedule` cron on `main` or by a PR carrying the `nightly` label. Concretely it adds `--match-all-labels` and turns off fast-fail like the `bypass-fastfail` label. +A **nightly** run includes `ft-short` but excludes `ft-long`, with fast-fail off. It is triggered either by the nightly `schedule` cron on `main` or by a PR carrying the `nightly` label; concretely it adds `--match-all-labels --exclude-labels ft-long` and turns off fast-fail like the `bypass-fastfail` label. + +`run-ci-all` and a manual `workflow_dispatch` add `--match-all-labels` without exclusions, so every enabled tag is included. If scope labels overlap, the precedence is `run-ci-all` > nightly > `run-ci-image`. **Dependencies / gating.** The job graph is `resolve-ci-image` → `stage-a-cpu` → all GPU stages (in parallel). GPU stages require `resolve-ci-image` to succeed; by default they also require `stage-a-cpu` to succeed, so a CPU-test failure short-circuits the expensive GPU fleet. The `bypass-fastfail` PR label relaxes only the `stage-a-cpu` failure gate and passes `--continue-on-error` to each stage; it does not bypass `resolve-ci-image`. `stage-b-cpu` has no dependency and runs alongside `stage-a-cpu`, outside the GPU gating path. diff --git a/docs/ci/01-label.md b/docs/ci/01-label.md index 5b641b6fdc..05909e211c 100644 --- a/docs/ci/01-label.md +++ b/docs/ci/01-label.md @@ -1,6 +1,6 @@ --- title: Labels -description: The three kinds of CI label — domain labels that gate tests, meta-labels that run everything, and bypass-fastfail. +description: The three kinds of CI label — domain labels that gate tests, scope labels that broaden selection, and bypass-fastfail. --- # Labels @@ -10,10 +10,12 @@ A label is a GitHub PR label that changes what CI runs or how it fails. Three ki | Kind | Example | Effect | |---|---|---| | Domain label | `run-ci-megatron` | selects which tests run | -| Meta-label | `run-ci-image`, `run-ci-all` | run the full suite, ignoring domain labels | +| Scope label | `run-ci-image` | run every enabled non-FT tag | +| Scope label | `nightly` | run every enabled tag except `ft-long`, with fast-fail disabled | +| Scope label | `run-ci-all` | run every enabled tag | | Behavior label | `bypass-fastfail` | opt out of fast-fail; one run surfaces every failure | -Only domain labels are declared by tests; the other two are workflow switches in `pr-test.yml`. +Only domain labels are declared by tests; scope and behavior labels are workflow switches in `pr-test.yml`. ## Domain labels: `register_*_ci(labels=...)` ↔ `run-ci-` @@ -34,13 +36,13 @@ Domain labels live in `tests/ci/labels.py` (`KNOWN_LABELS`); a `labels=[...]` va To add one: add the entry to `KNOWN_LABELS`, then create the matching `run-ci-` label on the PR. No workflow edit needed. -## Meta-labels: run everything +## Broad CI scopes -`run-ci-image` and `run-ci-all` are the same switch: both add `--match-all-labels` to run every enabled test in the suite, ignoring domain labels. Neither is in `KNOWN_LABELS`. +`run-ci-image` adds `--match-all-labels --exclude-labels ft-short ft-long`, so image validation ignores ordinary domain gating but does not schedule any FT tests. -A manual `workflow_dispatch` run — and the nightly `schedule` run on `main` — gets `--match-all-labels` too, because neither has PR labels to filter on. +A nightly run — the `schedule` on `main` or a PR carrying `nightly` — adds `--match-all-labels --exclude-labels ft-long`, so it includes `ft-short` but omits the FT soak tests. It also disables fast-fail on both levels. -The `nightly` label runs a PR as a nightly: `--match-all-labels` plus fast-fail bypass on both levels. +`run-ci-all` and a manual `workflow_dispatch` add `--match-all-labels` without exclusions, so every enabled tag is included. If scope labels overlap, the precedence is `run-ci-all` > nightly > `run-ci-image`. ## Registration and scan scope @@ -65,6 +67,6 @@ The `bypass-fastfail` PR label turns both off so one run surfaces every failure: - Cross-stage: each GPU stage's check becomes `(needs.stage-a-cpu.result == 'success' || (needs.stage-a-cpu.result == 'failure' && contains(..., 'bypass-fastfail')))`, so GPU stages run even after `stage-a-cpu` fails. - Within-stage: each stage adds `--continue-on-error` (drops `pytest -x`; sets `continue_on_error=True` for CUDA). The stage still ends red — it changes coverage, not the verdict. -A nightly run — the `schedule` cron, or a PR carrying the `nightly` label — bypasses fast-fail on both levels: the same cross-stage `if` and per-stage `--continue-on-error` match `github.event_name == 'schedule' || contains(..., 'nightly')`, because a nightly is meant to exercise every test and surface every failure (one datapoint per test), not stop at the first. +A nightly run — the `schedule` cron, or a PR carrying the `nightly` label — bypasses fast-fail on both levels: the same cross-stage `if` and per-stage `--continue-on-error` match `github.event_name == 'schedule' || contains(..., 'nightly')`, because a nightly is meant to exercise every enabled test except `ft-long` and surface every failure (one datapoint per test), not stop at the first. -Like the meta-labels, `bypass-fastfail` is matched directly in `pr-test.yml` and is not in `KNOWN_LABELS`. +Like the scope labels, `bypass-fastfail` is matched directly in `pr-test.yml` and is not in `KNOWN_LABELS`. diff --git a/tests/ci/labels.py b/tests/ci/labels.py index 26646a1e11..395ea5db30 100644 --- a/tests/ci/labels.py +++ b/tests/ci/labels.py @@ -11,9 +11,9 @@ The workflow does not need editing -- the generic stage job filters tests by labels at runtime. -The meta-labels `run-ci-image` / `run-ci-all` are intentionally NOT listed -here: they bypass the per-test labels filter and run the full suite via the -`--match-all-labels` flag (handled in run_suite.py). +The workflow-only scope labels are intentionally NOT listed here. `run-ci-all` +uses `--match-all-labels` without exclusions; `run-ci-image` and nightly pair +it with scope-specific `--exclude-labels` values (handled in run_suite.py). """ KNOWN_LABELS: dict[str, str] = { diff --git a/tests/ci/run_suite.py b/tests/ci/run_suite.py index a6ca287051..75c0fc3b7b 100644 --- a/tests/ci/run_suite.py +++ b/tests/ci/run_suite.py @@ -84,16 +84,18 @@ def filter_tests( nightly: bool = False, labels: set[str] | None = None, match_all_labels: bool = False, + exclude_labels: set[str] | None = None, ) -> tuple[list[CIRegistry], list[CIRegistry]]: """Filter registered tests down to the set that should run. The base predicate (hw / suite / nightly / disabled) is applied first. - Label selection then narrows further, with two modes: + Label selection then narrows further, with two modes followed by an + optional exclusion: * `match_all_labels=True`: ignore labels entirely -- every enabled test - that matches hw/suite/nightly runs. Used for the `run-ci-image` / - `run-ci-all` meta-labels and for `workflow_dispatch`. Precedence: this - mode wins even when `labels` is also passed. + that matches hw/suite/nightly runs. Used for broad CI scopes and for + `workflow_dispatch`. Precedence: this mode wins even when `labels` is + also passed. * `match_all_labels=False` (default): include only tests where `not test.labels or (set(test.labels) & labels)`. `labels` here is the already-stripped domain-label set produced by @@ -101,6 +103,9 @@ def filter_tests( omitted) is treated as always-run: it survives an empty PR-label set; a test with non-empty `labels` survives only when its labels intersect the PR-supplied set. + * `exclude_labels`: after either inclusion mode, remove tests carrying any + excluded label. This lets broad scopes omit a specific expensive class; + an exclusion always wins over inclusion. """ ci_tests = [t for t in ci_tests if t.backend == hw and t.suite == suite and t.nightly == nightly] @@ -113,6 +118,9 @@ def filter_tests( label_set: set[str] = labels or set() ci_tests = [t for t in ci_tests if not t.labels or (set(t.labels) & label_set)] + excluded_label_set = exclude_labels or set() + ci_tests = [t for t in ci_tests if not (set(t.labels) & excluded_label_set)] + enabled_tests = [t for t in ci_tests if t.disabled is None] skipped_tests = [t for t in ci_tests if t.disabled is not None] @@ -212,6 +220,7 @@ def run_a_suite(args): nightly, labels=stripped_labels, match_all_labels=args.match_all_labels, + exclude_labels=set(args.exclude_labels), ) if auto_partition_size: @@ -333,8 +342,18 @@ def main(): help=( "Bypass the labels filter and run every enabled test in the " "suite (subject to hw/suite/nightly/disabled). Set by the " - "workflow when the PR carries `run-ci-image` or `run-ci-all`, " - "and equivalently on `workflow_dispatch`." + "workflow for broad scopes; narrower scopes pair it with " + "`--exclude-labels`." + ), + ) + parser.add_argument( + "--exclude-labels", + nargs="*", + default=[], + help=( + "Exclude tests carrying any listed domain label after the normal " + "label selection. The workflow uses this with broad scopes that " + "intentionally omit an expensive test class." ), ) args = parser.parse_args() diff --git a/tests/ci/test/test_run_suite.py b/tests/ci/test/test_run_suite.py index b08afb5bf7..8670b35b0c 100644 --- a/tests/ci/test/test_run_suite.py +++ b/tests/ci/test/test_run_suite.py @@ -4,9 +4,8 @@ * `strip_run_ci_prefix`: empty input, prefix stripping, ignoring inputs without the `run-ci-` prefix (warning only). -* `filter_tests`: the six scenarios around `--labels` and - `--match-all-labels` with the new "empty labels means always run" - semantic. +* `filter_tests`: domain-label inclusion, match-all behavior, and broad-scope + exclusions with the "empty labels means always run" semantic. * `PER_COMMIT_SUITES`: locked to the new taxonomy including the always-run GPU bucket `stage-b-2-gpu-h200`. @@ -294,6 +293,57 @@ def test_case6_match_all_labels_wins_over_labels(self, cuda_h100_tests): assert _names(enabled_with_labels) == _names(enabled_without_labels) +# --- filter_tests: broad CI scope exclusions --------------------------------- + + +@pytest.fixture +def broad_scope_tests(): + return [ + _make("tests/e2e/always.py", labels=[]), + _make("tests/e2e/megatron.py", labels=["megatron"]), + _make("tests/e2e/ft/short.py", labels=["ft-short"]), + _make("tests/e2e/ft/long.py", labels=["ft-long", "long"]), + ] + + +class TestFilterTestsBroadScopes: + def test_image_scope_excludes_all_ft_labels(self, broad_scope_tests): + enabled, _ = filter_tests( + broad_scope_tests, + HWBackend.CUDA, + "stage-c-8-gpu-h100", + match_all_labels=True, + exclude_labels={"ft-short", "ft-long"}, + ) + assert _names(enabled) == { + "tests/e2e/always.py", + "tests/e2e/megatron.py", + } + + def test_nightly_scope_includes_ft_short_and_excludes_ft_long(self, broad_scope_tests): + enabled, _ = filter_tests( + broad_scope_tests, + HWBackend.CUDA, + "stage-c-8-gpu-h100", + match_all_labels=True, + exclude_labels={"ft-long"}, + ) + assert _names(enabled) == { + "tests/e2e/always.py", + "tests/e2e/megatron.py", + "tests/e2e/ft/short.py", + } + + def test_all_scope_includes_every_label(self, broad_scope_tests): + enabled, _ = filter_tests( + broad_scope_tests, + HWBackend.CUDA, + "stage-c-8-gpu-h100", + match_all_labels=True, + ) + assert _names(enabled) == _names(broad_scope_tests) + + # --- filter_tests: hw/suite/nightly partitioning still works ----------------