Repository navigation
Measure first-call and post-update costs of continuous rollout CUDA Graphs #6964
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: modal-torch-latest | ||
|
Check warning on line 1 in .github/workflows/modal-torch-latest.yml
|
||
| # This Required workflow selects tests on a no-secret GitHub runner, then tests | ||
| # the exact candidate SHA inside a no-secret Modal Sandbox. If that Sandbox | ||
| # cannot start within its bounded acquisition window, the same trusted | ||
| # controller makes one bounded G7.12 Spot acquisition pass across the | ||
| # configured AWS regions. Under | ||
| # pull_request_target, every GitHub-side checkout and Python entrypoint comes | ||
| # from the trusted base commit; merge queue runs resolve that trusted base from | ||
| # merge_group.base_sha. A GPU backend runs only for merge queue entries, the | ||
| # nightly scheduled full run, and manual runs, so plain PR events never spend | ||
| # GPU quota. Master receives no direct pushes (the merge queue lands every | ||
| # entry, and each merge_group run already tested the merged tree), so there is | ||
| # no push trigger; cross-entry interaction regressions are covered by the | ||
| # nightly full suite, which feeds regression triage | ||
| # (.github/workflows/nightly-triage.yml). See .github/workflows/TEST_SELECTION.md. | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| infrastructure_smoke: | ||
| description: Run only the parallel backend infrastructure smoke checks | ||
| required: false | ||
| default: false | ||
| type: boolean | ||
| torch_preset: | ||
| description: Modal PyTorch/CUDA image preset for manual runs | ||
| required: false | ||
| default: '2.10.0-cuda12.8' | ||
| type: choice | ||
| options: | ||
| - '2.7.1-cuda12.8' | ||
| - '2.8.0-cuda12.8' | ||
| - '2.9.1-cuda12.8' | ||
| - '2.10.0-cuda12.8' | ||
| - '2.11.0-cuda12.8' | ||
| transformers_source: | ||
| description: Hugging Face Transformers source for manual runs | ||
| required: false | ||
| default: 'git' | ||
| type: choice | ||
| options: | ||
| - 'requirements' | ||
| - 'git' | ||
| transformers_ref: | ||
| description: Hugging Face Transformers git ref to install when source is git | ||
| required: false | ||
| default: 'main' | ||
| type: string | ||
| # Nightly full-suite run of master. This is the only event whose failures feed | ||
| # automated regression triage; see .github/workflows/nightly-triage.yml. | ||
| schedule: | ||
| - cron: '0 10 * * *' | ||
| pull_request_target: | ||
| types: [opened, reopened, review_requested, ready_for_review, synchronize] | ||
| branches: | ||
| - master | ||
| # Required checks must also trigger on merge queue entries, otherwise the | ||
| # merge fails waiting for a status that is never reported. This is the only | ||
| # PR-related event that spends GPU quota. | ||
| merge_group: | ||
| types: [checks_requested] | ||
| branches: | ||
| - master | ||
| permissions: | ||
| contents: read | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} | ||
| cancel-in-progress: true | ||
| jobs: | ||
| modal-infrastructure-smoke: | ||
| name: modal-torch-latest / Modal infrastructure smoke | ||
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.infrastructure_smoke == true }} | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 15 | ||
| permissions: | ||
| contents: read | ||
| steps: | ||
| - name: Checkout trusted controller code | ||
| uses: actions/checkout@v7 | ||
| with: | ||
| ref: ${{ github.sha }} | ||
| fetch-depth: 1 | ||
| persist-credentials: false | ||
| lfs: false | ||
| submodules: false | ||
| - name: Install Python | ||
| uses: actions/setup-python@v7 | ||
| with: | ||
| python-version: "3.10" | ||
| cache: 'pip' | ||
| - name: Install trusted controller dependency | ||
| run: | | ||
| pip install uv | ||
| uv pip install --system modal==1.2.6 | ||
| - name: Check Modal OCI startup, two GPUs, and cleanup | ||
| env: | ||
| MODAL_TORCH_PRESET: ${{ inputs.torch_preset }} | ||
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | ||
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | ||
| run: python3 ci/torch_latest.py modal-infrastructure-smoke | ||
| aws-infrastructure-smoke: | ||
| name: modal-torch-latest / AWS infrastructure smoke / ${{ matrix.region }} | ||
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.infrastructure_smoke == true }} | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 35 | ||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| region: [us-east-1, us-east-2, us-west-2] | ||
| steps: | ||
| - name: Checkout trusted controller code | ||
| uses: actions/checkout@v7 | ||
| with: | ||
| ref: ${{ github.sha }} | ||
| fetch-depth: 1 | ||
| persist-credentials: false | ||
| lfs: false | ||
| submodules: false | ||
| - name: Install Python | ||
| uses: actions/setup-python@v7 | ||
| with: | ||
| python-version: "3.10" | ||
| - name: Configure AWS credentials for infrastructure smoke | ||
| id: aws-auth | ||
| env: | ||
| AWS_ROLE_ARN: ${{ secrets.AWS_MODAL_FALLBACK_ROLE_ARN }} | ||
| SMOKE_REGION: ${{ matrix.region }} | ||
| run: | | ||
| set -euo pipefail | ||
| test -n "$AWS_ROLE_ARN" | ||
| echo "::add-mask::$AWS_ROLE_ARN" | ||
| oidc_response=$(curl --fail --silent --show-error \ | ||
| --header "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ | ||
| "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com") | ||
| web_identity_token=$(jq -er '.value | select(type == "string" and length > 0)' <<< "$oidc_response") | ||
| echo "::add-mask::$web_identity_token" | ||
| read -r aws_access_key_value aws_secret_key_value aws_session_token_value <<< "$( | ||
| aws --no-cli-pager sts assume-role-with-web-identity \ | ||
| --role-arn "$AWS_ROLE_ARN" \ | ||
| --role-session-name "DeepSpeedCI-${GITHUB_RUN_ID}-${SMOKE_REGION}" \ | ||
| --web-identity-token "$web_identity_token" \ | ||
| --duration-seconds 3600 \ | ||
| --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \ | ||
| --output text | ||
| )" | ||
| test -n "$aws_access_key_value" | ||
| test -n "$aws_secret_key_value" | ||
| test -n "$aws_session_token_value" | ||
| echo "::add-mask::$aws_access_key_value" | ||
| echo "::add-mask::$aws_secret_key_value" | ||
| echo "::add-mask::$aws_session_token_value" | ||
| { | ||
| printf 'AWS_ACCESS_KEY_ID=%s\n' "$aws_access_key_value" | ||
| printf 'AWS_SECRET_ACCESS_KEY=%s\n' "$aws_secret_key_value" | ||
| printf 'AWS_SESSION_TOKEN=%s\n' "$aws_session_token_value" | ||
| } >> "$GITHUB_ENV" | ||
| - name: Check regional launch, SSM, two GPUs, container, and private proof | ||
| env: | ||
| AWS_MODAL_FALLBACK_CONFIG: ${{ secrets.AWS_MODAL_FALLBACK_CONFIG }} | ||
| DS_CI_AWS_RUN_SUFFIX: ${{ matrix.region }} | ||
| MODAL_TORCH_PRESET: ${{ inputs.torch_preset }} | ||
| run: python3 ci/torch_latest.py aws-infrastructure-smoke --region "${{ matrix.region }}" | ||
| - name: Reconcile regional infrastructure-smoke cleanup | ||
| if: ${{ always() && steps.aws-auth.outcome == 'success' }} | ||
| env: | ||
| AWS_MODAL_FALLBACK_CONFIG: ${{ secrets.AWS_MODAL_FALLBACK_CONFIG }} | ||
| DS_CI_AWS_RUN_SUFFIX: ${{ matrix.region }} | ||
| run: python3 ci/torch_latest.py cleanup-aws | ||
| collect-tests: | ||
| name: modal-torch-latest / collect tests | ||
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.infrastructure_smoke != true }} | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 20 | ||
| permissions: | ||
| contents: read | ||
| outputs: | ||
| mode: ${{ steps.select.outputs.mode }} | ||
| steps: | ||
| - name: Checkout trusted workflow code | ||
| uses: actions/checkout@v7 | ||
| with: | ||
| # Trusted base revision for every event: the merge-group commit must | ||
| # never control the selector that runs on this GitHub runner. | ||
| ref: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.sha }} | ||
| fetch-depth: 1 | ||
| persist-credentials: false | ||
| lfs: false | ||
| submodules: false | ||
| - name: Fetch candidate tree as data | ||
| if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group' | ||
| env: | ||
| HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | ||
| HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | ||
| BASE_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} | ||
| BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} | ||
| CANDIDATE_ROOT: ${{ runner.temp }}/deepspeed-candidate | ||
| run: | | ||
| python3 ci/torch_latest.py checkout-candidate \ | ||
| --head-repository "$HEAD_REPOSITORY" \ | ||
| --head-sha "$HEAD_SHA" \ | ||
| --base-repository "$BASE_REPOSITORY" \ | ||
| --base-sha "$BASE_SHA" \ | ||
| --destination "$CANDIDATE_ROOT" | ||
| - name: Self-test the trusted selector | ||
| run: python3 ci/test_tests_fetcher.py | ||
| - name: Select impacted tests | ||
| id: select | ||
| env: | ||
| EVENT_NAME: ${{ github.event_name }} | ||
| CANDIDATE_ROOT: ${{ runner.temp }}/deepspeed-candidate | ||
| run: | | ||
| if [ "$EVENT_NAME" = "pull_request_target" ] || [ "$EVENT_NAME" = "merge_group" ]; then | ||
| python3 ci/tests_fetcher.py \ | ||
| --workflow modal-torch-latest \ | ||
| --repo-root "$CANDIDATE_ROOT" \ | ||
| --base refs/ci/base | ||
| else | ||
| python3 ci/tests_fetcher.py \ | ||
| --workflow modal-torch-latest \ | ||
| --repo-root "$GITHUB_WORKSPACE" \ | ||
| --base "" | ||
| fi | ||
| - name: Validate test selection | ||
| env: | ||
| SELECTION_MODE: ${{ steps.select.outputs.mode }} | ||
| run: | | ||
| python3 ci/torch_latest.py validate-selection \ | ||
| --mode "$SELECTION_MODE" \ | ||
| --path ci/.test_selection/test_list.txt | ||
| - name: Upload test selection | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: modal-torch-latest-test-selection | ||
| path: ci/.test_selection/test_list.txt | ||
| include-hidden-files: true | ||
| retention-days: 3 | ||
| deploy: | ||
| name: modal-torch-latest / DeepSpeedAI CI | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 180 | ||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
| needs: collect-tests | ||
| # Preserve the Required check on failures and no-test selections. A selector | ||
| # failure enters this job and fails explicitly; mode=none skips GPU execution. | ||
| # pull_request_target runs skip the backend entirely (a skipped dependent | ||
| # job still satisfies the Required check), so the tests gate the merge queue | ||
| # entry instead of every PR update. | ||
| if: ${{ (github.event_name != 'workflow_dispatch' || inputs.infrastructure_smoke != true) && !cancelled() && (needs.collect-tests.result != 'success' || (needs.collect-tests.outputs.mode != 'none' && github.event_name != 'pull_request_target')) }} | ||
| steps: | ||
| - name: Fail if test selection failed | ||
| if: needs.collect-tests.result != 'success' | ||
| run: exit 1 | ||
| - name: Checkout trusted controller code | ||
| uses: actions/checkout@v7 | ||
| with: | ||
| # Trusted base revision for every event: the merge-group commit is | ||
| # only ever fetched by exact SHA inside the isolated backend. | ||
| ref: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.sha }} | ||
| fetch-depth: 1 | ||
| persist-credentials: false | ||
| lfs: false | ||
| submodules: false | ||
| - name: Install Python | ||
| uses: actions/setup-python@v7 | ||
| with: | ||
| python-version: "3.10" | ||
| cache: 'pip' | ||
| - name: Download test selection | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: modal-torch-latest-test-selection | ||
| path: ci/.test_selection | ||
| - name: Install trusted controller dependency | ||
| run: | | ||
| pip install uv | ||
| uv pip install --system modal==1.2.6 | ||
| - name: Try tests in isolated Modal Sandbox | ||
| id: modal | ||
| env: | ||
| DS_CI_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | ||
| DS_CI_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | ||
| DS_CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || '' }} | ||
| DS_TEST_SELECTION_MODE: ${{ needs.collect-tests.outputs.mode }} | ||
| DS_TEST_LIST_FILE: ci/.test_selection/test_list.txt | ||
| MODAL_TORCH_PRESET: ${{ github.event.inputs.torch_preset || '2.10.0-cuda12.8' }} | ||
| MODAL_TRANSFORMERS_SOURCE: ${{ github.event.inputs.transformers_source || 'git' }} | ||
| MODAL_TRANSFORMERS_REF: ${{ github.event.inputs.transformers_ref || 'main' }} | ||
| MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} | ||
| MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} | ||
| run: | | ||
| set +e | ||
| python3 ci/torch_latest.py controller | ||
| status=$? | ||
| set -e | ||
| echo "status=$status" >> "$GITHUB_OUTPUT" | ||
| if [ "$status" -eq 0 ]; then | ||
| echo "fallback=false" >> "$GITHUB_OUTPUT" | ||
| exit 0 | ||
| fi | ||
| if [ "$status" -eq 75 ]; then | ||
| echo "fallback=true" >> "$GITHUB_OUTPUT" | ||
| exit 0 | ||
| fi | ||
| exit "$status" | ||
| - name: Configure AWS credentials for capacity fallback | ||
| if: steps.modal.outputs.fallback == 'true' | ||
| env: | ||
| AWS_ROLE_ARN: ${{ secrets.AWS_MODAL_FALLBACK_ROLE_ARN }} | ||
| run: | | ||
| set -euo pipefail | ||
| test -n "$AWS_ROLE_ARN" | ||
| echo "::add-mask::$AWS_ROLE_ARN" | ||
| oidc_response=$(curl --fail --silent --show-error \ | ||
| --header "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ | ||
| "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com") | ||
| web_identity_token=$(jq -er '.value | select(type == "string" and length > 0)' <<< "$oidc_response") | ||
| echo "::add-mask::$web_identity_token" | ||
| read -r aws_access_key_value aws_secret_key_value aws_session_token_value <<< "$( | ||
| aws --no-cli-pager sts assume-role-with-web-identity \ | ||
| --role-arn "$AWS_ROLE_ARN" \ | ||
| --role-session-name "DeepSpeedCI-${GITHUB_RUN_ID}" \ | ||
| --web-identity-token "$web_identity_token" \ | ||
| --duration-seconds 7200 \ | ||
| --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \ | ||
| --output text | ||
| )" | ||
| test -n "$aws_access_key_value" | ||
| test -n "$aws_secret_key_value" | ||
| test -n "$aws_session_token_value" | ||
| echo "::add-mask::$aws_access_key_value" | ||
| echo "::add-mask::$aws_secret_key_value" | ||
| echo "::add-mask::$aws_session_token_value" | ||
| { | ||
| printf 'AWS_ACCESS_KEY_ID=%s\n' "$aws_access_key_value" | ||
| printf 'AWS_SECRET_ACCESS_KEY=%s\n' "$aws_secret_key_value" | ||
| printf 'AWS_SESSION_TOKEN=%s\n' "$aws_session_token_value" | ||
| printf 'AWS_REGION=us-east-1\n' | ||
| printf 'AWS_DEFAULT_REGION=us-east-1\n' | ||
| } >> "$GITHUB_ENV" | ||
| - name: Run tests on AWS capacity fallback | ||
| if: steps.modal.outputs.fallback == 'true' | ||
| env: | ||
| DS_CI_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | ||
| DS_CI_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | ||
| DS_CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || '' }} | ||
| DS_TEST_SELECTION_MODE: ${{ needs.collect-tests.outputs.mode }} | ||
| DS_TEST_LIST_FILE: ci/.test_selection/test_list.txt | ||
| MODAL_TORCH_PRESET: ${{ github.event.inputs.torch_preset || '2.10.0-cuda12.8' }} | ||
| MODAL_TRANSFORMERS_SOURCE: ${{ github.event.inputs.transformers_source || 'git' }} | ||
| MODAL_TRANSFORMERS_REF: ${{ github.event.inputs.transformers_ref || 'main' }} | ||
| AWS_MODAL_FALLBACK_CONFIG: ${{ secrets.AWS_MODAL_FALLBACK_CONFIG }} | ||
| run: python3 ci/torch_latest.py aws-controller | ||
| - name: Reconcile AWS capacity fallback cleanup | ||
| if: ${{ always() && steps.modal.outputs.fallback == 'true' }} | ||
| env: | ||
| AWS_MODAL_FALLBACK_CONFIG: ${{ secrets.AWS_MODAL_FALLBACK_CONFIG }} | ||
| GITHUB_RUN_ID: ${{ github.run_id }} | ||
| GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} | ||
| run: python3 ci/torch_latest.py cleanup-aws | ||
