Split deterministic v1 tests from live E2Es

.github/workflows/test.yml

@@ -19,13 +19,14 @@ on:
 env:
   HF_HOME: ${{ github.workspace }}/.cache/huggingface
 
+permissions:
+  contents: read
+
 jobs:
   test:
     name: Verifiers
     runs-on: ubuntu-latest
     env:
-      PRIME_API_KEY: ${{ secrets.PRIME_API_KEY }}
-      PRIME_TEAM_ID: ${{ secrets.PRIME_TEAM_ID }}
       HF_TOKEN: ${{ secrets.HF_TOKEN }}
     strategy:
       fail-fast: false
@@ -56,13 +57,51 @@ jobs:
       - name: Install dependencies
         run: uv sync
 
-      - name: Install prime
-        run: uv pip install prime
-
       - name: Run tests
         run: |
           uv run pytest tests/ -v --ignore=tests/v1 -m "not prime" --cov=verifiers --cov-report=xml --cov-report=term
 
-      - name: Run v1 tests
+      - name: Run deterministic v1 tests
+        run: |
+          uv run pytest tests/v1 -vv -n auto -m "not e2e" --cov=verifiers --cov-append --cov-report=xml --cov-report=term
+
+  v1-e2e:
+    name: V1 live E2E (Python 3.12)
+    if: >-
+      github.event_name != 'pull_request' ||
+      (github.event.pull_request.head.repo.full_name == github.repository &&
+      github.event.pull_request.user.login != 'dependabot[bot]')
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    env:
+      PRIME_API_KEY: ${{ secrets.PRIME_API_KEY }}
+      PRIME_TEAM_ID: ${{ secrets.PRIME_TEAM_ID }}
+      HF_TOKEN: ${{ secrets.HF_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Cache Hugging Face tokenizers
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.HF_HOME }}
+          key: hf-tokenizers-${{ runner.os }}-${{ hashFiles('uv.lock', 'tests/test_renderer_client.py', 'tests/test_renderer_e2e.py') }}
+          restore-keys: |
+            hf-tokenizers-${{ runner.os }}-
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run live v1 E2Es
         run: |
-          uv run pytest tests/v1 -vv -n auto -m "not prime and not modal" --cov=verifiers --cov-append --cov-report=xml --cov-report=term
+          uv run pytest tests/v1 -vv -n auto -m "e2e and not prime and not modal" --cov=verifiers --cov-report=xml --cov-report=term

tests/v1/conftest.py

@@ -17,7 +17,8 @@
 Every combination carries its axes' pytest marks, so subsets select with `-m`:
 
     uv run pytest tests/v1 -n auto                                # everything (needs modal setup)
-    uv run pytest tests/v1 -n auto -m "not prime and not modal"  # the CI set (host + docker only)
+    uv run pytest tests/v1 -n auto -m "not e2e"                   # deterministic CI matrix
+    uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal"  # live CI job
     uv run pytest tests/v1 -n auto -m docker                      # any case touching the docker runtime
     uv run pytest tests/v1 -n auto -m bash                        # only the bash harness
     uv run pytest tests/v1 -n auto -m prime                       # only prime (real sandboxes; local)
@@ -27,8 +28,8 @@
 harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `codex` / `claude_code`.
 A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact
 combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`).
-prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only
-— CI runs `-m "not prime and not modal"`.
+prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
+CI runs deterministic tests across the Python matrix and the remaining live E2Es once.
 """
 
 import os