evals.yml

name: Vally Evaluations

on:
  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'skills/**'
      - 'evals/**'
      - 'scripts/compile.js'
      - 'scripts/eval*.mjs'
      - 'package.json'
      - 'package-lock.json'
      - '.github/workflows/evals.yml'
  # Always report the aggregate check; skip expensive work for unrelated PRs.
  pull_request:
    types: [opened, reopened, synchronize]

permissions:
  contents: read

concurrency:
  group: vally-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    name: Detect evaluation changes
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      relevant: ${{ steps.filter.outputs.relevant }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: github.event_name == 'pull_request'
        with:
          persist-credentials: false
          fetch-depth: 2
      - name: Check evaluation inputs
        id: filter
        env:
          EVENT_NAME: ${{ github.event_name }}
        shell: bash
        run: |
          if [ "$EVENT_NAME" != "pull_request" ]; then
            echo "relevant=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          git diff --no-renames --name-only -z HEAD^1 HEAD -- > "$RUNNER_TEMP/vally-changed-files"
          relevant=false
          while IFS= read -r -d '' path; do
            case "$path" in
              skills/*|evals/*|scripts/compile.js|scripts/eval*.mjs|package.json|package-lock.json|.github/workflows/evals.yml)
                relevant=true
                break
                ;;
            esac
          done < "$RUNNER_TEMP/vally-changed-files"
          echo "relevant=$relevant" >> "$GITHUB_OUTPUT"

  validate:
    name: Validate specs and graders offline
    needs: changes
    if: needs.changes.outputs.relevant == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: '22.22.2'
          cache: npm
      - run: npm ci
      - run: npm run build
      - run: npm run eval:lint
      - run: npm run eval:test
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.11'
      - run: python -B -m unittest discover -s evals/cosmosdb-best-practices -p "test_*.py" -v

  eval:
    name: Run live evaluations
    needs: [changes, validate]
    # Fork and Dependabot PRs receive offline validation only.
    if: >-
      github.event_name != 'pull_request' ||
      (github.event.pull_request.head.repo.full_name == github.repository &&
       github.actor != 'dependabot[bot]' &&
       github.event.pull_request.user.login != 'dependabot[bot]')
    runs-on: ubuntu-latest
    timeout-minutes: 180
    permissions:
      contents: read
      actions: read
      copilot-requests: write
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: '22.22.2'
          cache: npm
      - run: npm ci
      - run: npm run build
      - name: Run Vally
        id: vally
        env:
          COPILOT_GITHUB_TOKEN: ${{ github.token }}
          GITHUB_TOKEN: ${{ github.token }}
        shell: bash
        run: |
          mkdir -p results
          npm run eval 2>&1 | tee results/vally.log
      - name: Publish failure summary
        if: always() && steps.vally.outcome != 'skipped'
        env:
          GH_TOKEN: ${{ github.token }}
          VALLY_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          VALLY_OUTCOME: ${{ steps.vally.outcome }}
          VALLY_LOG: results/vally.log
        shell: bash
        run: |
          if gh api --paginate --slurp \
            "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \
            > "$RUNNER_TEMP/vally-jobs.json"; then
            export VALLY_JOBS="$RUNNER_TEMP/vally-jobs.json"
          else
            echo "::warning::Could not read pipeline steps; linking to the workflow run instead."
          fi
          node scripts/eval-summary.mjs results
      - name: Upload reports and trajectories
        id: reports
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: vally-results-${{ github.run_attempt }}
          path: results/
          if-no-files-found: warn
          retention-days: 14
      - name: Link results artifact
        if: always() && steps.vally.outcome != 'skipped'
        env:
          ARTIFACT_URL: ${{ steps.reports.outputs.artifact-url }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        shell: bash
        run: |
          if [ -n "$ARTIFACT_URL" ]; then
            printf '\n[Download full results artifact](%s) (reports, grader evidence, trajectories, and logs).\n' \
              "$ARTIFACT_URL" >> "$GITHUB_STEP_SUMMARY"
          else
            echo "::warning::Results artifact is unavailable; check the upload step."
            printf '\n**Results artifact unavailable.** See the [workflow run](%s) for upload errors.\n' \
              "$RUN_URL" >> "$GITHUB_STEP_SUMMARY"
          fi

  gate:
    name: Vally evaluation gate
    needs: [changes, validate, eval]
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 5
    permissions:
      actions: read
    steps:
      - name: Report evaluation outcome
        env:
          GH_TOKEN: ${{ github.token }}
          CHANGES_RESULT: ${{ needs.changes.result }}
          RELEVANT: ${{ needs.changes.outputs.relevant }}
          VALIDATION_RESULT: ${{ needs.validate.result }}
          EVAL_RESULT: ${{ needs.eval.result }}
          OFFLINE_ONLY: >-
            ${{ github.event_name == 'pull_request' &&
                (github.event.pull_request.head.repo.full_name != github.repository ||
                 github.actor == 'dependabot[bot]' ||
                 github.event.pull_request.user.login == 'dependabot[bot]') }}
        shell: bash
        run: |
          fail() {
            echo "::error::$1"
            printf '%s\n' "$1" >> "$GITHUB_STEP_SUMMARY"
            if ! gh api --paginate \
              "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \
              --jq '.jobs[] | select(.conclusion == "failure") | . as $job | .steps[] | select(.conclusion == "failure") | "- [\($job.name) / \(.name)](\($job.html_url)#step:\(.number):1)"' \
              >> "$GITHUB_STEP_SUMMARY"; then
              echo "::warning::Could not link the failed pipeline steps."
            fi
            exit 1
          }
          [ "$CHANGES_RESULT" = success ] || fail "Change detection failed ($CHANGES_RESULT)."
          case "$RELEVANT" in
            false)
              echo "No evaluation inputs changed; offline and live evaluations skipped." >> "$GITHUB_STEP_SUMMARY"
              exit 0
              ;;
            true) ;;
            *) fail "Change detection returned an invalid decision." ;;
          esac
          [ "$VALIDATION_RESULT" = success ] || fail "Offline validation failed ($VALIDATION_RESULT)."
          if [ "$OFFLINE_ONLY" = true ]; then
            [ "$EVAL_RESULT" = skipped ] || fail "Live evaluations must be skipped for fork or Dependabot PRs."
            echo "Offline checks passed; live evaluations skipped for a fork or Dependabot PR." >> "$GITHUB_STEP_SUMMARY"
          else
            [ "$EVAL_RESULT" = success ] || fail "Live evaluations did not succeed ($EVAL_RESULT). See reports and artifacts."
            echo "Offline checks and live evaluations passed." >> "$GITHUB_STEP_SUMMARY"
          fi