pr-ai-validation.yml

name: PR AI Validation

on:
  pull_request_target:
    types: [opened, edited, synchronize, labeled, unlabeled]
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'PR number to validate (manual runs only)'
        required: false
        type: string

permissions:
  pull-requests: write
  issues: write
  contents: read
  copilot-requests: write

jobs:
  validate-pr:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    env:
      PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
      COPILOT_CLI_VERSION: 1.0.72-0
    steps:
      - name: Check if contributor is trusted
        uses: actions/github-script@v8
        with:
          script: |
            if (!context.payload.pull_request) {
              console.log('No pull request context (likely workflow_dispatch). Skipping.');
              return;
            }

            const authorAssociation = context.payload.pull_request.author_association;
            const trustedAssociations = ['COLLABORATOR', 'MEMBER', 'OWNER', 'CONTRIBUTOR'];

            if (trustedAssociations.includes(authorAssociation)) {
              console.log(`Trusted contributor (${authorAssociation}), proceeding...`);
              return;
            }

            const labels = context.payload.pull_request.labels.map(l => l.name);
            if (labels.includes('external-approved')) {
              console.log('External PR approved by maintainer, proceeding...');
              return;
            }

            console.log(`External contributor (${authorAssociation}) - requires approval`);
            const owner = context.repo.owner;
            const repo = context.repo.repo;
            const issue_number = context.payload.pull_request.number;
            const marker = '<!-- AI_VALIDATION_PENDING -->';
            const pendingBody = `${marker}\n## 🔒 AI Validation Pending\n\nThis PR is from an external contributor. AI validation will be performed after manual review.\n\nMaintainers: Add the \`external-approved\` label to enable AI validation.`;

            const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number, per_page: 100 });
            if (!comments.find(c => c.body && c.body.includes(marker))) {
              await github.rest.issues.createComment({ owner, repo, issue_number, body: pendingBody });
            }
            core.setFailed('External contributor - requires manual approval');

      - name: Checkout for prompt access
        uses: actions/checkout@v5
        with:
          # Keep the workflow and its regression test on the existing trusted event ref, never the PR head.
          sparse-checkout: |
            prompts/copilot-pr-desc-check.md
            .github/workflows/pr-ai-validation.yml
            scripts/pr-ai-validation.test.mjs
          sparse-checkout-cone-mode: false

      - name: Test PR validation data gathering
        run: node --test scripts/pr-ai-validation.test.mjs

      - name: Cache Copilot CLI download
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: copilot-cli-${{ runner.os }}-${{ env.COPILOT_CLI_VERSION }}

      - name: Install Copilot CLI
        shell: bash
        run: |
          # @github/copilot is a thin loader; the real CLI is a large platform-native binary shipped as
          # an optionalDependency. npm's optional-dep install is best-effort, so a transient registry/network
          # blip can leave the binary missing/incomplete while npm still exits 0. Harden the install so a
          # broken CLI is caught here rather than mid-validation:
          #   - pin the version (reproducible; the ~/.npm cache above serves it offline on the happy path);
          #   - retry the install and verify the binary actually runs (copilot --version) before proceeding;
          #   - clear the npm cache between retries so a poisoned/partial artifact is not reused.
          # (The oversized-prompt exit 126 is handled separately in the validation step via stdin.)
          install_ok=0
          for attempt in 1 2 3; do
            if npm install -g "@github/copilot@${COPILOT_CLI_VERSION}" && copilot --version >/dev/null 2>&1; then
              install_ok=1; break
            fi
            echo "::warning::Copilot CLI install/verify attempt ${attempt} failed; clearing cache and retrying..."
            npm cache clean --force >/dev/null 2>&1 || true
            sleep $((attempt * 5))
          done
          if [ "$install_ok" != "1" ]; then
            echo "::warning::Copilot CLI could not be verified after 3 attempts; validation will fall back."
          fi

      - name: Gather PR data
        id: gather
        uses: actions/github-script@v8
        with:
          script: |
            const fs = require('fs');
            let pr = context.payload.pull_request;
            if (!pr) {
              const num = Number(process.env.PR_NUMBER);
              if (!num) { core.setFailed('No pull_request context and no pr_number input provided.'); return; }
              pr = (await github.rest.pulls.get({
                owner: context.repo.owner, repo: context.repo.repo, pull_number: num
              })).data;
            }
            const prData = {
              number: pr.number,
              title: pr.title,
              body: pr.body || '',
              labels: pr.labels.map(l => l.name),
              head_ref: pr.head.ref,
              author: pr.user.login,
            };
            fs.writeFileSync('/tmp/pr-meta.json', JSON.stringify(prData, null, 2));

            let diff;
            try {
              ({ data: diff } = await github.rest.pulls.get({
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: pr.number,
                mediaType: { format: 'diff' }
              }));
            } catch (error) {
              const errors = error?.response?.data?.errors;
              if (error?.status !== 406 || !Array.isArray(errors) || !errors.some(
                item => item?.resource === 'PullRequest' && item?.field === 'diff' && item?.code === 'too_large'
              )) {
                throw error;
              }
              // GitHub rejects oversized raw diffs before local sampling; use API patches, never execute PR code.
              core.warning('Raw PR diff exceeds GitHub limits; sampling paginated file patches instead.');
              const lines = [];
              pages: for await (const { data: files } of github.paginate.iterator(github.rest.pulls.listFiles, {
                owner: context.repo.owner, repo: context.repo.repo, pull_number: pr.number, per_page: 100
              })) {
                for (const file of files) {
                  const unavailable = `[Patch unavailable: ${file.filename} (${file.status}, +${file.additions}/-${file.deletions})]`;
                  if (!file.patch) {
                    core.warning(unavailable);
                  }
                  const header = `diff --git a/${file.previous_filename || file.filename} b/${file.filename}`;
                  lines.push(...`${header}\n${file.patch || unavailable}`.split('\n').slice(0, 4000 - lines.length));
                  // Stop requesting pages as soon as the existing sample budget is filled.
                  if (lines.length === 4000) {
                    break pages;
                  }
                }
              }
              if (!lines.length) {
                throw new Error('No files returned for the oversized PR diff.');
              }
              diff = lines.join('\n');
            }
            const diffLines = (typeof diff === 'string' ? diff : '').split('\n').slice(0, 4000).join('\n');
            fs.writeFileSync('/tmp/pr-diff.txt', diffLines);
            console.log(`PR #${pr.number}: ${pr.title} (${prData.labels.join(', ')})`);

      - name: Run Copilot CLI validation
        id: validate
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        shell: bash
        run: |
          SYSTEM_PROMPT=$(cat prompts/copilot-pr-desc-check.md 2>/dev/null || echo "Validate this PR description.")
          PR_META=$(cat /tmp/pr-meta.json)
          # Cap the diff sample by BYTES as well as lines: `head -2000` bounds line count but not
          # size, and recompiled/generated files can have very long single lines that bloat the
          # prompt (token cost + latency). 100 KB keeps the full prompt comfortably bounded.
          # Separate reads avoid producer SIGPIPE under Actions' strict Bash pipefail handling.
          PR_DIFF_LINES=$(mktemp "$RUNNER_TEMP/pr-diff-lines.XXXXXX")
          head -2000 /tmp/pr-diff.txt > "$PR_DIFF_LINES"
          PR_DIFF=$(head -c 100000 "$PR_DIFF_LINES")
          rm "$PR_DIFF_LINES"

          # Write combined prompt to file (avoids shell escaping issues)
          {
            echo "$SYSTEM_PROMPT"
            echo ""
            echo "---"
            echo ""
            echo "## PR Metadata"
            echo "$PR_META"
            echo ""
            echo "## Diff (sample)"
            echo "$PR_DIFF"
            echo ""
            echo "Validate this PR against the rules. Return JSON only."
          } > /tmp/full-prompt.txt

          set +e
          MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5-mini}"
          RESULT=""; CLI_EXIT=1
          for MODEL in $MODELS; do
            for try in 1 2; do
              # Pass the prompt via stdin, NOT as a `-p "$(...)"` argv. On Linux a single argument is
              # capped at MAX_ARG_STRLEN (128 KB); large-diff PRs push the combined prompt past that,
              # making execve fail with E2BIG which surfaces as an instant exit 126 with no output.
              # stdin has no such limit, so this validates PRs of any diff size.
              RESULT=$(timeout 90 copilot --yolo \
                --model "$MODEL" < /tmp/full-prompt.txt 2>&1)
              CLI_EXIT=$?
              if [ $CLI_EXIT -eq 0 ] && [ -n "$RESULT" ]; then break 2; fi
              echo "::warning::Copilot model $MODEL attempt $try failed (exit $CLI_EXIT)"
              # 126/127 = binary could not be executed (transient install/exec flake) — brief backoff and retry.
              # Any other non-zero exit is deterministic for this model, so move on to the next one.
              if [ $CLI_EXIT -eq 126 ] || [ $CLI_EXIT -eq 127 ]; then sleep 5; else break; fi
            done
          done
          set -e

          if [ $CLI_EXIT -ne 0 ] || [ -z "$RESULT" ]; then
            echo "cli_success=false" >> "$GITHUB_OUTPUT"
            echo "::warning::Copilot CLI unavailable (last exit $CLI_EXIT) — using fallback"
            echo "----- Copilot CLI output (tail, for debugging) -----"
            printf '%s\n' "$RESULT" | tail -n 25
            echo "----------------------------------------------------"
          else
            echo "$RESULT" | python3 -c "
          import sys, json
          text = sys.stdin.read()
          # Find JSON in output
          start = text.find('{')
          end = text.rfind('}') + 1
          if start >= 0 and end > start:
              try:
                  data = json.loads(text[start:end])
                  json.dump(data, open('/tmp/validation-result.json', 'w'))
                  print('cli_success=true')
              except:
                  print('cli_success=false')
          else:
              print('cli_success=false')
          " >> "$GITHUB_OUTPUT"
          fi

      - name: Fallback to Logic App
        if: steps.validate.outputs.cli_success != 'true'
        id: fallback
        uses: actions/github-script@v8
        env:
          LOGIC_APP_URL: ${{ vars.PR_VALIDATION_LOGIC_APP_URL }}
        with:
          script: |
            const pr_number = Number(process.env.PR_NUMBER);
            if (!process.env.LOGIC_APP_URL) {
              console.log('No Logic App URL configured either. Skipping validation.');
              return;
            }
            try {
              const response = await fetch(process.env.LOGIC_APP_URL, {
                method: 'POST',
                headers: { 'Content-Type': 'application/json' },
                body: JSON.stringify({ pr_id: pr_number })
              });
              if (!response.ok) {
                // Validator infrastructure is unavailable (e.g. 502/503/504). Validation could not run,
                // which is not a defect in the PR — do not block it and do not post a false failure.
                // The check fails only on an actual negative verdict (see "Post result and manage labels").
                console.log(`Logic App unavailable (status ${response.status}). Skipping validation (non-blocking).`);
                return;
              }
              const result = await response.json();
              const fs = require('fs');
              fs.writeFileSync('/tmp/validation-result.json', JSON.stringify(result));
            } catch (error) {
              // Network error reaching the fallback — infrastructure unavailable, not a PR problem. Skip, don't block.
              console.log(`Logic App fallback unreachable: ${error.message}. Skipping validation (non-blocking).`);
              return;
            }

      - name: Post result and manage labels
        if: always()
        uses: actions/github-script@v8
        env:
          CLI_SUCCESS: ${{ steps.validate.outputs.cli_success }}
        with:
          script: |
            const fs = require('fs');
            const pr_number = Number(process.env.PR_NUMBER);
            if (!pr_number) return;

            const botCommentIdentifier = '## 🤖 AI PR Validation Report';
            let result;
            try {
              result = JSON.parse(fs.readFileSync('/tmp/validation-result.json', 'utf8'));
            } catch (e) {
              console.log('No validation result. Skipping.');
              return;
            }

            if (!result.hasOwnProperty('passes') || !result.hasOwnProperty('message')) {
              console.log('Invalid result format.');
              return;
            }

            const source = process.env.CLI_SUCCESS === 'true' ? 'Copilot CLI (claude-opus-4.8)' : 'Logic App (fallback)';
            const commentBody = `${botCommentIdentifier}\n\n${result.message}\n\n---\n*Powered by: ${source} | Last updated: ${new Date().toUTCString()}*`;

            const comments = await github.rest.issues.listComments({
              owner: context.repo.owner, repo: context.repo.repo, issue_number: pr_number
            });
            const botComment = comments.data.find(c => c.user.type === 'Bot' && c.body.includes(botCommentIdentifier));

            if (botComment) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner, repo: context.repo.repo,
                comment_id: botComment.id, body: commentBody
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner, repo: context.repo.repo,
                issue_number: pr_number, body: commentBody
              });
            }

            if (result.passes === true) {
              try { await github.rest.issues.addLabels({ owner: context.repo.owner, repo: context.repo.repo, issue_number: pr_number, labels: ['pr-validated'] }); } catch (e) {}
              for (const label of ['needs-pr-update', 'needs-title-update', 'needs-description-update']) {
                try { await github.rest.issues.removeLabel({ owner: context.repo.owner, repo: context.repo.repo, issue_number: pr_number, name: label }); } catch (e) {}
              }
            } else {
              try { await github.rest.issues.addLabels({ owner: context.repo.owner, repo: context.repo.repo, issue_number: pr_number, labels: ['needs-pr-update'] }); } catch (e) {}
              try { await github.rest.issues.removeLabel({ owner: context.repo.owner, repo: context.repo.repo, issue_number: pr_number, name: 'pr-validated' }); } catch (e) {}
              core.setFailed('PR validation failed. See the validation report comment.');
            }