Add sandbox reliability and feature matrix benchmark

.github/workflows/sandbox-features.yml

@@ -0,0 +1,208 @@
+name: Sandbox Feature Matrix
+
+on:
+  pull_request:
+    paths:
+      - 'src/sandbox/**'
+      - 'src/util/**'
+      - 'src/run.ts'
+      - 'src/merge-results.ts'
+      - '.github/workflows/sandbox-features.yml'
+      - 'package.json'
+  schedule:
+    - cron: '0 6 * * *'
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  bench:
+    name: Features ${{ matrix.provider }}
+    runs-on: namespace-profile-default
+    timeout-minutes: 25
+    permissions:
+      contents: read
+    concurrency:
+      group: sandbox-features-${{ github.event_name }}-${{ github.ref }}-${{ matrix.provider }}
+      cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        provider:
+          - archil
+          - blaxel
+          - cloudflare
+          - codesandbox
+          - daytona
+          - declaw
+          - e2b
+          - hopx
+          - isorun
+          - modal
+          - northflank
+          - runloop
+          - tensorlake
+          - upstash
+          - vercel
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+      - name: Install dependencies
+        run: npm ci
+      - name: Clear stale results from checkout
+        run: rm -rf results/
+      - name: Run feature matrix benchmark
+        env:
+          COMPUTESDK_API_KEY: ${{ github.event_name != 'pull_request' && secrets.COMPUTESDK_API_KEY || '' }}
+          ARCHIL_API_KEY: ${{ github.event_name != 'pull_request' && secrets.ARCHIL_API_KEY || '' }}
+          ARCHIL_REGION: ${{ github.event_name != 'pull_request' && secrets.ARCHIL_REGION || '' }}
+          ARCHIL_DISK_ID: ${{ github.event_name != 'pull_request' && secrets.ARCHIL_DISK_ID || '' }}
+          BL_API_KEY: ${{ github.event_name != 'pull_request' && secrets.BL_API_KEY || '' }}
+          BL_WORKSPACE: ${{ github.event_name != 'pull_request' && secrets.BL_WORKSPACE || '' }}
+          CLOUDFLARE_SANDBOX_URL: ${{ github.event_name != 'pull_request' && secrets.CLOUDFLARE_SANDBOX_URL || '' }}
+          CLOUDFLARE_SANDBOX_SECRET: ${{ github.event_name != 'pull_request' && secrets.CLOUDFLARE_SANDBOX_SECRET || '' }}
+          CSB_API_KEY: ${{ github.event_name != 'pull_request' && secrets.CSB_API_KEY || '' }}
+          DAYTONA_API_KEY: ${{ github.event_name != 'pull_request' && secrets.DAYTONA_API_KEY || '' }}
+          DECLAW_API_KEY: ${{ github.event_name != 'pull_request' && secrets.DECLAW_API_KEY || '' }}
+          E2B_API_KEY: ${{ github.event_name != 'pull_request' && secrets.E2B_API_KEY || '' }}
+          HOPX_API_KEY: ${{ github.event_name != 'pull_request' && secrets.HOPX_API_KEY || '' }}
+          ISORUN_API_KEY: ${{ github.event_name != 'pull_request' && secrets.ISORUN_API_KEY || '' }}
+          MODAL_TOKEN_ID: ${{ github.event_name != 'pull_request' && secrets.MODAL_TOKEN_ID || '' }}
+          MODAL_TOKEN_SECRET: ${{ github.event_name != 'pull_request' && secrets.MODAL_TOKEN_SECRET || '' }}
+          NORTHFLANK_TOKEN: ${{ github.event_name != 'pull_request' && secrets.NORTHFLANK_TOKEN || '' }}
+          NORTHFLANK_PROJECT_ID: ${{ github.event_name != 'pull_request' && secrets.NORTHFLANK_PROJECT_ID || '' }}
+          RUNLOOP_API_KEY: ${{ github.event_name != 'pull_request' && secrets.RUNLOOP_API_KEY || '' }}
+          TENSORLAKE_API_KEY: ${{ github.event_name != 'pull_request' && secrets.TENSORLAKE_API_KEY || '' }}
+          UPSTASH_BOX_API_KEY: ${{ github.event_name != 'pull_request' && secrets.UPSTASH_BOX_API_KEY || '' }}
+          VERCEL_TOKEN: ${{ github.event_name != 'pull_request' && secrets.VERCEL_TOKEN || '' }}
+          VERCEL_TEAM_ID: ${{ github.event_name != 'pull_request' && secrets.VERCEL_TEAM_ID || '' }}
+          VERCEL_PROJECT_ID: ${{ github.event_name != 'pull_request' && secrets.VERCEL_PROJECT_ID || '' }}
+        run: |
+          npm run bench:features -- --provider ${{ matrix.provider }}
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sandbox-features-${{ matrix.provider }}
+          path: results/
+          if-no-files-found: ignore
+          retention-days: 7
+
+  collect:
+    name: Collect Feature Matrix Results
+    runs-on: namespace-profile-default
+    needs: bench
+    if: always()
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: 'npm'
+      - name: Install dependencies
+        run: npm ci
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts/
+          pattern: sandbox-features-*
+      - name: Merge results
+        run: npx tsx src/merge-results.ts --input artifacts --mode sandbox-features
+      - name: Post results to PR
+        if: github.event_name == 'pull_request'
+        continue-on-error: true
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const latestPath = path.join('results', 'sandbox-features', 'latest.json');
+            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            let body = '## Sandbox Feature Matrix Results\n\n';
+
+            if (!fs.existsSync(latestPath)) {
+              body += '> No sandbox feature matrix results were generated.\n\n';
+            } else {
+              const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8'));
+              const results = data.results
+                .filter(r => !r.skipped)
+                .sort((a, b) => a.provider.localeCompare(b.provider));
+
+              if (results.length === 0) {
+                body += '> No sandbox feature matrix results were generated.\n\n';
+              } else {
+                body += '| Provider | Features | Node | Python | Docker | TCP | UDP | FS IO | CPU Ops/s |\n';
+                body += '|----------|----------|------|--------|--------|-----|-----|-------|-----------|\n';
+
+                for (const r of results) {
+                  const features = r.summary?.featureMatrix || {};
+                  const supported = Object.values(features).filter(f => f.supportRate === 1).length;
+                  const total = Object.keys(features).length;
+                  const probes = r.iterations?.[0]?.probes || {};
+                  const mark = (value) => value === true ? 'yes' : value === false ? 'no' : '--';
+                  const fsIo = probes.fsWrite1MbMs && probes.fsRead1MbMs ? `${probes.fsWrite1MbMs}/${probes.fsRead1MbMs}ms` : '--';
+                  const cpu = probes.cpuSha256OpsPerSecond ? Math.round(probes.cpuSha256OpsPerSecond).toLocaleString() : '--';
+                  body += `| ${r.provider} | ${supported}/${total} | ${mark(probes.hasNode)} | ${mark(probes.hasPython)} | ${mark(probes.dockerUsable)} | ${mark(probes.tcpExample)} | ${mark(probes.udpDns)} | ${fsIo} | ${cpu} |\n`;
+                }
+
+                body += '\n';
+              }
+            }
+
+            body += `---\n*[View full run](${runUrl}) · Results available as build artifacts*`;
+
+            const marker = '## Sandbox Feature Matrix Results';
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const existing = comments.find(c => c.body.startsWith(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body,
+              });
+            }
+      - name: Commit and push
+        if: github.event_name != 'pull_request'
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add results/sandbox-features/
+          git diff --cached --quiet && echo "No changes to commit" && exit 0
+          git commit -m "chore: update sandbox feature matrix results [skip ci]"
+
+          branch="${GITHUB_REF#refs/heads/}"
+          for attempt in 1 2 3 4 5; do
+            git fetch origin "${branch}"
+            git rebase "origin/${branch}" || { git rebase --abort; exit 1; }
+            if git push origin "HEAD:${branch}"; then
+              echo "Pushed on attempt ${attempt}"
+              exit 0
+            fi
+            echo "Push rejected (attempt ${attempt}); will rebase and retry"
+          done
+          echo "Failed to push after multiple attempts" >&2
+          exit 1