storage-concurrency-benchmarks.yml

name: Storage Concurrency Benchmark

# computesdk:secrets=NSC_TOKEN
# computesdk:secrets-env=declared

on:
  push:
    branches: [master]
    paths:
      - 'benchmarks/storage/storage-concurrency.*'
      - 'benchmarks/storage/report-storage-concurrency.ts'
      - 'benchmarks/storage/providers.ts'
  schedule:
    - cron: '0 13 * * 3' # Weekly on Wednesday at 13:00 UTC
  workflow_dispatch:
    inputs:
      provider:
        description: 'Optional provider name (for example aws-s3)'
        required: false
        default: ''
        type: string
      operations:
        description: 'GET operations per concurrency cell'
        required: false
        default: '1200'
        type: string
      seed_corpus:
        description: 'Seed missing corpus objects before benchmarking'
        required: false
        default: false
        type: boolean
      dry_run:
        description: 'Run without ingesting or committing results'
        required: false
        default: false
        type: boolean

concurrency:
  group: storage-concurrency-benchmarks
  cancel-in-progress: true

permissions:
  contents: read
  id-token: write

jobs:
  benchmark:
    name: Compare storage providers
    runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe;permissions.additional_grant=vault/object:*:reveal
    # runs-on: self-hosted
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: pnpm/action-setup@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 24
          cache: 'pnpm'
      # ComputeSDK runners get no GitHub OIDC token, so nscloud-setup's
      # `nsc auth exchange-github-token` cannot work there. Pre-seed the
      # workload token file it checks for when NSC_TOKEN is available;
      # accepts either the raw bearer token or a full token JSON document.
      # It also exports NSC_DOCKER_LOGIN/NSC_CONTAINER_REGISTRY so
      # nscloud-setup skips `nsc docker login`, which this token is not
      # granted for — the benchmarks only need vault access.
      - name: Seed Namespace token file
        run: |
          if [ -n "${NSC_TOKEN:-}" ] && [ ! -f /var/run/nsc/token.json ]; then
            mkdir -p /var/run/nsc
            case "$NSC_TOKEN" in
              \{*) printf '%s\n' "$NSC_TOKEN" > /var/run/nsc/token.json ;;
              *) printf '{"bearer_token":"%s"}\n' "$NSC_TOKEN" > /var/run/nsc/token.json ;;
            esac
            chmod 600 /var/run/nsc/token.json
            # The seeded token has no container-registry grant; tell
            # nscloud-setup to skip `nsc docker login` for later steps.
            echo "NSC_DOCKER_LOGIN=1" >> "$GITHUB_ENV"
            echo "NSC_CONTAINER_REGISTRY=nscr.io" >> "$GITHUB_ENV"
          fi
      - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12
      - name: Install dependencies
        run: pnpm install --frozen-lockfile --ignore-scripts
      - name: Build workspace packages
        # The @benchsdk/* packages are built from source and dist/ is not
        # committed. Build them in topological order so dependents like
        # @benchsdk/client can resolve @benchsdk/worker during their own build.
        run: pnpm -r --filter './packages/**' run build
      - name: Run storage concurrency benchmark
        env:
          STORAGE_CONCURRENCY_PROVIDER: ${{ inputs.provider || '' }}
          STORAGE_CONCURRENCY_OPERATIONS: ${{ inputs.operations || '1200' }}
          STORAGE_CONCURRENCY_SEED: ${{ inputs.seed_corpus || 'false' }}
          BENCHSDK_NO_INGEST: ${{ (github.event_name == 'push' || github.event.inputs.dry_run == 'true') && '1' || '' }}
        run: |
          . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|BENCHMARKS_PLATFORM_API_KEY)'
          PROVIDER_ARGS=()
          if [ -n "$STORAGE_CONCURRENCY_PROVIDER" ]; then
            PROVIDER_ARGS=(--provider "$STORAGE_CONCURRENCY_PROVIDER")
          fi

          if [ "$STORAGE_CONCURRENCY_SEED" = "true" ]; then
            pnpm bench:storage-concurrency:seed -- "${PROVIDER_ARGS[@]}"
          fi

          pnpm bench:storage-concurrency -- "${PROVIDER_ARGS[@]}"
      - name: Upload combined results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: storage-concurrency-results
          path: results/storage-concurrency/
          if-no-files-found: ignore
          retention-days: 14

  collect:
    name: Collect Results
    runs-on: ubuntu-latest
    permissions:
      contents: read
    needs: benchmark
    if: always()
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: pnpm/action-setup@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 24
          cache: 'pnpm'
      - name: Install dependencies
        run: pnpm install --frozen-lockfile --ignore-scripts
      - name: Build workspace packages
        # The @benchsdk/* packages are built from source and dist/ is not
        # committed. Build them in topological order so dependents like
        # @benchsdk/client can resolve @benchsdk/worker during their own build.
        run: pnpm -r --filter './packages/**' run build
      - name: Download combined results
        uses: actions/download-artifact@v4
        continue-on-error: true
        with:
          name: storage-concurrency-results
          path: artifacts/storage-concurrency/
      - name: Generate comparison report
        run: |
          if [ ! -f artifacts/storage-concurrency/latest.json ]; then
            echo 'No storage concurrency results found; skipping report.' >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          if [ ! -f benchmarks/storage/report-storage-concurrency.ts ] \
            || ! grep -q '"report:storage-concurrency"' package.json; then
            echo 'Storage concurrency report generator is not present on this ref; skipping report.' >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          mkdir -p results/storage-concurrency
          cp artifacts/storage-concurrency/latest.json results/storage-concurrency/latest.json
          pnpm report:storage-concurrency
          cat storage-concurrency.md >> "$GITHUB_STEP_SUMMARY"
      - name: Upload comparison report
        if: hashFiles('storage-concurrency.md', 'storage-concurrency.svg') != ''
        uses: actions/upload-artifact@v4
        with:
          name: storage-concurrency-report
          path: |
            storage-concurrency.md
            storage-concurrency.svg
            results/storage-concurrency/latest.json
          retention-days: 14