Repository navigation
Storage Concurrency Benchmark #56
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Storage Concurrency Benchmark | |
| # computesdk:secrets=NSC_TOKEN | |
| # computesdk:secrets-env=declared | |
| on: | |
| push: | |
| branches: [master] | |
| paths: | |
| - 'benchmarks/storage/storage-concurrency.*' | |
| - 'benchmarks/storage/report-storage-concurrency.ts' | |
| - 'benchmarks/storage/providers.ts' | |
| schedule: | |
| - cron: '0 13 * * 3' # Weekly on Wednesday at 13:00 UTC | |
| workflow_dispatch: | |
| inputs: | |
| provider: | |
| description: 'Optional provider name (for example aws-s3)' | |
| required: false | |
| default: '' | |
| type: string | |
| operations: | |
| description: 'GET operations per concurrency cell' | |
| required: false | |
| default: '1200' | |
| type: string | |
| seed_corpus: | |
| description: 'Seed missing corpus objects before benchmarking' | |
| required: false | |
| default: false | |
| type: boolean | |
| dry_run: | |
| description: 'Run without ingesting or committing results' | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: storage-concurrency-benchmarks | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| id-token: write | |
| jobs: | |
| benchmark: | |
| name: Compare storage providers | |
| runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe;permissions.additional_grant=vault/object:*:reveal | |
| # runs-on: self-hosted | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| # ComputeSDK runners get no GitHub OIDC token, so nscloud-setup's | |
| # `nsc auth exchange-github-token` cannot work there. Pre-seed the | |
| # workload token file it checks for when NSC_TOKEN is available; | |
| # accepts either the raw bearer token or a full token JSON document. | |
| # It also exports NSC_DOCKER_LOGIN/NSC_CONTAINER_REGISTRY so | |
| # nscloud-setup skips `nsc docker login`, which this token is not | |
| # granted for — the benchmarks only need vault access. | |
| - name: Seed Namespace token file | |
| run: | | |
| if [ -n "${NSC_TOKEN:-}" ] && [ ! -f /var/run/nsc/token.json ]; then | |
| mkdir -p /var/run/nsc | |
| case "$NSC_TOKEN" in | |
| \{*) printf '%s\n' "$NSC_TOKEN" > /var/run/nsc/token.json ;; | |
| *) printf '{"bearer_token":"%s"}\n' "$NSC_TOKEN" > /var/run/nsc/token.json ;; | |
| esac | |
| chmod 600 /var/run/nsc/token.json | |
| # The seeded token has no container-registry grant; tell | |
| # nscloud-setup to skip `nsc docker login` for later steps. | |
| echo "NSC_DOCKER_LOGIN=1" >> "$GITHUB_ENV" | |
| echo "NSC_CONTAINER_REGISTRY=nscr.io" >> "$GITHUB_ENV" | |
| fi | |
| - uses: namespacelabs/nscloud-setup@df198f982fcecfb8264bea3f1274b56a61b6dfdc # v0.0.12 | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile --ignore-scripts | |
| - name: Build workspace packages | |
| # The @benchsdk/* packages are built from source and dist/ is not | |
| # committed. Build them in topological order so dependents like | |
| # @benchsdk/client can resolve @benchsdk/worker during their own build. | |
| run: pnpm -r --filter './packages/**' run build | |
| - name: Run storage concurrency benchmark | |
| env: | |
| STORAGE_CONCURRENCY_PROVIDER: ${{ inputs.provider || '' }} | |
| STORAGE_CONCURRENCY_OPERATIONS: ${{ inputs.operations || '1200' }} | |
| STORAGE_CONCURRENCY_SEED: ${{ inputs.seed_corpus || 'false' }} | |
| BENCHSDK_NO_INGEST: ${{ (github.event_name == 'push' || github.event.inputs.dry_run == 'true') && '1' || '' }} | |
| run: | | |
| . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|BENCHMARKS_PLATFORM_API_KEY)' | |
| PROVIDER_ARGS=() | |
| if [ -n "$STORAGE_CONCURRENCY_PROVIDER" ]; then | |
| PROVIDER_ARGS=(--provider "$STORAGE_CONCURRENCY_PROVIDER") | |
| fi | |
| if [ "$STORAGE_CONCURRENCY_SEED" = "true" ]; then | |
| pnpm bench:storage-concurrency:seed -- "${PROVIDER_ARGS[@]}" | |
| fi | |
| pnpm bench:storage-concurrency -- "${PROVIDER_ARGS[@]}" | |
| - name: Upload combined results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: storage-concurrency-results | |
| path: results/storage-concurrency/ | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| collect: | |
| name: Collect Results | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| needs: benchmark | |
| if: always() | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 24 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile --ignore-scripts | |
| - name: Build workspace packages | |
| # The @benchsdk/* packages are built from source and dist/ is not | |
| # committed. Build them in topological order so dependents like | |
| # @benchsdk/client can resolve @benchsdk/worker during their own build. | |
| run: pnpm -r --filter './packages/**' run build | |
| - name: Download combined results | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| name: storage-concurrency-results | |
| path: artifacts/storage-concurrency/ | |
| - name: Generate comparison report | |
| run: | | |
| if [ ! -f artifacts/storage-concurrency/latest.json ]; then | |
| echo 'No storage concurrency results found; skipping report.' >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| if [ ! -f benchmarks/storage/report-storage-concurrency.ts ] \ | |
| || ! grep -q '"report:storage-concurrency"' package.json; then | |
| echo 'Storage concurrency report generator is not present on this ref; skipping report.' >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| mkdir -p results/storage-concurrency | |
| cp artifacts/storage-concurrency/latest.json results/storage-concurrency/latest.json | |
| pnpm report:storage-concurrency | |
| cat storage-concurrency.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload comparison report | |
| if: hashFiles('storage-concurrency.md', 'storage-concurrency.svg') != '' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: storage-concurrency-report | |
| path: | | |
| storage-concurrency.md | |
| storage-concurrency.svg | |
| results/storage-concurrency/latest.json | |
| retention-days: 14 |