From 04ab36bfc4bdabd4a4e20e3b5e847c00ff40def6 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 5 Oct 2026 11:43:29 -0700 Subject: [PATCH 1/4] docs: propose Playwright-native browser benchmarks --- PLAYWRIGHT_NATIVE_BENCHMARKS.md | 250 ++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 PLAYWRIGHT_NATIVE_BENCHMARKS.md diff --git a/PLAYWRIGHT_NATIVE_BENCHMARKS.md b/PLAYWRIGHT_NATIVE_BENCHMARKS.md new file mode 100644 index 00000000..56afb48d --- /dev/null +++ b/PLAYWRIGHT_NATIVE_BENCHMARKS.md @@ -0,0 +1,250 @@ +# Proposal: Playwright-native browser benchmarks + +This proposal requests ComputeSDK maintainer feedback before implementation, +following Noah's request for a proposal PR. It proposes two separate benchmarks +over `chromium.connect()`: **browser readiness** and **action throughput**. +It contains no benchmark results or performance claims. + +## Motivation and scope + +Providers exposing Playwright-native endpoints should be measurable alongside +the providers served by the existing CDP benchmarks. Keep the CDP suites, +identities, methodology, and historical results intact; give the new suites +separate identities and result series. This is a provider comparison under a +specified workload, not evidence that either protocol is inherently faster. + +[CONTRIBUTING.md](./CONTRIBUTING.md) asks contributors to discuss methodology +and wait for maintainer feedback before implementing changes, because historical +comparability matters. This documentation-only PR supplies that discussion for +approval or revision. It changes no APIs, types, dependencies, workflows, or +runtime code. + +## Initial participants and disclosure + +Propose **Momentic** and **Azure Playwright Workspaces** initially. Other providers +may participate if they expose a Playwright-native endpoint compatible with +`chromium.connect()` and can meet the same workload, settings, and lifecycle +requirements. A CDP endpoint alone does not qualify for these suites. + +The proposer is affiliated with Momentic. Momentic offers to fund Azure usage for +the comparison; the billing arrangement remains to be agreed before execution. +Funding does not determine inclusion, methodology, or interpretation of results. +Provider participation and sponsorship remain separate, as required by the +contribution guidance. + +## Execution and environment + +- Run **100 fresh sessions per provider per suite**, with `iterations: 100`, + `concurrency: 1`, and `groupBy: 'round'`. One runner executes one session at a + time: Momentic round 1, Azure round 1, Momentic round 2, Azure round 2, and so on. + Record provider order. Do not substitute parallel provider matrix jobs for + this initial interleaved comparison. +- Use separate benchmark slugs, run IDs/keys, and result series for readiness + and throughput. Do not reuse sessions between suites or rounds. Fresh means a + new logical provider session, context, and page, not a guarantee of a cold + physical host or empty provider cache. +- Require both providers' browsers to execute in the **same Azure region** and + keep the runner location fixed for the entire initial comparison. Agree the + exact region and runner location before implementation. Record the actual + browser regions, runner region/location, routing configuration, and evidence + of effective placement; a workspace or requested region alone is insufficient + if routing can select another region. +- Use Linux Chromium in headless mode and an explicitly configured + **1920 × 1080 viewport** in every fresh context. Fix other context settings + equally across participants and record them. Disable optional stealth, proxy, + video, trace, HAR, and provider recording features. Screenshots remain part of + the measured workload. +- Pin a common compatible Playwright client version and compatible provider + server versions. [Playwright's connection contract](https://playwright.dev/docs/api/class-browsertype#browser-type-connect) + requires matching client/server major and minor versions. Record the exact + client, server Playwright, and Chromium versions, plus runner OS and Node.js + version. Confirm provider support and version reporting before implementation; + do not silently change versions during a run. + +## Suite 1: browser readiness + +Prepare reusable credentials and provider clients before measurement. Workspace +creation, account setup, reusable token acquisition, and other one-time setup +are prerequisites, not readiness samples. Any authentication performed for each +individual session belongs inside that session's timed boundary. + +Start a monotonic timer **immediately before the first per-session provisioning +or connection request**, whichever comes first for that provider. Include all +per-session authentication, session provisioning and polling where required, +endpoint acquisition, `chromium.connect()`, `browser.newContext()` with the fixed +viewport, and `context.newPage()`. Stop immediately after +`page.goto('https://www.example.com', { waitUntil: 'load' })` completes +successfully. For a provider that provisions on connection, start immediately +before that connection request; do not omit provisioning hidden in the handshake. + +Report this elapsed interval as `readinessMs`, with component timings where +available. The end condition is the completed page load, not merely an open +socket. Each attempt uses a fresh context/page rather than a provider's default +page. Measure cleanup separately as `cleanupMs`, including context/browser close +and any provider release or termination; exclude it from `readinessMs`. + +## Suite 2: action throughput + +Provision and connect a fresh session, then create a fresh context/page using the +same settings. Record setup and cleanup separately; exclude both from the action +throughput denominator. Reuse the ten-action Wikipedia workload from +[browser-throughput.bench.ts](./benchmarks/browser/browser-throughput.bench.ts), +with **five repetitions per session: 50 sequential actions**. + +Before measured sessions begin, resolve and validate one concrete article URL +per round, using the same heading and article-body link requirements as the +workload. Freeze and publish the ordered URL list; both providers use the same +round's URL for all five repetitions. Require 100 validated inputs rather than +silently wrapping a short list or falling back to `Special:Random` during +measurement. URL preparation is outside session timing. Shared inputs reduce +page-selection variance; live Wikipedia content and network conditions can still +vary and should not be treated as isolated protocol overhead. + +| Action in each loop | Operation | +| --- | --- | +| 1 | `page.goto(sharedArticleUrl, { waitUntil: 'load' })` | +| 2 | `page.waitForSelector('#firstHeading')` | +| 3 | `page.screenshot()` with the fixed viewport | +| 4 | `page.textContent('#firstHeading')` | +| 5 | Wait for `#mw-content-text a[href*="/wiki/"]`, inspect links in document order, and click the first article link whose path after `/wiki/` has no namespace colon. Accept relative, protocol-relative, and absolute URL forms. | +| 6 | `page.waitForSelector('#firstHeading')` on the linked article | +| 7 | `page.screenshot()` | +| 8 | `page.textContent('#firstHeading')` | +| 9 | `page.goBack({ waitUntil: 'commit' })` | +| 10 | `page.waitForSelector('#firstHeading')` on the original article | + +The click's link discovery and selection remain inside that action's timing. +Keep the existing 30-second bound per action and the click's 10-second internal +selector wait. Preserve dependency skipping: if action 5 fails, record actions +6–10 as unsuccessful, skipped actions with zero duration, then continue with +the next repetition. Other action failures are recorded without aborting the +loop. Back navigation resolves at `commit`; the following selector wait confirms +the heading is present. + +Use the existing metric definitions: `taskMs` is the **sum of the individual +action durations**, and `actionsPerSecond = actionsCompleted / (taskMs / 1000)`. +A complete successful session has `actionsCompleted: 50`. Record the enclosing +`actions` step's elapsed time too, but do not substitute it for `taskMs` without +maintainer agreement. Each complete session contains ten screenshots; report +their latency separately from actions per second. + +## Integration and failure handling + +Follow the existing declarative +[config + task conventions](./WRITING_BENCHMARKS.md): each suite exports `config` +and `task`; the `bench run` runner owns scheduling and reporting. Participants +supply endpoint/authentication information, per-session provisioning where +required, and lifecycle cleanup. Provider-specific setup surrounds a shared +measured workload; it must not change the action sequence or timing definitions. +Do not assume the current CDP session interface already satisfies these needs. + +Use `step()` for setup/provisioning, connection and fresh context/page creation, +navigation or actions, and cleanup. Use `measure()` for readiness elapsed time, +action counts, rates, and screenshot metrics, and retain ordered action records +in task data. Capture readiness's start and stop explicitly across steps; the +runner's overall task duration includes cleanup and is not `readinessMs`. +Use the existing task error/data conventions to retain diagnostics and partial +records when a session-level operation fails. + +Bound provisioning/polling, per-session authentication, connection, context/page +creation, navigation, and every cleanup operation. Agree and record limits +before implementation. `chromium.connect()` needs an explicit timeout because +its default is unbounded. Attempt cleanup in `finally` after success or failure, +including partial setup. Closing a connection must not be assumed to release +provider resources: verify each provider's lifecycle semantics, retain session +identifiers as soon as available, and define how timed-out provisioning or +late-created sessions are reconciled. Record cleanup failures without masking +the original failure; prevent uncertain leaked resources from accumulating. + +Every started attempt remains in the reliability denominator, including +provisioning, authentication, connection, navigation, action, and cleanup +failures. Do not replace failed samples with successful retries. Report workload +success and cleanup success separately, as well as overall session success +(both succeeded). Readiness workload success requires the completed target load; +throughput workload success requires all 50 actions and no setup/session error. +Incomplete sessions never enter successful-session throughput summaries. +Raw failed/partial action records, elapsed failure times, timeout reasons, and +skipped dependencies remain available. A participant skipped for missing +credentials is unavailable, not a measured success; the planned comparison +requires both initial participants to be ready before launch. + +Separate SDK integration and package publication are follow-up work. The +contribution guidance describes provider integration in +[`computesdk/computesdk`](https://github.com/computesdk/computesdk), package +publication, and credentials for ongoing tests. Maintainers should confirm how +Playwright-native endpoint/authentication and cleanup requirements fit that +process before implementation; this PR proposes no new SDK API or package. + +## Results and methodology review + +Publish raw session and action records for all attempts, the shared URL list, +settings/version/region metadata, sample counts, and success rates. Remove +credentials and credential-bearing endpoints from public artifacts. Publish +**untrimmed median/P95/P99** timings for successful workload sessions, with +failures and cleanup outcomes visible alongside them. Do not turn a timeout into +a successful latency observation or report missing samples as zero latency. + +Report `readinessMs`, `taskMs`, per-action-type latency, and `cleanupMs` separately. +For throughput, summarize per-session actions per second and publish a separate +screenshot latency distribution over successful screenshots in complete +sessions. Retain the existing `screenshotMs` metric as each session's median +successful screenshot latency, clearly distinguishing its across-session +summary from the pooled screenshot distribution. Define median as the middle +value (mean of the middle two for an even count); use nearest-rank P95/P99 on +sorted samples and disclose sample sizes. With 100 attempted sessions, tail +estimates may rest on very few observations, especially after failures. + +Defer composite scores, ceilings, and weights to maintainer review. Initially +report distributions without a composite ranking. The runner supports omitting +`scoring`; use metric-based display and retain explicit workload-success flags. +Do not copy the existing 5% tail trimming or legacy result summaries into these +suites. If scoring is approved later, use `trim: 0` for the agreed untrimmed +metrics and preserve the explicit complete-session success criterion. + +## Correspondence with current executable behavior + +The proposal uses current code and workflows to resolve documentation drift: + +- [THROUGHPUT.md](./THROUGHPUT.md) describes five repetitions, but + [throughput-types.ts](./benchmarks/browser/throughput-types.ts) currently sets + `LOOPS_PER_SESSION = 1`: the executable workload is ten actions. This proposal + adopts the documented 50-action design explicitly; it does not claim the + current executable suite already runs 50 or change the existing CDP constant. +- The executable action loop uses shared URLs when supplied, supports three + article-link URL forms, and skips dependent actions after click failure. + Those behaviors take precedence over the older random-page selector and + failure descriptions in `THROUGHPUT.md`. +- [browser.bench.ts](./benchmarks/browser/browser.bench.ts) measures a CDP + lifecycle whose `totalMs` includes browser close and session release. The new + readiness boundary excludes cleanup and includes fresh context/page creation. +- Both browser configs default to two iterations; the + [throughput workflow](./.github/workflows/browser-throughput-benchmarks.yml) + supplies 100 for scheduled/manual runs and three for browser-code pushes. It + currently runs weekly, with providers in separate matrix jobs. Those settings + do not establish the proposed single-runner interleaving or runner location. +- [throughput-benchmark.ts](./benchmarks/browser/throughput-benchmark.ts) trims + tails and its legacy summaries can include incomplete sessions. The declarative + throughput scoring config requires complete action counts. The new suites + must explicitly filter workload success and publish untrimmed distributions, + rather than assuming both reporting paths have the same semantics. + +## Prerequisites and follow-up work + +Before launching implementation, obtain maintainer agreement on these timing +boundaries, the 50-action workload, reliability and summary definitions, and +provider integration requirements. Select and verify the shared Azure region +and fixed runner location, compatible pinned versions, credential setup, Azure +funding arrangement, lifecycle timeouts, and cleanup behavior for both providers. +These are outstanding prerequisites, not capabilities validated by this PR. + +Validate the eventual implementation with repeatable checks of timing boundaries, +identical round inputs, exact action counts, timeout/dependency behavior, partial +result retention, actual placement/settings, and resource cleanup after failures. +For this proposal, validation is limited to Markdown rendering and links, +methodology correspondence with source/workflows, metric definitions, whitespace +checks, and normal PR CI. No provider sessions or performance measurements are +part of this documentation change. + +Concurrency testing, multiple regions, composite scoring, provider/workspace +provisioning, SDK/package publication, and benchmark publication are follow-up +work after methodology review. From 9804b15a41edb008df4c6a441c1a178f2e9527cb Mon Sep 17 00:00:00 2001 From: Jason Song Date: Mon, 5 Oct 2026 12:23:46 -0700 Subject: [PATCH 2/4] feat: add Playwright-native browser benchmark suites --- .github/workflows/ci.yml | 18 ++ .gitignore | 6 +- PLAYWRIGHT_NATIVE_BENCHMARKS.md | 141 +++++++++-- benchmarks/.env.example | 8 + .../browser/playwright-native/native.test.ts | 224 ++++++++++++++++++ .../browser/playwright-native/prepare-urls.ts | 26 ++ .../browser/playwright-native/providers.ts | 185 +++++++++++++++ .../browser/playwright-native/results.ts | 48 ++++ .../browser/playwright-native/session.ts | 90 +++++++ .../browser/playwright-native/workload.ts | 70 ++++++ .../browser/playwright-readiness.bench.ts | 27 +++ .../browser/playwright-throughput.bench.ts | 51 ++++ package.json | 7 +- pnpm-lock.yaml | 10 + 14 files changed, 892 insertions(+), 19 deletions(-) create mode 100644 benchmarks/browser/playwright-native/native.test.ts create mode 100644 benchmarks/browser/playwright-native/prepare-urls.ts create mode 100644 benchmarks/browser/playwright-native/providers.ts create mode 100644 benchmarks/browser/playwright-native/results.ts create mode 100644 benchmarks/browser/playwright-native/session.ts create mode 100644 benchmarks/browser/playwright-native/workload.ts create mode 100644 benchmarks/browser/playwright-readiness.bench.ts create mode 100644 benchmarks/browser/playwright-throughput.bench.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b106448b..34688684 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,24 @@ jobs: - name: Test create-bench run: pnpm --filter create-bench run test + playwright-native: + name: Test Playwright-native benchmarks + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + - run: pnpm install --frozen-lockfile --ignore-scripts + - run: pnpm -r --filter './packages/**' run build + - name: Install fixture browser + run: node node_modules/playwright-native-core/cli.js install --with-deps chromium + - name: Exercise native suites against local fixtures + run: pnpm test:playwright-native + actionlint: name: Lint workflows runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index f5d264c7..3f6e6e45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ .env node_modules/ +.pnpm-store/ .DS_Store dist/ scale-run.log scale.mjs -fetch-raw-json.ts \ No newline at end of file +fetch-raw-json.ts +results/playwright-native/ +playwright-native-urls.json +playwright-native-environment.json diff --git a/PLAYWRIGHT_NATIVE_BENCHMARKS.md b/PLAYWRIGHT_NATIVE_BENCHMARKS.md index 56afb48d..0a189f4a 100644 --- a/PLAYWRIGHT_NATIVE_BENCHMARKS.md +++ b/PLAYWRIGHT_NATIVE_BENCHMARKS.md @@ -1,9 +1,9 @@ # Proposal: Playwright-native browser benchmarks -This proposal requests ComputeSDK maintainer feedback before implementation, -following Noah's request for a proposal PR. It proposes two separate benchmarks -over `chromium.connect()`: **browser readiness** and **action throughput**. -It contains no benchmark results or performance claims. +This proposal requests ComputeSDK maintainer feedback, following Noah's request +for a proposal PR. It includes runnable implementations of two separate +benchmarks over `chromium.connect()`: **browser readiness** and **action +throughput**. It contains no provider benchmark results or performance claims. ## Motivation and scope @@ -15,9 +15,11 @@ specified workload, not evidence that either protocol is inherently faster. [CONTRIBUTING.md](./CONTRIBUTING.md) asks contributors to discuss methodology and wait for maintainer feedback before implementing changes, because historical -comparability matters. This documentation-only PR supplies that discussion for -approval or revision. It changes no APIs, types, dependencies, workflows, or -runtime code. +comparability matters. This PR supplies that discussion for approval or revision, +alongside isolated native suites, local fixture validation, and run instructions. +The existing CDP suites and their dependency version remain unchanged. Approval +of the methodology and environment prerequisites is still required before a +published provider comparison. ## Initial participants and disclosure @@ -45,7 +47,7 @@ contribution guidance. physical host or empty provider cache. - Require both providers' browsers to execute in the **same Azure region** and keep the runner location fixed for the entire initial comparison. Agree the - exact region and runner location before implementation. Record the actual + exact region and runner location before a provider comparison. Record the actual browser regions, runner region/location, routing configuration, and evidence of effective placement; a workspace or requested region alone is insufficient if routing can select another region. @@ -58,7 +60,7 @@ contribution guidance. server versions. [Playwright's connection contract](https://playwright.dev/docs/api/class-browsertype#browser-type-connect) requires matching client/server major and minor versions. Record the exact client, server Playwright, and Chromium versions, plus runner OS and Node.js - version. Confirm provider support and version reporting before implementation; + version. Confirm provider support and version reporting before a provider comparison; do not silently change versions during a run. ## Suite 1: browser readiness @@ -148,7 +150,7 @@ records when a session-level operation fails. Bound provisioning/polling, per-session authentication, connection, context/page creation, navigation, and every cleanup operation. Agree and record limits -before implementation. `chromium.connect()` needs an explicit timeout because +before a provider comparison. `chromium.connect()` needs an explicit timeout because its default is unbounded. Attempt cleanup in `finally` after success or failure, including partial setup. Closing a connection must not be assumed to release provider resources: verify each provider's lifecycle semantics, retain session @@ -173,7 +175,7 @@ contribution guidance describes provider integration in [`computesdk/computesdk`](https://github.com/computesdk/computesdk), package publication, and credentials for ongoing tests. Maintainers should confirm how Playwright-native endpoint/authentication and cleanup requirements fit that -process before implementation; this PR proposes no new SDK API or package. +process before a provider comparison; this PR proposes no new SDK API or package. ## Results and methodology review @@ -230,21 +232,126 @@ The proposal uses current code and workflows to resolve documentation drift: ## Prerequisites and follow-up work -Before launching implementation, obtain maintainer agreement on these timing +Before launching a provider comparison, obtain maintainer agreement on these timing boundaries, the 50-action workload, reliability and summary definitions, and provider integration requirements. Select and verify the shared Azure region and fixed runner location, compatible pinned versions, credential setup, Azure funding arrangement, lifecycle timeouts, and cleanup behavior for both providers. -These are outstanding prerequisites, not capabilities validated by this PR. +These are outstanding prerequisites, not provider capabilities validated by the +local fixture tests. Validate the eventual implementation with repeatable checks of timing boundaries, identical round inputs, exact action counts, timeout/dependency behavior, partial result retention, actual placement/settings, and resource cleanup after failures. -For this proposal, validation is limited to Markdown rendering and links, -methodology correspondence with source/workflows, metric definitions, whitespace -checks, and normal PR CI. No provider sessions or performance measurements are -part of this documentation change. +The included test exercises the real native protocol and existing declarative +runner against local HTTP/TLS fixtures. It validates the 50-action workload, +round interleaving, load boundary, authentication headers, partial results, +lost-response reconciliation, cleanup failures, credential redaction, and +untrimmed summaries. This does not validate the deployed providers, their +placement, or their resource-release semantics. Concurrency testing, multiple regions, composite scoring, provider/workspace provisioning, SDK/package publication, and benchmark publication are follow-up work after methodology review. + +## Running the implementations + +The runnable entrypoints are +[playwright-readiness.bench.ts](./benchmarks/browser/playwright-readiness.bench.ts) +and [playwright-throughput.bench.ts](./benchmarks/browser/playwright-throughput.bench.ts). +They use a separate, exact `playwright-core@1.60.0` dependency alias; the existing +CDP dependency remains unchanged. Compatibility with deployed providers must be +verified before running a comparison. + +```bash +pnpm install --frozen-lockfile --ignore-scripts +pnpm -r --filter './packages/**' run build + +# Install the local browser used only to prepare inputs and run fixture tests. +node node_modules/playwright-native-core/cli.js install chromium +pnpm test:playwright-native + +# Resolve and DOM-validate the shared inputs before provider measurement. +pnpm bench:playwright-prepare-urls 100 playwright-native-urls.json + +# Run separately; --no-ingest writes local records without platform publication. +pnpm bench:playwright-readiness --no-ingest +pnpm bench:playwright-throughput --no-ingest +``` + +Store credentials in your shell or the ignored `benchmarks/.env`, never in Git: + +- `MOMENTIC_BROWSER_FLEET_URL`: the Browser Fleet API base URL. +- `MOMENTIC_BROWSER_FLEET_API_KEY`: a credential authorized to create, read, and + delete benchmark sessions. A normal Momentic app API key is not interchangeable. +- `AZURE_PLAYWRIGHT_SERVICE_URL`: the workspace browser endpoint ending in + `/playwrightworkspaces//browsers`. The adapter resolves the + [documented browser redirect](https://learn.microsoft.com/en-us/rest/api/playwright/dataplane/workspaces/get-browsers?view=rest-playwright-dataplane-2025-09-01) + inside measurement, then connects with bearer authentication and `os=Linux`. +- `AZURE_PLAYWRIGHT_ACCESS_TOKEN`: a reusable Entra or workspace access token + prepared before measurement; its authentication mode must already be enabled. +- `PLAYWRIGHT_NATIVE_URLS_FILE`: the prepared `playwright-native-urls.json` path. +- `PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE`: an operator-verified environment manifest + path. The manifest validates the shared region, compatible versions, and fixed + settings and records evidence; it does not discover actual placement itself. +- `PLAYWRIGHT_NATIVE_RESULTS_DIR`: optional local output directory (default + `results/playwright-native`). Each suite writes a fresh JSON file with all task + and action records, reliability denominators, and untrimmed summaries. + +Create an ignored `playwright-native-environment.json` only after verifying the +environment. Use this shape for **both** entries in `providers` (`momentic` and +`azure`), replacing the evidence placeholders with non-secret verification +references: + +```json +{ + "region": "", + "runnerLocation": "", + "providers": { + "momentic": { + "region": "", + "playwrightVersion": "1.60.0", + "os": "linux", + "headless": true, + "stealth": false, + "proxy": false, + "recording": false, + "placementEvidence": "", + "versionEvidence": "", + "cleanupEvidence": "" + }, + "azure": { + "region": "", + "playwrightVersion": "1.60.0", + "os": "linux", + "headless": true, + "stealth": false, + "proxy": false, + "recording": false, + "placementEvidence": "", + "versionEvidence": "", + "cleanupEvidence": "" + } + } +} +``` + +Momentic provisioning creates a session with an idempotency key and polls READY; +cleanup requests deletion and polls a terminal state. A lost creation response +is reconciled with the same key during cleanup, without replacing the failed +sample. Azure teardown closes the context and connection and relies on the +service's verified release semantics. +If Azure provisioning/connection fails without an owned connection to close, +cleanup is recorded as uncertain and further allocations stop for that +participant; the implementation does not claim it released an unknown resource. +Provisioning has a 120-second bound; +connection, context/page creation, and navigation have 30-second bounds; each +cleanup operation has a 15-second bound. After uncertain cleanup, the process +refuses further allocations for that participant and records unstarted tasks +separately from attempted-session reliability denominators. + +No real credentials are used by fixture tests. Local fixture credentials and +certificates are disposable, and the CI fixture job runs without provider +credentials. Shared input/setup files and generated result files are ignored by +Git. Actual Chromium versions are collected from each connected browser; +server versions and placement evidence are labeled by their source in raw data. diff --git a/benchmarks/.env.example b/benchmarks/.env.example index 04e66ca1..a464f46f 100644 --- a/benchmarks/.env.example +++ b/benchmarks/.env.example @@ -185,3 +185,11 @@ NEON_AI_GATEWAY_TOKEN=your_neon_ai_gateway_token NGROK_AI_GATEWAY_API_KEY=your_ngrok_ai_gateway_api_key LLMAPI_API_KEY=your_llmapi_api_key BLAZERAIL_API_KEY=your_blazerail_api_key +# Playwright-native browser benchmarks (keep real values in the ignored .env). +# MOMENTIC_BROWSER_FLEET_URL= +# MOMENTIC_BROWSER_FLEET_API_KEY= +# AZURE_PLAYWRIGHT_SERVICE_URL= +# AZURE_PLAYWRIGHT_ACCESS_TOKEN= +# PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE=playwright-native-environment.json +# PLAYWRIGHT_NATIVE_URLS_FILE=playwright-native-urls.json +# PLAYWRIGHT_NATIVE_RESULTS_DIR=results/playwright-native diff --git a/benchmarks/browser/playwright-native/native.test.ts b/benchmarks/browser/playwright-native/native.test.ts new file mode 100644 index 00000000..c468b2a5 --- /dev/null +++ b/benchmarks/browser/playwright-native/native.test.ts @@ -0,0 +1,224 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import https from 'node:https'; +import net from 'node:net'; +import { execFileSync, execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { mkdtempSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { chromium } from 'playwright-native-core'; +import { runBenchmark, type BenchmarkRunOutcome } from '@benchsdk/runner'; +import { distribution } from './results.js'; +import { articleUrl } from './workload.js'; + +// Plausible failures covered here: incomplete actions, wrong connection auth, +// lost create response, malformed READY response, refused connection, release +// failure, missing round input, leaked contexts, and trimmed/partial summaries. +test('native suites use real Playwright connections and preserve reliability records', { timeout: 120_000 }, async () => { + const directory = mkdtempSync(path.join(tmpdir(), 'native-bench-')); + const envNames = ['MOMENTIC_BROWSER_FLEET_URL', 'MOMENTIC_BROWSER_FLEET_API_KEY', 'AZURE_PLAYWRIGHT_SERVICE_URL', 'AZURE_PLAYWRIGHT_ACCESS_TOKEN', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE', 'PLAYWRIGHT_NATIVE_URLS_FILE', 'PLAYWRIGHT_NATIVE_RESULTS_DIR']; + const previous = new Map(envNames.map(key => [key, process.env[key]])); + execFileSync('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', path.join(directory, 'key.pem'), '-out', path.join(directory, 'cert.pem'), '-days', '1', '-subj', '/CN=localhost'], { stdio: 'ignore' }); + const listen = (server: http.Server | https.Server) => new Promise(resolve => { + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + resolve(address.port); + }); + }); + const fixtures = https.createServer({ key: readFileSync(path.join(directory, 'key.pem')), cert: readFileSync(path.join(directory, 'cert.pem')) }, (req, res) => { + if (req.url === '/slow.gif') { + setTimeout(() => { res.setHeader('Content-Type', 'image/gif'); res.end(Buffer.from('47494638396101000100800000000000ffffff21f90401000000002c00000000010001000002024401003b', 'hex')); }, 150); + return; + } + res.setHeader('Content-Type', 'text/html'); + if (req.headers.host?.includes('example.com')) { + res.end('Ready'); + return; + } + const link = req.url?.includes('Native_no_links') ? '' : 'Article'; + res.end(`

Fixture

Help${link}
`); + }); + const fixturePort = await listen(fixtures); + fixtures.on('connection', socket => socket.on('error', () => {})); + const sessions = new Map(); + const terminated = new Set(); + const order: string[] = []; + const websocketAuth: string[] = []; + let mode = ''; + let lost = false; + let endpoint = ''; + const api = http.createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://localhost'); + res.setHeader('Content-Type', 'application/json'); + if (url.pathname === '/browsers') { + assert.equal(req.headers.authorization, 'Bearer local-azure-credential'); + assert.equal(url.searchParams.get('os'), 'Linux'); + assert.equal(url.searchParams.get('api-version'), '2025-09-01'); + order.push('azure'); + res.writeHead(302, { Location: mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${apiPort}/connect/azure/${url.searchParams.get('runId')}` }); + res.end(); + return; + } + // Chromium may make unrelated startup requests through the fixture proxy. + if (!url.pathname.startsWith('/v1/sessions')) { res.writeHead(404); res.end('{}'); return; } + assert.equal(req.headers.authorization, 'Bearer local-momentic-credential'); + const id = url.pathname.split('/').at(-1); + if (req.method === 'POST') { + let body = ''; + for await (const chunk of req) body += chunk; + assert.equal(JSON.parse(body).playwrightVersion, '1.60.0'); + const key = String(req.headers['idempotency-key']); + if (!sessions.has(key)) { sessions.set(key, `session-${sessions.size}`); order.push('momentic'); } + const sessionId = sessions.get(key); + if (mode === 'lost-response' && !lost) { lost = true; req.socket.destroy(); return; } + if (mode === 'malformed-json') { res.end('{"connectToken":"local-unpublished-credential'); return; } + res.end(JSON.stringify({ id: sessionId, status: 'READY', playwrightVersion: '1.60.0', wsEndpoint: mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${apiPort}/connect/momentic/${sessionId}`, ...(mode !== 'malformed-ready' ? { connectToken: 'local-connect-credential' } : {}) })); + } else if (req.method === 'DELETE') { + if (mode === 'cleanup-failure') { res.writeHead(500); res.end('{}'); return; } + if (id) terminated.add(id); + res.writeHead(204); res.end(); + } else { + res.end(JSON.stringify({ id, status: terminated.has(id ?? '') ? 'TERMINATED' : 'READY' })); + } + }); + const sockets = new Set(); + api.on('connection', socket => { sockets.add(socket); socket.on('error', () => {}); socket.on('close', () => sockets.delete(socket)); }); + api.on('connect', (_req, socket, head) => { + const upstream = net.connect(fixturePort, '127.0.0.1', () => { + socket.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + if (head.length) upstream.write(head); + socket.pipe(upstream).pipe(socket); + }); + upstream.on('error', () => socket.destroy()); + socket.on('close', () => upstream.destroy()); + }); + api.on('upgrade', (req, socket, head) => { + const provider = req.url?.includes('/momentic/') ? 'momentic' : 'azure'; + const expected = provider === 'momentic' ? 'Bearer local-connect-credential' : 'Bearer local-azure-credential'; + if (req.headers.authorization !== expected) { socket.end('HTTP/1.1 401 Unauthorized\r\n\r\n'); return; } + websocketAuth.push(provider); + const target = new URL(endpoint); + const upstream = net.connect(Number(target.port), '127.0.0.1', () => { + const headers = Object.entries(req.headers).filter(([key]) => key !== 'host').map(([key, value]) => `${key}: ${value}`).join('\r\n'); + upstream.write(`GET ${target.pathname} HTTP/1.1\r\nHost: ${target.host}\r\n${headers}\r\n\r\n`); + if (head.length) upstream.write(head); + socket.pipe(upstream).pipe(socket); + }); + upstream.on('error', () => socket.destroy()); + socket.on('close', () => upstream.destroy()); + }); + const apiPort = await listen(api); + const server = await chromium.launchServer({ host: '127.0.0.1', headless: true, ...(process.env.PLAYWRIGHT_NATIVE_TEST_EXECUTABLE ? { executablePath: process.env.PLAYWRIGHT_NATIVE_TEST_EXECUTABLE } : {}), args: ['--ignore-certificate-errors'], proxy: { server: `http://127.0.0.1:${apiPort}` } }); + endpoint = server.wsEndpoint(); + try { + const environment = { region: 'local-fixture', runnerLocation: 'local-fixture', providers: Object.fromEntries(['momentic', 'azure'].map(name => [name, { region: 'local-fixture', playwrightVersion: '1.60.0', os: 'linux', headless: true, stealth: false, proxy: false, recording: false, placementEvidence: 'local fixture only', versionEvidence: 'local launchServer version', cleanupEvidence: 'fixture lifecycle verification' }])) }; + const urls = Array.from({ length: 100 }, (_, i) => `https://en.wikipedia.org/wiki/${i === 2 ? 'Native_no_links' : `Native_fixture_${i}`}`); + const environmentFile = path.join(directory, 'environment.json'); + const urlsFile = path.join(directory, 'urls.json'); + writeFileSync(environmentFile, JSON.stringify(environment)); + writeFileSync(urlsFile, JSON.stringify(urls)); + Object.assign(process.env, { MOMENTIC_BROWSER_FLEET_URL: `http://127.0.0.1:${apiPort}`, MOMENTIC_BROWSER_FLEET_API_KEY: 'local-momentic-credential', AZURE_PLAYWRIGHT_SERVICE_URL: `http://127.0.0.1:${apiPort}/browsers`, AZURE_PLAYWRIGHT_ACCESS_TOKEN: 'local-azure-credential', PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE: environmentFile, PLAYWRIGHT_NATIVE_URLS_FILE: urlsFile, PLAYWRIGHT_NATIVE_RESULTS_DIR: directory }); + const readiness = await import('../playwright-readiness.bench.js'); + const throughput = await import('../playwright-throughput.bench.js'); + const records = (outcome: BenchmarkRunOutcome) => outcome.participants.flatMap(p => p.records); + const ready = await runBenchmark({ ...readiness.config, iterations: 2 }, readiness.task, ['--no-ingest']); + assert.deepEqual(order, ['momentic', 'azure', 'momentic', 'azure']); + assert.deepEqual(websocketAuth, order); + for (const record of records(ready)) { + assert.equal(record.status, 'success'); + assert.equal(record.data?.sessionSuccess, true); + assert.ok(Number(record.data?.readinessMs) >= 150, 'readiness waits for the delayed subresource load'); + assert.ok(Number(record.latencyMs) >= Number(record.data?.readinessMs)); + } + order.length = 0; + const through = await runBenchmark({ ...throughput.config, iterations: 3 }, throughput.task, ['--no-ingest']); + assert.deepEqual(order, ['momentic', 'azure', 'momentic', 'azure', 'momentic', 'azure']); + for (const record of records(through)) { + const actions = record.data?.actions; + assert.ok(Array.isArray(actions)); + assert.equal(actions.length, 50); + const screenshots = actions.filter(a => a && typeof a === 'object' && !Array.isArray(a) && a.type === 'screenshot'); + if (record.taskIndex < 2) { + assert.equal(record.status, 'success'); + assert.equal(record.data?.actionsCompleted, 50); + assert.equal(screenshots.length, 10); + assert.equal(record.data?.articleUrl, urls[record.taskIndex]); + } else { + assert.equal(record.status, 'error'); + assert.equal(record.data?.actionsCompleted, 20); + assert.equal(actions.filter(a => a && typeof a === 'object' && !Array.isArray(a) && a.error === 'skipped: click failed').length, 25); + } + const sum = actions.reduce((total, a) => total + (a && typeof a === 'object' && !Array.isArray(a) ? Number(a.durationMs) : 0), 0); + assert.equal(record.data?.taskMs, sum); + assert.equal(record.data?.actionsPerSecond, Number(record.data?.actionsCompleted) / (sum / 1000)); + assert.equal(record.steps?.find(step => step.name === 'actions')?.data?.actionsPerSecond, record.data?.actionsPerSecond); + } + const outputFile = path.join(directory, 'throughput', readdirSync(path.join(directory, 'throughput'))[0]); + const raw = readFileSync(outputFile, 'utf8'); + for (const secret of ['local-momentic-credential', 'local-connect-credential', 'local-azure-credential']) assert.ok(!raw.includes(secret)); + const output = JSON.parse(raw); + assert.deepEqual(output.articleUrls, urls); + for (const summary of output.summaries) { + assert.equal(summary.attempts, 3); + assert.equal(summary.successRate, 2 / 3); + assert.equal(summary.metrics.actionsPerSecond.count, 2); + assert.equal(summary.perActionType.screenshot.count, 20); + } + for (const failureMode of ['lost-response', 'malformed-ready', 'refused-connection', 'cleanup-failure']) { + mode = failureMode; + order.length = 0; + const participants = readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })); + const failed = await runBenchmark({ ...readiness.config, participants, iterations: failureMode === 'cleanup-failure' ? 2 : 1 }, readiness.task, ['--no-ingest']); + assert.equal(records(failed)[0].status, 'error'); + assert.equal(records(failed)[0].data?.attemptStarted, true); + assert.equal(records(failed)[0].data?.cleanupSuccess, failureMode !== 'cleanup-failure'); + assert.ok(!JSON.stringify(failed).includes('private-query')); + if (failureMode === 'cleanup-failure') { + assert.equal(records(failed)[1].data?.attemptStarted, false); + assert.deepEqual(order, ['momentic'], 'uncertain cleanup stops new allocations'); + } else assert.equal(terminated.size, sessions.size); + } + mode = 'refused-connection'; + const azureFailure = await runBenchmark({ ...readiness.config, participants: readiness.config.participants.filter(p => p.name === 'azure').map(p => ({ ...p })), iterations: 2 }, readiness.task, ['--no-ingest']); + assert.equal(records(azureFailure)[0].data?.cleanupSuccess, false); + assert.equal(records(azureFailure)[1].data?.attemptStarted, false); + assert.ok(!JSON.stringify(azureFailure).includes('private-query')); + mode = 'malformed-json'; + const malformed = await runBenchmark({ ...readiness.config, participants: readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })), iterations: 1 }, readiness.task, ['--no-ingest']); + assert.equal(records(malformed)[0].data?.cleanupSuccess, false); + assert.ok(!JSON.stringify(malformed).includes('local-unpublished-credential')); + assert.throws(() => articleUrl('https://local-user:local-password@en.wikipedia.org/wiki/Fixture')); + mode = ''; + order.length = 0; + const missing = await runBenchmark({ ...throughput.config, iterations: 1 }, ctx => throughput.task({ ...ctx, taskIndex: 100 }), ['--no-ingest']); + assert.deepEqual(order, []); + assert.ok(records(missing).every(record => record.status === 'error' && record.data?.attemptStarted === false)); + assert.deepEqual(distribution(Array.from({ length: 100 }, (_, i) => i + 1)), { count: 100, median: 50.5, p95: 95, p99: 99 }); + assert.equal(distribution([]).median, null); + writeFileSync(urlsFile, JSON.stringify(urls.slice(0, 1))); + order.length = 0; + await assert.rejects(promisify(execFile)(process.execPath, ['node_modules/tsx/dist/cli.mjs', 'packages/benchsdk-runner/dist/bin.js', 'run', 'benchmarks/browser/playwright-throughput.bench.ts', '--no-ingest', '--iterations', '2'], { env: process.env }), /Prepare a validated article URL/); + assert.deepEqual(order, [], 'short input lists fail before any provider request'); + writeFileSync(urlsFile, JSON.stringify(urls)); + // Exercise the published CLI entrypoints, not only runBenchmark imports. + for (const entry of ['playwright-readiness', 'playwright-throughput']) { + const { stdout } = await promisify(execFile)(process.execPath, ['node_modules/tsx/dist/cli.mjs', 'packages/benchsdk-runner/dist/bin.js', 'run', `benchmarks/browser/${entry}.bench.ts`, '--no-ingest', '--iterations', '1'], { env: process.env }); + assert.ok(stdout.includes('All done. No platform run created.')); + assert.ok(!stdout.includes('FAILED')); + } + const audit = await chromium.connect(endpoint); + assert.equal(audit.contexts().length, 0, 'all benchmark contexts were closed'); + await audit.close(); + console.log('Verified real native connections, 50-action records, round interleaving, load boundary, auth, partial failures, cleanup recovery, secret-free artifacts, and untrimmed summaries.'); + } finally { + await server.close(); + for (const socket of sockets) socket.destroy(); + api.closeAllConnections(); fixtures.closeAllConnections(); + await Promise.all([new Promise(resolve => api.close(() => resolve())), new Promise(resolve => fixtures.close(() => resolve()))]); + for (const [key, value] of previous) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/benchmarks/browser/playwright-native/prepare-urls.ts b/benchmarks/browser/playwright-native/prepare-urls.ts new file mode 100644 index 00000000..b59b7688 --- /dev/null +++ b/benchmarks/browser/playwright-native/prepare-urls.ts @@ -0,0 +1,26 @@ +/** Resolve and DOM-validate shared Wikipedia inputs before measured sessions. */ +import { chromium } from 'playwright-native-core'; +import { writeFileSync } from 'node:fs'; +import { articleUrl, firstArticleLink } from './workload.js'; +import { withTimeout } from '../../src/util/timeout.js'; + +const count = Number(process.argv[2] ?? '100'); +const filename = process.argv[3] ?? 'playwright-native-urls.json'; +if (!Number.isInteger(count) || count < 1) throw new Error('Count must be a positive integer'); +const browser = await chromium.launch({ headless: true, timeout: 30_000 }); +try { + const page = await browser.newPage({ viewport: { width: 1920, height: 1080 } }); + page.setDefaultTimeout(30_000); + page.setDefaultNavigationTimeout(30_000); + const urls: string[] = []; + for (let i = 0; i < count; i++) { + await page.goto('https://en.wikipedia.org/wiki/Special:Random', { waitUntil: 'load' }); + const url = articleUrl(page.url()); + await page.waitForSelector('#firstHeading'); + await withTimeout((await firstArticleLink(page)).click(), 30_000, 'Article link validation timed out'); + await page.waitForSelector('#firstHeading'); + urls.push(url); + } + writeFileSync(filename, JSON.stringify(urls, null, 2)); + console.log(`Wrote ${urls.length} validated shared inputs to ${filename}`); +} finally { await withTimeout(browser.close(), 15_000, 'URL preparation cleanup timed out'); } diff --git a/benchmarks/browser/playwright-native/providers.ts b/benchmarks/browser/playwright-native/providers.ts new file mode 100644 index 00000000..b4fcd819 --- /dev/null +++ b/benchmarks/browser/playwright-native/providers.ts @@ -0,0 +1,185 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { readFileSync } from 'node:fs'; +import type { JsonObject } from '@benchsdk/api'; + +export const PLAYWRIGHT_VERSION = '1.60.0'; +export const PROVISION_TIMEOUT_MS = 120_000; +export const CONNECT_TIMEOUT_MS = 30_000; +export const CLEANUP_TIMEOUT_MS = 15_000; + +export interface NativeSession { + metadata: JsonObject; + provision(): Promise<{ endpoint: string; headers: Record }>; + cleanup(options: { connected: boolean }): Promise; +} + +export interface NativeParticipant { + name: string; + requiredEnvVars: string[]; + session(identity: string): NativeSession; +} + +export function requiredEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing ${name}`); + return value; +} + +function object(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Invalid provider response'); + return value as Record; +} + +function string(value: unknown): string { + if (typeof value !== 'string' || !value.trim()) throw new Error('Missing provider response field'); + return value; +} + +/** Only publish explicit, non-secret environment attestations. */ +function environment(provider: string): JsonObject { + const filename = requiredEnv('PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'); + let parsed: unknown; + try { parsed = JSON.parse(readFileSync(filename, 'utf8')); } + catch { throw new Error('Cannot read a valid Playwright-native environment manifest'); } + const manifest = object(parsed); + const entry = object(object(manifest.providers)[provider]); + const region = string(manifest.region); + const serverVersion = string(entry.playwrightVersion); + if (entry.region !== region || !/^1\.60\.\d+$/.test(serverVersion)) { + throw new Error('Both providers must attest to the shared region and Playwright 1.60.x'); + } + if (entry.os !== 'linux' || entry.headless !== true || entry.stealth !== false || entry.proxy !== false || entry.recording !== false) { + throw new Error('Environment must attest to headless Linux Chromium without optional features'); + } + return { + region, + runnerLocation: string(manifest.runnerLocation), + placementEvidence: string(entry.placementEvidence), + versionEvidence: string(entry.versionEvidence), + cleanupEvidence: string(entry.cleanupEvidence), + serverPlaywrightVersion: serverVersion, + serverPlaywrightVersionSource: 'operator-verified environment manifest', + clientPlaywrightVersion: PLAYWRIGHT_VERSION, + browserOs: 'linux', headless: true, viewport: { width: 1920, height: 1080 }, + stealth: false, proxy: false, recording: false, + metadataSource: 'operator-verified environment manifest', + }; +} + +export function safeError(error: unknown, secrets: string[] = []): string { + let message = error instanceof Error ? error.message : 'Operation failed'; + const values = [...secrets, ...Object.entries(process.env) + .filter(([key]) => /token|key|secret|password/i.test(key)) + .map(([, value]) => value ?? '')]; + for (const value of values) if (value) message = message.split(value).join('[redacted]'); + return message.replace(/(?:https?|wss?):\/\/[^\s<>]+/g, '[endpoint]') + .replace(/Bearer\s+[^\s]+/gi, 'Bearer [redacted]'); +} + +async function request(url: URL, init: RequestInit, signal: AbortSignal): Promise { + const response = await fetch(url, { ...init, signal, redirect: 'error' }); + if (!response.ok) throw new Error(`Provider request failed (HTTP ${response.status})`); + return response; +} + +async function sessionResponse(response: Response): Promise> { + // JSON parser diagnostics may include credential-bearing response snippets. + try { return object(await response.json()); } + catch { throw new Error('Provider returned invalid session JSON'); } +} + +export const nativeParticipants: NativeParticipant[] = [ + { + name: 'momentic', + requiredEnvVars: ['MOMENTIC_BROWSER_FLEET_URL', 'MOMENTIC_BROWSER_FLEET_API_KEY', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'], + session(identity) { + const base = new URL(requiredEnv('MOMENTIC_BROWSER_FLEET_URL')); + const headers = { Authorization: `Bearer ${requiredEnv('MOMENTIC_BROWSER_FLEET_API_KEY')}`, 'Content-Type': 'application/json', 'Idempotency-Key': identity }; + const metadata = environment('momentic'); + let sessionId: string | undefined; + let creationUncertain = false; + const create = async (signal: AbortSignal) => { + creationUncertain = true; + const response = await fetch(new URL('/v1/sessions', base), { + method: 'POST', headers, signal, redirect: 'error', + body: JSON.stringify({ browser: 'chromium', playwrightVersion: PLAYWRIGHT_VERSION, network: { mode: 'playwright' }, ttlSeconds: 3600, waitTimeoutMs: 0 }), + }); + if (!response.ok) { + // A definitive rejection did not allocate a session. A 5xx may have. + creationUncertain = response.status >= 500; + throw new Error(`Session creation failed (HTTP ${response.status})`); + } + const data = await sessionResponse(response); + sessionId = string(data.id); // Retain the ID before validating other fields. + metadata.sessionId = sessionId; + creationUncertain = false; + return data; + }; + const retrieve = async (signal: AbortSignal) => { + if (!sessionId) throw new Error('Session ID unavailable'); + return sessionResponse(await request(new URL(`/v1/sessions/${encodeURIComponent(sessionId)}`, base), { headers }, signal)); + }; + return { + metadata, + async provision() { + const signal = AbortSignal.timeout(PROVISION_TIMEOUT_MS); + let data = await create(signal); + while (data.status === 'QUEUED' || data.status === 'PROVISIONING') { + await delay(500, undefined, { signal }); + data = await retrieve(signal); + } + if (data.status !== 'READY') throw new Error('Momentic session did not become READY'); + if (data.playwrightVersion !== PLAYWRIGHT_VERSION) throw new Error('Momentic returned an incompatible Playwright version'); + metadata.serverPlaywrightVersion = string(data.playwrightVersion); + metadata.serverPlaywrightVersionSource = 'Momentic session response'; + return { endpoint: string(data.wsEndpoint), headers: { Authorization: `Bearer ${string(data.connectToken)}` } }; + }, + async cleanup() { + const signal = AbortSignal.timeout(CLEANUP_TIMEOUT_MS); + // Reconcile a lost POST response using the same key, never a new attempt. + if (!sessionId && creationUncertain) await create(signal); + if (!sessionId) return; + await request(new URL(`/v1/sessions/${encodeURIComponent(sessionId)}`, base), { method: 'DELETE', headers }, signal); + for (;;) { + const data = await retrieve(signal); + if (data.status === 'TERMINATED' || data.status === 'EXPIRED' || data.status === 'FAILED') return; + await delay(500, undefined, { signal }); + } + }, + }; + }, + }, + { + name: 'azure', + requiredEnvVars: ['AZURE_PLAYWRIGHT_SERVICE_URL', 'AZURE_PLAYWRIGHT_ACCESS_TOKEN', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'], + session(identity) { + const url = new URL(requiredEnv('AZURE_PLAYWRIGHT_SERVICE_URL')); + const headers = { Authorization: `Bearer ${requiredEnv('AZURE_PLAYWRIGHT_ACCESS_TOKEN')}` }; + const metadata = environment('azure'); + url.protocol = url.protocol === 'wss:' ? 'https:' : url.protocol === 'ws:' ? 'http:' : url.protocol; + url.searchParams.set('api-version', '2025-09-01'); + url.searchParams.set('os', 'Linux'); + url.searchParams.set('runId', identity); + let allocationUncertain = false; + return { + metadata: { ...metadata, sessionId: identity }, + async provision() { + // Resolve the documented redirect inside the measured interval. + allocationUncertain = true; + const response = await fetch(url, { headers, redirect: 'manual', signal: AbortSignal.timeout(PROVISION_TIMEOUT_MS) }); + if (response.status >= 400 && response.status < 500) allocationUncertain = false; + if (response.status !== 302 && response.status !== 307) throw new Error(`Azure browser redirect failed (HTTP ${response.status})`); + const endpoint = new URL(string(response.headers.get('location')), url); + if (endpoint.protocol !== 'wss:' && endpoint.protocol !== 'ws:') throw new Error('Azure did not return a Playwright WebSocket endpoint'); + return { endpoint: endpoint.href, headers }; + }, + // Azure owns termination on connection close; verify this before a comparison. + async cleanup({ connected }) { + if (allocationUncertain && !connected) { + throw new Error('Azure allocation cleanup is uncertain after failed provisioning/connection; verify resources before rerunning'); + } + }, + }; + }, + }, +]; diff --git a/benchmarks/browser/playwright-native/results.ts b/benchmarks/browser/playwright-native/results.ts new file mode 100644 index 00000000..4a367d28 --- /dev/null +++ b/benchmarks/browser/playwright-native/results.ts @@ -0,0 +1,48 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import type { BenchmarkRunOutcome } from '@benchsdk/runner'; +import { ACTION_TYPES } from '../throughput-types.js'; + +/** Untrimmed nearest-rank percentiles; missing observations are not zeroes. */ +export function distribution(values: number[]) { + const sorted = [...values].sort((a, b) => a - b); + const n = sorted.length; + if (!n) return { count: 0, median: null, p95: null, p99: null }; + const mid = Math.floor(n / 2); + return { + count: n, + median: n % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2, + p95: sorted[Math.ceil(n * 0.95) - 1], + p99: sorted[Math.ceil(n * 0.99) - 1], + }; +} + +export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readiness' | 'throughput', articleUrls?: string[]): string { + const summaries = outcome.participants.map(({ participant, records }) => { + const attempted = records.filter(record => record.data?.attemptStarted === true); + const complete = attempted.filter(record => record.data?.workloadSuccess === true); + const metrics = Object.fromEntries(['readinessMs', 'taskMs', 'actionsPerSecond', 'screenshotMs', 'cleanupMs'].map(key => [key, + distribution((key === 'cleanupMs' ? attempted : complete).flatMap(record => { + const value = record.data?.[key]; + return typeof value === 'number' && Number.isFinite(value) ? [value] : []; + })), + ])); + const perActionType = Object.fromEntries(ACTION_TYPES.map(type => [type, distribution(complete.flatMap(record => { + const actions = record.data?.actions; + if (!Array.isArray(actions)) return []; + return actions.flatMap(action => { + if (!action || typeof action !== 'object' || Array.isArray(action)) return []; + return action.type === type && action.success === true && typeof action.durationMs === 'number' ? [action.durationMs] : []; + }); + }))])); + const rate = (key: string) => attempted.length ? attempted.filter(record => record.data?.[key] === true).length / attempted.length : null; + return { participant, scheduled: records.length, attempts: attempted.length, workloadSuccessRate: rate('workloadSuccess'), cleanupSuccessRate: rate('cleanupSuccess'), successRate: rate('sessionSuccess'), metrics, perActionType }; + }); + const directory = path.resolve(process.env.PLAYWRIGHT_NATIVE_RESULTS_DIR ?? 'results/playwright-native', suite); + mkdirSync(directory, { recursive: true }); + const filename = path.join(directory, `${randomUUID()}.json`); + writeFileSync(filename, JSON.stringify({ suite, runId: outcome.runId, createdAt: new Date().toISOString(), config: outcome.config, ...(articleUrls ? { articleUrls } : {}), summaries, participants: outcome.participants }, null, 2)); + console.log(`Raw Playwright-native records and untrimmed summaries: ${filename}`); + return filename; +} diff --git a/benchmarks/browser/playwright-native/session.ts b/benchmarks/browser/playwright-native/session.ts new file mode 100644 index 00000000..99543ade --- /dev/null +++ b/benchmarks/browser/playwright-native/session.ts @@ -0,0 +1,90 @@ +import { chromium, type Page, type Browser, type BrowserContext } from 'playwright-native-core'; +import { randomUUID } from 'node:crypto'; +import { TaskError, type TaskContext } from '@benchsdk/runner'; +import type { JsonObject } from '@benchsdk/api'; +import { withTimeout } from '../../src/util/timeout.js'; +import { CLEANUP_TIMEOUT_MS, CONNECT_TIMEOUT_MS, nativeParticipants, safeError, type NativeParticipant } from './providers.js'; + +const unsafeParticipants = new WeakSet(); + +export const nativeConfig = { + iterations: 100, + concurrency: 1, + groupBy: 'round' as const, + participants: nativeParticipants, +}; + +/** Timer spans the first provider request through workload completion only. */ +export async function runSession( + ctx: TaskContext, + workload: (page: Page, startedAt: number, data: JsonObject) => Promise, +): Promise<{ data: JsonObject }> { + if (unsafeParticipants.has(ctx.participant)) { + throw new TaskError('Participant stopped after uncertain cleanup; verify resources before rerunning', { + code: 'CLEANUP_UNCERTAIN', data: { attemptStarted: false }, + }); + } + // Prepare reusable credentials and validate the environment before timing. + const session = ctx.participant.session(randomUUID()); + const data: JsonObject = { ...session.metadata, attemptStarted: true, workloadSuccess: false, cleanupSuccess: false }; + let browser: Browser | undefined; + let context: BrowserContext | undefined; + let secrets: string[] = []; + let failure: string | undefined; + const cleanupErrors: string[] = []; + const guarded = async (fn: () => Promise): Promise => { + try { return await fn(); } catch (error) { throw new Error(safeError(error, secrets)); } + }; + const startedAt = performance.now(); + try { + const connection = await ctx.step('provision', () => guarded(() => session.provision())); + Object.assign(data, session.metadata); + secrets = Object.values(connection.headers); + browser = await ctx.step('connect', () => guarded(() => chromium.connect(connection.endpoint, { headers: connection.headers, timeout: CONNECT_TIMEOUT_MS }))); + data.chromiumVersion = browser.version(); + data.runnerOs = process.platform; + data.nodeVersion = process.version; + const connected = browser; + const page = await ctx.step('page', () => guarded(async () => { + context = await withTimeout(connected.newContext({ viewport: { width: 1920, height: 1080 } }), CONNECT_TIMEOUT_MS, 'Context creation timed out'); + context.setDefaultTimeout(30_000); + context.setDefaultNavigationTimeout(30_000); + return withTimeout(context.newPage(), CONNECT_TIMEOUT_MS, 'Page creation timed out'); + })); + await workload(page, startedAt, data); + data.workloadSuccess = true; + } catch (error) { + failure = safeError(error, secrets); + data.failedAfterMs = performance.now() - startedAt; + } finally { + const cleanupStart = performance.now(); + const cleanup = async (name: string, fn: () => Promise) => { + try { + await ctx.step(name, () => guarded(() => withTimeout(fn(), CLEANUP_TIMEOUT_MS, `${name} timed out`)), { reportConcurrency: false }); + } catch (error) { cleanupErrors.push(safeError(error, secrets)); } + }; + if (context) { + const activeContext = context; + await cleanup('close-context', () => activeContext.close()); + } + if (browser) { + const activeBrowser = browser; + await cleanup('close-browser', () => activeBrowser.close()); + } + await cleanup('release', () => session.cleanup({ connected: browser !== undefined })); + data.cleanupMs = performance.now() - cleanupStart; + data.cleanupSuccess = cleanupErrors.length === 0; + Object.assign(data, session.metadata); + if (cleanupErrors.length) { + data.cleanupErrors = cleanupErrors; + unsafeParticipants.add(ctx.participant); + } + } + data.sessionSuccess = data.workloadSuccess === true && data.cleanupSuccess === true; + if (failure) data.errorMessage = failure; + ctx.measure(data); + if (failure || cleanupErrors.length) { + throw new TaskError(failure ?? 'Session cleanup failed', { code: 'PLAYWRIGHT_NATIVE_ERROR', data }); + } + return { data }; +} diff --git a/benchmarks/browser/playwright-native/workload.ts b/benchmarks/browser/playwright-native/workload.ts new file mode 100644 index 00000000..c1faea02 --- /dev/null +++ b/benchmarks/browser/playwright-native/workload.ts @@ -0,0 +1,70 @@ +import { readFileSync } from 'node:fs'; +import type { Page, ElementHandle } from 'playwright-native-core'; +import { withTimeout } from '../../src/util/timeout.js'; +import type { ActionResult, ActionType } from '../throughput-types.js'; +import { requiredEnv, safeError } from './providers.js'; + +export const ACTIONS_PER_SESSION = 50; +export const ACTION_TIMEOUT_MS = 30_000; +export const ARTICLE_LINK_SELECTOR = '#mw-content-text a[href*="/wiki/"]'; + +export function articleUrl(value: unknown): string { + if (typeof value !== 'string') throw new Error('Article URL must be a string'); + const url = new URL(value); + const article = url.pathname.slice('/wiki/'.length); + if (url.origin !== 'https://en.wikipedia.org' || url.username || url.password || !url.pathname.startsWith('/wiki/') || !article || decodeURIComponent(article).includes(':') || url.search || url.hash) { + throw new Error('Expected a concrete Wikipedia article URL'); + } + return url.href; +} + +export function loadArticleUrls(): string[] { + const parsed: unknown = JSON.parse(readFileSync(requiredEnv('PLAYWRIGHT_NATIVE_URLS_FILE'), 'utf8')); + if (!Array.isArray(parsed) || parsed.length === 0) throw new Error('Expected a nonempty array of validated article URLs'); + return parsed.map(articleUrl); +} + +export async function firstArticleLink(page: Page): Promise { + await page.waitForSelector(ARTICLE_LINK_SELECTOR, { timeout: 10_000 }); + const links = await page.$$(ARTICLE_LINK_SELECTOR); + for (const link of links) { + const href = await link.getAttribute('href'); + const match = href?.match(/\/wiki\/([^#]*)/); + if (match && !match[1].includes(':')) return link; + } + throw new Error('No article body link found on page'); +} + +/** Same ten operations as CDP, repeated five times for both native providers. */ +export async function runActionLoop(page: Page, url: string, actions: ActionResult[]): Promise { + for (let loop = 0; loop < 5; loop++) { + const operations: [ActionType, () => Promise][] = [ + ['navigate', () => page.goto(url, { waitUntil: 'load' })], + ['waitForSelector', () => page.waitForSelector('#firstHeading')], + ['screenshot', () => page.screenshot()], + ['textContent', () => page.textContent('#firstHeading')], + ['click', async () => { await (await firstArticleLink(page)).click(); }], + ['waitForSelector', () => page.waitForSelector('#firstHeading')], + ['screenshot', () => page.screenshot()], + ['textContent', () => page.textContent('#firstHeading')], + ['goBack', () => page.goBack({ waitUntil: 'commit' })], + ['waitForSelector', () => page.waitForSelector('#firstHeading')], + ]; + let clickFailed = false; + for (const [offset, [type, operation]] of operations.entries()) { + const index = loop * 10 + offset + 1; + if (clickFailed && offset >= 5) { + actions.push({ index, type, durationMs: 0, success: false, error: 'skipped: click failed' }); + continue; + } + const start = performance.now(); + try { + await withTimeout(operation(), ACTION_TIMEOUT_MS, 'Action timed out'); + actions.push({ index, type, durationMs: performance.now() - start, success: true }); + } catch (error) { + actions.push({ index, type, durationMs: performance.now() - start, success: false, error: safeError(error) }); + if (offset === 4) clickFailed = true; + } + } + } +} diff --git a/benchmarks/browser/playwright-readiness.bench.ts b/benchmarks/browser/playwright-readiness.bench.ts new file mode 100644 index 00000000..f9f80412 --- /dev/null +++ b/benchmarks/browser/playwright-readiness.bench.ts @@ -0,0 +1,27 @@ +/** First per-session request → example.com load; cleanup is measured separately. */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import { nativeConfig, runSession } from './playwright-native/session.js'; +import { writeNativeResults } from './playwright-native/results.js'; + +export const config = defineBenchmarkConfig({ + ...nativeConfig, + benchmarkSlug: 'playwright-native-readiness', + benchmarkName: 'Playwright-native Browser Readiness', + display: { + metrics: [ + { key: 'readinessMs', label: 'Browser readiness', unit: 'ms', direction: 'lower-better' }, + { key: 'cleanupMs', label: 'Cleanup', unit: 'ms', direction: 'lower-better' }, + ], + overview: { defaultMetric: 'readinessMs', defaultLayout: 'chart' }, + }, + onComplete: outcome => { writeNativeResults(outcome, 'readiness'); }, +}); + +export const task = defineTask(ctx => runSession(ctx, async (page, startedAt, data) => { + await ctx.step('navigate', async () => { + await page.goto('https://www.example.com', { waitUntil: 'load' }); + data.readinessMs = performance.now() - startedAt; + ctx.measure({ readinessMs: data.readinessMs }); + }); +})); diff --git a/benchmarks/browser/playwright-throughput.bench.ts b/benchmarks/browser/playwright-throughput.bench.ts new file mode 100644 index 00000000..8e404e15 --- /dev/null +++ b/benchmarks/browser/playwright-throughput.bench.ts @@ -0,0 +1,51 @@ +/** Five ten-action loops; setup and cleanup excluded from action throughput. */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask, TaskError, parseCliArgs } from '@benchsdk/runner'; +import { nativeConfig, runSession } from './playwright-native/session.js'; +import { distribution, writeNativeResults } from './playwright-native/results.js'; +import { ACTIONS_PER_SESSION, loadArticleUrls, runActionLoop } from './playwright-native/workload.js'; +import type { ActionResult } from './throughput-types.js'; + +// Load the frozen validated inputs before any measured provider request. +const urls = loadArticleUrls(); +const iterationArgs = process.argv.flatMap((arg, index) => arg.startsWith('--iterations=') ? [arg] : arg === '--iterations' ? [arg, process.argv[index + 1] ?? ''] : []); +const iterations = parseCliArgs(iterationArgs).iterations ?? nativeConfig.iterations; +if (urls.length < iterations) throw new Error('Prepare a validated article URL for every requested iteration before measurement'); + +export const config = defineBenchmarkConfig({ + ...nativeConfig, + benchmarkSlug: 'playwright-native-throughput', + benchmarkName: 'Playwright-native Browser Throughput', + display: { + metrics: [ + { key: 'actionsPerSecond', label: 'Actions/s', unit: '/s', direction: 'higher-better' }, + { key: 'taskMs', label: 'Sum of action durations', unit: 'ms', direction: 'lower-better' }, + { key: 'screenshotMs', label: 'Session median screenshot', unit: 'ms', direction: 'lower-better' }, + { key: 'cleanupMs', label: 'Cleanup', unit: 'ms', direction: 'lower-better' }, + ], + overview: { defaultMetric: 'actionsPerSecond', defaultLayout: 'chart' }, + }, + onComplete: outcome => { writeNativeResults(outcome, 'throughput', urls); }, +}); + +export const task = defineTask(async ctx => { + const url = urls[ctx.taskIndex]; + if (!url) throw new TaskError('Validated URL list is shorter than the requested run; prepare all inputs first', { code: 'MISSING_INPUT', data: { attemptStarted: false } }); + const actions: ActionResult[] = []; + return runSession(ctx, async (page, _startedAt, data) => { + data.articleUrl = url; + await ctx.step('actions', async () => { + try { + await runActionLoop(page, url, actions); + } finally { + const taskMs = actions.reduce((sum, action) => sum + action.durationMs, 0); + const actionsCompleted = actions.filter(action => action.success).length; + const screenshotMs = distribution(actions.filter(action => action.type === 'screenshot' && action.success).map(action => action.durationMs)).median; + const metrics = { taskMs, actionsCompleted, actionsPerSecond: taskMs > 0 ? actionsCompleted / (taskMs / 1000) : 0, ...(screenshotMs !== null ? { screenshotMs } : {}) }; + Object.assign(data, { actions: actions.map(action => ({ ...action })) }, metrics); + ctx.measure(metrics); + } + }); + if (data.actionsCompleted !== ACTIONS_PER_SESSION) throw new Error('Incomplete action session'); + }); +}); diff --git a/package.json b/package.json index ec168cd1..e5441d57 100644 --- a/package.json +++ b/package.json @@ -147,7 +147,11 @@ "bench:ai-gateway-model-index:ngrok": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider ngrok", "bench:ai-gateway-model-index:llmapi": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider llmapi", "bench:ai-gateway-model-index:novita": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider novita", - "bench:ai-gateway-model-index:github-copilot": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider github-copilot" + "bench:ai-gateway-model-index:github-copilot": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider github-copilot", + "bench:playwright-readiness": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/playwright-readiness.bench.ts", + "bench:playwright-throughput": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/playwright-throughput.bench.ts", + "bench:playwright-prepare-urls": "tsx benchmarks/browser/playwright-native/prepare-urls.ts", + "test:playwright-native": "tsx --test benchmarks/browser/playwright-native/native.test.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.1152.0", @@ -223,6 +227,7 @@ "p-limit": "^7.3.3", "pg": "^8.23.1", "playwright-core": "^1.64.1", + "playwright-native-core": "npm:playwright-core@1.60.0", "sql.js": "^1.15.0", "tensorlake": "0.5.148" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 21e87d30..585cb71a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -227,6 +227,9 @@ importers: playwright-core: specifier: ^1.64.1 version: 1.64.1 + playwright-native-core: + specifier: npm:playwright-core@1.60.0 + version: playwright-core@1.60.0 sql.js: specifier: ^1.15.0 version: 1.15.0 @@ -4797,6 +4800,11 @@ packages: platform@1.3.6: resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + playwright-core@1.60.0: + resolution: {integrity: sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==} + engines: {node: '>=18'} + hasBin: true + playwright-core@1.64.1: resolution: {integrity: sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ==} engines: {node: '>=20'} @@ -10941,6 +10949,8 @@ snapshots: platform@1.3.6: {} + playwright-core@1.60.0: {} + playwright-core@1.64.1: {} playwright@1.64.1: From 6d84fe3e43ef22ac6accf888cbb3233a6b1a64a5 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 8 Oct 2026 16:15:24 -0700 Subject: [PATCH 3/4] Simplify Playwright-native benchmark implementation --- .../browser/playwright-native/native.test.ts | 14 +++++-- .../browser/playwright-native/prepare-urls.ts | 3 +- .../browser/playwright-native/providers.ts | 8 ++-- .../browser/playwright-native/results.ts | 13 ++++--- .../browser/playwright-native/session.ts | 37 +++++++++---------- .../browser/playwright-native/workload.ts | 32 ++++++++-------- .../browser/playwright-readiness.bench.ts | 2 +- .../browser/playwright-throughput.bench.ts | 12 ++++-- 8 files changed, 66 insertions(+), 55 deletions(-) diff --git a/benchmarks/browser/playwright-native/native.test.ts b/benchmarks/browser/playwright-native/native.test.ts index c468b2a5..bf66126a 100644 --- a/benchmarks/browser/playwright-native/native.test.ts +++ b/benchmarks/browser/playwright-native/native.test.ts @@ -13,9 +13,6 @@ import { runBenchmark, type BenchmarkRunOutcome } from '@benchsdk/runner'; import { distribution } from './results.js'; import { articleUrl } from './workload.js'; -// Plausible failures covered here: incomplete actions, wrong connection auth, -// lost create response, malformed READY response, refused connection, release -// failure, missing round input, leaked contexts, and trimmed/partial summaries. test('native suites use real Playwright connections and preserve reliability records', { timeout: 120_000 }, async () => { const directory = mkdtempSync(path.join(tmpdir(), 'native-bench-')); const envNames = ['MOMENTIC_BROWSER_FLEET_URL', 'MOMENTIC_BROWSER_FLEET_API_KEY', 'AZURE_PLAYWRIGHT_SERVICE_URL', 'AZURE_PLAYWRIGHT_ACCESS_TOKEN', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE', 'PLAYWRIGHT_NATIVE_URLS_FILE', 'PLAYWRIGHT_NATIVE_RESULTS_DIR']; @@ -47,6 +44,7 @@ test('native suites use real Playwright connections and preserve reliability rec const terminated = new Set(); const order: string[] = []; const websocketAuth: string[] = []; + const azureRunIds = new Set(); let mode = ''; let lost = false; let endpoint = ''; @@ -57,6 +55,9 @@ test('native suites use real Playwright connections and preserve reliability rec assert.equal(req.headers.authorization, 'Bearer local-azure-credential'); assert.equal(url.searchParams.get('os'), 'Linux'); assert.equal(url.searchParams.get('api-version'), '2025-09-01'); + const runId = url.searchParams.get('runId'); + assert.ok(runId); + azureRunIds.add(runId); order.push('azure'); res.writeHead(302, { Location: mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${apiPort}/connect/azure/${url.searchParams.get('runId')}` }); res.end(); @@ -127,6 +128,11 @@ test('native suites use real Playwright connections and preserve reliability rec const ready = await runBenchmark({ ...readiness.config, iterations: 2 }, readiness.task, ['--no-ingest']); assert.deepEqual(order, ['momentic', 'azure', 'momentic', 'azure']); assert.deepEqual(websocketAuth, order); + for (const { participant, records } of ready.participants) { + if (participant !== 'azure') continue; + assert.deepEqual(records.map(record => record.data?.runId), [...azureRunIds]); + assert.ok(records.every(record => record.data?.sessionId === undefined), 'run IDs must not be reported as service session IDs'); + } for (const record of records(ready)) { assert.equal(record.status, 'success'); assert.equal(record.data?.sessionSuccess, true); @@ -212,7 +218,7 @@ test('native suites use real Playwright connections and preserve reliability rec const audit = await chromium.connect(endpoint); assert.equal(audit.contexts().length, 0, 'all benchmark contexts were closed'); await audit.close(); - console.log('Verified real native connections, 50-action records, round interleaving, load boundary, auth, partial failures, cleanup recovery, secret-free artifacts, and untrimmed summaries.'); + console.log('Native benchmark end-to-end checks passed.'); } finally { await server.close(); for (const socket of sockets) socket.destroy(); diff --git a/benchmarks/browser/playwright-native/prepare-urls.ts b/benchmarks/browser/playwright-native/prepare-urls.ts index b59b7688..9963125f 100644 --- a/benchmarks/browser/playwright-native/prepare-urls.ts +++ b/benchmarks/browser/playwright-native/prepare-urls.ts @@ -3,13 +3,14 @@ import { chromium } from 'playwright-native-core'; import { writeFileSync } from 'node:fs'; import { articleUrl, firstArticleLink } from './workload.js'; import { withTimeout } from '../../src/util/timeout.js'; +import { VIEWPORT } from './providers.js'; const count = Number(process.argv[2] ?? '100'); const filename = process.argv[3] ?? 'playwright-native-urls.json'; if (!Number.isInteger(count) || count < 1) throw new Error('Count must be a positive integer'); const browser = await chromium.launch({ headless: true, timeout: 30_000 }); try { - const page = await browser.newPage({ viewport: { width: 1920, height: 1080 } }); + const page = await browser.newPage({ viewport: VIEWPORT }); page.setDefaultTimeout(30_000); page.setDefaultNavigationTimeout(30_000); const urls: string[] = []; diff --git a/benchmarks/browser/playwright-native/providers.ts b/benchmarks/browser/playwright-native/providers.ts index b4fcd819..b94462a8 100644 --- a/benchmarks/browser/playwright-native/providers.ts +++ b/benchmarks/browser/playwright-native/providers.ts @@ -6,6 +6,7 @@ export const PLAYWRIGHT_VERSION = '1.60.0'; export const PROVISION_TIMEOUT_MS = 120_000; export const CONNECT_TIMEOUT_MS = 30_000; export const CLEANUP_TIMEOUT_MS = 15_000; +export const VIEWPORT = { width: 1920, height: 1080 }; export interface NativeSession { metadata: JsonObject; @@ -35,7 +36,6 @@ function string(value: unknown): string { return value; } -/** Only publish explicit, non-secret environment attestations. */ function environment(provider: string): JsonObject { const filename = requiredEnv('PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'); let parsed: unknown; @@ -60,7 +60,7 @@ function environment(provider: string): JsonObject { serverPlaywrightVersion: serverVersion, serverPlaywrightVersionSource: 'operator-verified environment manifest', clientPlaywrightVersion: PLAYWRIGHT_VERSION, - browserOs: 'linux', headless: true, viewport: { width: 1920, height: 1080 }, + browserOs: 'linux', headless: true, viewport: VIEWPORT, stealth: false, proxy: false, recording: false, metadataSource: 'operator-verified environment manifest', }; @@ -155,14 +155,14 @@ export const nativeParticipants: NativeParticipant[] = [ session(identity) { const url = new URL(requiredEnv('AZURE_PLAYWRIGHT_SERVICE_URL')); const headers = { Authorization: `Bearer ${requiredEnv('AZURE_PLAYWRIGHT_ACCESS_TOKEN')}` }; - const metadata = environment('azure'); + const metadata = { ...environment('azure'), runId: identity }; url.protocol = url.protocol === 'wss:' ? 'https:' : url.protocol === 'ws:' ? 'http:' : url.protocol; url.searchParams.set('api-version', '2025-09-01'); url.searchParams.set('os', 'Linux'); url.searchParams.set('runId', identity); let allocationUncertain = false; return { - metadata: { ...metadata, sessionId: identity }, + metadata, async provision() { // Resolve the documented redirect inside the measured interval. allocationUncertain = true; diff --git a/benchmarks/browser/playwright-native/results.ts b/benchmarks/browser/playwright-native/results.ts index 4a367d28..80c289af 100644 --- a/benchmarks/browser/playwright-native/results.ts +++ b/benchmarks/browser/playwright-native/results.ts @@ -18,7 +18,7 @@ export function distribution(values: number[]) { }; } -export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readiness' | 'throughput', articleUrls?: string[]): string { +export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readiness' | 'throughput', articleUrls?: string[]): void { const summaries = outcome.participants.map(({ participant, records }) => { const attempted = records.filter(record => record.data?.attemptStarted === true); const complete = attempted.filter(record => record.data?.workloadSuccess === true); @@ -28,14 +28,18 @@ export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readine return typeof value === 'number' && Number.isFinite(value) ? [value] : []; })), ])); - const perActionType = Object.fromEntries(ACTION_TYPES.map(type => [type, distribution(complete.flatMap(record => { + const actions = complete.flatMap(record => { const actions = record.data?.actions; if (!Array.isArray(actions)) return []; return actions.flatMap(action => { if (!action || typeof action !== 'object' || Array.isArray(action)) return []; - return action.type === type && action.success === true && typeof action.durationMs === 'number' ? [action.durationMs] : []; + if (action.success !== true || typeof action.durationMs !== 'number') return []; + return [{ type: action.type, durationMs: action.durationMs }]; }); - }))])); + }); + const perActionType = Object.fromEntries(ACTION_TYPES.map(type => [type, + distribution(actions.filter(action => action.type === type).map(action => action.durationMs)), + ])); const rate = (key: string) => attempted.length ? attempted.filter(record => record.data?.[key] === true).length / attempted.length : null; return { participant, scheduled: records.length, attempts: attempted.length, workloadSuccessRate: rate('workloadSuccess'), cleanupSuccessRate: rate('cleanupSuccess'), successRate: rate('sessionSuccess'), metrics, perActionType }; }); @@ -44,5 +48,4 @@ export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readine const filename = path.join(directory, `${randomUUID()}.json`); writeFileSync(filename, JSON.stringify({ suite, runId: outcome.runId, createdAt: new Date().toISOString(), config: outcome.config, ...(articleUrls ? { articleUrls } : {}), summaries, participants: outcome.participants }, null, 2)); console.log(`Raw Playwright-native records and untrimmed summaries: ${filename}`); - return filename; } diff --git a/benchmarks/browser/playwright-native/session.ts b/benchmarks/browser/playwright-native/session.ts index 99543ade..2183bdea 100644 --- a/benchmarks/browser/playwright-native/session.ts +++ b/benchmarks/browser/playwright-native/session.ts @@ -1,9 +1,9 @@ import { chromium, type Page, type Browser, type BrowserContext } from 'playwright-native-core'; import { randomUUID } from 'node:crypto'; -import { TaskError, type TaskContext } from '@benchsdk/runner'; +import { TaskError, type TaskContext, type TaskStepOptions } from '@benchsdk/runner'; import type { JsonObject } from '@benchsdk/api'; import { withTimeout } from '../../src/util/timeout.js'; -import { CLEANUP_TIMEOUT_MS, CONNECT_TIMEOUT_MS, nativeParticipants, safeError, type NativeParticipant } from './providers.js'; +import { CLEANUP_TIMEOUT_MS, CONNECT_TIMEOUT_MS, VIEWPORT, nativeParticipants, safeError, type NativeParticipant } from './providers.js'; const unsafeParticipants = new WeakSet(); @@ -24,7 +24,7 @@ export async function runSession( code: 'CLEANUP_UNCERTAIN', data: { attemptStarted: false }, }); } - // Prepare reusable credentials and validate the environment before timing. + // Read credentials and validate settings before starting the timer. const session = ctx.participant.session(randomUUID()); const data: JsonObject = { ...session.metadata, attemptStarted: true, workloadSuccess: false, cleanupSuccess: false }; let browser: Browser | undefined; @@ -32,25 +32,27 @@ export async function runSession( let secrets: string[] = []; let failure: string | undefined; const cleanupErrors: string[] = []; - const guarded = async (fn: () => Promise): Promise => { - try { return await fn(); } catch (error) { throw new Error(safeError(error, secrets)); } - }; + // Redact errors before the runner records step diagnostics. + const step = (name: string, fn: () => Promise, options?: Pick) => + ctx.step(name, async () => { + try { return await fn(); } + catch (error) { throw new Error(safeError(error, secrets)); } + }, options); const startedAt = performance.now(); try { - const connection = await ctx.step('provision', () => guarded(() => session.provision())); - Object.assign(data, session.metadata); + const connection = await step('provision', () => session.provision()); secrets = Object.values(connection.headers); - browser = await ctx.step('connect', () => guarded(() => chromium.connect(connection.endpoint, { headers: connection.headers, timeout: CONNECT_TIMEOUT_MS }))); + browser = await step('connect', () => chromium.connect(connection.endpoint, { headers: connection.headers, timeout: CONNECT_TIMEOUT_MS })); data.chromiumVersion = browser.version(); data.runnerOs = process.platform; data.nodeVersion = process.version; const connected = browser; - const page = await ctx.step('page', () => guarded(async () => { - context = await withTimeout(connected.newContext({ viewport: { width: 1920, height: 1080 } }), CONNECT_TIMEOUT_MS, 'Context creation timed out'); + const page = await step('page', async () => { + context = await withTimeout(connected.newContext({ viewport: VIEWPORT }), CONNECT_TIMEOUT_MS, 'Context creation timed out'); context.setDefaultTimeout(30_000); context.setDefaultNavigationTimeout(30_000); return withTimeout(context.newPage(), CONNECT_TIMEOUT_MS, 'Page creation timed out'); - })); + }); await workload(page, startedAt, data); data.workloadSuccess = true; } catch (error) { @@ -60,16 +62,11 @@ export async function runSession( const cleanupStart = performance.now(); const cleanup = async (name: string, fn: () => Promise) => { try { - await ctx.step(name, () => guarded(() => withTimeout(fn(), CLEANUP_TIMEOUT_MS, `${name} timed out`)), { reportConcurrency: false }); + await step(name, () => withTimeout(fn(), CLEANUP_TIMEOUT_MS, `${name} timed out`), { reportConcurrency: false }); } catch (error) { cleanupErrors.push(safeError(error, secrets)); } }; - if (context) { - const activeContext = context; - await cleanup('close-context', () => activeContext.close()); - } - if (browser) { - const activeBrowser = browser; - await cleanup('close-browser', () => activeBrowser.close()); + for (const [name, resource] of [['close-context', context], ['close-browser', browser]] as const) { + if (resource) await cleanup(name, () => resource.close()); } await cleanup('release', () => session.cleanup({ connected: browser !== undefined })); data.cleanupMs = performance.now() - cleanupStart; diff --git a/benchmarks/browser/playwright-native/workload.ts b/benchmarks/browser/playwright-native/workload.ts index c1faea02..131b2a83 100644 --- a/benchmarks/browser/playwright-native/workload.ts +++ b/benchmarks/browser/playwright-native/workload.ts @@ -4,7 +4,8 @@ import { withTimeout } from '../../src/util/timeout.js'; import type { ActionResult, ActionType } from '../throughput-types.js'; import { requiredEnv, safeError } from './providers.js'; -export const ACTIONS_PER_SESSION = 50; +const LOOPS_PER_SESSION = 5; +export const ACTIONS_PER_SESSION = LOOPS_PER_SESSION * 10; export const ACTION_TIMEOUT_MS = 30_000; export const ARTICLE_LINK_SELECTOR = '#mw-content-text a[href*="/wiki/"]'; @@ -35,24 +36,23 @@ export async function firstArticleLink(page: Page): Promise { throw new Error('No article body link found on page'); } -/** Same ten operations as CDP, repeated five times for both native providers. */ export async function runActionLoop(page: Page, url: string, actions: ActionResult[]): Promise { - for (let loop = 0; loop < 5; loop++) { - const operations: [ActionType, () => Promise][] = [ - ['navigate', () => page.goto(url, { waitUntil: 'load' })], - ['waitForSelector', () => page.waitForSelector('#firstHeading')], - ['screenshot', () => page.screenshot()], - ['textContent', () => page.textContent('#firstHeading')], - ['click', async () => { await (await firstArticleLink(page)).click(); }], - ['waitForSelector', () => page.waitForSelector('#firstHeading')], - ['screenshot', () => page.screenshot()], - ['textContent', () => page.textContent('#firstHeading')], - ['goBack', () => page.goBack({ waitUntil: 'commit' })], - ['waitForSelector', () => page.waitForSelector('#firstHeading')], - ]; + const operations: [ActionType, () => Promise][] = [ + ['navigate', () => page.goto(url, { waitUntil: 'load' })], + ['waitForSelector', () => page.waitForSelector('#firstHeading')], + ['screenshot', () => page.screenshot()], + ['textContent', () => page.textContent('#firstHeading')], + ['click', async () => { await (await firstArticleLink(page)).click(); }], + ['waitForSelector', () => page.waitForSelector('#firstHeading')], + ['screenshot', () => page.screenshot()], + ['textContent', () => page.textContent('#firstHeading')], + ['goBack', () => page.goBack({ waitUntil: 'commit' })], + ['waitForSelector', () => page.waitForSelector('#firstHeading')], + ]; + for (let loop = 0; loop < LOOPS_PER_SESSION; loop++) { let clickFailed = false; for (const [offset, [type, operation]] of operations.entries()) { - const index = loop * 10 + offset + 1; + const index = loop * operations.length + offset + 1; if (clickFailed && offset >= 5) { actions.push({ index, type, durationMs: 0, success: false, error: 'skipped: click failed' }); continue; diff --git a/benchmarks/browser/playwright-readiness.bench.ts b/benchmarks/browser/playwright-readiness.bench.ts index f9f80412..683a9e81 100644 --- a/benchmarks/browser/playwright-readiness.bench.ts +++ b/benchmarks/browser/playwright-readiness.bench.ts @@ -15,7 +15,7 @@ export const config = defineBenchmarkConfig({ ], overview: { defaultMetric: 'readinessMs', defaultLayout: 'chart' }, }, - onComplete: outcome => { writeNativeResults(outcome, 'readiness'); }, + onComplete: outcome => writeNativeResults(outcome, 'readiness'), }); export const task = defineTask(ctx => runSession(ctx, async (page, startedAt, data) => { diff --git a/benchmarks/browser/playwright-throughput.bench.ts b/benchmarks/browser/playwright-throughput.bench.ts index 8e404e15..cd6f9819 100644 --- a/benchmarks/browser/playwright-throughput.bench.ts +++ b/benchmarks/browser/playwright-throughput.bench.ts @@ -8,7 +8,11 @@ import type { ActionResult } from './throughput-types.js'; // Load the frozen validated inputs before any measured provider request. const urls = loadArticleUrls(); -const iterationArgs = process.argv.flatMap((arg, index) => arg.startsWith('--iterations=') ? [arg] : arg === '--iterations' ? [arg, process.argv[index + 1] ?? ''] : []); +const iterationArgs = process.argv.flatMap((arg, index) => { + if (arg.startsWith('--iterations=')) return [arg]; + if (arg === '--iterations') return [arg, process.argv[index + 1] ?? '']; + return []; +}); const iterations = parseCliArgs(iterationArgs).iterations ?? nativeConfig.iterations; if (urls.length < iterations) throw new Error('Prepare a validated article URL for every requested iteration before measurement'); @@ -25,10 +29,10 @@ export const config = defineBenchmarkConfig({ ], overview: { defaultMetric: 'actionsPerSecond', defaultLayout: 'chart' }, }, - onComplete: outcome => { writeNativeResults(outcome, 'throughput', urls); }, + onComplete: outcome => writeNativeResults(outcome, 'throughput', urls), }); -export const task = defineTask(async ctx => { +export const task = defineTask(ctx => { const url = urls[ctx.taskIndex]; if (!url) throw new TaskError('Validated URL list is shorter than the requested run; prepare all inputs first', { code: 'MISSING_INPUT', data: { attemptStarted: false } }); const actions: ActionResult[] = []; @@ -42,7 +46,7 @@ export const task = defineTask(async c const actionsCompleted = actions.filter(action => action.success).length; const screenshotMs = distribution(actions.filter(action => action.type === 'screenshot' && action.success).map(action => action.durationMs)).median; const metrics = { taskMs, actionsCompleted, actionsPerSecond: taskMs > 0 ? actionsCompleted / (taskMs / 1000) : 0, ...(screenshotMs !== null ? { screenshotMs } : {}) }; - Object.assign(data, { actions: actions.map(action => ({ ...action })) }, metrics); + Object.assign(data, metrics, { actions }); ctx.measure(metrics); } }); From e0a29591e9fed4e94c558a42f21434a866374630 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 8 Oct 2026 21:03:37 -0700 Subject: [PATCH 4/4] Fix native benchmark cleanup and comparison preflight --- PLAYWRIGHT_NATIVE_BENCHMARKS.md | 12 ++- ...tive.test.ts => playwright-native.test.ts} | 75 +++++++++++++++++-- ...are-urls.ts => playwright-prepare-urls.ts} | 6 +- .../providers.ts => playwright-providers.ts} | 28 ++++--- .../browser/playwright-readiness.bench.ts | 25 ++++++- .../results.ts => playwright-results.ts} | 9 ++- .../session.ts => playwright-session.ts} | 32 +++++--- .../browser/playwright-throughput.bench.ts | 40 ++++++---- benchmarks/browser/playwright-types.ts | 13 ++++ .../workload.ts => playwright-workload.ts} | 6 +- package.json | 4 +- 11 files changed, 189 insertions(+), 61 deletions(-) rename benchmarks/browser/{playwright-native/native.test.ts => playwright-native.test.ts} (76%) rename benchmarks/browser/{playwright-native/prepare-urls.ts => playwright-prepare-urls.ts} (87%) rename benchmarks/browser/{playwright-native/providers.ts => playwright-providers.ts} (87%) rename benchmarks/browser/{playwright-native/results.ts => playwright-results.ts} (78%) rename benchmarks/browser/{playwright-native/session.ts => playwright-session.ts} (75%) create mode 100644 benchmarks/browser/playwright-types.ts rename benchmarks/browser/{playwright-native/workload.ts => playwright-workload.ts} (94%) diff --git a/PLAYWRIGHT_NATIVE_BENCHMARKS.md b/PLAYWRIGHT_NATIVE_BENCHMARKS.md index 0a189f4a..ec06bbd6 100644 --- a/PLAYWRIGHT_NATIVE_BENCHMARKS.md +++ b/PLAYWRIGHT_NATIVE_BENCHMARKS.md @@ -279,6 +279,12 @@ pnpm bench:playwright-readiness --no-ingest pnpm bench:playwright-throughput --no-ingest ``` +The default commands require credentials for both providers and fail before any +provider request if either is unavailable. For an explicit single-provider +diagnostic, pass `--provider momentic` or `--provider azure`. Raw diagnostic +outputs are labeled `provider-diagnostic` and list unavailable participants +separately from attempted-session denominators. + Store credentials in your shell or the ignored `benchmarks/.env`, never in Git: - `MOMENTIC_BROWSER_FLEET_URL`: the Browser Fleet API base URL. @@ -346,8 +352,10 @@ cleanup is recorded as uncertain and further allocations stop for that participant; the implementation does not claim it released an unknown resource. Provisioning has a 120-second bound; connection, context/page creation, and navigation have 30-second bounds; each -cleanup operation has a 15-second bound. After uncertain cleanup, the process -refuses further allocations for that participant and records unstarted tasks +cleanup operation has a 15-second bound. Timed-out context creation is retained +and reconciled within that cleanup bound; late contexts are explicitly closed. +Unresolved context ownership makes cleanup uncertain. After uncertain cleanup, +the process refuses further allocations for that participant and records unstarted tasks separately from attempted-session reliability denominators. No real credentials are used by fixture tests. Local fixture credentials and diff --git a/benchmarks/browser/playwright-native/native.test.ts b/benchmarks/browser/playwright-native.test.ts similarity index 76% rename from benchmarks/browser/playwright-native/native.test.ts rename to benchmarks/browser/playwright-native.test.ts index bf66126a..2e9d6982 100644 --- a/benchmarks/browser/playwright-native/native.test.ts +++ b/benchmarks/browser/playwright-native.test.ts @@ -8,12 +8,13 @@ import { promisify } from 'node:util'; import { mkdtempSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { chromium } from 'playwright-native-core'; +import { chromium, type BrowserContext, type BrowserContextOptions } from 'playwright-native-core'; +import { withTimeout } from '../src/util/timeout.js'; import { runBenchmark, type BenchmarkRunOutcome } from '@benchsdk/runner'; -import { distribution } from './results.js'; -import { articleUrl } from './workload.js'; +import { distribution } from './playwright-results.js'; +import { articleUrl } from './playwright-workload.js'; -test('native suites use real Playwright connections and preserve reliability records', { timeout: 120_000 }, async () => { +test('native suites use real Playwright connections and preserve reliability records', { timeout: 180_000 }, async t => { const directory = mkdtempSync(path.join(tmpdir(), 'native-bench-')); const envNames = ['MOMENTIC_BROWSER_FLEET_URL', 'MOMENTIC_BROWSER_FLEET_API_KEY', 'AZURE_PLAYWRIGHT_SERVICE_URL', 'AZURE_PLAYWRIGHT_ACCESS_TOKEN', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE', 'PLAYWRIGHT_NATIVE_URLS_FILE', 'PLAYWRIGHT_NATIVE_RESULTS_DIR']; const previous = new Map(envNames.map(key => [key, process.env[key]])); @@ -76,7 +77,7 @@ test('native suites use real Playwright connections and preserve reliability rec const sessionId = sessions.get(key); if (mode === 'lost-response' && !lost) { lost = true; req.socket.destroy(); return; } if (mode === 'malformed-json') { res.end('{"connectToken":"local-unpublished-credential'); return; } - res.end(JSON.stringify({ id: sessionId, status: 'READY', playwrightVersion: '1.60.0', wsEndpoint: mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${apiPort}/connect/momentic/${sessionId}`, ...(mode !== 'malformed-ready' ? { connectToken: 'local-connect-credential' } : {}) })); + res.end(JSON.stringify({ id: sessionId, status: 'READY', playwrightVersion: mode === 'compatible-patch' ? '1.60.1' : mode === 'incompatible-version' ? '1.61.0' : '1.60.0', wsEndpoint: mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${apiPort}/connect/momentic/${sessionId}`, ...(mode !== 'malformed-ready' ? { connectToken: 'local-connect-credential' } : {}) })); } else if (req.method === 'DELETE') { if (mode === 'cleanup-failure') { res.writeHead(500); res.end('{}'); return; } if (id) terminated.add(id); @@ -122,8 +123,8 @@ test('native suites use real Playwright connections and preserve reliability rec writeFileSync(environmentFile, JSON.stringify(environment)); writeFileSync(urlsFile, JSON.stringify(urls)); Object.assign(process.env, { MOMENTIC_BROWSER_FLEET_URL: `http://127.0.0.1:${apiPort}`, MOMENTIC_BROWSER_FLEET_API_KEY: 'local-momentic-credential', AZURE_PLAYWRIGHT_SERVICE_URL: `http://127.0.0.1:${apiPort}/browsers`, AZURE_PLAYWRIGHT_ACCESS_TOKEN: 'local-azure-credential', PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE: environmentFile, PLAYWRIGHT_NATIVE_URLS_FILE: urlsFile, PLAYWRIGHT_NATIVE_RESULTS_DIR: directory }); - const readiness = await import('../playwright-readiness.bench.js'); - const throughput = await import('../playwright-throughput.bench.js'); + const readiness = await import('./playwright-readiness.bench.js'); + const throughput = await import('./playwright-throughput.bench.js'); const records = (outcome: BenchmarkRunOutcome) => outcome.participants.flatMap(p => p.records); const ready = await runBenchmark({ ...readiness.config, iterations: 2 }, readiness.task, ['--no-ingest']); assert.deepEqual(order, ['momentic', 'azure', 'momentic', 'azure']); @@ -173,7 +174,7 @@ test('native suites use real Playwright connections and preserve reliability rec assert.equal(summary.metrics.actionsPerSecond.count, 2); assert.equal(summary.perActionType.screenshot.count, 20); } - for (const failureMode of ['lost-response', 'malformed-ready', 'refused-connection', 'cleanup-failure']) { + for (const failureMode of ['lost-response', 'malformed-ready', 'refused-connection', 'incompatible-version', 'cleanup-failure']) { mode = failureMode; order.length = 0; const participants = readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })); @@ -187,6 +188,10 @@ test('native suites use real Playwright connections and preserve reliability rec assert.deepEqual(order, ['momentic'], 'uncertain cleanup stops new allocations'); } else assert.equal(terminated.size, sessions.size); } + mode = 'compatible-patch'; + const compatible = await runBenchmark({ ...readiness.config, participants: readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })), iterations: 1 }, readiness.task, ['--no-ingest']); + assert.equal(records(compatible)[0].status, 'success'); + assert.equal(records(compatible)[0].data?.serverPlaywrightVersion, '1.60.1'); mode = 'refused-connection'; const azureFailure = await runBenchmark({ ...readiness.config, participants: readiness.config.participants.filter(p => p.name === 'azure').map(p => ({ ...p })), iterations: 2 }, readiness.task, ['--no-ingest']); assert.equal(records(azureFailure)[0].data?.cleanupSuccess, false); @@ -215,6 +220,60 @@ test('native suites use real Playwright connections and preserve reliability rec assert.ok(stdout.includes('All done. No platform run created.')); assert.ok(!stdout.includes('FAILED')); } + // Missing credentials must fail the comparison before any allocation. + const diagnosticEnv = { ...process.env }; + delete diagnosticEnv.AZURE_PLAYWRIGHT_ACCESS_TOKEN; + for (const entry of ['playwright-readiness', 'playwright-throughput']) { + const command = ['node_modules/tsx/dist/cli.mjs', 'packages/benchsdk-runner/dist/bin.js', 'run', `benchmarks/browser/${entry}.bench.ts`, '--no-ingest', '--iterations=1']; + order.length = 0; + await assert.rejects(promisify(execFile)(process.execPath, command, { env: diagnosticEnv }), /Unavailable Playwright participant\(s\): azure: AZURE_PLAYWRIGHT_ACCESS_TOKEN/); + assert.deepEqual(order, []); + const { stdout } = await promisify(execFile)(process.execPath, [...command, '--provider=momentic'], { env: diagnosticEnv }); + assert.deepEqual(order, ['momentic']); + const match = stdout.match(/Raw Playwright-native records and untrimmed summaries: (.+)/); + assert.ok(match); + const diagnostic = JSON.parse(readFileSync(match[1].trim(), 'utf8')); + assert.equal(diagnostic.mode, 'provider-diagnostic'); + assert.deepEqual(diagnostic.unavailableParticipants, [{ name: 'azure', missing: ['AZURE_PLAYWRIGHT_ACCESS_TOKEN'] }]); + } + // Delay returning an actual remote context beyond the 30-second deadline. + // Cover reconciliation within cleanup, and ownership still unresolved at its bound. + for (const delayMs of [31_000, 46_000]) { + const client = await chromium.connect(endpoint); + const newContext = client.newContext.bind(client); + let pending: Promise | undefined; + let closed: Promise | undefined; + const contextMock = t.mock.method(client, 'newContext', (options: BrowserContextOptions) => { + pending = newContext(options).then(async created => { + closed = new Promise(resolve => created.once('close', () => resolve())); + await new Promise(resolve => setTimeout(resolve, delayMs)); + return created; + }); + return pending; + }); + const connectMock = t.mock.method(chromium, 'connect', async () => client); + const closeMock = delayMs === 46_000 ? t.mock.method(client, 'close', async () => { throw new Error('fixture disconnect failed'); }) : undefined; + try { + order.length = 0; + const participants = readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })); + const late = await runBenchmark({ ...readiness.config, participants, iterations: delayMs === 46_000 ? 2 : 1 }, readiness.task, ['--no-ingest']); + const [first, second] = records(late); + assert.equal(first.status, 'error'); + assert.match(String(first.data?.errorMessage), /Context creation timed out/); + assert.equal(first.data?.cleanupSuccess, delayMs === 31_000); + if (delayMs === 46_000) { + assert.equal(second.data?.attemptStarted, false); + assert.deepEqual(order, ['momentic'], 'unresolved context ownership stops allocations'); + } + assert.ok(pending && closed); + await pending; + await withTimeout(closed, 5_000, 'Late context was not closed'); + assert.equal(client.contexts().length, 0); + } finally { + contextMock.mock.restore(); connectMock.mock.restore(); closeMock?.mock.restore(); + await client.close(); + } + } const audit = await chromium.connect(endpoint); assert.equal(audit.contexts().length, 0, 'all benchmark contexts were closed'); await audit.close(); diff --git a/benchmarks/browser/playwright-native/prepare-urls.ts b/benchmarks/browser/playwright-prepare-urls.ts similarity index 87% rename from benchmarks/browser/playwright-native/prepare-urls.ts rename to benchmarks/browser/playwright-prepare-urls.ts index 9963125f..35899986 100644 --- a/benchmarks/browser/playwright-native/prepare-urls.ts +++ b/benchmarks/browser/playwright-prepare-urls.ts @@ -1,9 +1,9 @@ /** Resolve and DOM-validate shared Wikipedia inputs before measured sessions. */ import { chromium } from 'playwright-native-core'; import { writeFileSync } from 'node:fs'; -import { articleUrl, firstArticleLink } from './workload.js'; -import { withTimeout } from '../../src/util/timeout.js'; -import { VIEWPORT } from './providers.js'; +import { articleUrl, firstArticleLink } from './playwright-workload.js'; +import { withTimeout } from '../src/util/timeout.js'; +import { VIEWPORT } from './playwright-providers.js'; const count = Number(process.argv[2] ?? '100'); const filename = process.argv[3] ?? 'playwright-native-urls.json'; diff --git a/benchmarks/browser/playwright-native/providers.ts b/benchmarks/browser/playwright-providers.ts similarity index 87% rename from benchmarks/browser/playwright-native/providers.ts rename to benchmarks/browser/playwright-providers.ts index b94462a8..2a3660f1 100644 --- a/benchmarks/browser/playwright-native/providers.ts +++ b/benchmarks/browser/playwright-providers.ts @@ -1,6 +1,8 @@ import { setTimeout as delay } from 'node:timers/promises'; import { readFileSync } from 'node:fs'; import type { JsonObject } from '@benchsdk/api'; +import { parseCliArgs, selectParticipants, filterParticipantsByEnv } from '@benchsdk/runner'; +import type { PlaywrightProviderConfig } from './playwright-types.js'; export const PLAYWRIGHT_VERSION = '1.60.0'; export const PROVISION_TIMEOUT_MS = 120_000; @@ -8,16 +10,20 @@ export const CONNECT_TIMEOUT_MS = 30_000; export const CLEANUP_TIMEOUT_MS = 15_000; export const VIEWPORT = { width: 1920, height: 1080 }; -export interface NativeSession { - metadata: JsonObject; - provision(): Promise<{ endpoint: string; headers: Record }>; - cleanup(options: { connected: boolean }): Promise; -} +// Parse only flags owned by the suites; the runner handles the rest of the CLI. +export const playwrightArgs = parseCliArgs(process.argv.flatMap((arg, index) => { + if (/^--(?:provider|iterations)=/.test(arg)) return [arg]; + if (arg === '--provider' || arg === '--iterations') return [arg, process.argv[index + 1] ?? '']; + return []; +})); -export interface NativeParticipant { - name: string; - requiredEnvVars: string[]; - session(identity: string): NativeSession; +export function requireProviderCredentials(): void { + const selected = selectParticipants(playwrightProviders, playwrightArgs.providers); + const { skipped } = filterParticipantsByEnv(selected); + if (skipped.length) { + const missing = skipped.map(({ name, missing }) => `${name}: ${missing.join(', ')}`).join('; '); + throw new Error(`Unavailable Playwright participant(s): ${missing}. Supply credentials for every selected provider; use --provider for an explicit diagnostic run.`); + } } export function requiredEnv(name: string): string { @@ -88,7 +94,7 @@ async function sessionResponse(response: Response): Promise writeNativeResults(outcome, 'readiness'), }); -export const task = defineTask(ctx => runSession(ctx, async (page, startedAt, data) => { +export const task = defineTask(ctx => runSession(ctx, async (page, startedAt, data) => { await ctx.step('navigate', async () => { await page.goto('https://www.example.com', { waitUntil: 'load' }); data.readinessMs = performance.now() - startedAt; diff --git a/benchmarks/browser/playwright-native/results.ts b/benchmarks/browser/playwright-results.ts similarity index 78% rename from benchmarks/browser/playwright-native/results.ts rename to benchmarks/browser/playwright-results.ts index 80c289af..b64200fd 100644 --- a/benchmarks/browser/playwright-native/results.ts +++ b/benchmarks/browser/playwright-results.ts @@ -1,8 +1,9 @@ import { mkdirSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import { randomUUID } from 'node:crypto'; -import type { BenchmarkRunOutcome } from '@benchsdk/runner'; -import { ACTION_TYPES } from '../throughput-types.js'; +import { filterParticipantsByEnv, type BenchmarkRunOutcome } from '@benchsdk/runner'; +import { playwrightProviders } from './playwright-providers.js'; +import { ACTION_TYPES } from './throughput-types.js'; /** Untrimmed nearest-rank percentiles; missing observations are not zeroes. */ export function distribution(values: number[]) { @@ -19,6 +20,8 @@ export function distribution(values: number[]) { } export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readiness' | 'throughput', articleUrls?: string[]): void { + const comparison = playwrightProviders.every(provider => outcome.participants.some(p => p.participant === provider.name)); + const { skipped: unavailableParticipants } = filterParticipantsByEnv(playwrightProviders); const summaries = outcome.participants.map(({ participant, records }) => { const attempted = records.filter(record => record.data?.attemptStarted === true); const complete = attempted.filter(record => record.data?.workloadSuccess === true); @@ -46,6 +49,6 @@ export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readine const directory = path.resolve(process.env.PLAYWRIGHT_NATIVE_RESULTS_DIR ?? 'results/playwright-native', suite); mkdirSync(directory, { recursive: true }); const filename = path.join(directory, `${randomUUID()}.json`); - writeFileSync(filename, JSON.stringify({ suite, runId: outcome.runId, createdAt: new Date().toISOString(), config: outcome.config, ...(articleUrls ? { articleUrls } : {}), summaries, participants: outcome.participants }, null, 2)); + writeFileSync(filename, JSON.stringify({ suite, mode: comparison ? 'comparison' : 'provider-diagnostic', unavailableParticipants, runId: outcome.runId, createdAt: new Date().toISOString(), config: outcome.config, ...(articleUrls ? { articleUrls } : {}), summaries, participants: outcome.participants }, null, 2)); console.log(`Raw Playwright-native records and untrimmed summaries: ${filename}`); } diff --git a/benchmarks/browser/playwright-native/session.ts b/benchmarks/browser/playwright-session.ts similarity index 75% rename from benchmarks/browser/playwright-native/session.ts rename to benchmarks/browser/playwright-session.ts index 2183bdea..48d3eaf8 100644 --- a/benchmarks/browser/playwright-native/session.ts +++ b/benchmarks/browser/playwright-session.ts @@ -2,21 +2,15 @@ import { chromium, type Page, type Browser, type BrowserContext } from 'playwrig import { randomUUID } from 'node:crypto'; import { TaskError, type TaskContext, type TaskStepOptions } from '@benchsdk/runner'; import type { JsonObject } from '@benchsdk/api'; -import { withTimeout } from '../../src/util/timeout.js'; -import { CLEANUP_TIMEOUT_MS, CONNECT_TIMEOUT_MS, VIEWPORT, nativeParticipants, safeError, type NativeParticipant } from './providers.js'; +import { withTimeout } from '../src/util/timeout.js'; +import { CLEANUP_TIMEOUT_MS, CONNECT_TIMEOUT_MS, VIEWPORT, safeError } from './playwright-providers.js'; +import type { PlaywrightProviderConfig } from './playwright-types.js'; -const unsafeParticipants = new WeakSet(); - -export const nativeConfig = { - iterations: 100, - concurrency: 1, - groupBy: 'round' as const, - participants: nativeParticipants, -}; +const unsafeParticipants = new WeakSet(); /** Timer spans the first provider request through workload completion only. */ export async function runSession( - ctx: TaskContext, + ctx: TaskContext, workload: (page: Page, startedAt: number, data: JsonObject) => Promise, ): Promise<{ data: JsonObject }> { if (unsafeParticipants.has(ctx.participant)) { @@ -29,6 +23,8 @@ export async function runSession( const data: JsonObject = { ...session.metadata, attemptStarted: true, workloadSuccess: false, cleanupSuccess: false }; let browser: Browser | undefined; let context: BrowserContext | undefined; + let pendingContext: Promise | undefined; + let closeLateContext = false; let secrets: string[] = []; let failure: string | undefined; const cleanupErrors: string[] = []; @@ -48,7 +44,13 @@ export async function runSession( data.nodeVersion = process.version; const connected = browser; const page = await step('page', async () => { - context = await withTimeout(connected.newContext({ viewport: VIEWPORT }), CONNECT_TIMEOUT_MS, 'Context creation timed out'); + // Keep ownership even when the caller's timeout wins the race. + pendingContext = connected.newContext({ viewport: VIEWPORT }).then(async created => { + context = created; + if (closeLateContext) await withTimeout(created.close(), CLEANUP_TIMEOUT_MS, 'Late context cleanup timed out'); + return created; + }); + context = await withTimeout(pendingContext, CONNECT_TIMEOUT_MS, 'Context creation timed out'); context.setDefaultTimeout(30_000); context.setDefaultNavigationTimeout(30_000); return withTimeout(context.newPage(), CONNECT_TIMEOUT_MS, 'Page creation timed out'); @@ -65,6 +67,12 @@ export async function runSession( await step(name, () => withTimeout(fn(), CLEANUP_TIMEOUT_MS, `${name} timed out`), { reportConcurrency: false }); } catch (error) { cleanupErrors.push(safeError(error, secrets)); } }; + const creation = pendingContext; + if (creation && !context) { + // A rejected creation owns no context; an unresolved one makes cleanup uncertain. + await cleanup('reconcile-context', () => creation.catch(() => undefined)); + } + closeLateContext = true; for (const [name, resource] of [['close-context', context], ['close-browser', browser]] as const) { if (resource) await cleanup(name, () => resource.close()); } diff --git a/benchmarks/browser/playwright-throughput.bench.ts b/benchmarks/browser/playwright-throughput.bench.ts index cd6f9819..68227eb5 100644 --- a/benchmarks/browser/playwright-throughput.bench.ts +++ b/benchmarks/browser/playwright-throughput.bench.ts @@ -1,38 +1,52 @@ /** Five ten-action loops; setup and cleanup excluded from action throughput. */ import '../src/env.js'; -import { defineBenchmarkConfig, defineTask, TaskError, parseCliArgs } from '@benchsdk/runner'; -import { nativeConfig, runSession } from './playwright-native/session.js'; -import { distribution, writeNativeResults } from './playwright-native/results.js'; -import { ACTIONS_PER_SESSION, loadArticleUrls, runActionLoop } from './playwright-native/workload.js'; +import { defineBenchmarkConfig, defineTask, TaskError, mergeConfig } from '@benchsdk/runner'; +import { runSession } from './playwright-session.js'; +import { playwrightProviders, requireProviderCredentials, playwrightArgs } from './playwright-providers.js'; +import type { PlaywrightProviderConfig } from './playwright-types.js'; +import { distribution, writeNativeResults } from './playwright-results.js'; +import { ACTIONS_PER_SESSION, loadArticleUrls, runActionLoop } from './playwright-workload.js'; import type { ActionResult } from './throughput-types.js'; // Load the frozen validated inputs before any measured provider request. const urls = loadArticleUrls(); -const iterationArgs = process.argv.flatMap((arg, index) => { - if (arg.startsWith('--iterations=')) return [arg]; - if (arg === '--iterations') return [arg, process.argv[index + 1] ?? '']; - return []; -}); -const iterations = parseCliArgs(iterationArgs).iterations ?? nativeConfig.iterations; -if (urls.length < iterations) throw new Error('Prepare a validated article URL for every requested iteration before measurement'); + +requireProviderCredentials(); export const config = defineBenchmarkConfig({ - ...nativeConfig, benchmarkSlug: 'playwright-native-throughput', benchmarkName: 'Playwright-native Browser Throughput', + iterations: 100, + concurrency: 1, + groupBy: 'round', + participants: playwrightProviders, display: { metrics: [ { key: 'actionsPerSecond', label: 'Actions/s', unit: '/s', direction: 'higher-better' }, + { key: 'actionsCompleted', label: 'Actions completed', direction: 'higher-better', decimals: 0 }, { key: 'taskMs', label: 'Sum of action durations', unit: 'ms', direction: 'lower-better' }, { key: 'screenshotMs', label: 'Session median screenshot', unit: 'ms', direction: 'lower-better' }, { key: 'cleanupMs', label: 'Cleanup', unit: 'ms', direction: 'lower-better' }, ], + steps: [ + { key: 'provision', label: 'Provision session' }, + { key: 'connect', label: 'Connect Playwright' }, + { key: 'page', label: 'Create context and page' }, + { key: 'actions', label: 'Actions' }, + { key: 'reconcile-context', label: 'Reconcile pending context' }, + { key: 'close-context', label: 'Close context' }, + { key: 'close-browser', label: 'Close browser' }, + { key: 'release', label: 'Release session' }, + ], overview: { defaultMetric: 'actionsPerSecond', defaultLayout: 'chart' }, }, onComplete: outcome => writeNativeResults(outcome, 'throughput', urls), }); -export const task = defineTask(ctx => { +const iterations = mergeConfig(config, playwrightArgs).iterations; +if (urls.length < iterations) throw new Error('Prepare a validated article URL for every requested iteration before measurement'); + +export const task = defineTask(ctx => { const url = urls[ctx.taskIndex]; if (!url) throw new TaskError('Validated URL list is shorter than the requested run; prepare all inputs first', { code: 'MISSING_INPUT', data: { attemptStarted: false } }); const actions: ActionResult[] = []; diff --git a/benchmarks/browser/playwright-types.ts b/benchmarks/browser/playwright-types.ts new file mode 100644 index 00000000..d917ae20 --- /dev/null +++ b/benchmarks/browser/playwright-types.ts @@ -0,0 +1,13 @@ +import type { JsonObject } from '@benchsdk/api'; + +export interface PlaywrightSession { + metadata: JsonObject; + provision(): Promise<{ endpoint: string; headers: Record }>; + cleanup(options: { connected: boolean }): Promise; +} + +export interface PlaywrightProviderConfig { + name: string; + requiredEnvVars: string[]; + session(identity: string): PlaywrightSession; +} diff --git a/benchmarks/browser/playwright-native/workload.ts b/benchmarks/browser/playwright-workload.ts similarity index 94% rename from benchmarks/browser/playwright-native/workload.ts rename to benchmarks/browser/playwright-workload.ts index 131b2a83..9bd2a3b2 100644 --- a/benchmarks/browser/playwright-native/workload.ts +++ b/benchmarks/browser/playwright-workload.ts @@ -1,8 +1,8 @@ import { readFileSync } from 'node:fs'; import type { Page, ElementHandle } from 'playwright-native-core'; -import { withTimeout } from '../../src/util/timeout.js'; -import type { ActionResult, ActionType } from '../throughput-types.js'; -import { requiredEnv, safeError } from './providers.js'; +import { withTimeout } from '../src/util/timeout.js'; +import type { ActionResult, ActionType } from './throughput-types.js'; +import { requiredEnv, safeError } from './playwright-providers.js'; const LOOPS_PER_SESSION = 5; export const ACTIONS_PER_SESSION = LOOPS_PER_SESSION * 10; diff --git a/package.json b/package.json index e5441d57..7b0d3703 100644 --- a/package.json +++ b/package.json @@ -150,8 +150,8 @@ "bench:ai-gateway-model-index:github-copilot": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider github-copilot", "bench:playwright-readiness": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/playwright-readiness.bench.ts", "bench:playwright-throughput": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/playwright-throughput.bench.ts", - "bench:playwright-prepare-urls": "tsx benchmarks/browser/playwright-native/prepare-urls.ts", - "test:playwright-native": "tsx --test benchmarks/browser/playwright-native/native.test.ts" + "bench:playwright-prepare-urls": "tsx benchmarks/browser/playwright-prepare-urls.ts", + "test:playwright-native": "tsx --test benchmarks/browser/playwright-native.test.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.1152.0",