Add Playwright-native browser readiness and throughput benchmarks

.github/workflows/ci.yml

@@ -47,6 +47,24 @@ jobs:
       - name: Test create-bench
         run: pnpm --filter create-bench run test
 
+  playwright-native:
+    name: Test Playwright-native benchmarks
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+          cache: 'pnpm'
+      - run: pnpm install --frozen-lockfile --ignore-scripts
+      - run: pnpm -r --filter './packages/**' run build
+      - name: Install fixture browser
+        run: node node_modules/playwright-native-core/cli.js install --with-deps chromium
+      - name: Exercise native suites against local fixtures
+        run: pnpm test:playwright-native
+
   actionlint:
     name: Lint workflows
     runs-on: ubuntu-latest

.gitignore

@@ -1,7 +1,11 @@
 .env
 node_modules/
+.pnpm-store/
 .DS_Store
 dist/
 scale-run.log
 scale.mjs
-fetch-raw-json.ts
+fetch-raw-json.ts
+results/playwright-native/
+playwright-native-urls.json
+playwright-native-environment.json

benchmarks/.env.example

@@ -185,3 +185,11 @@ NEON_AI_GATEWAY_TOKEN=your_neon_ai_gateway_token
 NGROK_AI_GATEWAY_API_KEY=your_ngrok_ai_gateway_api_key
 LLMAPI_API_KEY=your_llmapi_api_key
 BLAZERAIL_API_KEY=your_blazerail_api_key
+# Playwright-native browser benchmarks (keep real values in the ignored .env).
+# MOMENTIC_BROWSER_FLEET_URL=
+# MOMENTIC_BROWSER_FLEET_API_KEY=
+# AZURE_PLAYWRIGHT_SERVICE_URL=
+# AZURE_PLAYWRIGHT_ACCESS_TOKEN=
+# PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE=playwright-native-environment.json
+# PLAYWRIGHT_NATIVE_URLS_FILE=playwright-native-urls.json
+# PLAYWRIGHT_NATIVE_RESULTS_DIR=results/playwright-native

benchmarks/browser/playwright-prepare-urls.ts

@@ -0,0 +1,27 @@
+/** Resolve and DOM-validate shared Wikipedia inputs before measured sessions. */
+import { chromium } from 'playwright-native-core';
+import { writeFileSync } from 'node:fs';
+import { articleUrl, firstArticleLink } from './playwright-workload.js';
+import { withTimeout } from '../src/util/timeout.js';
+import { VIEWPORT } from './playwright-providers.js';
+
+const count = Number(process.argv[2] ?? '100');
+const filename = process.argv[3] ?? 'playwright-native-urls.json';
+if (!Number.isInteger(count) || count < 1) throw new Error('Count must be a positive integer');
+const browser = await chromium.launch({ headless: true, timeout: 30_000 });
+try {
+  const page = await browser.newPage({ viewport: VIEWPORT });
+  page.setDefaultTimeout(30_000);
+  page.setDefaultNavigationTimeout(30_000);
+  const urls: string[] = [];
+  for (let i = 0; i < count; i++) {
+    await page.goto('https://en.wikipedia.org/wiki/Special:Random', { waitUntil: 'load' });
+    const url = articleUrl(page.url());
+    await page.waitForSelector('#firstHeading');
+    await withTimeout((await firstArticleLink(page)).click(), 30_000, 'Article link validation timed out');
+    await page.waitForSelector('#firstHeading');
+    urls.push(url);
+  }
+  writeFileSync(filename, JSON.stringify(urls, null, 2));
+  console.log(`Wrote ${urls.length} validated shared inputs to ${filename}`);
+} finally { await withTimeout(browser.close(), 15_000, 'URL preparation cleanup timed out'); }

benchmarks/browser/playwright-providers.ts

@@ -0,0 +1,191 @@
+import { setTimeout as delay } from 'node:timers/promises';
+import { readFileSync } from 'node:fs';
+import type { JsonObject } from '@benchsdk/api';
+import { parseCliArgs, selectParticipants, filterParticipantsByEnv } from '@benchsdk/runner';
+import type { PlaywrightProviderConfig } from './playwright-types.js';
+
+export const PLAYWRIGHT_VERSION = '1.60.0';
+export const PROVISION_TIMEOUT_MS = 120_000;
+export const CONNECT_TIMEOUT_MS = 30_000;
+export const CLEANUP_TIMEOUT_MS = 15_000;
+export const VIEWPORT = { width: 1920, height: 1080 };
+
+// Parse only flags owned by the suites; the runner handles the rest of the CLI.
+export const playwrightArgs = parseCliArgs(process.argv.flatMap((arg, index) => {
+  if (/^--(?:provider|iterations)=/.test(arg)) return [arg];
+  if (arg === '--provider' || arg === '--iterations') return [arg, process.argv[index + 1] ?? ''];
+  return [];
+}));
+
+export function requireProviderCredentials(): void {
+  const selected = selectParticipants(playwrightProviders, playwrightArgs.providers);
+  const { skipped } = filterParticipantsByEnv(selected);
+  if (skipped.length) {
+    const missing = skipped.map(({ name, missing }) => `${name}: ${missing.join(', ')}`).join('; ');
+    throw new Error(`Unavailable Playwright participant(s): ${missing}. Supply credentials for every selected provider; use --provider for an explicit diagnostic run.`);
+  }
+}
+
+export function requiredEnv(name: string): string {
+  const value = process.env[name]?.trim();
+  if (!value) throw new Error(`Missing ${name}`);
+  return value;
+}
+
+function object(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Invalid provider response');
+  return value as Record<string, unknown>;
+}
+
+function string(value: unknown): string {
+  if (typeof value !== 'string' || !value.trim()) throw new Error('Missing provider response field');
+  return value;
+}
+
+function environment(provider: string): JsonObject {
+  const filename = requiredEnv('PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE');
+  let parsed: unknown;
+  try { parsed = JSON.parse(readFileSync(filename, 'utf8')); }
+  catch { throw new Error('Cannot read a valid Playwright-native environment manifest'); }
+  const manifest = object(parsed);
+  const entry = object(object(manifest.providers)[provider]);
+  const region = string(manifest.region);
+  const serverVersion = string(entry.playwrightVersion);
+  if (entry.region !== region || !/^1\.60\.\d+$/.test(serverVersion)) {
+    throw new Error('Both providers must attest to the shared region and Playwright 1.60.x');
+  }
+  if (entry.os !== 'linux' || entry.headless !== true || entry.stealth !== false || entry.proxy !== false || entry.recording !== false) {
+    throw new Error('Environment must attest to headless Linux Chromium without optional features');
+  }
+  return {
+    region,
+    runnerLocation: string(manifest.runnerLocation),
+    placementEvidence: string(entry.placementEvidence),
+    versionEvidence: string(entry.versionEvidence),
+    cleanupEvidence: string(entry.cleanupEvidence),
+    serverPlaywrightVersion: serverVersion,
+    serverPlaywrightVersionSource: 'operator-verified environment manifest',
+    clientPlaywrightVersion: PLAYWRIGHT_VERSION,
+    browserOs: 'linux', headless: true, viewport: VIEWPORT,
+    stealth: false, proxy: false, recording: false,
+    metadataSource: 'operator-verified environment manifest',
+  };
+}
+
+export function safeError(error: unknown, secrets: string[] = []): string {
+  let message = error instanceof Error ? error.message : 'Operation failed';
+  const values = [...secrets, ...Object.entries(process.env)
+    .filter(([key]) => /token|key|secret|password/i.test(key))
+    .map(([, value]) => value ?? '')];
+  for (const value of values) if (value) message = message.split(value).join('[redacted]');
+  return message.replace(/(?:https?|wss?):\/\/[^\s<>]+/g, '[endpoint]')
+    .replace(/Bearer\s+[^\s]+/gi, 'Bearer [redacted]');
+}
+
+async function request(url: URL, init: RequestInit, signal: AbortSignal): Promise<Response> {
+  const response = await fetch(url, { ...init, signal, redirect: 'error' });
+  if (!response.ok) throw new Error(`Provider request failed (HTTP ${response.status})`);
+  return response;
+}
+
+async function sessionResponse(response: Response): Promise<Record<string, unknown>> {
+  // JSON parser diagnostics may include credential-bearing response snippets.
+  try { return object(await response.json()); }
+  catch { throw new Error('Provider returned invalid session JSON'); }
+}
+
+export const playwrightProviders: PlaywrightProviderConfig[] = [
+  {
+    name: 'momentic',
+    requiredEnvVars: ['MOMENTIC_BROWSER_FLEET_URL', 'MOMENTIC_BROWSER_FLEET_API_KEY', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'],
+    session(identity) {
+      const base = new URL(requiredEnv('MOMENTIC_BROWSER_FLEET_URL'));
+      const headers = { Authorization: `Bearer ${requiredEnv('MOMENTIC_BROWSER_FLEET_API_KEY')}`, 'Content-Type': 'application/json', 'Idempotency-Key': identity };
+      const metadata = environment('momentic');
+      let sessionId: string | undefined;
+      let creationUncertain = false;
+      const create = async (signal: AbortSignal) => {
+        creationUncertain = true;
+        const response = await fetch(new URL('/v1/sessions', base), {
+          method: 'POST', headers, signal, redirect: 'error',
+          body: JSON.stringify({ browser: 'chromium', playwrightVersion: PLAYWRIGHT_VERSION, network: { mode: 'playwright' }, ttlSeconds: 3600, waitTimeoutMs: 0 }),
+        });
+        if (!response.ok) {
+          // A definitive rejection did not allocate a session. A 5xx may have.
+          creationUncertain = response.status >= 500;
+          throw new Error(`Session creation failed (HTTP ${response.status})`);
+        }
+        const data = await sessionResponse(response);
+        sessionId = string(data.id); // Retain the ID before validating other fields.
+        metadata.sessionId = sessionId;
+        creationUncertain = false;
+        return data;
+      };
+      const retrieve = async (signal: AbortSignal) => {
+        if (!sessionId) throw new Error('Session ID unavailable');
+        return sessionResponse(await request(new URL(`/v1/sessions/${encodeURIComponent(sessionId)}`, base), { headers }, signal));
+      };
+      return {
+        metadata,
+        async provision() {
+          const signal = AbortSignal.timeout(PROVISION_TIMEOUT_MS);
+          let data = await create(signal);
+          while (data.status === 'QUEUED' || data.status === 'PROVISIONING') {
+            await delay(500, undefined, { signal });
+            data = await retrieve(signal);
+          }
+          if (data.status !== 'READY') throw new Error('Momentic session did not become READY');
+          if (typeof data.playwrightVersion !== 'string' || !/^1\.60\.\d+$/.test(data.playwrightVersion)) throw new Error('Momentic returned an incompatible Playwright version');
+          metadata.serverPlaywrightVersion = string(data.playwrightVersion);
+          metadata.serverPlaywrightVersionSource = 'Momentic session response';
+          return { endpoint: string(data.wsEndpoint), headers: { Authorization: `Bearer ${string(data.connectToken)}` } };
+        },
+        async cleanup() {
+          const signal = AbortSignal.timeout(CLEANUP_TIMEOUT_MS);
+          // Reconcile a lost POST response using the same key, never a new attempt.
+          if (!sessionId && creationUncertain) await create(signal);
+          if (!sessionId) return;
+          await request(new URL(`/v1/sessions/${encodeURIComponent(sessionId)}`, base), { method: 'DELETE', headers }, signal);
+          for (;;) {
+            const data = await retrieve(signal);
+            if (data.status === 'TERMINATED' || data.status === 'EXPIRED' || data.status === 'FAILED') return;
+            await delay(500, undefined, { signal });
+          }
+        },
+      };
+    },
+  },
+  {
+    name: 'azure',
+    requiredEnvVars: ['AZURE_PLAYWRIGHT_SERVICE_URL', 'AZURE_PLAYWRIGHT_ACCESS_TOKEN', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'],
+    session(identity) {
+      const url = new URL(requiredEnv('AZURE_PLAYWRIGHT_SERVICE_URL'));
+      const headers = { Authorization: `Bearer ${requiredEnv('AZURE_PLAYWRIGHT_ACCESS_TOKEN')}` };
+      const metadata = { ...environment('azure'), runId: identity };
+      url.protocol = url.protocol === 'wss:' ? 'https:' : url.protocol === 'ws:' ? 'http:' : url.protocol;
+      url.searchParams.set('api-version', '2025-09-01');
+      url.searchParams.set('os', 'Linux');
+      url.searchParams.set('runId', identity);
+      let allocationUncertain = false;
+      return {
+        metadata,
+        async provision() {
+          // Resolve the documented redirect inside the measured interval.
+          allocationUncertain = true;
+          const response = await fetch(url, { headers, redirect: 'manual', signal: AbortSignal.timeout(PROVISION_TIMEOUT_MS) });
+          if (response.status >= 400 && response.status < 500) allocationUncertain = false;
+          if (response.status !== 302 && response.status !== 307) throw new Error(`Azure browser redirect failed (HTTP ${response.status})`);
+          const endpoint = new URL(string(response.headers.get('location')), url);
+          if (endpoint.protocol !== 'wss:' && endpoint.protocol !== 'ws:') throw new Error('Azure did not return a Playwright WebSocket endpoint');
+          return { endpoint: endpoint.href, headers };
+        },
+        // Azure owns termination on connection close; verify this before a comparison.
+        async cleanup({ connected }) {
+          if (allocationUncertain && !connected) {
+            throw new Error('Azure allocation cleanup is uncertain after failed provisioning/connection; verify resources before rerunning');
+          }
+        },
+      };
+    },
+  },
+];

benchmarks/browser/playwright-readiness.bench.ts

@@ -0,0 +1,44 @@
+/** First per-session request → example.com load; cleanup is measured separately. */
+import '../src/env.js';
+import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner';
+import { runSession } from './playwright-session.js';
+import { playwrightProviders, requireProviderCredentials } from './playwright-providers.js';
+import type { PlaywrightProviderConfig } from './playwright-types.js';
+import { writeNativeResults } from './playwright-results.js';
+
+requireProviderCredentials();
+
+export const config = defineBenchmarkConfig({
+  benchmarkSlug: 'playwright-native-readiness',
+  benchmarkName: 'Playwright-native Browser Readiness',
+  iterations: 100,
+  concurrency: 1,
+  groupBy: 'round',
+  participants: playwrightProviders,
+  display: {
+    metrics: [
+      { key: 'readinessMs', label: 'Browser readiness', unit: 'ms', direction: 'lower-better' },
+      { key: 'cleanupMs', label: 'Cleanup', unit: 'ms', direction: 'lower-better' },
+    ],
+    steps: [
+      { key: 'provision', label: 'Provision session' },
+      { key: 'connect', label: 'Connect Playwright' },
+      { key: 'page', label: 'Create context and page' },
+      { key: 'navigate', label: 'Navigate page' },
+      { key: 'reconcile-context', label: 'Reconcile pending context' },
+      { key: 'close-context', label: 'Close context' },
+      { key: 'close-browser', label: 'Close browser' },
+      { key: 'release', label: 'Release session' },
+    ],
+    overview: { defaultMetric: 'readinessMs', defaultLayout: 'chart' },
+  },
+  onComplete: outcome => writeNativeResults(outcome, 'readiness'),
+});
+
+export const task = defineTask<PlaywrightProviderConfig>(ctx => runSession(ctx, async (page, startedAt, data) => {
+  await ctx.step('navigate', async () => {
+    await page.goto('https://www.example.com', { waitUntil: 'load' });
+    data.readinessMs = performance.now() - startedAt;
+    ctx.measure({ readinessMs: data.readinessMs });
+  });
+}));

benchmarks/browser/playwright-results.ts

@@ -0,0 +1,54 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { filterParticipantsByEnv, type BenchmarkRunOutcome } from '@benchsdk/runner';
+import { playwrightProviders } from './playwright-providers.js';
+import { ACTION_TYPES } from './throughput-types.js';
+
+/** Untrimmed nearest-rank percentiles; missing observations are not zeroes. */
+export function distribution(values: number[]) {
+  const sorted = [...values].sort((a, b) => a - b);
+  const n = sorted.length;
+  if (!n) return { count: 0, median: null, p95: null, p99: null };
+  const mid = Math.floor(n / 2);
+  return {
+    count: n,
+    median: n % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2,
+    p95: sorted[Math.ceil(n * 0.95) - 1],
+    p99: sorted[Math.ceil(n * 0.99) - 1],
+  };
+}
+
+export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readiness' | 'throughput', articleUrls?: string[]): void {
+  const comparison = playwrightProviders.every(provider => outcome.participants.some(p => p.participant === provider.name));
+  const { skipped: unavailableParticipants } = filterParticipantsByEnv(playwrightProviders);
+  const summaries = outcome.participants.map(({ participant, records }) => {
+    const attempted = records.filter(record => record.data?.attemptStarted === true);
+    const complete = attempted.filter(record => record.data?.workloadSuccess === true);
+    const metrics = Object.fromEntries(['readinessMs', 'taskMs', 'actionsPerSecond', 'screenshotMs', 'cleanupMs'].map(key => [key,
+      distribution((key === 'cleanupMs' ? attempted : complete).flatMap(record => {
+        const value = record.data?.[key];
+        return typeof value === 'number' && Number.isFinite(value) ? [value] : [];
+      })),
+    ]));
+    const actions = complete.flatMap(record => {
+      const actions = record.data?.actions;
+      if (!Array.isArray(actions)) return [];
+      return actions.flatMap(action => {
+        if (!action || typeof action !== 'object' || Array.isArray(action)) return [];
+        if (action.success !== true || typeof action.durationMs !== 'number') return [];
+        return [{ type: action.type, durationMs: action.durationMs }];
+      });
+    });
+    const perActionType = Object.fromEntries(ACTION_TYPES.map(type => [type,
+      distribution(actions.filter(action => action.type === type).map(action => action.durationMs)),
+    ]));
+    const rate = (key: string) => attempted.length ? attempted.filter(record => record.data?.[key] === true).length / attempted.length : null;
+    return { participant, scheduled: records.length, attempts: attempted.length, workloadSuccessRate: rate('workloadSuccess'), cleanupSuccessRate: rate('cleanupSuccess'), successRate: rate('sessionSuccess'), metrics, perActionType };
+  });
+  const directory = path.resolve(process.env.PLAYWRIGHT_NATIVE_RESULTS_DIR ?? 'results/playwright-native', suite);
+  mkdirSync(directory, { recursive: true });
+  const filename = path.join(directory, `${randomUUID()}.json`);
+  writeFileSync(filename, JSON.stringify({ suite, mode: comparison ? 'comparison' : 'provider-diagnostic', unavailableParticipants, runId: outcome.runId, createdAt: new Date().toISOString(), config: outcome.config, ...(articleUrls ? { articleUrls } : {}), summaries, participants: outcome.participants }, null, 2));
+  console.log(`Raw Playwright-native records and untrimmed summaries: ${filename}`);
+}