Skip to content

Commit 3677286

Browse files
committed
Fix native benchmark cleanup and comparison preflight
1 parent 4f527c5 commit 3677286

11 files changed

Lines changed: 189 additions & 61 deletions

‎PLAYWRIGHT_NATIVE_BENCHMARKS.md‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,12 @@ pnpm bench:playwright-readiness --no-ingest
279279
pnpm bench:playwright-throughput --no-ingest
280280
```
281281

282+
The default commands require credentials for both providers and fail before any
283+
provider request if either is unavailable. For an explicit single-provider
284+
diagnostic, pass `--provider momentic` or `--provider azure`. Raw diagnostic
285+
outputs are labeled `provider-diagnostic` and list unavailable participants
286+
separately from attempted-session denominators.
287+
282288
Store credentials in your shell or the ignored `benchmarks/.env`, never in Git:
283289

284290
- `MOMENTIC_BROWSER_FLEET_URL`: the Browser Fleet API base URL.
@@ -346,8 +352,10 @@ cleanup is recorded as uncertain and further allocations stop for that
346352
participant; the implementation does not claim it released an unknown resource.
347353
Provisioning has a 120-second bound;
348354
connection, context/page creation, and navigation have 30-second bounds; each
349-
cleanup operation has a 15-second bound. After uncertain cleanup, the process
350-
refuses further allocations for that participant and records unstarted tasks
355+
cleanup operation has a 15-second bound. Timed-out context creation is retained
356+
and reconciled within that cleanup bound; late contexts are explicitly closed.
357+
Unresolved context ownership makes cleanup uncertain. After uncertain cleanup,
358+
the process refuses further allocations for that participant and records unstarted tasks
351359
separately from attempted-session reliability denominators.
352360

353361
No real credentials are used by fixture tests. Local fixture credentials and

benchmarks/browser/playwright-native/native.test.ts renamed to benchmarks/browser/playwright-native.test.ts

Lines changed: 67 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@ import { promisify } from 'node:util';
88
import { mkdtempSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs';
99
import { tmpdir } from 'node:os';
1010
import path from 'node:path';
11-
import { chromium } from 'playwright-native-core';
11+
import { chromium, type BrowserContext, type BrowserContextOptions } from 'playwright-native-core';
12+
import { withTimeout } from '../src/util/timeout.js';
1213
import { runBenchmark, type BenchmarkRunOutcome } from '@benchsdk/runner';
13-
import { distribution } from './results.js';
14-
import { articleUrl } from './workload.js';
14+
import { distribution } from './playwright-results.js';
15+
import { articleUrl } from './playwright-workload.js';
1516

16-
test('native suites use real Playwright connections and preserve reliability records', { timeout: 120_000 }, async () => {
17+
test('native suites use real Playwright connections and preserve reliability records', { timeout: 180_000 }, async t => {
1718
const directory = mkdtempSync(path.join(tmpdir(), 'native-bench-'));
1819
const envNames = ['MOMENTIC_BROWSER_FLEET_URL', 'MOMENTIC_BROWSER_FLEET_API_KEY', 'AZURE_PLAYWRIGHT_SERVICE_URL', 'AZURE_PLAYWRIGHT_ACCESS_TOKEN', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE', 'PLAYWRIGHT_NATIVE_URLS_FILE', 'PLAYWRIGHT_NATIVE_RESULTS_DIR'];
1920
const previous = new Map(envNames.map(key => [key, process.env[key]]));
@@ -76,7 +77,7 @@ test('native suites use real Playwright connections and preserve reliability rec
7677
const sessionId = sessions.get(key);
7778
if (mode === 'lost-response' && !lost) { lost = true; req.socket.destroy(); return; }
7879
if (mode === 'malformed-json') { res.end('{"connectToken":"local-unpublished-credential'); return; }
79-
res.end(JSON.stringify({ id: sessionId, status: 'READY', playwrightVersion: '1.60.0', wsEndpoint: mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${apiPort}/connect/momentic/${sessionId}`, ...(mode !== 'malformed-ready' ? { connectToken: 'local-connect-credential' } : {}) }));
80+
res.end(JSON.stringify({ id: sessionId, status: 'READY', playwrightVersion: mode === 'compatible-patch' ? '1.60.1' : mode === 'incompatible-version' ? '1.61.0' : '1.60.0', wsEndpoint: mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${apiPort}/connect/momentic/${sessionId}`, ...(mode !== 'malformed-ready' ? { connectToken: 'local-connect-credential' } : {}) }));
8081
} else if (req.method === 'DELETE') {
8182
if (mode === 'cleanup-failure') { res.writeHead(500); res.end('{}'); return; }
8283
if (id) terminated.add(id);
@@ -122,8 +123,8 @@ test('native suites use real Playwright connections and preserve reliability rec
122123
writeFileSync(environmentFile, JSON.stringify(environment));
123124
writeFileSync(urlsFile, JSON.stringify(urls));
124125
Object.assign(process.env, { MOMENTIC_BROWSER_FLEET_URL: `http://127.0.0.1:${apiPort}`, MOMENTIC_BROWSER_FLEET_API_KEY: 'local-momentic-credential', AZURE_PLAYWRIGHT_SERVICE_URL: `http://127.0.0.1:${apiPort}/browsers`, AZURE_PLAYWRIGHT_ACCESS_TOKEN: 'local-azure-credential', PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE: environmentFile, PLAYWRIGHT_NATIVE_URLS_FILE: urlsFile, PLAYWRIGHT_NATIVE_RESULTS_DIR: directory });
125-
const readiness = await import('../playwright-readiness.bench.js');
126-
const throughput = await import('../playwright-throughput.bench.js');
126+
const readiness = await import('./playwright-readiness.bench.js');
127+
const throughput = await import('./playwright-throughput.bench.js');
127128
const records = (outcome: BenchmarkRunOutcome) => outcome.participants.flatMap(p => p.records);
128129
const ready = await runBenchmark({ ...readiness.config, iterations: 2 }, readiness.task, ['--no-ingest']);
129130
assert.deepEqual(order, ['momentic', 'azure', 'momentic', 'azure']);
@@ -173,7 +174,7 @@ test('native suites use real Playwright connections and preserve reliability rec
173174
assert.equal(summary.metrics.actionsPerSecond.count, 2);
174175
assert.equal(summary.perActionType.screenshot.count, 20);
175176
}
176-
for (const failureMode of ['lost-response', 'malformed-ready', 'refused-connection', 'cleanup-failure']) {
177+
for (const failureMode of ['lost-response', 'malformed-ready', 'refused-connection', 'incompatible-version', 'cleanup-failure']) {
177178
mode = failureMode;
178179
order.length = 0;
179180
const participants = readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p }));
@@ -187,6 +188,10 @@ test('native suites use real Playwright connections and preserve reliability rec
187188
assert.deepEqual(order, ['momentic'], 'uncertain cleanup stops new allocations');
188189
} else assert.equal(terminated.size, sessions.size);
189190
}
191+
mode = 'compatible-patch';
192+
const compatible = await runBenchmark({ ...readiness.config, participants: readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })), iterations: 1 }, readiness.task, ['--no-ingest']);
193+
assert.equal(records(compatible)[0].status, 'success');
194+
assert.equal(records(compatible)[0].data?.serverPlaywrightVersion, '1.60.1');
190195
mode = 'refused-connection';
191196
const azureFailure = await runBenchmark({ ...readiness.config, participants: readiness.config.participants.filter(p => p.name === 'azure').map(p => ({ ...p })), iterations: 2 }, readiness.task, ['--no-ingest']);
192197
assert.equal(records(azureFailure)[0].data?.cleanupSuccess, false);
@@ -215,6 +220,60 @@ test('native suites use real Playwright connections and preserve reliability rec
215220
assert.ok(stdout.includes('All done. No platform run created.'));
216221
assert.ok(!stdout.includes('FAILED'));
217222
}
223+
// Missing credentials must fail the comparison before any allocation.
224+
const diagnosticEnv = { ...process.env };
225+
delete diagnosticEnv.AZURE_PLAYWRIGHT_ACCESS_TOKEN;
226+
for (const entry of ['playwright-readiness', 'playwright-throughput']) {
227+
const command = ['node_modules/tsx/dist/cli.mjs', 'packages/benchsdk-runner/dist/bin.js', 'run', `benchmarks/browser/${entry}.bench.ts`, '--no-ingest', '--iterations=1'];
228+
order.length = 0;
229+
await assert.rejects(promisify(execFile)(process.execPath, command, { env: diagnosticEnv }), /Unavailable Playwright participant\(s\): azure: AZURE_PLAYWRIGHT_ACCESS_TOKEN/);
230+
assert.deepEqual(order, []);
231+
const { stdout } = await promisify(execFile)(process.execPath, [...command, '--provider=momentic'], { env: diagnosticEnv });
232+
assert.deepEqual(order, ['momentic']);
233+
const match = stdout.match(/Raw Playwright-native records and untrimmed summaries: (.+)/);
234+
assert.ok(match);
235+
const diagnostic = JSON.parse(readFileSync(match[1].trim(), 'utf8'));
236+
assert.equal(diagnostic.mode, 'provider-diagnostic');
237+
assert.deepEqual(diagnostic.unavailableParticipants, [{ name: 'azure', missing: ['AZURE_PLAYWRIGHT_ACCESS_TOKEN'] }]);
238+
}
239+
// Delay returning an actual remote context beyond the 30-second deadline.
240+
// Cover reconciliation within cleanup, and ownership still unresolved at its bound.
241+
for (const delayMs of [31_000, 46_000]) {
242+
const client = await chromium.connect(endpoint);
243+
const newContext = client.newContext.bind(client);
244+
let pending: Promise<BrowserContext> | undefined;
245+
let closed: Promise<void> | undefined;
246+
const contextMock = t.mock.method(client, 'newContext', (options: BrowserContextOptions) => {
247+
pending = newContext(options).then(async created => {
248+
closed = new Promise<void>(resolve => created.once('close', () => resolve()));
249+
await new Promise(resolve => setTimeout(resolve, delayMs));
250+
return created;
251+
});
252+
return pending;
253+
});
254+
const connectMock = t.mock.method(chromium, 'connect', async () => client);
255+
const closeMock = delayMs === 46_000 ? t.mock.method(client, 'close', async () => { throw new Error('fixture disconnect failed'); }) : undefined;
256+
try {
257+
order.length = 0;
258+
const participants = readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p }));
259+
const late = await runBenchmark({ ...readiness.config, participants, iterations: delayMs === 46_000 ? 2 : 1 }, readiness.task, ['--no-ingest']);
260+
const [first, second] = records(late);
261+
assert.equal(first.status, 'error');
262+
assert.match(String(first.data?.errorMessage), /Context creation timed out/);
263+
assert.equal(first.data?.cleanupSuccess, delayMs === 31_000);
264+
if (delayMs === 46_000) {
265+
assert.equal(second.data?.attemptStarted, false);
266+
assert.deepEqual(order, ['momentic'], 'unresolved context ownership stops allocations');
267+
}
268+
assert.ok(pending && closed);
269+
await pending;
270+
await withTimeout(closed, 5_000, 'Late context was not closed');
271+
assert.equal(client.contexts().length, 0);
272+
} finally {
273+
contextMock.mock.restore(); connectMock.mock.restore(); closeMock?.mock.restore();
274+
await client.close();
275+
}
276+
}
218277
const audit = await chromium.connect(endpoint);
219278
assert.equal(audit.contexts().length, 0, 'all benchmark contexts were closed');
220279
await audit.close();

benchmarks/browser/playwright-native/prepare-urls.ts renamed to benchmarks/browser/playwright-prepare-urls.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
/** Resolve and DOM-validate shared Wikipedia inputs before measured sessions. */
22
import { chromium } from 'playwright-native-core';
33
import { writeFileSync } from 'node:fs';
4-
import { articleUrl, firstArticleLink } from './workload.js';
5-
import { withTimeout } from '../../src/util/timeout.js';
6-
import { VIEWPORT } from './providers.js';
4+
import { articleUrl, firstArticleLink } from './playwright-workload.js';
5+
import { withTimeout } from '../src/util/timeout.js';
6+
import { VIEWPORT } from './playwright-providers.js';
77

88
const count = Number(process.argv[2] ?? '100');
99
const filename = process.argv[3] ?? 'playwright-native-urls.json';

benchmarks/browser/playwright-native/providers.ts renamed to benchmarks/browser/playwright-providers.ts

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,29 @@
11
import { setTimeout as delay } from 'node:timers/promises';
22
import { readFileSync } from 'node:fs';
33
import type { JsonObject } from '@benchsdk/api';
4+
import { parseCliArgs, selectParticipants, filterParticipantsByEnv } from '@benchsdk/runner';
5+
import type { PlaywrightProviderConfig } from './playwright-types.js';
46

57
export const PLAYWRIGHT_VERSION = '1.60.0';
68
export const PROVISION_TIMEOUT_MS = 120_000;
79
export const CONNECT_TIMEOUT_MS = 30_000;
810
export const CLEANUP_TIMEOUT_MS = 15_000;
911
export const VIEWPORT = { width: 1920, height: 1080 };
1012

11-
export interface NativeSession {
12-
metadata: JsonObject;
13-
provision(): Promise<{ endpoint: string; headers: Record<string, string> }>;
14-
cleanup(options: { connected: boolean }): Promise<void>;
15-
}
13+
// Parse only flags owned by the suites; the runner handles the rest of the CLI.
14+
export const playwrightArgs = parseCliArgs(process.argv.flatMap((arg, index) => {
15+
if (/^--(?:provider|iterations)=/.test(arg)) return [arg];
16+
if (arg === '--provider' || arg === '--iterations') return [arg, process.argv[index + 1] ?? ''];
17+
return [];
18+
}));
1619

17-
export interface NativeParticipant {
18-
name: string;
19-
requiredEnvVars: string[];
20-
session(identity: string): NativeSession;
20+
export function requireProviderCredentials(): void {
21+
const selected = selectParticipants(playwrightProviders, playwrightArgs.providers);
22+
const { skipped } = filterParticipantsByEnv(selected);
23+
if (skipped.length) {
24+
const missing = skipped.map(({ name, missing }) => `${name}: ${missing.join(', ')}`).join('; ');
25+
throw new Error(`Unavailable Playwright participant(s): ${missing}. Supply credentials for every selected provider; use --provider for an explicit diagnostic run.`);
26+
}
2127
}
2228

2329
export function requiredEnv(name: string): string {
@@ -88,7 +94,7 @@ async function sessionResponse(response: Response): Promise<Record<string, unkno
8894
catch { throw new Error('Provider returned invalid session JSON'); }
8995
}
9096

91-
export const nativeParticipants: NativeParticipant[] = [
97+
export const playwrightProviders: PlaywrightProviderConfig[] = [
9298
{
9399
name: 'momentic',
94100
requiredEnvVars: ['MOMENTIC_BROWSER_FLEET_URL', 'MOMENTIC_BROWSER_FLEET_API_KEY', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'],
@@ -129,7 +135,7 @@ export const nativeParticipants: NativeParticipant[] = [
129135
data = await retrieve(signal);
130136
}
131137
if (data.status !== 'READY') throw new Error('Momentic session did not become READY');
132-
if (data.playwrightVersion !== PLAYWRIGHT_VERSION) throw new Error('Momentic returned an incompatible Playwright version');
138+
if (typeof data.playwrightVersion !== 'string' || !/^1\.60\.\d+$/.test(data.playwrightVersion)) throw new Error('Momentic returned an incompatible Playwright version');
133139
metadata.serverPlaywrightVersion = string(data.playwrightVersion);
134140
metadata.serverPlaywrightVersionSource = 'Momentic session response';
135141
return { endpoint: string(data.wsEndpoint), headers: { Authorization: `Bearer ${string(data.connectToken)}` } };

‎benchmarks/browser/playwright-readiness.bench.ts‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,41 @@
11
/** First per-session request → example.com load; cleanup is measured separately. */
22
import '../src/env.js';
33
import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner';
4-
import { nativeConfig, runSession } from './playwright-native/session.js';
5-
import { writeNativeResults } from './playwright-native/results.js';
4+
import { runSession } from './playwright-session.js';
5+
import { playwrightProviders, requireProviderCredentials } from './playwright-providers.js';
6+
import type { PlaywrightProviderConfig } from './playwright-types.js';
7+
import { writeNativeResults } from './playwright-results.js';
8+
9+
requireProviderCredentials();
610

711
export const config = defineBenchmarkConfig({
8-
...nativeConfig,
912
benchmarkSlug: 'playwright-native-readiness',
1013
benchmarkName: 'Playwright-native Browser Readiness',
14+
iterations: 100,
15+
concurrency: 1,
16+
groupBy: 'round',
17+
participants: playwrightProviders,
1118
display: {
1219
metrics: [
1320
{ key: 'readinessMs', label: 'Browser readiness', unit: 'ms', direction: 'lower-better' },
1421
{ key: 'cleanupMs', label: 'Cleanup', unit: 'ms', direction: 'lower-better' },
1522
],
23+
steps: [
24+
{ key: 'provision', label: 'Provision session' },
25+
{ key: 'connect', label: 'Connect Playwright' },
26+
{ key: 'page', label: 'Create context and page' },
27+
{ key: 'navigate', label: 'Navigate page' },
28+
{ key: 'reconcile-context', label: 'Reconcile pending context' },
29+
{ key: 'close-context', label: 'Close context' },
30+
{ key: 'close-browser', label: 'Close browser' },
31+
{ key: 'release', label: 'Release session' },
32+
],
1633
overview: { defaultMetric: 'readinessMs', defaultLayout: 'chart' },
1734
},
1835
onComplete: outcome => writeNativeResults(outcome, 'readiness'),
1936
});
2037

21-
export const task = defineTask<typeof nativeConfig.participants[number]>(ctx => runSession(ctx, async (page, startedAt, data) => {
38+
export const task = defineTask<PlaywrightProviderConfig>(ctx => runSession(ctx, async (page, startedAt, data) => {
2239
await ctx.step('navigate', async () => {
2340
await page.goto('https://www.example.com', { waitUntil: 'load' });
2441
data.readinessMs = performance.now() - startedAt;

benchmarks/browser/playwright-native/results.ts renamed to benchmarks/browser/playwright-results.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import { mkdirSync, writeFileSync } from 'node:fs';
22
import path from 'node:path';
33
import { randomUUID } from 'node:crypto';
4-
import type { BenchmarkRunOutcome } from '@benchsdk/runner';
5-
import { ACTION_TYPES } from '../throughput-types.js';
4+
import { filterParticipantsByEnv, type BenchmarkRunOutcome } from '@benchsdk/runner';
5+
import { playwrightProviders } from './playwright-providers.js';
6+
import { ACTION_TYPES } from './throughput-types.js';
67

78
/** Untrimmed nearest-rank percentiles; missing observations are not zeroes. */
89
export function distribution(values: number[]) {
@@ -19,6 +20,8 @@ export function distribution(values: number[]) {
1920
}
2021

2122
export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readiness' | 'throughput', articleUrls?: string[]): void {
23+
const comparison = playwrightProviders.every(provider => outcome.participants.some(p => p.participant === provider.name));
24+
const { skipped: unavailableParticipants } = filterParticipantsByEnv(playwrightProviders);
2225
const summaries = outcome.participants.map(({ participant, records }) => {
2326
const attempted = records.filter(record => record.data?.attemptStarted === true);
2427
const complete = attempted.filter(record => record.data?.workloadSuccess === true);
@@ -46,6 +49,6 @@ export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readine
4649
const directory = path.resolve(process.env.PLAYWRIGHT_NATIVE_RESULTS_DIR ?? 'results/playwright-native', suite);
4750
mkdirSync(directory, { recursive: true });
4851
const filename = path.join(directory, `${randomUUID()}.json`);
49-
writeFileSync(filename, JSON.stringify({ suite, runId: outcome.runId, createdAt: new Date().toISOString(), config: outcome.config, ...(articleUrls ? { articleUrls } : {}), summaries, participants: outcome.participants }, null, 2));
52+
writeFileSync(filename, JSON.stringify({ suite, mode: comparison ? 'comparison' : 'provider-diagnostic', unavailableParticipants, runId: outcome.runId, createdAt: new Date().toISOString(), config: outcome.config, ...(articleUrls ? { articleUrls } : {}), summaries, participants: outcome.participants }, null, 2));
5053
console.log(`Raw Playwright-native records and untrimmed summaries: ${filename}`);
5154
}

0 commit comments

Comments
 (0)