@@ -8,12 +8,13 @@ import { promisify } from 'node:util';
88import { mkdtempSync , readFileSync , writeFileSync , readdirSync , rmSync } from 'node:fs' ;
99import { tmpdir } from 'node:os' ;
1010import path from 'node:path' ;
11- import { chromium } from 'playwright-native-core' ;
11+ import { chromium , type BrowserContext , type BrowserContextOptions } from 'playwright-native-core' ;
12+ import { withTimeout } from '../src/util/timeout.js' ;
1213import { runBenchmark , type BenchmarkRunOutcome } from '@benchsdk/runner' ;
13- import { distribution } from './results.js' ;
14- import { articleUrl } from './workload.js' ;
14+ import { distribution } from './playwright- results.js' ;
15+ import { articleUrl } from './playwright- workload.js' ;
1516
16- test ( 'native suites use real Playwright connections and preserve reliability records' , { timeout : 120_000 } , async ( ) => {
17+ test ( 'native suites use real Playwright connections and preserve reliability records' , { timeout : 180_000 } , async t => {
1718 const directory = mkdtempSync ( path . join ( tmpdir ( ) , 'native-bench-' ) ) ;
1819 const envNames = [ 'MOMENTIC_BROWSER_FLEET_URL' , 'MOMENTIC_BROWSER_FLEET_API_KEY' , 'AZURE_PLAYWRIGHT_SERVICE_URL' , 'AZURE_PLAYWRIGHT_ACCESS_TOKEN' , 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE' , 'PLAYWRIGHT_NATIVE_URLS_FILE' , 'PLAYWRIGHT_NATIVE_RESULTS_DIR' ] ;
1920 const previous = new Map ( envNames . map ( key => [ key , process . env [ key ] ] ) ) ;
@@ -76,7 +77,7 @@ test('native suites use real Playwright connections and preserve reliability rec
7677 const sessionId = sessions . get ( key ) ;
7778 if ( mode === 'lost-response' && ! lost ) { lost = true ; req . socket . destroy ( ) ; return ; }
7879 if ( mode === 'malformed-json' ) { res . end ( '{"connectToken":"local-unpublished-credential' ) ; return ; }
79- res . end ( JSON . stringify ( { id : sessionId , status : 'READY' , playwrightVersion : '1.60.0' , wsEndpoint : mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${ apiPort } /connect/momentic/${ sessionId } ` , ...( mode !== 'malformed-ready' ? { connectToken : 'local-connect-credential' } : { } ) } ) ) ;
80+ res . end ( JSON . stringify ( { id : sessionId , status : 'READY' , playwrightVersion : mode === 'compatible-patch' ? '1.60.1' : mode === 'incompatible-version' ? '1.61.0' : '1.60.0' , wsEndpoint : mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${ apiPort } /connect/momentic/${ sessionId } ` , ...( mode !== 'malformed-ready' ? { connectToken : 'local-connect-credential' } : { } ) } ) ) ;
8081 } else if ( req . method === 'DELETE' ) {
8182 if ( mode === 'cleanup-failure' ) { res . writeHead ( 500 ) ; res . end ( '{}' ) ; return ; }
8283 if ( id ) terminated . add ( id ) ;
@@ -122,8 +123,8 @@ test('native suites use real Playwright connections and preserve reliability rec
122123 writeFileSync ( environmentFile , JSON . stringify ( environment ) ) ;
123124 writeFileSync ( urlsFile , JSON . stringify ( urls ) ) ;
124125 Object . assign ( process . env , { MOMENTIC_BROWSER_FLEET_URL : `http://127.0.0.1:${ apiPort } ` , MOMENTIC_BROWSER_FLEET_API_KEY : 'local-momentic-credential' , AZURE_PLAYWRIGHT_SERVICE_URL : `http://127.0.0.1:${ apiPort } /browsers` , AZURE_PLAYWRIGHT_ACCESS_TOKEN : 'local-azure-credential' , PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE : environmentFile , PLAYWRIGHT_NATIVE_URLS_FILE : urlsFile , PLAYWRIGHT_NATIVE_RESULTS_DIR : directory } ) ;
125- const readiness = await import ( '.. /playwright-readiness.bench.js' ) ;
126- const throughput = await import ( '.. /playwright-throughput.bench.js' ) ;
126+ const readiness = await import ( './playwright-readiness.bench.js' ) ;
127+ const throughput = await import ( './playwright-throughput.bench.js' ) ;
127128 const records = ( outcome : BenchmarkRunOutcome ) => outcome . participants . flatMap ( p => p . records ) ;
128129 const ready = await runBenchmark ( { ...readiness . config , iterations : 2 } , readiness . task , [ '--no-ingest' ] ) ;
129130 assert . deepEqual ( order , [ 'momentic' , 'azure' , 'momentic' , 'azure' ] ) ;
@@ -173,7 +174,7 @@ test('native suites use real Playwright connections and preserve reliability rec
173174 assert . equal ( summary . metrics . actionsPerSecond . count , 2 ) ;
174175 assert . equal ( summary . perActionType . screenshot . count , 20 ) ;
175176 }
176- for ( const failureMode of [ 'lost-response' , 'malformed-ready' , 'refused-connection' , 'cleanup-failure' ] ) {
177+ for ( const failureMode of [ 'lost-response' , 'malformed-ready' , 'refused-connection' , 'incompatible-version' , ' cleanup-failure'] ) {
177178 mode = failureMode ;
178179 order . length = 0 ;
179180 const participants = readiness . config . participants . filter ( p => p . name === 'momentic' ) . map ( p => ( { ...p } ) ) ;
@@ -187,6 +188,10 @@ test('native suites use real Playwright connections and preserve reliability rec
187188 assert . deepEqual ( order , [ 'momentic' ] , 'uncertain cleanup stops new allocations' ) ;
188189 } else assert . equal ( terminated . size , sessions . size ) ;
189190 }
191+ mode = 'compatible-patch' ;
192+ const compatible = await runBenchmark ( { ...readiness . config , participants : readiness . config . participants . filter ( p => p . name === 'momentic' ) . map ( p => ( { ...p } ) ) , iterations : 1 } , readiness . task , [ '--no-ingest' ] ) ;
193+ assert . equal ( records ( compatible ) [ 0 ] . status , 'success' ) ;
194+ assert . equal ( records ( compatible ) [ 0 ] . data ?. serverPlaywrightVersion , '1.60.1' ) ;
190195 mode = 'refused-connection' ;
191196 const azureFailure = await runBenchmark ( { ...readiness . config , participants : readiness . config . participants . filter ( p => p . name === 'azure' ) . map ( p => ( { ...p } ) ) , iterations : 2 } , readiness . task , [ '--no-ingest' ] ) ;
192197 assert . equal ( records ( azureFailure ) [ 0 ] . data ?. cleanupSuccess , false ) ;
@@ -215,6 +220,60 @@ test('native suites use real Playwright connections and preserve reliability rec
215220 assert . ok ( stdout . includes ( 'All done. No platform run created.' ) ) ;
216221 assert . ok ( ! stdout . includes ( 'FAILED' ) ) ;
217222 }
223+ // Missing credentials must fail the comparison before any allocation.
224+ const diagnosticEnv = { ...process . env } ;
225+ delete diagnosticEnv . AZURE_PLAYWRIGHT_ACCESS_TOKEN ;
226+ for ( const entry of [ 'playwright-readiness' , 'playwright-throughput' ] ) {
227+ const command = [ 'node_modules/tsx/dist/cli.mjs' , 'packages/benchsdk-runner/dist/bin.js' , 'run' , `benchmarks/browser/${ entry } .bench.ts` , '--no-ingest' , '--iterations=1' ] ;
228+ order . length = 0 ;
229+ await assert . rejects ( promisify ( execFile ) ( process . execPath , command , { env : diagnosticEnv } ) , / U n a v a i l a b l e P l a y w r i g h t p a r t i c i p a n t \( s \) : a z u r e : A Z U R E _ P L A Y W R I G H T _ A C C E S S _ T O K E N / ) ;
230+ assert . deepEqual ( order , [ ] ) ;
231+ const { stdout } = await promisify ( execFile ) ( process . execPath , [ ...command , '--provider=momentic' ] , { env : diagnosticEnv } ) ;
232+ assert . deepEqual ( order , [ 'momentic' ] ) ;
233+ const match = stdout . match ( / R a w P l a y w r i g h t - n a t i v e r e c o r d s a n d u n t r i m m e d s u m m a r i e s : ( .+ ) / ) ;
234+ assert . ok ( match ) ;
235+ const diagnostic = JSON . parse ( readFileSync ( match [ 1 ] . trim ( ) , 'utf8' ) ) ;
236+ assert . equal ( diagnostic . mode , 'provider-diagnostic' ) ;
237+ assert . deepEqual ( diagnostic . unavailableParticipants , [ { name : 'azure' , missing : [ 'AZURE_PLAYWRIGHT_ACCESS_TOKEN' ] } ] ) ;
238+ }
239+ // Delay returning an actual remote context beyond the 30-second deadline.
240+ // Cover reconciliation within cleanup, and ownership still unresolved at its bound.
241+ for ( const delayMs of [ 31_000 , 46_000 ] ) {
242+ const client = await chromium . connect ( endpoint ) ;
243+ const newContext = client . newContext . bind ( client ) ;
244+ let pending : Promise < BrowserContext > | undefined ;
245+ let closed : Promise < void > | undefined ;
246+ const contextMock = t . mock . method ( client , 'newContext' , ( options : BrowserContextOptions ) => {
247+ pending = newContext ( options ) . then ( async created => {
248+ closed = new Promise < void > ( resolve => created . once ( 'close' , ( ) => resolve ( ) ) ) ;
249+ await new Promise ( resolve => setTimeout ( resolve , delayMs ) ) ;
250+ return created ;
251+ } ) ;
252+ return pending ;
253+ } ) ;
254+ const connectMock = t . mock . method ( chromium , 'connect' , async ( ) => client ) ;
255+ const closeMock = delayMs === 46_000 ? t . mock . method ( client , 'close' , async ( ) => { throw new Error ( 'fixture disconnect failed' ) ; } ) : undefined ;
256+ try {
257+ order . length = 0 ;
258+ const participants = readiness . config . participants . filter ( p => p . name === 'momentic' ) . map ( p => ( { ...p } ) ) ;
259+ const late = await runBenchmark ( { ...readiness . config , participants, iterations : delayMs === 46_000 ? 2 : 1 } , readiness . task , [ '--no-ingest' ] ) ;
260+ const [ first , second ] = records ( late ) ;
261+ assert . equal ( first . status , 'error' ) ;
262+ assert . match ( String ( first . data ?. errorMessage ) , / C o n t e x t c r e a t i o n t i m e d o u t / ) ;
263+ assert . equal ( first . data ?. cleanupSuccess , delayMs === 31_000 ) ;
264+ if ( delayMs === 46_000 ) {
265+ assert . equal ( second . data ?. attemptStarted , false ) ;
266+ assert . deepEqual ( order , [ 'momentic' ] , 'unresolved context ownership stops allocations' ) ;
267+ }
268+ assert . ok ( pending && closed ) ;
269+ await pending ;
270+ await withTimeout ( closed , 5_000 , 'Late context was not closed' ) ;
271+ assert . equal ( client . contexts ( ) . length , 0 ) ;
272+ } finally {
273+ contextMock . mock . restore ( ) ; connectMock . mock . restore ( ) ; closeMock ?. mock . restore ( ) ;
274+ await client . close ( ) ;
275+ }
276+ }
218277 const audit = await chromium . connect ( endpoint ) ;
219278 assert . equal ( audit . contexts ( ) . length , 0 , 'all benchmark contexts were closed' ) ;
220279 await audit . close ( ) ;
0 commit comments