From a311531a238af52a8951e0a7984b0588cc7c1be7 Mon Sep 17 00:00:00 2001 From: Adrian de la Rosa Date: Thu, 14 May 2026 12:58:13 +0200 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=91=B7=20Split=20unit-bs=20CI=20job?= =?UTF-8?q?=20per=20browser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 11 +++++++---- scripts/test/ci-bs.ts | 36 +++++++++++++++++++++++++++--------- test/browsers.conf.d.ts | 1 + test/unit/browsers.conf.ts | 5 +++++ test/unit/karma.bs.conf.js | 16 +++++++++++++--- 5 files changed, 53 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed32496a49..9d80aa03f1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -303,15 +303,18 @@ unit-bs: - .base-configuration - .bs-allowed-branches interruptible: true - resource_group: browserstack + resource_group: browserstack-$BS_BROWSER + parallel: + matrix: + - BS_BROWSER: [edge, firefox, safari-desktop, chrome-desktop, chrome-mobile] artifacts: reports: - junit: test-report/unit-bs/*.xml + junit: test-report/unit-bs-$BS_BROWSER/*.xml script: - yarn - - node scripts/test/ci-bs.ts test:unit + - node scripts/test/ci-bs.ts test:unit --browser $BS_BROWSER after_script: - - node ./scripts/test/export-test-result.ts unit-bs + - node ./scripts/test/export-test-result.ts unit-bs-$BS_BROWSER script-tests: extends: diff --git a/scripts/test/ci-bs.ts b/scripts/test/ci-bs.ts index 9b233c13dc..303547f3a8 100644 --- a/scripts/test/ci-bs.ts +++ b/scripts/test/ci-bs.ts @@ -1,3 +1,4 @@ +import { parseArgs } from 'node:util' import { printLog, runMain } from '../lib/executionUtils.ts' import { command } from '../lib/command.ts' import { fetchPR, getLastCommonCommit, LOCAL_BRANCH } from '../lib/gitUtils.ts' @@ -16,9 +17,20 @@ const RELEVANT_FILE_PATTERNS = [ ] runMain(async () => { - const testCommand = process.argv[2] + const { + values: { browser }, + positionals, + } = parseArgs({ + args: process.argv.slice(2), + options: { + browser: { type: 'string' }, + }, + allowPositionals: true, + }) + + const testCommand = positionals[0] if (!testCommand) { - throw new Error('Usage: ci-bs.ts ') + throw new Error('Usage: ci-bs.ts [--browser ]') } const pr = await fetchPR(LOCAL_BRANCH!) @@ -30,13 +42,19 @@ runMain(async () => { return } - command`yarn ${testCommand}:bs` - .withEnvironment({ - BS_USERNAME: getBrowserStackUsername(), - BS_ACCESS_KEY: getBrowserStackAccessKey(), - }) - .withLogs() - .run() + const environment: Record = { + BS_USERNAME: getBrowserStackUsername(), + BS_ACCESS_KEY: getBrowserStackAccessKey(), + } + + if (browser) { + environment.BS_BROWSER = browser + // Override CI_JOB_NAME so getTestReportDirectory() produces a per-browser path + // (e.g. test-report/unit-bs-firefox/) instead of sharing test-report/unit-bs/ across all browsers. + environment.CI_JOB_NAME = `unit-bs-${browser}` + } + + command`yarn ${testCommand}:bs`.withEnvironment(environment).withLogs().run() }) function hasRelevantChanges(baseCommit: string): boolean { diff --git a/test/browsers.conf.d.ts b/test/browsers.conf.d.ts index 231cc71575..1bc8f4189b 100644 --- a/test/browsers.conf.d.ts +++ b/test/browsers.conf.d.ts @@ -1,4 +1,5 @@ export interface BrowserConfiguration { + id?: string sessionName: string name: string version?: string diff --git a/test/unit/browsers.conf.ts b/test/unit/browsers.conf.ts index 11db782976..fd0bb06664 100644 --- a/test/unit/browsers.conf.ts +++ b/test/unit/browsers.conf.ts @@ -9,6 +9,7 @@ export const OLDEST_BROWSER_ECMA_VERSION = 2017 export const browserConfigurations: BrowserConfiguration[] = [ { + id: 'edge', sessionName: 'Edge', name: 'Edge', version: '80.0', @@ -16,6 +17,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: '11', }, { + id: 'firefox', sessionName: 'Firefox', name: 'Firefox', version: '78.0', @@ -23,6 +25,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: '11', }, { + id: 'safari-desktop', sessionName: 'Safari desktop', name: 'Safari', version: '14.0', @@ -30,6 +33,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: 'Big Sur', }, { + id: 'chrome-desktop', sessionName: 'Chrome desktop', name: 'Chrome', version: '80.0', @@ -37,6 +41,7 @@ export const browserConfigurations: BrowserConfiguration[] = [ osVersion: '11', }, { + id: 'chrome-mobile', sessionName: 'Chrome mobile', name: 'chrome', os: 'android', diff --git a/test/unit/karma.bs.conf.js b/test/unit/karma.bs.conf.js index c6b4314a8c..e6f1438a29 100644 --- a/test/unit/karma.bs.conf.js +++ b/test/unit/karma.bs.conf.js @@ -2,6 +2,16 @@ import { getBuildInfos } from '../envUtils.ts' import { browserConfigurations } from './browsers.conf.ts' import karmaBaseConf from './karma.base.conf.js' +const selectedBrowser = process.env.BS_BROWSER +const filteredConfigurations = selectedBrowser + ? browserConfigurations.filter((configuration) => configuration.id === selectedBrowser) + : browserConfigurations + +if (selectedBrowser && filteredConfigurations.length === 0) { + const availableIds = browserConfigurations.map((c) => c.id).join(', ') + throw new Error(`Unknown BS_BROWSER "${selectedBrowser}". Available: ${availableIds}`) +} + // eslint-disable-next-line import/no-default-export export default function (config) { config.set({ @@ -13,8 +23,8 @@ export default function (config) { ], plugins: [...karmaBaseConf.plugins, 'karma-browserstack-launcher'], reporters: [...karmaBaseConf.reporters, 'BrowserStack'], - browsers: browserConfigurations.map((configuration) => configuration.sessionName), - concurrency: 5, + browsers: filteredConfigurations.map((configuration) => configuration.sessionName), + concurrency: 1, browserDisconnectTolerance: 3, captureTimeout: 2 * 60 * 1000, browserStack: { @@ -25,7 +35,7 @@ export default function (config) { video: false, }, customLaunchers: Object.fromEntries( - browserConfigurations.map((configuration) => [ + filteredConfigurations.map((configuration) => [ configuration.sessionName, // See https://github.com/karma-runner/karma-browserstack-launcher#per-browser-options { From a1b4dc5e8e064b8d84884c8b09c29bca4791ead9 Mon Sep 17 00:00:00 2001 From: Adrian de la Rosa Date: Wed, 20 May 2026 12:20:52 +0200 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=91=B7=20Address=20review=20feedback:?= =?UTF-8?q?=20avoid=20CI=5FJOB=5FNAME=20mutation,=20check=20BS=20session?= =?UTF-8?q?=20capacity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 4 ++-- scripts/test/bs-wrapper.ts | 21 ++++++++++++--------- scripts/test/ci-bs.ts | 3 --- test/unit/karma.base.conf.js | 1 + test/unit/karma.bs.conf.js | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d80aa03f1..bcd1a06b6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -309,12 +309,12 @@ unit-bs: - BS_BROWSER: [edge, firefox, safari-desktop, chrome-desktop, chrome-mobile] artifacts: reports: - junit: test-report/unit-bs-$BS_BROWSER/*.xml + junit: test-report/unit-bs/*.xml script: - yarn - node scripts/test/ci-bs.ts test:unit --browser $BS_BROWSER after_script: - - node ./scripts/test/export-test-result.ts unit-bs-$BS_BROWSER + - node ./scripts/test/export-test-result.ts unit-bs script-tests: extends: diff --git a/scripts/test/bs-wrapper.ts b/scripts/test/bs-wrapper.ts index 58e5d860ee..2ba8107483 100644 --- a/scripts/test/bs-wrapper.ts +++ b/scripts/test/bs-wrapper.ts @@ -1,7 +1,7 @@ -// This wrapper script ensures that no other test is running in BrowserStack before launching the +// This wrapper script waits until a BrowserStack parallel session is available before launching the // test command, to avoid overloading the service and making tests more flaky than necessary. This -// is also handled by the CI (exclusive lock on the "browserstack" resource), but it is helpful when -// launching tests outside of the CI. +// is also handled by the CI (resource groups), but it is helpful when launching tests outside of +// the CI. // // It used to re-run the test command based on its output (in particular, when the BrowserStack // session failed to be created), but we observed that: @@ -21,7 +21,7 @@ import { browserStackRequest } from '../lib/bsUtils.ts' const AVAILABILITY_CHECK_DELAY = 30_000 const NO_OUTPUT_TIMEOUT = 5 * 60_000 -const BS_BUILD_URL = 'https://api.browserstack.com/automate/builds.json?status=running' +const BS_PLAN_URL = 'https://api.browserstack.com/automate/plan.json' const bsLocal = new browserStack.Local() @@ -44,15 +44,18 @@ runMain(async () => { }) async function waitForAvailability(): Promise { - while (await hasRunningBuild()) { - printLog('Other build running, waiting...') + while (await isAtCapacity()) { + printLog('All BrowserStack sessions occupied, waiting...') await timeout(AVAILABILITY_CHECK_DELAY) } } -async function hasRunningBuild(): Promise { - const builds = (await browserStackRequest(BS_BUILD_URL)) as any[] - return builds.length > 0 +async function isAtCapacity(): Promise { + const plan = (await browserStackRequest(BS_PLAN_URL)) as { + parallel_sessions_running: number + parallel_sessions_max_allowed: number + } + return plan.parallel_sessions_running >= plan.parallel_sessions_max_allowed } function startBsLocal(): Promise { diff --git a/scripts/test/ci-bs.ts b/scripts/test/ci-bs.ts index 303547f3a8..c3adba131c 100644 --- a/scripts/test/ci-bs.ts +++ b/scripts/test/ci-bs.ts @@ -49,9 +49,6 @@ runMain(async () => { if (browser) { environment.BS_BROWSER = browser - // Override CI_JOB_NAME so getTestReportDirectory() produces a per-browser path - // (e.g. test-report/unit-bs-firefox/) instead of sharing test-report/unit-bs/ across all browsers. - environment.CI_JOB_NAME = `unit-bs-${browser}` } command`yarn ${testCommand}:bs`.withEnvironment(environment).withLogs().run() diff --git a/test/unit/karma.base.conf.js b/test/unit/karma.base.conf.js index ad92a9db9c..b1e733a04a 100644 --- a/test/unit/karma.base.conf.js +++ b/test/unit/karma.base.conf.js @@ -76,6 +76,7 @@ export default { }, junitReporter: { outputDir: testReportDirectory, + outputFile: process.env.BS_BROWSER ? `results-${process.env.BS_BROWSER}.xml` : 'results.xml', }, singleRun: true, webpack: { diff --git a/test/unit/karma.bs.conf.js b/test/unit/karma.bs.conf.js index e6f1438a29..28589d17da 100644 --- a/test/unit/karma.bs.conf.js +++ b/test/unit/karma.bs.conf.js @@ -24,7 +24,7 @@ export default function (config) { plugins: [...karmaBaseConf.plugins, 'karma-browserstack-launcher'], reporters: [...karmaBaseConf.reporters, 'BrowserStack'], browsers: filteredConfigurations.map((configuration) => configuration.sessionName), - concurrency: 1, + concurrency: filteredConfigurations.length, browserDisconnectTolerance: 3, captureTimeout: 2 * 60 * 1000, browserStack: { From de471cc3f9ee639fb773a5c0649cd0fdeffbc718 Mon Sep 17 00:00:00 2001 From: Adrian de la Rosa Date: Wed, 20 May 2026 14:49:13 +0200 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=91=B7=20Isolate=20BrowserStackLocal?= =?UTF-8?q?=20tunnels=20with=20unique=20localIdentifier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/test/bs-wrapper.ts | 5 ++++- test/unit/karma.bs.conf.js | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/test/bs-wrapper.ts b/scripts/test/bs-wrapper.ts index 2ba8107483..983a5567cc 100644 --- a/scripts/test/bs-wrapper.ts +++ b/scripts/test/bs-wrapper.ts @@ -13,6 +13,7 @@ // after killing it. There might be a better way of prematurely aborting the test command if we need // to in the future. +import { randomUUID } from 'node:crypto' import { spawn, type ChildProcess } from 'node:child_process' import browserStack from 'browserstack-local' import { printLog, runMain, timeout, printError } from '../lib/executionUtils.ts' @@ -24,6 +25,7 @@ const NO_OUTPUT_TIMEOUT = 5 * 60_000 const BS_PLAN_URL = 'https://api.browserstack.com/automate/plan.json' const bsLocal = new browserStack.Local() +const localIdentifier = `browser-sdk-${randomUUID()}` runMain(async () => { if (command`git tag --points-at HEAD`.run()) { @@ -65,8 +67,8 @@ function startBsLocal(): Promise { bsLocal.start( { key: process.env.BS_ACCESS_KEY, + localIdentifier, forceLocal: true, - forceKill: true, onlyAutomate: true, }, (error?: Error) => { @@ -100,6 +102,7 @@ function runTests(): Promise { ...process.env, FORCE_COLOR: 'true', BROWSER_STACK: 'true', + BROWSERSTACK_LOCAL_IDENTIFIER: localIdentifier, }, }) diff --git a/test/unit/karma.bs.conf.js b/test/unit/karma.bs.conf.js index 28589d17da..f7af94ddbe 100644 --- a/test/unit/karma.bs.conf.js +++ b/test/unit/karma.bs.conf.js @@ -30,6 +30,7 @@ export default function (config) { browserStack: { username: process.env.BS_USERNAME, accessKey: process.env.BS_ACCESS_KEY, + localIdentifier: process.env.BROWSERSTACK_LOCAL_IDENTIFIER, project: 'browser sdk unit', build: getBuildInfos(), video: false, From 9ba5afb59d72573ee5c1a5138317ca525b5700bf Mon Sep 17 00:00:00 2001 From: Adrian de la Rosa Date: Wed, 20 May 2026 18:15:49 +0200 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=91=B7=20Remove=20outputFile=20overri?= =?UTF-8?q?de,=20default=20naming=20is=20already=20unique=20per=20browser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/unit/karma.base.conf.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/karma.base.conf.js b/test/unit/karma.base.conf.js index b1e733a04a..ad92a9db9c 100644 --- a/test/unit/karma.base.conf.js +++ b/test/unit/karma.base.conf.js @@ -76,7 +76,6 @@ export default { }, junitReporter: { outputDir: testReportDirectory, - outputFile: process.env.BS_BROWSER ? `results-${process.env.BS_BROWSER}.xml` : 'results.xml', }, singleRun: true, webpack: {