Run automated conformance suite in CI

.github/workflows/conformance.yml

@@ -0,0 +1,69 @@
+name: Automated Conformance
+
+on:
+  workflow_dispatch:
+  workflow_call:
+  pull_request:
+    paths:
+      - 'toolbox/fdc3-conformance/src/**'
+      - 'toolbox/fdc3-for-web/fdc3-web-impl/src/**'
+      - 'packages/fdc3-agent-proxy/src/**'
+      - 'toolbox/fdc3-conformance/e2e/**'
+      - 'toolbox/fdc3-conformance/playwright.config.ts'
+      - 'toolbox/fdc3-conformance/package.json'
+      - '.github/workflows/conformance.yml'
+  push:
+    branches:
+      - main
+    paths:
+      - 'toolbox/fdc3-conformance/src/**'
+      - 'toolbox/fdc3-for-web/fdc3-web-impl/src/**'
+      - 'packages/fdc3-agent-proxy/src/**'
+      - 'toolbox/fdc3-conformance/e2e/**'
+      - 'toolbox/fdc3-conformance/playwright.config.ts'
+      - 'toolbox/fdc3-conformance/package.json'
+      - '.github/workflows/conformance.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Harden runner
+        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
+        with:
+          egress-policy: audit
+
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 24.18.0
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Chromium
+        run: npx playwright install --with-deps chromium
+
+      - name: Build workspaces
+        run: npm run build
+
+      - name: Run automated conformance tests
+        run: npm run test:ci --workspace fdc3-conformance
+
+      - name: Upload conformance report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: conformance-report-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            toolbox/fdc3-conformance/playwright-report/
+            toolbox/fdc3-conformance/test-results/
+          if-no-files-found: ignore
+          retention-days: 7

.gitignore

@@ -34,6 +34,8 @@ tsconfig.tsbuildinfo
 **/html-report/
 packages/fdc3-security/junit.xml
 **/html-test-results/
+toolbox/fdc3-conformance/playwright-report/
+toolbox/fdc3-conformance/test-results/
 toolbox/fdc3-example-apps/directory/static/generated
 toolbox/fdc3-example-apps/directory/.vite
 toolbox/fdc3-example-apps/front-end-apps/*/.vite

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 * Added CI dependency checks for the root package and every npm workspace, with documented baselines of existing unused-dependency findings.
 * Added conformance coverage for `ChannelError.NoChannelFound`, `ChannelError.MalformedContext`, and `ChannelError.InvalidArguments`. ([#1779](https://github.com/finos/FDC3/issues/1779))
 * Added conformance coverage verifying that Desktop Agent methods continue to work when destructured from the `fdc3` object. ([#1778](https://github.com/finos/FDC3/issues/1778))
+* Added a separate, path-filtered Playwright contribution check that runs all non-manual conformance tests against the FDC3 for Web reference Desktop Agent, with manual and reusable workflow entry points. ([#2045](https://github.com/finos/FDC3/issues/2045))
 * Added standalone Workbench examples for the FDC3 2.2 `fdc3.action`, `fdc3.fileAttachment`, `fdc3.message`, `fdc3.orderList`, `fdc3.tradeList`, and `fdc3.timeRange` context types. ([#1949](https://github.com/finos/FDC3/pull/1949))
 * Added advanced conformance tests (`fdc3.intentListenerConflict`) covering intent listener conflicts, verifying that `addIntentListener`/`addIntentListenerWithContext` reject with `ResolveError.IntentListenerConflict` for conflicting listeners (unfiltered, or overlapping context types) and allow non-overlapping filtered listeners, listeners for different intents, and re-adding after `unsubscribe()`. Added the corresponding test definitions to the "Avoiding Adding Multiple Intent Listeners" section of the Intents conformance docs.
 * Added a classification field to Instrument context type ([#1665](https://github.com/finos/FDC3/pull/1665))

toolbox/fdc3-conformance/README.md

@@ -87,6 +87,35 @@ Successful runs look something like this:
 
 <img src="static/running.png" alt="Success" width="400px" />
 
+#### Running In CI
+
+The automated (non-manual) conformance pack can be run against the FDC3 for Web
+reference Desktop Agent with Playwright. Build the workspaces, install Chromium,
+and run the CI script from the repository root:
+
+```sh
+npm run build
+npx playwright install chromium
+npm run test:ci --workspace fdc3-conformance
+```
+
+Playwright starts the conformance app and reference Desktop Agent, launches the
+runner inside the Desktop Agent, and fails when any automated conformance test
+fails. Tests that require a human to use the intent resolver or channel selector
+remain excluded from this pack.
+
+The separate **Automated Conformance** workflow runs on pull requests and pushes
+to `main` that change the conformance, web-implementation, or agent-proxy source
+directories, or the conformance harness and workflow itself. Unrelated pull
+requests still run the ordinary tests and coverage without this browser pack.
+Once the workflow is on the default branch, maintainers can also use
+**Run workflow** in GitHub Actions, or call
+`.github/workflows/conformance.yml` from another workflow with `workflow_call`.
+Manual and reusable runs do not depend on the path filters. Reports and failure
+traces are retained as workflow artifacts for seven days. Since path-filtered
+workflows do not run on every pull request, this check should not be required
+unconditionally by branch protection.
+
 ### Joining The Conformance Program
 
 If you've had a clean run of all the tests locally, why not join the conformance program?
@@ -100,4 +129,3 @@ Once you have followed these steps, you will be allowed to display the FDC3 Comp
 ### Which Desktop Agents Are Conformant?
 
 We publish the details of conformant desktop agents on the [FDC3 Home Page](https://fdc3.finos.org#conformance).  Please check there to find out who FINOS has certified! 
-

toolbox/fdc3-conformance/e2e/conformance.spec.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2026 Future Edge Group FZE
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { expect, test } from '@playwright/test';
+import { assertConformancePassed } from './conformanceResults';
+
+test('runs all automated conformance tests against the reference Desktop Agent', async ({ page }) => {
+  await page.goto('/static/da/index.html');
+
+  const conformanceApp = page.locator('.da-app-card').filter({ hasText: 'FDC3 Conformance Framework' });
+  await expect(conformanceApp).toBeVisible();
+  await conformanceApp.getByRole('button', { name: 'Start' }).click();
+
+  const runner = page.frameLocator('#app-frames iframe').first();
+  const runButton = runner.getByRole('button', { name: 'Run', exact: true }).first();
+  await expect(runButton).toBeVisible();
+  await runner.locator('#testSuite').selectOption({ label: 'All' });
+  await runButton.click();
+
+  const results = runner.locator('#mocha');
+  await expect(results).toHaveAttribute('data-conformance-status', /passed|failed/, {
+    timeout: 9 * 60 * 1000,
+  });
+
+  const status = await results.getAttribute('data-conformance-status');
+  const passes = Number(await results.getAttribute('data-conformance-passes'));
+  const failures = Number(await results.getAttribute('data-conformance-failures'));
+  const tests = Number(await results.getAttribute('data-conformance-tests'));
+  const failureMessages = JSON.parse((await results.getAttribute('data-conformance-failure-messages')) ?? '[]');
+
+  assertConformancePassed({ status, passes, failures, tests, failureMessages });
+});

toolbox/fdc3-conformance/e2e/conformanceResults.spec.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2026 Future Edge Group FZE
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { expect, test } from '@playwright/test';
+import { assertConformancePassed } from './conformanceResults';
+
+test.describe('conformance result gate', () => {
+  test('accepts a completed passing run', () => {
+    expect(() =>
+      assertConformancePassed({
+        status: 'passed',
+        passes: 42,
+        failures: 0,
+        tests: 42,
+        failureMessages: [],
+      })
+    ).not.toThrow();
+  });
+
+  test('rejects a failed or timed-out conformance test', () => {
+    expect(() =>
+      assertConformancePassed({
+        status: 'failed',
+        passes: 41,
+        failures: 1,
+        tests: 42,
+        failureMessages: ['Desktop Agent test: Timeout of 10000ms exceeded'],
+      })
+    ).toThrow(/1 conformance test\(s\) failed[\s\S]*Timeout of 10000ms exceeded/);
+  });
+
+  test('rejects a run that ends without completing every test', () => {
+    expect(() =>
+      assertConformancePassed({
+        status: 'passed',
+        passes: 41,
+        failures: 0,
+        tests: 42,
+        failureMessages: [],
+      })
+    ).toThrow(/run was incomplete: 41 of 42 test\(s\) completed/);
+  });
+});

toolbox/fdc3-conformance/e2e/conformanceResults.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2026 Future Edge Group FZE
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export interface ConformanceResults {
+  status: string | null;
+  passes: number;
+  failures: number;
+  tests: number;
+  failureMessages: string[];
+}
+
+export function assertConformancePassed(results: ConformanceResults): void {
+  const errors: string[] = [];
+
+  if (results.status !== 'passed') {
+    errors.push(`status was ${JSON.stringify(results.status)}, expected "passed"`);
+  }
+  if (results.passes <= 0) {
+    errors.push('the automated conformance pack did not execute any passing tests');
+  }
+  if (results.failures !== 0) {
+    errors.push(`${results.failures} conformance test(s) failed`);
+  }
+  if (results.passes + results.failures !== results.tests) {
+    errors.push(`the run was incomplete: ${results.passes + results.failures} of ${results.tests} test(s) completed`);
+  }
+
+  if (errors.length > 0) {
+    throw new Error([...errors, ...results.failureMessages].join('\n'));
+  }
+}

toolbox/fdc3-conformance/package.json

@@ -14,6 +14,7 @@
     "build": "vite build && npm run copy && shx cp ../../node_modules/mocha/mocha.css ./dist/lib/mocha.css",
     "serve": "npx http-server ./dist -p 3001 -c-1 --cors",
     "dev": "npm run clean && npm run build && npm run serve",
+    "test:ci": "playwright test",
     "clean": "rimraf dist"
   },
   "dependencies": {
@@ -27,6 +28,7 @@
     "util": "^0.12.5"
   },
   "devDependencies": {
+    "@playwright/test": "^1.62.1",
     "@rollup/plugin-inject": "^5.0.5",
     "@types/chai": "^5.2.3",
     "@types/mocha": "^10.0.10",

toolbox/fdc3-conformance/playwright.config.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2026 Future Edge Group FZE
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './e2e',
+  timeout: 10 * 60 * 1000,
+  expect: {
+    timeout: 30 * 1000,
+  },
+  fullyParallel: false,
+  forbidOnly: Boolean(process.env.CI),
+  retries: process.env.CI ? 1 : 0,
+  workers: 1,
+  reporter: process.env.CI ? [['line'], ['html', { open: 'never' }]] : 'list',
+  use: {
+    baseURL: 'http://127.0.0.1:4000',
+    channel: 'chromium',
+    screenshot: 'only-on-failure',
+    trace: 'retain-on-failure',
+  },
+  webServer: [
+    {
+      command: 'npm run dev',
+      url: 'http://127.0.0.1:3001/apps/app/index.html',
+      timeout: 2 * 60 * 1000,
+      reuseExistingServer: !process.env.CI,
+    },
+    {
+      command: 'npm run dev --workspace @finos/demo',
+      url: 'http://127.0.0.1:4000/static/da/index.html',
+      timeout: 2 * 60 * 1000,
+      reuseExistingServer: !process.env.CI,
+    },
+  ],
+});