diff --git a/.github/workflows/handbook-check.yaml b/.github/workflows/handbook-check.yaml index be7cf9f4e..b13f8aebc 100644 --- a/.github/workflows/handbook-check.yaml +++ b/.github/workflows/handbook-check.yaml @@ -1,5 +1,6 @@ # PR check: build handbook image (no push) and run container smoke tests. -# Draft PRs are skipped; ready_for_review re-triggers the check. +# A second, Node-only job checks e2e/*.spec.ts ↔ metadata.json parity. +# Draft PRs skip the image job; ready_for_review re-triggers the check. name: Handbook Build Check @@ -7,7 +8,7 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review] paths: - - 'e2e/screenshots/**' + - 'e2e/**' - 'scripts/handbook/**' - 'tailwind.config.js' - 'src/static/assets/**' @@ -142,3 +143,22 @@ jobs: docker stop handbook-smoke docker rm handbook-smoke + + spec-metadata-parity: + name: "Handbook coverage: spec ↔ metadata parity" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Test the parity checker + run: node --test scripts/handbook/check-parity.test.js + + - name: Check spec ↔ metadata parity + run: node scripts/handbook/check-parity.js diff --git a/scripts/handbook/check-parity.js b/scripts/handbook/check-parity.js new file mode 100644 index 000000000..cf7f3f862 --- /dev/null +++ b/scripts/handbook/check-parity.js @@ -0,0 +1,296 @@ +#!/usr/bin/env node +/** + * Handbook coverage: every screenshot spec under e2e/*.spec.ts must have a + * metadata.json key equal to its file name without .spec.ts, and every + * spec-metadata key must name an existing spec file. + * + * Screenshot spec = top-level e2e/*.spec.ts whose contents call + * toHaveScreenshot(. Nested trees (helpers/, synpress/, wallet-setup/) + * and non-.spec.ts files are ignored, except the one confirmed extra + * path in EXTRA_SPEC_REL_PATHS. + * + * metadata.json has two kinds of top-level keys (same split as build.js): + * - spec/screenshot-group metadata (title + description) + * - "docs": markdown title overrides, not a spec name + * + * Usage: + * node scripts/handbook/check-parity.js + * node scripts/handbook/check-parity.js --e2e-dir --metadata + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const SCREENSHOT_CALL = 'toHaveScreenshot('; +const SPEC_SUFFIX = '.spec.ts'; + +// Same reserved key as scripts/handbook/build.js (orphan loop ~667): +// if (key === 'docs') continue; +// build.js reads metadata.docs as path → { title } overrides for markdown +// ("Optional title overrides via metadata.json → docs[relSrc].title"). +// That is documentation metadata, not a screenshot-group / spec-metadata +// entry, so it must not be required to name an e2e/*.spec.ts file. +const DOCS_METADATA_KEY = 'docs'; + +// Historical screenshot-group keys whose names are not the spec file. +// Each mapping was confirmed against spec contents and the metadata +// description — not inferred from the key string alone. +const ALIAS_MAP = { + // debug-session-switch.spec.ts writes + // e2e/screenshots/bug-session-1-account1.png and + // bug-session-2-account2.png; metadata describes the session/account + // switch. Key is the screenshot prefix, not the spec file name. + 'bug-session': 'debug-session-switch', + // subpages-test.spec.ts writes e2e/screenshots/subpage-{name}.png for + // Buy/Sell/Swap/Transactions (and Account/Settings); metadata describes + // those subpages. Key is the screenshot prefix. + 'subpage': 'subpages-test', + // swap-bitcoin-to-lightning.spec.ts writes + // baseline/swap-btc-to-ln-01-loaded.png and -02-complete.png; + // metadata: "Swap von Bitcoin zu Lightning (geladen und abgeschlossen)." + 'swap-btc-to-ln': 'swap-bitcoin-to-lightning', + // swap-lightning-to-bitcoin.spec.ts writes + // baseline/swap-ln-to-btc-01-loaded.png and -02-complete.png; + // metadata: "Swap von Lightning zu Bitcoin." + 'swap-ln-to-btc': 'swap-lightning-to-bitcoin', +}; + +// Confirmed non-top-level screenshot spec. metadata "sell-complete" +// describes the MetaMask end-to-end sell in this file (toHaveScreenshot of +// sell page, amount, tx, etherscan for two wallets). Only this file — not +// a recursive scan of e2e/synpress/. +const EXTRA_SPEC_REL_PATHS = ['synpress/sell-complete.spec.ts']; + +function fail(message) { + console.error(message); + process.exit(1); +} + +function parseArgs(argv) { + const out = {}; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--e2e-dir' || arg === '--metadata') { + const value = argv[i + 1]; + if (value === undefined) { + fail(arg + ' requires a path argument'); + } + if (arg === '--e2e-dir') { + out.e2eDir = value; + } else { + out.metadataPath = value; + } + i += 1; + } else if (arg === '--help' || arg === '-h') { + out.help = true; + } else { + fail('Unknown argument: ' + arg); + } + } + return out; +} + +function specKey(fileName) { + return path.basename(fileName).slice(0, -SPEC_SUFFIX.length); +} + +function listTopLevelSpecs(e2eDir) { + if (!fs.existsSync(e2eDir) || !fs.statSync(e2eDir).isDirectory()) { + return []; + } + return fs + .readdirSync(e2eDir) + .filter((name) => name.endsWith(SPEC_SUFFIX)) + .filter((name) => fs.statSync(path.join(e2eDir, name)).isFile()) + .sort(); +} + +function listKnownExtraSpecs(e2eDir) { + return EXTRA_SPEC_REL_PATHS.filter((rel) => { + const full = path.join(e2eDir, rel); + return fs.existsSync(full) && fs.statSync(full).isFile(); + }); +} + +function loadMetadata(metadataPath) { + if (!fs.existsSync(metadataPath)) { + throw new Error('metadata.json not found: ' + metadataPath); + } + let parsed; + try { + parsed = JSON.parse(fs.readFileSync(metadataPath, 'utf8')); + } catch (err) { + throw new Error( + 'metadata.json is not valid JSON (' + + metadataPath + + '): ' + + (err && err.message ? err.message : String(err)), + ); + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('metadata.json must be a JSON object: ' + metadataPath); + } + return parsed; +} + +function keysMatch(specName, metadataKey) { + if (specName === metadataKey) { + return true; + } + return ALIAS_MAP[metadataKey] === specName; +} + +function formatReport(result) { + const lines = [ + 'Handbook spec ↔ metadata parity', + '', + 'Top-level specs: ' + result.topLevelSpecs.length, + 'Extra specs: ' + result.extraSpecs.length, + 'Screenshot specs that call toHaveScreenshot(: ' + result.screenshotSpecs.length, + 'Metadata keys: ' + result.metadataKeys.length, + ]; + + if (result.vacuous) { + lines.push('', result.message); + return lines.join('\n'); + } + + if (result.missing.length) { + lines.push('', 'Screenshot specs missing a metadata key:'); + for (const fileName of result.missing) { + lines.push(' - e2e/' + fileName + ' (expected key "' + specKey(fileName) + '")'); + } + } + + if (result.orphans.length) { + lines.push('', 'Metadata keys without a matching e2e/.spec.ts:'); + for (const key of result.orphans) { + lines.push(' - "' + key + '"'); + } + } + + if (result.ok) { + lines.push('', 'PASS'); + } else { + lines.push('', 'FAIL'); + } + return lines.join('\n'); +} + +function checkParity({ e2eDir, metadataPath }) { + if (!e2eDir) { + throw new Error('e2eDir is required'); + } + if (!metadataPath) { + throw new Error('metadataPath is required'); + } + + const topLevelSpecs = listTopLevelSpecs(e2eDir); + if (topLevelSpecs.length === 0) { + const result = { + ok: false, + vacuous: true, + topLevelSpecs, + extraSpecs: [], + specFiles: [], + screenshotSpecs: [], + metadataKeys: [], + missing: [], + orphans: [], + message: + 'No e2e/*.spec.ts files found — refusing to pass on an empty input set.', + }; + result.report = formatReport(result); + return result; + } + + const extraSpecs = listKnownExtraSpecs(e2eDir); + const specFiles = topLevelSpecs.concat(extraSpecs); + const metadata = loadMetadata(metadataPath); + const metadataKeys = Object.keys(metadata).sort(); + const specNameByFile = new Map(specFiles.map((fileName) => [fileName, specKey(fileName)])); + const specNames = new Set(specNameByFile.values()); + + const screenshotSpecs = specFiles.filter((fileName) => { + const contents = fs.readFileSync(path.join(e2eDir, fileName), 'utf8'); + return contents.includes(SCREENSHOT_CALL); + }); + + const missing = screenshotSpecs.filter((fileName) => { + const name = specNameByFile.get(fileName); + return !metadataKeys.some((key) => keysMatch(name, key)); + }); + + const orphans = metadataKeys.filter((key) => { + if (key === DOCS_METADATA_KEY) { + return false; + } + return !Array.from(specNames).some((name) => keysMatch(name, key)); + }); + + const result = { + ok: missing.length === 0 && orphans.length === 0, + vacuous: false, + topLevelSpecs, + extraSpecs, + specFiles, + screenshotSpecs, + metadataKeys, + missing, + orphans, + message: '', + }; + result.report = formatReport(result); + return result; +} + +function defaultPaths() { + const repoRoot = process.env.HANDBOOK_REPO_ROOT || path.resolve(__dirname, '..', '..'); + return { + e2eDir: path.join(repoRoot, 'e2e'), + metadataPath: path.join(repoRoot, 'scripts', 'handbook', 'metadata.json'), + }; +} + +function main(argv) { + const args = parseArgs(argv); + if (args.help) { + console.log( + 'Usage: node scripts/handbook/check-parity.js [--e2e-dir ] [--metadata ]', + ); + return 0; + } + const defaults = defaultPaths(); + let result; + try { + result = checkParity({ + e2eDir: args.e2eDir || defaults.e2eDir, + metadataPath: args.metadataPath || defaults.metadataPath, + }); + } catch (err) { + fail(err && err.message ? err.message : String(err)); + } + if (result.ok) { + console.log(result.report); + return 0; + } + console.error(result.report); + return 1; +} + +module.exports = { + checkParity, + formatReport, + keysMatch, + specKey, + ALIAS_MAP, + DOCS_METADATA_KEY, + EXTRA_SPEC_REL_PATHS, + main, +}; + +if (require.main === module) { + process.exit(main(process.argv.slice(2))); +} diff --git a/scripts/handbook/check-parity.test.js b/scripts/handbook/check-parity.test.js new file mode 100644 index 000000000..c70d9db90 --- /dev/null +++ b/scripts/handbook/check-parity.test.js @@ -0,0 +1,183 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { test, after } = require('node:test'); + +const { checkParity } = require('./check-parity.js'); + +const SCRIPT = path.join(__dirname, 'check-parity.js'); +const TMP_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'handbook-parity-')); + +after(() => { + fs.rmSync(TMP_ROOT, { recursive: true, force: true }); +}); + +function writeFixture(name, files) { + const root = path.join(TMP_ROOT, name); + fs.rmSync(root, { recursive: true, force: true }); + fs.mkdirSync(root, { recursive: true }); + for (const [rel, contents] of Object.entries(files)) { + const full = path.join(root, rel); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, contents); + } + return { + e2eDir: path.join(root, 'e2e'), + metadataPath: path.join(root, 'metadata.json'), + root, + }; +} + +const SCREENSHOT_SPEC = ` +const { test, expect } = require('@playwright/test'); +test('shot', async ({ page }) => { + await expect(page).toHaveScreenshot('page.png'); +}); +`; + +const PLAIN_SPEC = ` +const { test } = require('@playwright/test'); +test('no shot', async ({ page }) => { + await page.goto('/'); +}); +`; + +function runCli(fixture) { + return spawnSync(process.execPath, [SCRIPT, '--e2e-dir', fixture.e2eDir, '--metadata', fixture.metadataPath], { + encoding: 'utf8', + }); +} + +test('(a) spec plus matching metadata key passes', () => { + const fixture = writeFixture('a-pass', { + 'e2e/buy-process.spec.ts': SCREENSHOT_SPEC, + 'metadata.json': JSON.stringify({ 'buy-process': { title: 'Buy' } }), + }); + const result = checkParity(fixture); + assert.equal(result.ok, true); + assert.equal(result.vacuous, false); + assert.deepEqual(result.missing, []); + assert.deepEqual(result.orphans, []); + assert.equal(runCli(fixture).status, 0); +}); + +test('(b) screenshot spec without metadata key fails and names the file', () => { + const fixture = writeFixture('b-missing', { + 'e2e/new-screen.spec.ts': SCREENSHOT_SPEC, + 'metadata.json': JSON.stringify({}), + }); + const result = checkParity(fixture); + assert.equal(result.ok, false); + assert.ok(result.missing.includes('new-screen.spec.ts')); + assert.match(result.report, /new-screen\.spec\.ts/); + const cli = runCli(fixture); + assert.notEqual(cli.status, 0); + assert.match(cli.stderr, /new-screen\.spec\.ts/); +}); + +test('(c) metadata key without a matching spec file fails as an orphan', () => { + const fixture = writeFixture('c-orphan', { + 'e2e/buy-process.spec.ts': SCREENSHOT_SPEC, + 'metadata.json': JSON.stringify({ + 'buy-process': { title: 'Buy' }, + buy: { title: 'orphan substring' }, + }), + }); + const result = checkParity(fixture); + assert.equal(result.ok, false); + assert.ok(result.orphans.includes('buy')); + assert.match(result.report, /"buy"/); + const cli = runCli(fixture); + assert.notEqual(cli.status, 0); + assert.match(cli.stderr, /"buy"/); +}); + +test('(d) spec without toHaveScreenshot and without a key passes', () => { + const fixture = writeFixture('d-plain', { + 'e2e/check-console.spec.ts': PLAIN_SPEC, + 'metadata.json': JSON.stringify({}), + }); + const result = checkParity(fixture); + assert.equal(result.ok, true); + assert.deepEqual(result.screenshotSpecs, []); + assert.equal(runCli(fixture).status, 0); +}); + +test('(e) empty e2e directory fails instead of passing vacuously', () => { + const fixture = writeFixture('e-empty', { + 'e2e/.keep': '', + 'metadata.json': JSON.stringify({}), + }); + fs.unlinkSync(path.join(fixture.e2eDir, '.keep')); + const result = checkParity(fixture); + assert.equal(result.ok, false); + assert.equal(result.vacuous, true); + assert.match(result.report, /empty input set/); + const cli = runCli(fixture); + assert.notEqual(cli.status, 0); + assert.match(cli.stderr, /empty input set/); +}); + +test('(f) reserved docs key is documentation metadata, not an orphan spec key', () => { + const fixture = writeFixture('f-docs', { + 'e2e/buy-process.spec.ts': SCREENSHOT_SPEC, + 'metadata.json': JSON.stringify({ + 'buy-process': { title: 'Buy' }, + docs: { 'README.md': { title: 'Readme' } }, + }), + }); + const result = checkParity(fixture); + assert.equal(result.ok, true); + assert.ok(result.metadataKeys.includes('docs')); + assert.ok(!result.orphans.includes('docs')); + assert.equal(runCli(fixture).status, 0); +}); + +test('(f2) reserved docs key does not satisfy a screenshot spec that has no key', () => { + const fixture = writeFixture('f-docs-not-a-cover', { + 'e2e/new-screen.spec.ts': SCREENSHOT_SPEC, + 'metadata.json': JSON.stringify({ + docs: { 'README.md': { title: 'Readme' } }, + }), + }); + const result = checkParity(fixture); + assert.equal(result.ok, false); + assert.ok(result.missing.includes('new-screen.spec.ts')); + assert.ok(!result.orphans.includes('docs')); +}); + +test('(g) historical alias maps a metadata key onto a differently named spec', () => { + const fixture = writeFixture('g-alias', { + 'e2e/swap-bitcoin-to-lightning.spec.ts': SCREENSHOT_SPEC, + 'metadata.json': JSON.stringify({ + 'swap-btc-to-ln': { title: 'Swap BTC to LN' }, + }), + }); + const result = checkParity(fixture); + assert.equal(result.ok, true, result.report); + assert.deepEqual(result.missing, []); + assert.deepEqual(result.orphans, []); + assert.ok(result.screenshotSpecs.includes('swap-bitcoin-to-lightning.spec.ts')); + assert.equal(runCli(fixture).status, 0); +}); + +test('(h) sell-complete is the one confirmed extra spec path, not a synpress glob', () => { + const fixture = writeFixture('h-extra', { + 'e2e/buy-process.spec.ts': PLAIN_SPEC, + 'e2e/synpress/sell-complete.spec.ts': SCREENSHOT_SPEC, + 'e2e/synpress/other.spec.ts': SCREENSHOT_SPEC, + 'metadata.json': JSON.stringify({ + 'sell-complete': { title: 'Sell complete' }, + }), + }); + const result = checkParity(fixture); + assert.equal(result.ok, true, result.report); + assert.ok(result.extraSpecs.includes('synpress/sell-complete.spec.ts')); + assert.ok(!result.extraSpecs.includes('synpress/other.spec.ts')); + assert.ok(!result.screenshotSpecs.includes('synpress/other.spec.ts')); + assert.equal(runCli(fixture).status, 0); +}); diff --git a/scripts/handbook/metadata.json b/scripts/handbook/metadata.json index 0659cd83a..9c9709cc2 100644 --- a/scripts/handbook/metadata.json +++ b/scripts/handbook/metadata.json @@ -147,6 +147,10 @@ "title": "Support-Dashboard Übersicht", "description": "Support-Dashboard-Übersicht und Statistiken." }, + "support-issue-receiver-iban": { + "title": "Support: Empfänger-IBAN prüfen", + "description": "Support-Issue-Formular: Empfänger-IBAN-Prüfung in den Zuständen leer, laufend, erkannt, nicht zugeordnet, ungültig, nicht verfügbar und Login erforderlich." + }, "subpage": { "title": "Unterseiten", "description": "Buy-, Sell-, Swap- und Transaktions-Unterseiten."