diff --git a/package.json b/package.json index ad5c797b..bb497283 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "@percy/config": "^1.31.10", "axios": "1.17.0", "cross-spawn": "^7.0.3", + "form-data": "4.0.5", "glob-to-regexp": "^0.4.1", "qs": "^6.11.0", "react-router-dom": "7.13.1" diff --git a/src/common.js b/src/common.js index 72adec9a..a01842ed 100644 --- a/src/common.js +++ b/src/common.js @@ -48,4 +48,10 @@ export const flags = [{ name: 'partial', description: 'Marks the build as a partial build', validate: () => (process.env.PERCY_PARTIAL_BUILD ||= '1') +}, { + name: 'upload-bundle', + description: 'Upload the built Storybook (directory mode) to Percy for per-build hosting', + percyrc: 'storybook.uploadBundle', + type: 'boolean', + default: false }]; diff --git a/src/config.js b/src/config.js index 4f523d91..bc1520a8 100644 --- a/src/config.js +++ b/src/config.js @@ -179,6 +179,13 @@ export const configSchema = { } ], properties: { + // Opt-in to per-build Storybook hosting (PER-8973). When true (and running in + // directory mode), the SDK uploads the built storybook-static bundle to Percy after + // the snapshot run. Default off so an SDK upgrade never changes upload behavior. + uploadBundle: { + type: 'boolean', + default: false + }, docs: { type: 'object', default: {}, diff --git a/src/snapshots.js b/src/snapshots.js index 5549c8d7..2aaf2146 100644 --- a/src/snapshots.js +++ b/src/snapshots.js @@ -119,6 +119,20 @@ function shardSnapshots(snapshots, { shardSize, shardCount, shardIndex }) { return snapshots.splice(size * shardIndex, size); } +// Recursively encode booleans in Storybook's canonical `!true`/`!false` URL form. +// encodeStoryArgs leaves booleans as-is (fine for capture, which applies args via +// channel events), but a URL deep-link needs the banged form to decode to a boolean. +function bangBooleans(value) { + if (typeof value === 'boolean') return `!${value}`; + if (Array.isArray(value)) return value.map(bangBooleans); + if (value && Object.getPrototypeOf(value) === Object.prototype) { + return Object.entries(value).reduce((acc, [k, v]) => ( + Object.assign(acc, { [k]: bangBooleans(v) }) + ), {}); + } + return value; +} + // Transforms a set of pre-encoded args into a single query parameter value function buildStorybookArgsParam(args) { let argsParam = qs.stringify(args, { @@ -231,14 +245,25 @@ function mapStorybookSnapshots(stories, { previewUrl, flags, config, globalDocSe // remove filter options and generate story snapshot URLs return snapshots.map(({ skip, include, exclude, ...story }) => { + let argsParam = story.args && buildStorybookArgsParam(story.args); + let globalsParam = story.globals && buildStorybookArgsParam(story.globals); let url = `${previewUrl}?id=${story.id}`; - if (story.args) url += `&args=${buildStorybookArgsParam(story.args)}`; - if (story.globals) url += `&globals=${buildStorybookArgsParam(story.globals)}`; + if (argsParam) url += `&args=${argsParam}`; + if (globalsParam) url += `&globals=${globalsParam}`; for (let [k, v] of Object.entries(story.queryParams ?? {})) url += `&${k}=${v}`; if (!story.queryParams?.viewMode) { url += `&viewMode=${viewModeFor(story)}`; } - return Object.assign(story, { url }); + // Carry the story identity with the snapshot (see the `storybook` property of the + // snapshot schema in @percy/core). Persisted by the API so the review UI can + // deep-link the hosted bundle to the exact variant this snapshot captured. + // Booleans use Storybook's canonical `!true`/`!false` URL form — the manager + // decodes those to real booleans, while a plain `false` would apply as the + // truthy string "false". + let storybook = { id: story.id }; + if (story.args) storybook.args = buildStorybookArgsParam(bangBooleans(story.args)); + if (story.globals) storybook.globals = buildStorybookArgsParam(bangBooleans(story.globals)); + return Object.assign(story, { url, storybook }); }); } diff --git a/src/storybook.js b/src/storybook.js index b541f35d..c185f235 100644 --- a/src/storybook.js +++ b/src/storybook.js @@ -33,7 +33,7 @@ export const storybook = command('storybook', { StorybookConfig.configSchema ] } -}, async function*({ percy, args, flags, exit }) { +}, async function*({ percy, args, flags, exit, log }) { if (!percy) exit(0, 'Percy is disabled'); let { takeStorybookSnapshots } = yield import('./snapshots.js'); let { createServer } = yield import('@percy/cli-command/utils'); @@ -43,6 +43,21 @@ export const storybook = command('storybook', { baseUrl: args.url ?? server?.address(), flags }); + + // Per-build Storybook hosting (PER-8973). Opt-in via `storybook.uploadBundle` + // (.percy.yml) or `--upload-bundle`, directory mode only (URL mode has no on-disk + // bundle). Best-effort: it runs after the snapshot run is the source of truth and can + // never fail the build. + if (args.serve && percy.config.storybook?.uploadBundle) { + yield* percy.yield.flush(); // ensure the build has been created (delayUploads: true) + let { uploadStorybookBundle } = yield import('./upload-bundle.js'); + yield uploadStorybookBundle({ + percy, + log, + directory: args.serve, + buildId: percy.build?.id + }); + } }); export default storybook; diff --git a/src/upload-bundle.js b/src/upload-bundle.js new file mode 100644 index 00000000..81755085 --- /dev/null +++ b/src/upload-bundle.js @@ -0,0 +1,196 @@ +/** + * Per-build Storybook hosting upload (PER-8973) — content-addressable, via the verifier + * middleman (Flow B). + * + * Flow (see the PER-8973 design comments): + * 1. Walk the built storybook-static dir, SHA-256 every file -> manifest [{path, sha, size}]. + * 2. POST /check -> the API returns the MISSING shas + the verifier upload URL + a + * short-lived upload token (dedup: already-stored files are skipped). + * 3. POST each missing file's bytes to the verifier (bounded parallelism), with the upload + * token + declared sha. The verifier re-hashes and writes the CAS blob only if it matches. + * Bytes never touch percy-api; nothing unverified reaches GCS. + * 4. POST /commit with the manifest (path -> sha) to finalize the build. + * + * Talks to percy-api via axios (percy.client.post is JSON-only), reusing percy.client.token + * + apiUrl. Best-effort: every failure is logged and swallowed; on terminal failure a + * beacon marks the bundle failed so it surfaces on the review page. Never throws / exits nonzero. + */ + +import { createHash } from 'node:crypto'; +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import path from 'node:path'; +import axios from 'axios'; + +export const MAX_FILE_BYTES = 50 * 1024 * 1024; // per-file cap (matches API MAX_FILE_BYTES) +export const MAX_BUNDLE_BYTES = 1024 * 1024 * 1024; // per-build total cap +export const MAX_FILE_COUNT = 5000; +const UPLOAD_CONCURRENCY = 12; + +function sha256Hex(buf) { + return createHash('sha256').update(buf).digest('hex'); +} + +// Recursively list files under dir as { relPath (posix), absPath, size }. +function walkFiles(dir) { + const out = []; + const walk = (abs, rel) => { + for (const entry of readdirSync(abs, { withFileTypes: true })) { + const childAbs = path.join(abs, entry.name); + const childRel = rel ? `${rel}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + walk(childAbs, childRel); + } else if (entry.isFile()) { + out.push({ relPath: childRel, absPath: childAbs, size: statSync(childAbs).size }); + } + // symlinks/other types are skipped — only regular files are hosted + } + }; + walk(dir, ''); + return out; +} + +// Build the manifest by hashing every file. Returns { entries, totalBytes }. +function buildManifest(dir) { + const entries = walkFiles(dir).map(f => { + const bytes = readFileSync(f.absPath); + return { path: f.relPath, sha256: sha256Hex(bytes), size: f.size, absPath: f.absPath }; + }); + const totalBytes = entries.reduce((n, e) => n + e.size, 0); + return { entries, totalBytes }; +} + +function endpoint(apiUrl, buildId, suffix) { + return `${apiUrl}/builds/${buildId}/storybook_bundle${suffix}`; +} + +function authHeaders(token, extra = {}) { + return { ...extra, Authorization: `Token token=${token}` }; +} + +async function sendFailureBeacon({ apiUrl, token, buildId, reason, log }) { + try { + const FormData = (await import('form-data')).default; + const form = new FormData(); + form.append('state_hint', 'failed'); + form.append('reason', reason); + await axios.post(endpoint(apiUrl, buildId, ''), form, { + headers: authHeaders(token, form.getHeaders()) + }); + } catch { + log?.debug?.('Storybook bundle failure beacon could not be delivered'); + } +} + +// Run tasks with bounded concurrency; rejects on the first failure. +async function runBounded(items, limit, fn) { + const queue = [...items]; + const workers = Array.from({ length: Math.min(limit, queue.length) }, async () => { + while (queue.length) { + const item = queue.shift(); + await fn(item); + } + }); + await Promise.all(workers); +} + +/** + * @param {Object} args + * @param {Object} args.percy percy core instance (.client has apiUrl + token) + * @param {Object} args.log logger (info/warn/debug) + * @param {string} args.directory absolute path to the static Storybook build dir + * @param {string|number} args.buildId + * @param {Object} [args.caps] override {maxFileBytes, maxBundleBytes, maxFileCount} (tests) + */ +export async function uploadStorybookBundle({ percy, log, directory, buildId, caps = {} }) { + const maxFileBytes = caps.maxFileBytes ?? MAX_FILE_BYTES; + const maxBundleBytes = caps.maxBundleBytes ?? MAX_BUNDLE_BYTES; + const maxFileCount = caps.maxFileCount ?? MAX_FILE_COUNT; + const apiUrl = percy?.client?.apiUrl; + // Resolve the token the same way the client does for requests: explicit token, then + // PERCY_TOKEN env (this.env.token), then config. Reading the raw `client.token` misses the + // common env case (token only lives in this.env.token) and silently skips the upload. + const token = percy?.client?.getToken?.(false) ?? percy?.client?.token; + if (!apiUrl || !token || !buildId || !directory) { + log?.warn?.( + 'Storybook bundle upload skipped: missing ' + + `apiUrl=${!!apiUrl} token=${!!token} buildId=${!!buildId} directory=${!!directory}` + ); + return; + } + + let entries, totalBytes; + try { + ({ entries, totalBytes } = buildManifest(directory)); + } catch (err) { + log?.warn?.(`Storybook bundle upload skipped: could not read build dir: ${err.message}`); + await sendFailureBeacon({ apiUrl, token, buildId, reason: 'client_failed', log }); + return; + } + + // Client-side quota pre-check (the API enforces the same before signing). + const oversize = entries.find(e => e.size > maxFileBytes); + if (entries.length > maxFileCount || totalBytes > maxBundleBytes || oversize) { + log?.warn?.( + 'Storybook bundle exceeds hosting limits ' + + `(${entries.length} files, ${Math.round(totalBytes / 1024 / 1024)} MB) and was not uploaded. ` + + 'Trim addons or large static assets, or contact support to raise the limit.' + ); + await sendFailureBeacon({ apiUrl, token, buildId, reason: 'size_cap', log }); + return; + } + + try { + // (1) /check — one call: which shas are missing + the verifier upload URL + upload token. + const checkRes = await axios.post( + endpoint(apiUrl, buildId, '/check'), + { files: entries.map(e => ({ path: e.path, sha256: e.sha256, size: e.size })) }, + { headers: authHeaders(token) } + ); + const missing = checkRes.data?.missing || []; + const uploadUrl = checkRes.data?.upload_url; + const uploadToken = checkRes.data?.upload_token; + const bySha = new Map(entries.map(e => [e.sha256, e])); + + if (missing.length && (!uploadUrl || !uploadToken)) { + throw new Error('Storybook /check did not return an upload endpoint (upload_url/upload_token)'); + } + + // (2)+(3) POST each missing blob to the verifier middleman (Flow B). It re-hashes the bytes + // and writes them to the content-addressed store only if they match — bytes never touch the + // API, and nothing unverified reaches GCS. + await runBounded(missing, UPLOAD_CONCURRENCY, async ({ sha256 }) => { + const entry = bySha.get(sha256); + if (!entry) return; + await axios.post(uploadUrl, readFileSync(entry.absPath), { + headers: { + 'X-Upload-Token': uploadToken, + 'X-Expected-Sha': sha256, + 'Content-Type': 'application/octet-stream' + }, + maxContentLength: Infinity, + maxBodyLength: Infinity + }); + }); + + // (4) /commit — finalize the build with the manifest. + await axios.post( + endpoint(apiUrl, buildId, '/commit'), + { manifest: entries.map(e => ({ path: e.path, sha256: e.sha256 })) }, + { headers: authHeaders(token) } + ); + + log?.info?.(`Uploaded Storybook bundle for build #${buildId} (${entries.length} files, ${missing.length} new)`); + } catch (err) { + const status = err.response?.status; + const detail = status ? `HTTP ${status}` : err.message; + // A 403 feature_disabled is terminal but not a client upload failure worth a beacon. + if (status === 403) { + log?.warn?.(`Storybook bundle upload skipped: ${detail}`); + return; + } + log?.warn?.(`Storybook bundle upload failed: ${detail}`); + await sendFailureBeacon({ apiUrl, token, buildId, reason: 'client_failed', log }); + } +} + +export default uploadStorybookBundle; diff --git a/test/.storybook/args.stories.js b/test/.storybook/args.stories.js index 9d4b186d..cd81f7b0 100644 --- a/test/.storybook/args.stories.js +++ b/test/.storybook/args.stories.js @@ -58,7 +58,8 @@ export default { hsla: 'hsla(120, 80%, 30%, .5)', shortHex: '#c6c', longHex: '#a907cf', - alphaHex: '#a907cf9f' + alphaHex: '#a907cf9f', + bool: true } }] } diff --git a/test/storybook.test.js b/test/storybook.test.js index 124411f7..257075f2 100644 --- a/test/storybook.test.js +++ b/test/storybook.test.js @@ -436,6 +436,27 @@ describe('percy storybook', () => { expect(logger.stderr).toEqual([]); }); + it('attaches the story identity to each snapshot', async () => { + // eslint-disable-next-line import/no-extraneous-dependencies + let { Percy } = await import('@percy/core'); + spyOn(Percy.prototype, 'snapshot').and.callThrough(); + + await storybook(['http://localhost:9000', '--dry-run', '--include=Args']); + + let options = Percy.prototype.snapshot.calls.allArgs().flat().flat(); + let byName = Object.fromEntries(options.map(o => [o.name, o.storybook])); + + // plain story: id only + expect(byName.Args).toEqual({ id: 'args--args' }); + // additionalSnapshots variant: same id, its own encoded args + expect(byName['Custom Args']).toEqual({ + id: 'args--args', + args: 'text:Snapshot+custom+args;style.font:1rem+sans-serif' + }); + // booleans use Storybook's canonical !true/!false URL form + expect(byName['Special Args'].args).toContain('bool:!true'); + }); + it('excludes stories from snapshots with --exclude', async () => { // Args and Mixed are excluded by default via story-level exclude, but global // --exclude switches shouldSkipStory to use config filters and disregards @@ -579,7 +600,7 @@ describe('percy storybook', () => { 'date:!date(2022-01-01T00:00:00.000Z);' + 'rgb:!rgb(20,30,40);rgba:!rgba(20,30,40,.5);' + 'hsl:!hsl(120,80,30);hsla:!hsla(120,80,30,.5);' + - 'shortHex:!hex(c6c);longHex:!hex(a907cf);alphaHex:!hex(a907cf9f)&viewMode=story' + 'shortHex:!hex(c6c);longHex:!hex(a907cf);alphaHex:!hex(a907cf9f);bool:true&viewMode=story' ])); }); diff --git a/test/upload-bundle.test.js b/test/upload-bundle.test.js new file mode 100644 index 00000000..f86d341d --- /dev/null +++ b/test/upload-bundle.test.js @@ -0,0 +1,140 @@ +import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import axios from 'axios'; +import { uploadStorybookBundle, MAX_FILE_COUNT } from '../src/upload-bundle.js'; + +const sha256 = s => createHash('sha256').update(s).digest('hex'); +const UPLOAD_URL = 'http://verifier.test/verify-upload'; + +describe('uploadStorybookBundle (CAS)', () => { + let directory; + let percy; + let log; + let postSpy; + let putSpy; + let indexSha; + let appSha; + + // Route axios.post by URL: /check returns the missing set + verifier upload url/token; + // verify-upload, /commit, and the beacon all resolve. + function stubPost({ missing }) { + postSpy = spyOn(axios, 'post').and.callFake((url) => { + if (url.endsWith('/check')) { + return Promise.resolve({ data: { missing, upload_url: UPLOAD_URL, upload_token: 'tok' } }); + } + return Promise.resolve({ status: 201 }); // verify-upload, /commit, or beacon + }); + } + + beforeEach(() => { + directory = mkdtempSync(path.join(tmpdir(), 'sb-cas-')); + writeFileSync(path.join(directory, 'index.html'), 'A'); + mkdirSync(path.join(directory, 'assets')); + writeFileSync(path.join(directory, 'assets', 'app.js'), 'console.log(1)'); + indexSha = sha256('A'); + appSha = sha256('console.log(1)'); + + log = { info: jasmine.createSpy('info'), warn: jasmine.createSpy('warn'), debug: jasmine.createSpy('debug') }; + percy = { client: { apiUrl: 'http://localhost:9090/api/v1', token: 'TEST_TOKEN' }, build: { id: 42 } }; + putSpy = spyOn(axios, 'put').and.returnValue(Promise.resolve({ status: 200 })); + }); + + afterEach(() => rmSync(directory, { recursive: true, force: true })); + + it('hashes files, checks, POSTs only missing blobs to the verifier, then commits', async () => { + stubPost({ missing: [{ sha256: indexSha }, { sha256: appSha }] }); + + await uploadStorybookBundle({ percy, log, directory, buildId: 42 }); + + const checkCall = postSpy.calls.all().find(c => c.args[0].endsWith('/check')); + expect(checkCall.args[1].files.map(f => f.sha256).sort()).toEqual([indexSha, appSha].sort()); + + // Both missing blobs POSTed to the verifier with the upload token + declared sha. + const uploadCalls = postSpy.calls.all().filter(c => c.args[0] === UPLOAD_URL); + expect(uploadCalls.length).toBe(2); + expect(uploadCalls.every(c => c.args[2].headers['X-Upload-Token'] === 'tok')).toBe(true); + expect(uploadCalls.map(c => c.args[2].headers['X-Expected-Sha']).sort()).toEqual([indexSha, appSha].sort()); + + // Commit sent the full manifest (path -> sha). + const commitCall = postSpy.calls.all().find(c => c.args[0].endsWith('/commit')); + expect(commitCall.args[1].manifest.map(e => e.path).sort()).toEqual(['assets/app.js', 'index.html']); + expect(log.info).toHaveBeenCalled(); + expect(log.warn).not.toHaveBeenCalled(); + }); + + it('uploads only the missing shas (dedup) — already-stored files are skipped', async () => { + stubPost({ missing: [{ sha256: appSha }] }); + + await uploadStorybookBundle({ percy, log, directory, buildId: 42 }); + + const uploadCalls = postSpy.calls.all().filter(c => c.args[0] === UPLOAD_URL); + expect(uploadCalls.length).toBe(1); + expect(uploadCalls[0].args[2].headers['X-Expected-Sha']).toBe(appSha); + expect(postSpy.calls.all().some(c => c.args[0].endsWith('/commit'))).toBe(true); + }); + + describe('input guards', () => { + it('skips (no /check) when buildId is missing', async () => { + stubPost({ missing: [] }); + await uploadStorybookBundle({ percy, log, directory, buildId: null }); + expect(postSpy).not.toHaveBeenCalled(); + expect(putSpy).not.toHaveBeenCalled(); + }); + + it('skips when percy.client lacks apiUrl/token', async () => { + stubPost({ missing: [] }); + percy.client = {}; + await uploadStorybookBundle({ percy, log, directory, buildId: 42 }); + expect(postSpy).not.toHaveBeenCalled(); + }); + }); + + describe('size cap', () => { + it('beacons size_cap and does not /check when over the (injected) file-count cap', async () => { + postSpy = spyOn(axios, 'post').and.returnValue(Promise.resolve({ status: 201 })); + await uploadStorybookBundle({ percy, log, directory, buildId: 42, caps: { maxFileCount: 1 } }); + + // No /check, no PUTs — only the beacon. + expect(postSpy).toHaveBeenCalledTimes(1); + expect(postSpy.calls.first().args[0]).toMatch(/\/storybook_bundle$/); + expect(postSpy.calls.first().args[1].getBuffer().toString()).toContain('size_cap'); + expect(putSpy).not.toHaveBeenCalled(); + }); + }); + + describe('terminal 403 on /check', () => { + it('does not beacon (feature disabled is not a client failure)', async () => { + const err = new Error('forbidden'); err.response = { status: 403 }; + postSpy = spyOn(axios, 'post').and.callFake((url) => { + if (url.endsWith('/check')) return Promise.reject(err); + return Promise.resolve({ status: 201 }); + }); + await uploadStorybookBundle({ percy, log, directory, buildId: 42 }); + const beacon = postSpy.calls.all().find(c => c.args[0].endsWith('/storybook_bundle')); + expect(beacon).toBeUndefined(); + expect(log.warn).toHaveBeenCalledWith('Storybook bundle upload skipped: HTTP 403'); + }); + }); + + describe('failure beacon', () => { + it('beacons client_failed when a blob upload to the verifier fails', async () => { + postSpy = spyOn(axios, 'post').and.callFake((url) => { + if (url.endsWith('/check')) { + return Promise.resolve({ data: { missing: [{ sha256: indexSha }], upload_url: UPLOAD_URL, upload_token: 'tok' } }); + } + if (url === UPLOAD_URL) return Promise.reject(new Error('ECONNRESET')); + return Promise.resolve({ status: 201 }); // /commit or beacon + }); + await uploadStorybookBundle({ percy, log, directory, buildId: 42 }); + const beacon = postSpy.calls.all().find(c => c.args[0].endsWith('/storybook_bundle')); + expect(beacon).toBeDefined(); + expect(beacon.args[1].getBuffer().toString()).toContain('client_failed'); + }); + }); + + it('exposes MAX_FILE_COUNT', () => { + expect(MAX_FILE_COUNT).toBe(5000); + }); +}); diff --git a/yarn.lock b/yarn.lock index 8d1b9987..dc6a7314 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6773,7 +6773,7 @@ fork-ts-checker-webpack-plugin@^9.1.0: semver "^7.3.5" tapable "^2.2.1" -form-data@^4.0.5: +form-data@4.0.5, form-data@^4.0.5: version "4.0.5" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.5.tgz#b49e48858045ff4cbf6b03e1805cebcad3679053" integrity sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==