diff --git a/src/tests/streams/AGENTS.md b/src/tests/streams/AGENTS.md index 3ef31a8163b..4d14e8cd4e5 100644 --- a/src/tests/streams/AGENTS.md +++ b/src/tests/streams/AGENTS.md @@ -3,7 +3,8 @@ Streams test suite, organized WPT-style: one subdirectory per functional area (`identity/`, `encoding/`, `compression/`, `digest/`, `strategies/`, `readable/`, `readable-byte/`, `writable/`, `transform/`, `piping/`, -`inspect/`, `r2-patterns/`, `iocontext/`). Every +`inspect/`, `r2-patterns/`, `iocontext/`, `cache/`, `htmlrewriter/`, +`formdata/`). Every test here runs against **both** streams implementations — the legacy C++ one (`src/workerd/api/streams/`) and the TypeScript one (`src/per_isolate/webstreams/`) — to prove parity. A test that only makes diff --git a/src/tests/streams/cache/AGENTS.md b/src/tests/streams/cache/AGENTS.md new file mode 100644 index 00000000000..808ffc8ff62 --- /dev/null +++ b/src/tests/streams/cache/AGENTS.md @@ -0,0 +1,35 @@ +# Cache API × streams + +The Cache API consuming and producing stream bodies under both stream +implementations. **The tests are the normative artifact.** The general +Cache API surface (headers, vary, purge, instrumentation) is owned by +`src/workerd/api/tests/cache-*`; this suite owns the STREAMS +interaction only. + +## Infrastructure + +Both cells wire `cacheApiOutbound` to a loopback `cache-backend` worker +(cache-backend.js). A cache.put() arrives there as a PUT whose body is +the SERIALIZED HTTP RESPONSE (status line + headers + CRLFCRLF + body); +the backend splits at the header boundary, verifies the continuous +prime-modulus byte pattern over the body, extracts the serialized +head's Content-Length, and records everything. The test worker reads +the record back through its MOCK service binding (/last-put). + +## Coverage (parity — no divergences observed) + +| Test | Shape | +| --- | --- | +| `putJsValueStreamBody` | value stream of Uint8Array chunks, byte-exact at the backend | +| `putJsByteStreamBody` | 64 KiB chunked byte stream | +| `putFixedLengthStreamBody` | FixedLengthStream body; the declared length arrives as a concrete Content-Length in the serialized head | +| `putIdentityStreamBody` | identity body fed by a concurrent writer | +| `putLargeStreamBody` | 1 MiB chunked, byte-exact | +| `putDisturbedBodyRejects` / `putLockedBodyRejects` | TypeError before any backend traffic | +| `putErroringBodyRejects` | source error rejects the put | +| `concurrentClonePuts` | the migrated cache-put-stream-test.js regression: clone + concurrent puts over a live TransformStream body, 1 MiB | +| `matchBodyIsReadableStream` | a HIT body streams out and drains via a reader | + +Consumed source (deleted): cache-put-stream-test.js. The backend +worker takes no compatibilityDate — wd_test injects `--compat-date`, +and a worker-level date conflicts with it. diff --git a/src/tests/streams/cache/BUILD.bazel b/src/tests/streams/cache/BUILD.bazel new file mode 100644 index 00000000000..513185ee1a2 --- /dev/null +++ b/src/tests/streams/cache/BUILD.bazel @@ -0,0 +1,19 @@ +load("//:build/wd_test.bzl", "wd_test") + +# The cache streams suite runs the same test modules under two configs: +# cache-cpp against the C++ implementation, cache-ts against the +# TypeScript implementation. Both wire a loopback cache backend that +# records puts for integrity verification. + +cache_suite_srcs = glob(["*.js"]) + ["cache-modules.capnp"] + +wd_test( + src = "cache-cpp.wd-test", + data = cache_suite_srcs, +) + +wd_test( + src = "cache-ts.wd-test", + args = ["--experimental"], + data = cache_suite_srcs, +) diff --git a/src/tests/streams/cache/cache-backend.js b/src/tests/streams/cache/cache-backend.js new file mode 100644 index 00000000000..bb6e1e527db --- /dev/null +++ b/src/tests/streams/cache/cache-backend.js @@ -0,0 +1,77 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Recording cache backend: a cache.put() arrives as a PUT whose body +// is the SERIALIZED HTTP RESPONSE (status line + headers + CRLFCRLF + +// body). The backend splits at the header boundary, verifies the +// continuous byte pattern over the BODY, extracts the serialized +// head's Content-Length, and records it all so the test worker can +// verify integrity via its MOCK service binding (/last-put). GETs +// serve the cache.match protocol (only-if-cached → HIT for +// /cached-resource, else 504 MISS). + +const PATTERN_MODULUS = 251; +let lastPut = null; + +function findHeaderBoundary(bytes) { + for (let i = 0; i + 3 < bytes.byteLength; i++) { + if ( + bytes[i] === 13 && + bytes[i + 1] === 10 && + bytes[i + 2] === 13 && + bytes[i + 3] === 10 + ) { + return i + 4; + } + } + return -1; +} + +export default { + async fetch(request) { + const url = new URL(request.url); + + if (request.method === 'PUT') { + const bytes = new Uint8Array(await request.arrayBuffer()); + const bodyStart = findHeaderBoundary(bytes); + const head = new TextDecoder().decode(bytes.subarray(0, bodyStart)); + const declaredLength = /content-length:\s*(\d+)/i.exec(head)?.[1] ?? null; + let patternOk = bodyStart >= 0; + for (let i = bodyStart; i < bytes.byteLength; i++) { + if (bytes[i] !== (i - bodyStart) % PATTERN_MODULUS) { + patternOk = false; + break; + } + } + lastPut = { + url: url.href, + byteLength: bytes.byteLength - bodyStart, + declaredLength, + patternOk, + }; + return new Response(null, { status: 204 }); + } + + if (request.method === 'GET') { + if (url.pathname === '/last-put') { + return Response.json(lastPut); + } + const cacheControl = request.headers.get('cache-control'); + if (cacheControl?.includes('only-if-cached')) { + if (url.pathname.includes('cached-resource')) { + return new Response('Cached content', { + status: 200, + headers: { 'CF-Cache-Status': 'HIT' }, + }); + } + return new Response(null, { + status: 504, + headers: { 'CF-Cache-Status': 'MISS' }, + }); + } + } + + return new Response('Not Found', { status: 404 }); + }, +}; diff --git a/src/tests/streams/cache/cache-cpp.wd-test b/src/tests/streams/cache/cache-cpp.wd-test new file mode 100644 index 00000000000..e0e5e1322f0 --- /dev/null +++ b/src/tests/streams/cache/cache-cpp.wd-test @@ -0,0 +1,33 @@ +using Workerd = import "/workerd/workerd.capnp"; +using CacheModules = import "cache-modules.capnp"; + +# Cache API × streams against the C++ implementation. The cache backend +# is a loopback worker that records each put (byte count, pattern +# integrity, declared Content-Length); the MOCK binding lets tests read +# the record back. + +const unitTests :Workerd.Config = ( + services = [ + ( name = "streams-cache-cpp", + worker = ( + modules = CacheModules.modules, + cacheApiOutbound = "cache-backend", + compatibilityFlags = [ + "nodejs_compat", + "streams_enable_constructors", + "transformstream_enable_standard_constructor", + ], + bindings = [ + ( name = "MOCK", service = "cache-backend" ), + ], + ) + ), + ( name = "cache-backend", + worker = ( + modules = [ + (name = "backend", esModule = embed "cache-backend.js"), + ], + ) + ), + ], +); diff --git a/src/tests/streams/cache/cache-modules.capnp b/src/tests/streams/cache/cache-modules.capnp new file mode 100644 index 00000000000..fce626153ee --- /dev/null +++ b/src/tests/streams/cache/cache-modules.capnp @@ -0,0 +1,9 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# The single source of truth for the cache suite's test modules. + +const modules :List(Workerd.Worker.Module) = [ + (name = "main", esModule = embed "main.js"), + (name = "which-impl", esModule = embed "which-impl.js"), + (name = "cache-streams", esModule = embed "cache-streams.js"), +]; diff --git a/src/tests/streams/cache/cache-streams.js b/src/tests/streams/cache/cache-streams.js new file mode 100644 index 00000000000..b39de7a02d3 --- /dev/null +++ b/src/tests/streams/cache/cache-streams.js @@ -0,0 +1,222 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// The Cache API consuming stream bodies: cache.put() must drain +// whatever body shape the response carries into the cache backend. The +// backend records what arrived (byte count, pattern integrity, +// declared Content-Length) and the MOCK binding reports it back. + +import { strictEqual, ok, rejects } from 'node:assert'; + +const PATTERN_MODULUS = 251; + +function patternChunk(offset, length) { + const chunk = new Uint8Array(length); + for (let i = 0; i < length; i++) { + chunk[i] = (offset + i) % PATTERN_MODULUS; + } + return chunk; +} + +function patternedByteSource(total, chunkLength) { + let offset = 0; + return new ReadableStream({ + type: 'bytes', + pull(c) { + const length = Math.min(chunkLength, total - offset); + c.enqueue(patternChunk(offset, length)); + offset += length; + if (offset >= total) c.close(); + }, + }); +} + +async function lastPut(env) { + const response = await env.MOCK.fetch('http://cache-backend/last-put'); + return response.json(); +} + +// A JS value stream of patterned Uint8Array chunks. +export const putJsValueStreamBody = { + async test(ctrl, env) { + const rs = new ReadableStream({ + start(c) { + c.enqueue(patternChunk(0, 100)); + c.enqueue(patternChunk(100, 100)); + c.close(); + }, + }); + await caches.default.put( + 'https://example.com/value-stream', + new Response(rs) + ); + const record = await lastPut(env); + strictEqual(record.byteLength, 200); + strictEqual(record.patternOk, true); + }, +}; + +// A JS byte stream, chunked. +export const putJsByteStreamBody = { + async test(ctrl, env) { + await caches.default.put( + 'https://example.com/byte-stream', + new Response(patternedByteSource(64 * 1024, 4 * 1024)) + ); + const record = await lastPut(env); + strictEqual(record.byteLength, 64 * 1024); + strictEqual(record.patternOk, true); + }, +}; + +// A FixedLengthStream body: the declared length must surface as a +// concrete Content-Length on the serialized put. +export const putFixedLengthStreamBody = { + async test(ctrl, env) { + const fls = new FixedLengthStream(300); + const putP = caches.default.put( + 'https://example.com/fixed-length', + new Response(fls.readable) + ); + const writer = fls.writable.getWriter(); + await writer.write(patternChunk(0, 300)); + await writer.close(); + await putP; + const record = await lastPut(env); + strictEqual(record.byteLength, 300); + strictEqual(record.patternOk, true); + strictEqual(record.declaredLength, '300'); + }, +}; + +// An IdentityTransformStream body fed by a concurrent writer. +export const putIdentityStreamBody = { + async test(ctrl, env) { + const its = new IdentityTransformStream(); + const putP = caches.default.put( + 'https://example.com/identity', + new Response(its.readable) + ); + const writer = its.writable.getWriter(); + await writer.write(patternChunk(0, 512)); + await writer.write(patternChunk(512, 512)); + await writer.close(); + await putP; + const record = await lastPut(env); + strictEqual(record.byteLength, 1024); + strictEqual(record.patternOk, true); + }, +}; + +// A LARGE chunked body: 1 MiB through the put pipeline, byte-exact. +export const putLargeStreamBody = { + async test(ctrl, env) { + await caches.default.put( + 'https://example.com/large', + new Response(patternedByteSource(1024 * 1024, 64 * 1024)) + ); + const record = await lastPut(env); + strictEqual(record.byteLength, 1024 * 1024); + strictEqual(record.patternOk, true); + }, +}; + +// A partially-read (disturbed) body must be rejected by put(). +export const putDisturbedBodyRejects = { + async test() { + const rs = patternedByteSource(100, 50); + const response = new Response(rs); + const reader = response.body.getReader(); + await reader.read(); + reader.releaseLock(); + await rejects( + caches.default.put('https://example.com/disturbed', response), + { name: 'TypeError' } + ); + }, +}; + +// A locked body must be rejected by put(). +export const putLockedBodyRejects = { + async test() { + const response = new Response(patternedByteSource(100, 50)); + response.body.getReader(); // lock, never read + await rejects(caches.default.put('https://example.com/locked', response), { + name: 'TypeError', + }); + }, +}; + +// An erroring body stream rejects the put with the source's error. +export const putErroringBodyRejects = { + async test() { + const boom = new Error('boom'); + const rs = new ReadableStream({ + start(c) { + c.enqueue(new Uint8Array(16)); + }, + pull(c) { + c.error(boom); + }, + }); + await rejects( + caches.default.put('https://example.com/erroring', new Response(rs)), + (e) => e === boom || e.name === 'Error' + ); + }, +}; + +// The seed regression, migrated from cache-put-stream-test.js: +// response.clone() over a TransformStream body with CONCURRENT puts of +// both halves must complete while the writer feeds 1 MiB. +export const concurrentClonePuts = { + async test() { + const { readable, writable } = new TransformStream(); + const response = new Response(readable); + const clone = response.clone(); + + const puts = Promise.all([ + caches.default.put('https://example.com/clone-1', response), + caches.default.put('https://example.com/clone-2', clone), + ]); + + const writer = writable.getWriter(); + const write = (async () => { + const chunk = new Uint8Array(64 * 1024); + for (let i = 0; i < 16; ++i) { + await writer.write(chunk); + } + await writer.close(); + })(); + + const result = await Promise.race([ + Promise.all([puts, write]).then(() => 'completed'), + scheduler.wait(5000).then(() => 'timed out'), + ]); + strictEqual(result, 'completed'); + }, +}; + +// cache.match streaming out of the backend: a HIT body arrives as a +// readable stream consumable by a reader. +export const matchBodyIsReadableStream = { + async test() { + const response = await caches.default.match( + 'https://example.com/cached-resource' + ); + ok(response); + ok(response.body instanceof ReadableStream); + const reader = response.body.getReader(); + const chunks = []; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + chunks.push(...value); + } + strictEqual( + new TextDecoder().decode(new Uint8Array(chunks)), + 'Cached content' + ); + }, +}; diff --git a/src/tests/streams/cache/cache-ts.wd-test b/src/tests/streams/cache/cache-ts.wd-test new file mode 100644 index 00000000000..0978a80d9c1 --- /dev/null +++ b/src/tests/streams/cache/cache-ts.wd-test @@ -0,0 +1,35 @@ +using Workerd = import "/workerd/workerd.capnp"; +using CacheModules = import "cache-modules.capnp"; + +# The SAME modules against the TypeScript implementation: put() must +# drain TS-implemented bodies through the cache serialization path (the +# bulk-drain conduit's production consumer). + +const unitTests :Workerd.Config = ( + services = [ + ( name = "streams-cache-ts", + worker = ( + modules = CacheModules.modules, + cacheApiOutbound = "cache-backend", + compatibilityFlags = [ + "nodejs_compat", + "typescript_implemented_streams", + "experimental", + ], + bindings = [ + ( name = "MOCK", service = "cache-backend" ), + ], + ) + ), + ( name = "cache-backend", + worker = ( + modules = [ + (name = "backend", esModule = embed "cache-backend.js"), + ], + ) + ), + ], + autogates = [ + "workerd-autogate-per-isolate-javascript-bootstrap", + ], +); diff --git a/src/tests/streams/cache/main.js b/src/tests/streams/cache/main.js new file mode 100644 index 00000000000..3a2e979efef --- /dev/null +++ b/src/tests/streams/cache/main.js @@ -0,0 +1,19 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Entry point for the cache streams suite. Explicit named re-exports +// only. + +export { + putJsValueStreamBody, + putJsByteStreamBody, + putFixedLengthStreamBody, + putIdentityStreamBody, + putLargeStreamBody, + putDisturbedBodyRejects, + putLockedBodyRejects, + putErroringBodyRejects, + concurrentClonePuts, + matchBodyIsReadableStream, +} from 'cache-streams'; diff --git a/src/tests/streams/cache/which-impl.js b/src/tests/streams/cache/which-impl.js new file mode 100644 index 00000000000..1242400172b --- /dev/null +++ b/src/tests/streams/cache/which-impl.js @@ -0,0 +1,14 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Distinguishes which streams implementation this worker is running, so +// tests can pin each side of a deliberate divergence exactly. +export const usingTsImpl = + globalThis.Cloudflare.compatibilityFlags['typescript_implemented_streams']; + +// True in the transform-cpp-pedantic cell: the dateless opt-in pedantic_wpt +// flag aligns the C++ finish-operation coordination (abort/close/cancel +// races) with the spec. +export const pedanticWpt = + globalThis.Cloudflare.compatibilityFlags['pedantic_wpt']; diff --git a/src/tests/streams/formdata/AGENTS.md b/src/tests/streams/formdata/AGENTS.md new file mode 100644 index 00000000000..e161af8340f --- /dev/null +++ b/src/tests/streams/formdata/AGENTS.md @@ -0,0 +1,23 @@ +# FormData × streams + +Multipart parsing FROM streamed bodies and FormData serialized INTO a +stream body — under both stream implementations. **The tests are the +normative artifact.** The general FormData surface (W3C API matrix, +urlencoded, entry semantics) is owned by `src/workerd/api/tests/ +form-data-test.js`; this suite owns the STREAMS interaction only. + +Both cells set `formdata_parser_supports_files` so multipart file +entries parse as File objects. + +## Coverage (parity — no divergences observed) + +| Test | Shape | +| --- | --- | +| `parseMultipartSingleChunk` | whole body in one stream chunk | +| `parseMultipartAwkwardChunkSplits` | chunk boundaries inside the boundary marker, inside a header, inside a value, inside the closing marker | +| `parseMultipartBytewiseChunks` | every byte its own chunk (worst-case reassembly) | +| `parseFilesFromStreamedMultipart` | File entries out of a streamed body; content read back via file.text() | +| `parseMultipartFromIdentityStream` | identity body fed by a concurrent writer | +| `parseLargeStreamedMultipart` | 100 fields + a 256 KiB file, 8 KiB chunks | +| `erroringBodyRejectsFormData` | source error rejects formData() | +| `serializedFormDataBodyRoundTrips` | Response(FormData) body drained as a stream and reparsed (boundary from the generated content-type) | diff --git a/src/tests/streams/formdata/BUILD.bazel b/src/tests/streams/formdata/BUILD.bazel new file mode 100644 index 00000000000..58e97dd7362 --- /dev/null +++ b/src/tests/streams/formdata/BUILD.bazel @@ -0,0 +1,18 @@ +load("//:build/wd_test.bzl", "wd_test") + +# The formdata streams suite runs the same test modules under two +# configs: formdata-cpp against the C++ implementation, formdata-ts +# against the TypeScript implementation. + +formdata_suite_srcs = glob(["*.js"]) + ["formdata-modules.capnp"] + +wd_test( + src = "formdata-cpp.wd-test", + data = formdata_suite_srcs, +) + +wd_test( + src = "formdata-ts.wd-test", + args = ["--experimental"], + data = formdata_suite_srcs, +) diff --git a/src/tests/streams/formdata/formdata-cpp.wd-test b/src/tests/streams/formdata/formdata-cpp.wd-test new file mode 100644 index 00000000000..bc0fd7c1c6d --- /dev/null +++ b/src/tests/streams/formdata/formdata-cpp.wd-test @@ -0,0 +1,22 @@ +using Workerd = import "/workerd/workerd.capnp"; +using FormdataModules = import "formdata-modules.capnp"; + +# FormData × streams against the C++ implementation. +# formdata_parser_supports_files makes multipart file entries parse as +# File objects (as in the api/tests form-data configs). + +const unitTests :Workerd.Config = ( + services = [ + ( name = "streams-formdata-cpp", + worker = ( + modules = FormdataModules.modules, + compatibilityFlags = [ + "nodejs_compat", + "formdata_parser_supports_files", + "streams_enable_constructors", + "transformstream_enable_standard_constructor", + ], + ) + ), + ], +); diff --git a/src/tests/streams/formdata/formdata-modules.capnp b/src/tests/streams/formdata/formdata-modules.capnp new file mode 100644 index 00000000000..058befb0bd3 --- /dev/null +++ b/src/tests/streams/formdata/formdata-modules.capnp @@ -0,0 +1,9 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# The single source of truth for the formdata suite's test modules. + +const modules :List(Workerd.Worker.Module) = [ + (name = "main", esModule = embed "main.js"), + (name = "which-impl", esModule = embed "which-impl.js"), + (name = "multipart-streams", esModule = embed "multipart-streams.js"), +]; diff --git a/src/tests/streams/formdata/formdata-ts.wd-test b/src/tests/streams/formdata/formdata-ts.wd-test new file mode 100644 index 00000000000..9b6752c4b6f --- /dev/null +++ b/src/tests/streams/formdata/formdata-ts.wd-test @@ -0,0 +1,24 @@ +using Workerd = import "/workerd/workerd.capnp"; +using FormdataModules = import "formdata-modules.capnp"; + +# The SAME modules against the TypeScript implementation: multipart +# parsing must drain TS-implemented byte-stream bodies. + +const unitTests :Workerd.Config = ( + services = [ + ( name = "streams-formdata-ts", + worker = ( + modules = FormdataModules.modules, + compatibilityFlags = [ + "nodejs_compat", + "formdata_parser_supports_files", + "typescript_implemented_streams", + "experimental", + ], + ) + ), + ], + autogates = [ + "workerd-autogate-per-isolate-javascript-bootstrap", + ], +); diff --git a/src/tests/streams/formdata/main.js b/src/tests/streams/formdata/main.js new file mode 100644 index 00000000000..62716ab79b3 --- /dev/null +++ b/src/tests/streams/formdata/main.js @@ -0,0 +1,17 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Entry point for the formdata streams suite. Explicit named +// re-exports only. + +export { + parseMultipartSingleChunk, + parseMultipartAwkwardChunkSplits, + parseMultipartBytewiseChunks, + parseFilesFromStreamedMultipart, + parseMultipartFromIdentityStream, + parseLargeStreamedMultipart, + erroringBodyRejectsFormData, + serializedFormDataBodyRoundTrips, +} from 'multipart-streams'; diff --git a/src/tests/streams/formdata/multipart-streams.js b/src/tests/streams/formdata/multipart-streams.js new file mode 100644 index 00000000000..7b2099a1d90 --- /dev/null +++ b/src/tests/streams/formdata/multipart-streams.js @@ -0,0 +1,226 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// FormData × streams: multipart parsing FROM streamed bodies (with +// chunk boundaries deliberately split inside multipart markers) and +// FormData serialized INTO a body consumed as a stream. + +import { strictEqual, ok, rejects } from 'node:assert'; + +const enc = new TextEncoder(); +const BOUNDARY = 'streamsuiteboundary'; + +function multipartBody(fields) { + const parts = []; + for (const [name, value, filename] of fields) { + parts.push(`--${BOUNDARY}\r\n`); + if (filename !== undefined) { + parts.push( + `Content-Disposition: form-data; name="${name}"; filename="${filename}"\r\n` + + 'Content-Type: application/octet-stream\r\n\r\n' + ); + } else { + parts.push(`Content-Disposition: form-data; name="${name}"\r\n\r\n`); + } + parts.push(`${value}\r\n`); + } + parts.push(`--${BOUNDARY}--`); + return parts.join(''); +} + +function requestWithStreamedBody(body, chunkAt) { + // Split the body at the given offsets, streaming each slice as its + // own byte chunk. + const offsets = [0, ...chunkAt, body.length]; + const chunks = []; + for (let i = 0; i + 1 < offsets.length; i++) { + chunks.push(body.slice(offsets[i], offsets[i + 1])); + } + let i = 0; + const rs = new ReadableStream({ + type: 'bytes', + pull(c) { + c.enqueue(enc.encode(chunks[i++])); + if (i >= chunks.length) c.close(); + }, + }); + return new Request('http://example.org/', { + method: 'POST', + body: rs, + headers: { + 'content-type': `multipart/form-data; boundary=${BOUNDARY}`, + }, + }); +} + +// Baseline: the whole multipart body in ONE stream chunk. +export const parseMultipartSingleChunk = { + async test() { + const body = multipartBody([ + ['alpha', 'one'], + ['beta', 'two'], + ]); + const form = await requestWithStreamedBody(body, []).formData(); + strictEqual(form.get('alpha'), 'one'); + strictEqual(form.get('beta'), 'two'); + }, +}; + +// Chunk boundaries INSIDE the multipart markers: mid-boundary, +// mid-header, and mid-value splits must all reassemble. +export const parseMultipartAwkwardChunkSplits = { + async test() { + const body = multipartBody([ + ['alpha', 'one'], + ['beta', 'two two two'], + ['gamma', 'three'], + ]); + // Split inside the first boundary marker, inside a + // Content-Disposition header, and inside a value. + const splits = [ + 3, // inside '--streamsuiteboundary' + body.indexOf('name="beta"') + 4, // inside a header + body.indexOf('two two two') + 5, // inside a value + body.lastIndexOf('--') + 1, // inside the closing marker + ].sort((a, b) => a - b); + const form = await requestWithStreamedBody(body, splits).formData(); + strictEqual(form.get('alpha'), 'one'); + strictEqual(form.get('beta'), 'two two two'); + strictEqual(form.get('gamma'), 'three'); + }, +}; + +// EVERY byte its own chunk (worst-case reassembly) over a small form. +export const parseMultipartBytewiseChunks = { + async test() { + const body = multipartBody([['key', 'val']]); + const splits = Array.from({ length: body.length - 1 }, (_, i) => i + 1); + const form = await requestWithStreamedBody(body, splits).formData(); + strictEqual(form.get('key'), 'val'); + }, +}; + +// File entries parsed out of a streamed multipart body, with the file +// content read back via file.text() (a Blob-backed re-stream). +export const parseFilesFromStreamedMultipart = { + async test() { + const body = multipartBody([ + ['doc', 'file-content-here', 'doc.txt'], + ['plain', 'not-a-file'], + ]); + const form = await requestWithStreamedBody(body, [ + body.indexOf('file-content') + 6, + ]).formData(); + const file = form.get('doc'); + ok(file instanceof File); + strictEqual(file.name, 'doc.txt'); + strictEqual(await file.text(), 'file-content-here'); + strictEqual(form.get('plain'), 'not-a-file'); + }, +}; + +// An IdentityTransformStream body fed concurrently parses the same way. +export const parseMultipartFromIdentityStream = { + async test() { + const body = multipartBody([['field', 'value']]); + const its = new IdentityTransformStream(); + const request = new Request('http://example.org/', { + method: 'POST', + body: its.readable, + headers: { + 'content-type': `multipart/form-data; boundary=${BOUNDARY}`, + }, + }); + const formP = request.formData(); + const writer = its.writable.getWriter(); + const mid = Math.floor(body.length / 2); + await writer.write(enc.encode(body.slice(0, mid))); + await writer.write(enc.encode(body.slice(mid))); + await writer.close(); + const form = await formP; + strictEqual(form.get('field'), 'value'); + }, +}; + +// A LARGE streamed multipart body: 100 fields plus a 256 KiB file. +export const parseLargeStreamedMultipart = { + async test() { + const fields = Array.from({ length: 100 }, (_, i) => [ + `field${i}`, + `value-${i}`, + ]); + const bigContent = 'x'.repeat(256 * 1024); + fields.push(['big', bigContent, 'big.bin']); + const body = multipartBody(fields); + const splits = []; + for (let at = 8 * 1024; at < body.length; at += 8 * 1024) splits.push(at); + const form = await requestWithStreamedBody(body, splits).formData(); + strictEqual(form.get('field0'), 'value-0'); + strictEqual(form.get('field99'), 'value-99'); + const file = form.get('big'); + ok(file instanceof File); + strictEqual((await file.text()).length, bigContent.length); + }, +}; + +// An erroring body stream rejects formData() parsing. +export const erroringBodyRejectsFormData = { + async test() { + const boom = new Error('boom'); + const rs = new ReadableStream({ + start(c) { + c.enqueue(enc.encode(`--${BOUNDARY}\r\n`)); + }, + pull(c) { + c.error(boom); + }, + }); + const request = new Request('http://example.org/', { + method: 'POST', + body: rs, + headers: { + 'content-type': `multipart/form-data; boundary=${BOUNDARY}`, + }, + }); + await rejects( + request.formData(), + (e) => e === boom || /boom/.test(e.message) + ); + }, +}; + +// FormData serialized INTO a body: the response body is a readable +// stream whose content reparses to the same form. +export const serializedFormDataBodyRoundTrips = { + async test() { + const form = new FormData(); + form.append('alpha', 'one'); + form.append('beta', 'two'); + form.append('file', new File(['file-bytes'], 'f.txt')); + const response = new Response(form); + ok(response.body instanceof ReadableStream); + const reader = response.body.getReader(); + const parts = []; + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + parts.push(value); + } + const total = parts.reduce((n, p) => n + p.byteLength, 0); + const bytes = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + bytes.set(part, offset); + offset += part.byteLength; + } + // Reparse through a fresh Response carrying the original + // content-type (with its generated boundary). + const reparsed = await new Response(bytes, { + headers: { 'content-type': response.headers.get('content-type') }, + }).formData(); + strictEqual(reparsed.get('alpha'), 'one'); + strictEqual(reparsed.get('beta'), 'two'); + strictEqual(await reparsed.get('file').text(), 'file-bytes'); + }, +}; diff --git a/src/tests/streams/formdata/which-impl.js b/src/tests/streams/formdata/which-impl.js new file mode 100644 index 00000000000..1242400172b --- /dev/null +++ b/src/tests/streams/formdata/which-impl.js @@ -0,0 +1,14 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Distinguishes which streams implementation this worker is running, so +// tests can pin each side of a deliberate divergence exactly. +export const usingTsImpl = + globalThis.Cloudflare.compatibilityFlags['typescript_implemented_streams']; + +// True in the transform-cpp-pedantic cell: the dateless opt-in pedantic_wpt +// flag aligns the C++ finish-operation coordination (abort/close/cancel +// races) with the spec. +export const pedanticWpt = + globalThis.Cloudflare.compatibilityFlags['pedantic_wpt']; diff --git a/src/tests/streams/htmlrewriter/AGENTS.md b/src/tests/streams/htmlrewriter/AGENTS.md new file mode 100644 index 00000000000..b234bcdd2d3 --- /dev/null +++ b/src/tests/streams/htmlrewriter/AGENTS.md @@ -0,0 +1,29 @@ +# HTMLRewriter × streams + +HTMLRewriter consuming stream bodies, producing a stream body, and +reading streamed replacement content — under both stream +implementations. **The tests are the normative artifact.** The general +rewriter surface (selectors, handler types, comments/doctype/text +tokens, async handlers) is owned by `src/workerd/api/tests/ +htmlrewriter-test.js`; this suite owns the STREAMS interaction only. + +Unlike the api/tests rewriter files (which pin the pre-fixup behavior +via `original-transform-stream-backpressure`), the cpp cell here runs +under the MODERN `fixup-transform-stream-backpressure`. + +## Coverage (parity — no divergences observed) + +| Test | Shape | +| --- | --- | +| `passthroughJsValueStream` / `passthroughJsByteStream` | no-handler passthrough over JS value/byte stream bodies | +| `handlerAcrossChunkBoundaries` | chunks split MID-TAG; the parser reassembles and the handler mutates both elements | +| `rewrittenBodyIsReadableStream` | output body drained incrementally via a reader | +| `contentFromReadableStream` | element.replace(ReadableStream) — streamed replacement content | +| `identityStreamBody` | identity body fed by a concurrent writer | +| `cancelDoesNotReachSource` | PARITY PIN: cancelling the transformed body does NOT invoke the source's cancel hook (contrast pipeTo); demand simply stops (bounded) | +| `erroringSourceRejectsConsumption` | source error surfaces from .text() | +| `largeDocumentThroughHandler` | 8192 elements / ~300 KiB through a counting handler, byte-exact output length | + +The api/tests htmlrewriter-transform-cancel-test.js (cancel-before-read +×50 UAF regression) stays where it is, per the security-regression +policy. diff --git a/src/tests/streams/htmlrewriter/BUILD.bazel b/src/tests/streams/htmlrewriter/BUILD.bazel new file mode 100644 index 00000000000..538da1ee3d5 --- /dev/null +++ b/src/tests/streams/htmlrewriter/BUILD.bazel @@ -0,0 +1,18 @@ +load("//:build/wd_test.bzl", "wd_test") + +# The htmlrewriter streams suite runs the same test modules under two +# configs: htmlrewriter-cpp against the C++ implementation, +# htmlrewriter-ts against the TypeScript implementation. + +htmlrewriter_suite_srcs = glob(["*.js"]) + ["htmlrewriter-modules.capnp"] + +wd_test( + src = "htmlrewriter-cpp.wd-test", + data = htmlrewriter_suite_srcs, +) + +wd_test( + src = "htmlrewriter-ts.wd-test", + args = ["--experimental"], + data = htmlrewriter_suite_srcs, +) diff --git a/src/tests/streams/htmlrewriter/htmlrewriter-cpp.wd-test b/src/tests/streams/htmlrewriter/htmlrewriter-cpp.wd-test new file mode 100644 index 00000000000..4919ce71aff --- /dev/null +++ b/src/tests/streams/htmlrewriter/htmlrewriter-cpp.wd-test @@ -0,0 +1,24 @@ +using Workerd = import "/workerd/workerd.capnp"; +using HtmlrewriterModules = import "htmlrewriter-modules.capnp"; + +# HTMLRewriter × streams against the C++ implementation. Runs under the +# MODERN transform backpressure (fixup-transform-stream-backpressure), +# unlike the api/tests htmlrewriter files, which pin the pre-fixup +# behavior via original-transform-stream-backpressure. + +const unitTests :Workerd.Config = ( + services = [ + ( name = "streams-htmlrewriter-cpp", + worker = ( + modules = HtmlrewriterModules.modules, + compatibilityFlags = [ + "nodejs_compat", + "streams_enable_constructors", + "transformstream_enable_standard_constructor", + "fixup-transform-stream-backpressure", + "html_rewriter_treats_esi_include_as_void_tag", + ], + ) + ), + ], +); diff --git a/src/tests/streams/htmlrewriter/htmlrewriter-modules.capnp b/src/tests/streams/htmlrewriter/htmlrewriter-modules.capnp new file mode 100644 index 00000000000..b57bd947eab --- /dev/null +++ b/src/tests/streams/htmlrewriter/htmlrewriter-modules.capnp @@ -0,0 +1,9 @@ +using Workerd = import "/workerd/workerd.capnp"; + +# The single source of truth for the htmlrewriter suite's test modules. + +const modules :List(Workerd.Worker.Module) = [ + (name = "main", esModule = embed "main.js"), + (name = "which-impl", esModule = embed "which-impl.js"), + (name = "rewrite-streams", esModule = embed "rewrite-streams.js"), +]; diff --git a/src/tests/streams/htmlrewriter/htmlrewriter-ts.wd-test b/src/tests/streams/htmlrewriter/htmlrewriter-ts.wd-test new file mode 100644 index 00000000000..7107c50455d --- /dev/null +++ b/src/tests/streams/htmlrewriter/htmlrewriter-ts.wd-test @@ -0,0 +1,23 @@ +using Workerd = import "/workerd/workerd.capnp"; +using HtmlrewriterModules = import "htmlrewriter-modules.capnp"; + +# The SAME modules against the TypeScript implementation: the rewriter +# must consume TS-implemented bodies and streamed replacement content. + +const unitTests :Workerd.Config = ( + services = [ + ( name = "streams-htmlrewriter-ts", + worker = ( + modules = HtmlrewriterModules.modules, + compatibilityFlags = [ + "nodejs_compat", + "typescript_implemented_streams", + "experimental", + ], + ) + ), + ], + autogates = [ + "workerd-autogate-per-isolate-javascript-bootstrap", + ], +); diff --git a/src/tests/streams/htmlrewriter/main.js b/src/tests/streams/htmlrewriter/main.js new file mode 100644 index 00000000000..3c0b55039a3 --- /dev/null +++ b/src/tests/streams/htmlrewriter/main.js @@ -0,0 +1,18 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Entry point for the htmlrewriter streams suite. Explicit named +// re-exports only. + +export { + passthroughJsValueStream, + passthroughJsByteStream, + handlerAcrossChunkBoundaries, + rewrittenBodyIsReadableStream, + contentFromReadableStream, + identityStreamBody, + cancelDoesNotReachSource, + erroringSourceRejectsConsumption, + largeDocumentThroughHandler, +} from 'rewrite-streams'; diff --git a/src/tests/streams/htmlrewriter/rewrite-streams.js b/src/tests/streams/htmlrewriter/rewrite-streams.js new file mode 100644 index 00000000000..f944c7e0337 --- /dev/null +++ b/src/tests/streams/htmlrewriter/rewrite-streams.js @@ -0,0 +1,224 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// HTMLRewriter consuming and producing stream bodies: the rewriter +// reads whatever body shape the response carries, parses across +// arbitrary chunk boundaries, and its OUTPUT is itself a readable +// stream. Content insertion FROM a ReadableStream (the streaming +// replacement extension) reads that stream through the same machinery. + +import { strictEqual, ok, rejects } from 'node:assert'; + +const enc = new TextEncoder(); + +function chunkedByteSource(chunks) { + let i = 0; + return new ReadableStream({ + type: 'bytes', + pull(c) { + c.enqueue(enc.encode(chunks[i++])); + if (i >= chunks.length) c.close(); + }, + }); +} + +// Passthrough (no handlers) over a JS VALUE stream body. +export const passthroughJsValueStream = { + async test() { + const rs = new ReadableStream({ + start(c) { + c.enqueue(enc.encode('
')); + c.enqueue(enc.encode('hello')); + c.close(); + }, + }); + const result = await new HTMLRewriter().transform(new Response(rs)).text(); + strictEqual(result, 'hello'); + }, +}; + +// Passthrough over a JS BYTE stream body. +export const passthroughJsByteStream = { + async test() { + const result = await new HTMLRewriter() + .transform( + new Response(chunkedByteSource(['hi', ''])) + ) + .text(); + strictEqual(result, 'hi'); + }, +}; + +// A handler over a document whose chunks split MID-TAG: the parser +// must reassemble the tag across stream chunk boundaries. +export const handlerAcrossChunkBoundaries = { + async test() { + const seen = []; + const result = await new HTMLRewriter() + .on('div', { + element(element) { + seen.push(element.getAttribute('id')); + element.setAttribute('data-seen', 'yes'); + }, + }) + .transform( + new Response( + chunkedByteSource(['x', '
']))); + ok(transformed.body instanceof ReadableStream); + const reader = transformed.body.getReader(); + let total = ''; + const dec = new TextDecoder(); + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + total += dec.decode(value, { stream: true }); + } + total += dec.decode(); + strictEqual(total, 'strong
'); + }, +}; + +// Streaming replacement: element content inserted FROM a +// ReadableStream. +export const contentFromReadableStream = { + async test() { + const contentStream = new ReadableStream({ + start(c) { + c.enqueue(enc.encode('streamed ')); + c.enqueue(enc.encode('content')); + c.close(); + }, + }); + const result = await new HTMLRewriter() + .on('div', { + element(element) { + element.replace(contentStream); + }, + }) + .transform(new Response('id')); + await writer.write(enc.encode('entity
')); + await writer.close(); + strictEqual(await resultP, 'identity
'); + }, +}; + +// Cancelling the transformed body mid-stream: PARITY — the cancel does +// NOT propagate to the source stream's cancel hook (contrast pipeTo, +// which delivers the reason). The rewriter simply stops pulling; the +// source is left readable and un-canceled (bounded observation). +export const cancelDoesNotReachSource = { + async test() { + let cancelReason = 'not-called'; + let pulls = 0; + let controller; + const rs = new ReadableStream({ + start(c) { + controller = c; + }, + pull() { + pulls++; + }, + cancel(reason) { + cancelReason = String(reason); + }, + }); + const transformed = new HTMLRewriter().transform(new Response(rs)); + const reader = transformed.body.getReader(); + controller.enqueue(enc.encode('first
')); + await reader.read(); + await reader.cancel('done early'); + await scheduler.wait(100); + strictEqual(cancelReason, 'not-called'); + const pullsAtCancel = pulls; + await scheduler.wait(50); + strictEqual(pulls, pullsAtCancel); // no further demand either + }, +}; + +// An erroring source stream rejects the transformed body's +// consumption. +export const erroringSourceRejectsConsumption = { + async test() { + const boom = new Error('boom'); + const rs = new ReadableStream({ + start(c) { + c.enqueue(enc.encode('x')); + }, + pull(c) { + c.error(boom); + }, + }); + await rejects( + new HTMLRewriter().transform(new Response(rs)).text(), + (e) => e === boom || /boom/.test(e.message) + ); + }, +}; + +// A LARGE document (256 KiB of repeated elements) through a counting +// handler, byte-exact output length. +export const largeDocumentThroughHandler = { + async test() { + const UNIT = '