diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dc8bd2c6..5cbae383 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,7 @@ updates: directory: "/" schedule: interval: daily + - package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: daily diff --git a/.github/workflows/nodejs-test.yml b/.github/workflows/nodejs-test.yml index c905ad27..ea5c91aa 100644 --- a/.github/workflows/nodejs-test.yml +++ b/.github/workflows/nodejs-test.yml @@ -39,6 +39,8 @@ jobs: steps: - uses: actions/checkout@v7 + with: + submodules: true - name: Use Node.js ${{ matrix.node }} uses: actions/setup-node@v6 with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..df94c73b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "test/fixtures/html5lib-tests"] + path = test/fixtures/html5lib-tests + url = https://github.com/html5lib/html5lib-tests diff --git a/src/html5lib.spec.ts b/src/html5lib.spec.ts new file mode 100644 index 00000000..3ba17a29 --- /dev/null +++ b/src/html5lib.spec.ts @@ -0,0 +1,233 @@ +// eslint-disable-next-line unicorn/name-replacements -- html5lib is the proper name of the upstream test suite. +import { existsSync, readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { + DecodingMode, + decodeHTML, + decodeHTMLAttribute, + EntityDecoder, +} from "./decode.js"; +import { htmlDecodeTree } from "./generated/decode-data-html.js"; + +/** + * Conformance tests against the WHATWG html5lib-tests tokenizer fixtures, + * vendored as a git submodule at `test/fixtures/html5lib-tests` (kept + * up-to-date by dependabot). + * + * The fixtures drive a full HTML tokenizer; this library only decodes + * character references. Tests are therefore filtered mechanically: + * + * Included: + * + * 1. Data-state tests (no `initialStates` beyond `Data state`, no + * `lastStartTag`) whose output consists solely of `Character` tokens and + * whose input contains neither `<` (would require tag-context modelling) + * nor `\r` (would require input-stream preprocessing). The concatenated + * character data must equal `decodeHTML(input)`, and also the output of + * a streaming `EntityDecoder` fed one character per chunk. + * 2. Tests emitting exactly one `` start tag with a single `a` + * attribute — the fixtures' template for attribute-value character + * reference tests. The expected attribute value must equal + * `decodeHTMLAttribute` (and the streaming decoder in attribute mode) + * applied to the raw attribute value extracted from the input. + * + * Everything else is excluded, as it requires tokenizer context that this + * library does not model. `errors` arrays are ignored: this library does + * not emit tokenizer parse errors. + */ + +interface TokenizerTest { + description: string; + input: string; + output: [type: string, ...data: unknown[]][]; + initialStates?: string[]; + lastStartTag?: string; + doubleEscaped?: boolean; +} + +const FIXTURE_FILES = [ + "entities", + "namedEntities", + "numericEntities", + "pendingSpecChanges", +]; + +/** Template used by the fixtures for attribute-value entity tests. */ +const ATTRIBUTE_TEMPLATE = /^$/s; + +/** + * Guards against a submodule update silently causing the filter to exclude + * everything. As of html5lib-tests 224991e the filter includes 4626 tests. + */ +const MINIMUM_INCLUDED_TESTS = 4000; + +/** + * Decodes a string by feeding an `EntityDecoder` one character per chunk, + * exercising the streaming state machine across chunk boundaries. + * @param input The string to decode. + * @param mode The decoding mode to use for each entity. + * @returns The decoded string. + */ +function decodeStreaming(input: string, mode: DecodingMode): string { + let result = ""; + const decoder = new EntityDecoder(htmlDecodeTree, (codePoint) => { + result += String.fromCodePoint(codePoint); + }); + + let index = 0; + while (index < input.length) { + const amp = input.indexOf("&", index); + if (amp === -1) { + result += input.slice(index); + break; + } + result += input.slice(index, amp); + + decoder.startEntity(mode); + let consumed = -1; + for ( + let cursor = amp + 1; + consumed === -1 && cursor < input.length; + cursor++ + ) { + consumed = decoder.write(input[cursor], 0); + } + if (consumed === -1) consumed = decoder.end(); + + if (consumed === 0) { + result += "&"; + index = amp + 1; + } else { + index = amp + consumed; + } + } + + return result; +} + +const fixtureDirectory = new URL( + "../test/fixtures/html5lib-tests/tokenizer/", + import.meta.url, +); + +interface FixtureCase { + description: string; + run: () => void; +} + +/** + * Applies the mechanical filter documented above and maps each included + * fixture to an executable assertion. + * @param tests The tests of a fixture file. + * @returns The runnable cases for the file. + */ +function collectCases(tests: TokenizerTest[]): FixtureCase[] { + const cases: FixtureCase[] = []; + + for (const test of tests) { + const states = test.initialStates ?? ["Data state"]; + if ( + states.length !== 1 || + states[0] !== "Data state" || + test.lastStartTag !== undefined + ) { + continue; + } + + const unescape = (value: string): string => + test.doubleEscaped ? (JSON.parse(`"${value}"`) as string) : value; + const input = unescape(test.input); + + if ( + test.output.every(([type]) => type === "Character") && + !input.includes("<") && + !input.includes("\r") + ) { + const expected = test.output + .map(([, data]) => unescape(data as string)) + .join(""); + + cases.push({ + description: test.description, + run: () => { + expect(decodeHTML(input)).toBe(expected); + expect(decodeStreaming(input, DecodingMode.Legacy)).toBe( + expected, + ); + }, + }); + } else if ( + test.output.length === 1 && + test.output[0][0] === "StartTag" + ) { + const match = ATTRIBUTE_TEMPLATE.exec(input); + const attributes = test.output[0][2] as Record; + if ( + match && + test.output[0][1] === "h" && + Object.keys(attributes).length === 1 && + "a" in attributes + ) { + const attributeValue = match[2]; + const expected = unescape(attributes["a"]); + + cases.push({ + description: test.description, + run: () => { + expect(decodeHTMLAttribute(attributeValue)).toBe( + expected, + ); + expect( + decodeStreaming( + attributeValue, + DecodingMode.Attribute, + ), + ).toBe(expected); + }, + }); + } + } + } + + return cases; +} + +if (existsSync(fixtureDirectory)) { + describe("html5lib-tests tokenizer entity fixtures", () => { + let includedCount = 0; + + for (const file of FIXTURE_FILES) { + const { tests } = JSON.parse( + readFileSync(new URL(`${file}.test`, fixtureDirectory), "utf8"), + ) as { tests: TokenizerTest[] }; + + const cases = collectCases(tests); + includedCount += cases.length; + + if (cases.length > 0) { + describe(file, () => { + for (const { description, run } of cases) { + it(description, run); + } + }); + } + } + + it("should not have filtered out most tests", () => { + expect(includedCount).toBeGreaterThanOrEqual( + MINIMUM_INCLUDED_TESTS, + ); + }); + }); +} else if (process.env["CI"]) { + throw new Error( + "html5lib-tests fixtures are missing. Run `git submodule update --init` to fetch them.", + ); +} else { + console.warn( + "Skipping html5lib-tests fixtures: submodule not initialized. Run `git submodule update --init` to enable them.", + ); + it.skip("html5lib-tests submodule not initialized", () => { + // Skipped: fixtures are unavailable. + }); +} diff --git a/test/fixtures/html5lib-tests b/test/fixtures/html5lib-tests new file mode 160000 index 00000000..224991ec --- /dev/null +++ b/test/fixtures/html5lib-tests @@ -0,0 +1 @@ +Subproject commit 224991ec10db04f056a89eed8b0bd8695fd2950e