Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 42 additions & 3 deletions bin/smoke/mutation-reproof.smoke.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ function makeRepo(mutate: (root: string) => void): { root: string; base: string;
"",
].join("\n"));
writeFileSync(join(root, "smoke", "mutations", `${name}.mutations.json`), JSON.stringify({
suite: `${name}.suite.mjs`,
command: `node ${name}.suite.mjs`,
mutations: [{
name: `the ${name} cap is removed`,
Expand Down Expand Up @@ -228,7 +229,45 @@ try {
);
}

// 5. A selected fixture whose suite is ALREADY RED before any mutation. a.mjs changes (so a's
// 5. A suite-only change must select the fixture that names that suite, even though neither its
// config nor guarded source changed. The other suite is the negative arm: changing a file whose
// name merely looks like a suite must not select either fixture.
{
const { root, base, head } = build((r) => {
writeFileSync(join(r, "a.suite.mjs"), [
"// suite-only change; a.mjs and the fixture config are untouched",
"import { capped_a } from './a.mjs';",
"if (capped_a(100) !== 32) { console.error('✗ FAIL: the a cap holds'); process.exit(1); }",
"console.log('✓ the a cap holds');",
"",
].join("\n"));
git(r, ["add", "a.suite.mjs"]);
git(r, ["commit", "--quiet", "-m", "change only a's suite"]);
});
const { status, out } = scan(root, base, head);
check(
"a suite-only change selects exactly the fixture that names that suite",
status === 0
&& eq(selectedPaths(out), ["smoke/mutations/a.mutations.json"])
&& JSON.stringify(okCounts(out)) === JSON.stringify({ discriminated: 1, preRed: 0, inconclusive: 0 }),
`status=${status} selected=${JSON.stringify(selectedPaths(out))} counts=${JSON.stringify(okCounts(out))}\n${out}`,
);
}
{
const { root, base, head } = build((r) => {
writeFileSync(join(r, "unrelated.suite.mjs"), "// not named by any fixture\n");
git(r, ["add", "unrelated.suite.mjs"]);
git(r, ["commit", "--quiet", "-m", "change an unrelated suite"]);
});
const { status, out } = scan(root, base, head);
check(
"an unrelated suite selects no fixture",
status === 0 && selectedPaths(out).length === 0,
`status=${status} selected=${JSON.stringify(selectedPaths(out))}\n${out}`,
);
}

// 6. A selected fixture whose suite is ALREADY RED before any mutation. a.mjs changes (so a's
// fixture is selected) but a's suite fails unconditionally, so mutation-proof refuses at its
// baseline (exit 4). That is the suite's own defect, not this diff's — the #1279/sandbox-guard
// shape the reviewer reproduced. The gate must report it as PRE-RED, name exactly that fixture,
Expand Down Expand Up @@ -263,7 +302,7 @@ try {
);
}

// 6. INCONCLUSIVE (not a timeout — deterministic): a mutation that leaves the suite exiting 0 but
// 7. INCONCLUSIVE (not a timeout — deterministic): a mutation that leaves the suite exiting 0 but
// never printing its named assertion. mutation-proof grades that INCONCLUSIVE, "a green status
// is not a pass". The gate must report it as INCONCLUSIVE, not fail, and NOT collapse it into
// SURVIVED (false blocker) or KILLED (false clearance). This is the outcome the reviewer flagged
Expand Down Expand Up @@ -303,7 +342,7 @@ try {
);
}

// 7. A real all-clear that reaches the OK summary (a fixture whose mutant is genuinely killed).
// 8. A real all-clear that reaches the OK summary (a fixture whose mutant is genuinely killed).
// Its counts must read `discriminated > 0, pre-red 0, inconclusive 0` — the positive contrast to
// cases 5 and 6, so "the gate worked and everything was clean" is not the same output as "every
// selected fixture was pre-red or inconclusive".
Expand Down
10 changes: 9 additions & 1 deletion bin/smoke/mutations/mutation-reproof.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"suite": "bin/smoke/mutation-reproof.smoke.ts",
"guard": "changed guarded source re-proves its fixture; a deleted or renamed-away source is a loud dangling failure naming the exact fixture; both sides of a rename enter the changed set",
"guard": "changed fixture config, suite, or guarded source re-proves its fixture; a deleted or renamed-away source is a loud dangling failure naming the exact fixture; both sides of a rename enter the changed set",
"command": "pnpm smoke:mutation-reproof",
"grades": "tool",
"proveWith": "pnpm mutation-proof --config bin/smoke/mutations/mutation-reproof.json",
Expand All @@ -21,6 +21,14 @@
"expectRed": "the gate FAILS on a renamed-away guarded source and names exactly the dangling fixture",
"cell": "the gate FAILS on a renamed-away guarded source and names exactly the dangling fixture"
},
{
"name": "a changed configured suite is ignored by changed-set selection",
"file": "scripts/mutation-reproof.mjs",
"find": " || (typeof suite === \"string\" && changed.has(suite))\n",
"replace": "",
"expectRed": "a suite-only change selects exactly the fixture that names that suite",
"cell": "a suite-only change selects exactly the fixture that names that suite"
},
{
"name": "a dangling fixture is no longer refused, so a deleted source stops being a loud failure",
"file": "scripts/mutation-reproof.mjs",
Expand Down
7 changes: 4 additions & 3 deletions scripts/mutation-reproof.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ function loadCorpus(root, paths) {
errors.push(`${path}: no top-level "mutations" array`);
continue;
}
fixtures.push({ path, mutations: config.mutations });
fixtures.push({ path, suite: config.suite, mutations: config.mutations });
}
return { fixtures, errors };
}
Expand Down Expand Up @@ -158,7 +158,8 @@ if (errors.length) {

const { changed, diffSize } = a.all ? { changed: new Set(), diffSize: 0 } : changedSet(root, a.base, head);

let selected = fixtures.filter(({ path, mutations }) => a.all || changed.has(path)
let selected = fixtures.filter(({ path, suite, mutations }) => a.all || changed.has(path)
|| (typeof suite === "string" && changed.has(suite))
|| mutations.some((mutation) => typeof mutation?.file === "string" && changed.has(mutation.file)));
if (shard) selected = selected.filter(({ path }) => shardOf(path, Number(shard[2])) === Number(shard[1]));

Expand Down Expand Up @@ -186,7 +187,7 @@ console.log(`mutation reproof: ${selected.length} fixture(s) selected from ${fix
? " for a full sweep"
: ` (diff ${diffSize} record(s), ${changed.size} changed path(s), corpus ${fixtures.length})`));
if (selected.length === 0) {
console.log("No mutation fixtures to re-prove: no fixture config or guarded source intersects the diff.");
console.log("No mutation fixtures to re-prove: no fixture config, suite, or guarded source intersects the diff.");
process.exit(0);
}
console.log(`selected fixture paths:\n${selected.map(({ path }) => ` ${path}`).join("\n")}`);
Expand Down
Loading