From 2e8ac69c425c59eebe716fbca86308186c1f12d0 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 22 Sep 2026 06:08:00 -0400 Subject: [PATCH 1/7] fix: enforce canonical probability, answer, and confidence invariants --- docs/hardening/01-answer-invariants.md | 18 +++++++++++ src/contract/decide.ts | 11 ++++++- tests/answer-invariants.test.ts | 42 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 docs/hardening/01-answer-invariants.md create mode 100644 tests/answer-invariants.test.ts diff --git a/docs/hardening/01-answer-invariants.md b/docs/hardening/01-answer-invariants.md new file mode 100644 index 0000000..103627d --- /dev/null +++ b/docs/hardening/01-answer-invariants.md @@ -0,0 +1,18 @@ +# Answer invariants (adversarial finding 1) + +`probability` is the probability of yes and the only source of truth for an +answer triple. The decision table now requires a finite probability in [0, 1], +a finite confidence in [0.5, 1], direction exactly `p >= 0.5 ? "yes" : "no"`, +and confidence exactly `Math.max(p, 1 - p)`. JSON round-trips of canonical +answers preserve these equalities. Rounded or inconsistent adapter values +must be normalized before constructing a review, never silently trusted. + +Malformed or inconsistent stored answer triples become `proposal_only` with +the question named; canonical answers retain the existing verdicts. Provider +parsing still belongs to the host: malformed raw responses become null answers +and `unavailable`. This change does not prematurely extract the upstream +transport or change any question, threshold, or authority boundary. + +Regression coverage: opposite-probability favorable labels, fabricated +confidence at p=0.5, finite out-of-range values, and 625 canonical combinations. +No live model call or new calibration claim is involved. diff --git a/src/contract/decide.ts b/src/contract/decide.ts index b52bed7..0dc33de 100644 --- a/src/contract/decide.ts +++ b/src/contract/decide.ts @@ -54,8 +54,17 @@ export function unfavorable( const out: string[] = []; for (const id of REVIEW_QUESTION_IDS) { const a = answers[id]; - if (!a || !Number.isFinite(a.probability) || !Number.isFinite(a.confidence)) + if ( + !a || + !Number.isFinite(a.probability) || a.probability < 0 || a.probability > 1 || + !Number.isFinite(a.confidence) || a.confidence < 0.5 || a.confidence > 1 + ) out.push(`${id}: no usable answer`); + else if ( + a.answer !== (a.probability >= 0.5 ? "yes" : "no") || + a.confidence !== Math.max(a.probability, 1 - a.probability) + ) + out.push(`${id}: answer or confidence disagrees with probability`); else if (a.answer !== FAVORABLE[id]) out.push(`${id}: ${a.answer} (${percent(a.confidence)})`); else if (a.confidence < threshold) diff --git a/tests/answer-invariants.test.ts b/tests/answer-invariants.test.ts new file mode 100644 index 0000000..8038b3c --- /dev/null +++ b/tests/answer-invariants.test.ts @@ -0,0 +1,42 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { decide, FAVORABLE, JEV_MODEL, REVIEW_QUESTION_IDS, type ReviewAnswer, type ReviewAnswers } from "../src"; + +const ok = { ok: true, errors: [] }; +const answer = (p: number): ReviewAnswer => ({ probability: p, answer: p >= 0.5 ? "yes" : "no", confidence: Math.max(p, 1 - p) }); +function favorable(): ReviewAnswers { + return Object.fromEntries(REVIEW_QUESTION_IDS.map(id => [id, answer(FAVORABLE[id] === "yes" ? 0.95 : 0.05)])) as ReviewAnswers; +} +const review = (answers: ReviewAnswers) => ({ model: JEV_MODEL, answers, error: null, latencyMs: 1, source: "mock" as const }); + +test("contradictory answer triples cannot permit", () => { + for (const id of REVIEW_QUESTION_IDS) { + const a = favorable(); + a[id] = { probability: FAVORABLE[id] === "yes" ? 0.01 : 0.99, answer: FAVORABLE[id], confidence: 0.99 }; + assert.equal(decide(ok, review(a)).verdict, "proposal_only", id); + a[id] = { probability: 0.5, answer: FAVORABLE[id], confidence: 0.99 }; + assert.equal(decide(ok, review(a)).verdict, "proposal_only", id); + } +}); + +test("finite but out-of-range probabilities and confidence cannot permit", () => { + for (const p of [-1, 2, Number.MAX_VALUE, -Number.MIN_VALUE]) { + const a = favorable(); + a.addresses_task = answer(p); + assert.equal(decide(ok, review(a)).verdict, "proposal_only"); + } + for (const confidence of [-1, 0.49, 1.01, 2, Number.NaN, Infinity]) { + const a = favorable(); + a.addresses_task.confidence = confidence; + assert.equal(decide(ok, review(a)).verdict, "proposal_only"); + } +}); + +test("canonical probability grid preserves the v1 decision table", () => { + const values = [0, 0.1, 0.5, 0.9, 1]; + for (const p of values) for (const q of values) for (const r of values) for (const s of values) { + const answers = { addresses_task: answer(p), evidence_supports: answer(q), unrelated_changes: answer(r), needs_clarification: answer(s) }; + const expected = p >= 0.8 && q >= 0.8 && r <= 0.2 && s <= 0.2 ? "permit" : "proposal_only"; + assert.equal(decide(ok, review(answers)).verdict, expected); + } +}); From 2ea567f6a63544978da2c745fb243994162213e8 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 22 Sep 2026 06:08:03 -0400 Subject: [PATCH 2/7] fix: prevent failed reviews from retaining favorable answers --- docs/hardening/02-review-failure.md | 14 ++++++++++++++ src/contract/decide.ts | 2 +- src/contract/types.ts | 11 +++++++---- tests/decide.test.ts | 5 ++++- tests/review-failure.test.ts | 28 ++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 docs/hardening/02-review-failure.md create mode 100644 tests/review-failure.test.ts diff --git a/docs/hardening/02-review-failure.md b/docs/hardening/02-review-failure.md new file mode 100644 index 0000000..8f37561 --- /dev/null +++ b/docs/hardening/02-review-failure.md @@ -0,0 +1,14 @@ +# Failed reviews (adversarial finding 2) + +A successful `JevReview` has non-null answers and `error: null`. A failed review +has null answers and an error string. The type is a discriminated union so an +adapter cannot accidentally retain answers alongside an error without a type +error. Runtime checks remain essential at serialization boundaries. + +After validation, any non-null review error yields `unavailable`, even when +favorable answers survived from an earlier attempt. Validation failure still +wins and yields `reject`. No fallback to the benchmark arm is permitted. +The four questions, threshold, and legitimate successful verdicts are unchanged. + +Adapters returning null answers without an error must now provide a diagnostic +string, such as `no answers returned`. Receipts retain schema version 1. diff --git a/src/contract/decide.ts b/src/contract/decide.ts index 0dc33de..c38cbfd 100644 --- a/src/contract/decide.ts +++ b/src/contract/decide.ts @@ -109,7 +109,7 @@ export function decide( reason: "Jev review was not performed. Treated as proposal-only, never as safe.", }; - if (jev.answers === null) + if (jev.answers === null || jev.error !== null) return { verdict: "unavailable", reason: `Jev review unavailable: ${jev.error ?? "no answers returned"}. Treated as proposal-only, never as safe.`, diff --git a/src/contract/types.ts b/src/contract/types.ts index 0c515d8..3266ae4 100644 --- a/src/contract/types.ts +++ b/src/contract/types.ts @@ -73,15 +73,18 @@ export interface ValidationResult { export type JevSource = "jev" | "mock"; -export interface JevReview { +interface JevReviewMetadata { model: string; - /** Null whenever the provider failed, timed out, or answered malformed. */ - answers: ReviewAnswers | null; - error: string | null; latencyMs: number; source: JevSource; } +/** A failed review cannot retain answers from an earlier request. */ +export type JevReview = JevReviewMetadata & ( + | { answers: ReviewAnswers; error: null } + | { answers: null; error: string } +); + export type ReviewArm = "good" | "bad"; export type ReviewMode = "base" | "plus_jev"; diff --git a/tests/decide.test.ts b/tests/decide.test.ts index 767eea4..160795b 100644 --- a/tests/decide.test.ts +++ b/tests/decide.test.ts @@ -31,7 +31,10 @@ function favorable(confidence = 0.95): ReviewAnswers { } function review(answers: ReviewAnswers | null, error: string | null = null): JevReview { - return { model: JEV_MODEL, answers, error, latencyMs: 1, source: "mock" }; + if (answers === null) + return { model: JEV_MODEL, answers: null, error: error ?? "no answers returned", latencyMs: 1, source: "mock" }; + if (error !== null) throw Error("Successful test reviews cannot contain errors."); + return { model: JEV_MODEL, answers, error: null, latencyMs: 1, source: "mock" }; } test("validation failure is reject and Jev is never needed", () => { diff --git a/tests/review-failure.test.ts b/tests/review-failure.test.ts new file mode 100644 index 0000000..48f21f3 --- /dev/null +++ b/tests/review-failure.test.ts @@ -0,0 +1,28 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { decide, JEV_MODEL, type JevReview, type ReviewAnswers } from "../src"; +const answers: ReviewAnswers = { + addresses_task: { probability: 1, answer: "yes", confidence: 1 }, + evidence_supports: { probability: 1, answer: "yes", confidence: 1 }, + unrelated_changes: { probability: 0, answer: "no", confidence: 1 }, + needs_clarification: { probability: 0, answer: "no", confidence: 1 }, +}; +const metadata = { model: JEV_MODEL, latencyMs: 1, source: "mock" as const }; + +test("a provider error wins over retained favorable answers", () => { + for (const error of ["synthetic timeout", "", "cancelled"]) { + // @ts-expect-error A success with an error is intentionally unrepresentable. + const mixed: JevReview = { ...metadata, answers, error }; + const result = decide({ ok: true, errors: [] }, mixed); + assert.equal(result.verdict, "unavailable"); + assert.match(result.reason, /never as safe/); + } +}); + +test("failure, success, and validation precedence remain distinct", () => { + const failed: JevReview = { ...metadata, answers: null, error: "synthetic failure" }; + const success: JevReview = { ...metadata, answers, error: null }; + assert.equal(decide({ ok: true, errors: [] }, failed).verdict, "unavailable"); + assert.equal(decide({ ok: true, errors: [] }, success).verdict, "permit"); + assert.equal(decide({ ok: false, errors: ["out of scope"] }, failed).verdict, "reject"); +}); From 8de6851b1e3a629ee71d025df70cdaaedb839fef Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 22 Sep 2026 06:08:06 -0400 Subject: [PATCH 3/7] fix: fail closed on malformed runtime validation and review inputs --- docs/hardening/03-runtime-boundary.md | 21 +++++++++++ src/contract/decide.ts | 51 +++++++++++++++------------ src/contract/input.ts | 27 ++++++++++++++ tests/runtime-inputs.test.ts | 40 +++++++++++++++++++++ 4 files changed, 116 insertions(+), 23 deletions(-) create mode 100644 docs/hardening/03-runtime-boundary.md create mode 100644 src/contract/input.ts create mode 100644 tests/runtime-inputs.test.ts diff --git a/docs/hardening/03-runtime-boundary.md b/docs/hardening/03-runtime-boundary.md new file mode 100644 index 0000000..b3a091a --- /dev/null +++ b/docs/hardening/03-runtime-boundary.md @@ -0,0 +1,21 @@ +# Runtime input boundary (adversarial finding 3) + +`decide`, `decideBase`, and `unfavorable` accept `unknown` at runtime. Schema-like +TypeScript interfaces are not validation of deserialized input. A validation +success requires `ok === true` and a present empty string-array error list. +Anything else rejects, including truthy strings and contradictory errors. + +A missing or malformed review envelope yields `unavailable`; malformed or +missing individual answers within a present answers object yield +`proposal_only`. This preserves the existing distinction for partial reviews. +Validation failure wins over review inspection. An invalid threshold still +throws as a deliberate configuration error; it is never clamped. + +Plain records are copied through own data-property descriptors. Inherited +fields, class instances, and accessors are rejected. This boundary is for +JSON/plain data, not adversarial same-process JavaScript: proxies and malicious +array accessors require process isolation and are not a supported trust domain. + +Review metadata is shape-checked, not authenticated. The host still pins the +expected model/source, verifies provenance, binds snapshots, and authorizes +any action. `permit` remains evidence only. No provider calls or I/O are added. diff --git a/src/contract/decide.ts b/src/contract/decide.ts index c38cbfd..e3d98dc 100644 --- a/src/contract/decide.ts +++ b/src/contract/decide.ts @@ -6,17 +6,16 @@ * four favorable, each ≥ threshold → permit * anything else → proposal_only (a human sees it) * - * Extracted verbatim from TypeSafeAI/typesafe-playground `lib/harness/decide.ts` + * Originally extracted from TypeSafeAI/typesafe-playground `lib/harness/decide.ts` * (branch feat/proposal-review, commit 245167d). */ import { REVIEW_QUESTION_IDS, - type JevReview, type ReviewAnswer, type ReviewQuestionId, type ReviewVerdict, - type ValidationResult, } from "./types"; +import { dataRecord, validationFailure } from "./input"; /** * Minimum probability mass on the favorable side before a question counts as @@ -47,17 +46,18 @@ function assertThreshold(threshold: number) { /** Which questions fall short, and why, for the receipt. */ export function unfavorable( - answers: NonNullable, + answers: unknown, threshold = REVIEW_CONFIDENCE_THRESHOLD, ): string[] { assertThreshold(threshold); const out: string[] = []; + const entries = dataRecord(answers); for (const id of REVIEW_QUESTION_IDS) { - const a = answers[id]; + const a = dataRecord(entries?.[id]); if ( !a || - !Number.isFinite(a.probability) || a.probability < 0 || a.probability > 1 || - !Number.isFinite(a.confidence) || a.confidence < 0.5 || a.confidence > 1 + typeof a.probability !== "number" || !Number.isFinite(a.probability) || a.probability < 0 || a.probability > 1 || + typeof a.confidence !== "number" || !Number.isFinite(a.confidence) || a.confidence < 0.5 || a.confidence > 1 ) out.push(`${id}: no usable answer`); else if ( @@ -78,12 +78,9 @@ export function unfavorable( const percent = (v: number) => `${Math.round(v * 100)}%`; /** Validate-only verdict for the base arm: no reviewer, so only scope checks. */ -export function decideBase(validation: ValidationResult): Decision { - if (!validation.ok) - return { - verdict: "reject", - reason: `Validation failed: ${validation.errors.join("; ")}`, - }; +export function decideBase(validation: unknown): Decision { + const failure = validationFailure(validation); + if (failure !== null) return { verdict: "reject", reason: failure }; return { verdict: "permit", reason: @@ -92,29 +89,37 @@ export function decideBase(validation: ValidationResult): Decision { } export function decide( - validation: ValidationResult, + validation: unknown, /** Null means no review ran; that is unavailable, never permit. */ - jev: JevReview | null, + jev: unknown, threshold = REVIEW_CONFIDENCE_THRESHOLD, ): Decision { assertThreshold(threshold); - if (!validation.ok) - return { - verdict: "reject", - reason: `Validation failed: ${validation.errors.join("; ")}`, - }; + const failure = validationFailure(validation); + if (failure !== null) return { verdict: "reject", reason: failure }; if (jev === null) return { verdict: "unavailable", reason: "Jev review was not performed. Treated as proposal-only, never as safe.", }; - if (jev.answers === null || jev.error !== null) + const review = dataRecord(jev); + if ( + !review || typeof review.model !== "string" || review.model.length === 0 || + (review.source !== "jev" && review.source !== "mock") || + typeof review.latencyMs !== "number" || !Number.isFinite(review.latencyMs) || review.latencyMs < 0 || + (review.error !== null && typeof review.error !== "string") || + (review.answers !== null && dataRecord(review.answers) === null) + ) return { + verdict: "unavailable", + reason: "Jev review malformed. Treated as proposal-only, never as safe.", + }; + if (review.answers === null || review.error !== null) return { verdict: "unavailable", - reason: `Jev review unavailable: ${jev.error ?? "no answers returned"}. Treated as proposal-only, never as safe.`, + reason: `Jev review unavailable: ${review.error ?? "no answers returned"}. Treated as proposal-only, never as safe.`, }; - const misses = unfavorable(jev.answers, threshold); + const misses = unfavorable(review.answers, threshold); if (misses.length) return { verdict: "proposal_only", diff --git a/src/contract/input.ts b/src/contract/input.ts new file mode 100644 index 0000000..9f03b38 --- /dev/null +++ b/src/contract/input.ts @@ -0,0 +1,27 @@ +/** Read plain data properties without invoking accessors. Not a plugin sandbox. */ +export function dataRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) return null; + const out: Record = Object.create(null); + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== "string") return null; + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor || !("value" in descriptor)) return null; + out[key] = descriptor.value; + } + return out; +} + +/** A success requires a real boolean and an empty, well-formed error list. */ +export function validationFailure(value: unknown): string | null { + const validation = dataRecord(value); + if (!validation || typeof validation.ok !== "boolean" || !Array.isArray(validation.errors)) + return "Validation failed: malformed validation result."; + const errors: unknown[] = Array.from(validation.errors); + if (!errors.every((error): error is string => typeof error === "string")) + return "Validation failed: malformed validation errors."; + if (!validation.ok || errors.length > 0) + return `Validation failed: ${errors.join("; ") || "validator did not succeed"}`; + return null; +} diff --git a/tests/runtime-inputs.test.ts b/tests/runtime-inputs.test.ts new file mode 100644 index 0000000..2b5b208 --- /dev/null +++ b/tests/runtime-inputs.test.ts @@ -0,0 +1,40 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { decide, decideBase, unfavorable, JEV_MODEL } from "../src"; +const ok = { ok: true, errors: [] }; +const review = () => ({ + model: JEV_MODEL, error: null, source: "mock", latencyMs: 1, + answers: { + addresses_task: { probability: 1, answer: "yes", confidence: 1 }, + evidence_supports: { probability: 1, answer: "yes", confidence: 1 }, + unrelated_changes: { probability: 0, answer: "no", confidence: 1 }, + needs_clarification: { probability: 0, answer: "no", confidence: 1 }, + }, +}); + +test("malformed validation and contradictory errors fail closed", () => { + for (const input of [undefined, null, [], {}, { ok: "false", errors: [] }, { ok: 1, errors: [] }, { ok: true }, { ok: true, errors: ["failed"] }, { ok: true, errors: [1] }, { ok: true, errors: Array(1) }, { ok: false }]) { + assert.equal(decide(input, review()).verdict, "reject"); + assert.equal(decideBase(input).verdict, "reject"); + } +}); + +test("missing and malformed review envelopes are unavailable rather than throwing", () => { + for (const input of [undefined, null, [], {}, { ...review(), answers: undefined }, { ...review(), error: undefined }, { ...review(), source: "other" }, { ...review(), latencyMs: Infinity }, { ...review(), answers: [] }]) { + assert.equal(decide(ok, input).verdict, "unavailable"); + } + assert.equal(decide(ok, { ...review(), answers: {} }).verdict, "proposal_only"); + assert.equal(unfavorable(undefined).length, 4); +}); + +test("inherited data and accessors are not trusted as review fields", () => { + assert.equal(decide(Object.create(ok), review()).verdict, "reject"); + assert.equal(decide(ok, Object.create(review())).verdict, "unavailable"); + const accessor = Object.defineProperty({}, "answers", { get() { throw Error("must not run"); } }); + assert.equal(decide(ok, accessor).verdict, "unavailable"); +}); + +test("JSON round-trips remain usable and invalid configuration still throws", () => { + assert.equal(decide(JSON.parse(JSON.stringify(ok)), JSON.parse(JSON.stringify(review()))).verdict, "permit"); + assert.throws(() => decide(undefined, undefined, 0.49), /between 0.5 and 1/); +}); From d4248802f33ba77e8cc1cb82c7452d3ec061d53c Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 22 Sep 2026 06:08:08 -0400 Subject: [PATCH 4/7] fix: freeze exported question IDs, directions, and tool metadata --- docs/hardening/04-policy-metadata.md | 12 ++++++++++++ src/contract/decide.ts | 4 ++-- src/contract/types.ts | 6 +++--- tests/policy-metadata.test.ts | 29 ++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 docs/hardening/04-policy-metadata.md create mode 100644 tests/policy-metadata.test.ts diff --git a/docs/hardening/04-policy-metadata.md b/docs/hardening/04-policy-metadata.md new file mode 100644 index 0000000..1a23837 --- /dev/null +++ b/docs/hardening/04-policy-metadata.md @@ -0,0 +1,12 @@ +# Immutable policy metadata (adversarial finding 4) + +`FAVORABLE`, `REVIEW_QUESTION_IDS`, and `PROPOSAL_TOOLS` are readonly in +TypeScript and frozen at runtime. Consumers cannot change global policy by +mutating metadata or shortening the question list to make evaluation vacuous. +All values are primitive strings, so shallow freezing fully covers these +structures. Hosts changing policy must do so through a reviewed versioned +change, not mutation. This is an API invariant, not a malicious-plugin sandbox. + +Verdict impact: no change for normal consumers; attempts to mutate metadata +fail. Tests cover compile-time immutability, runtime mutation attempts, and +empty-answer degradation before and after attempted mutation. diff --git a/src/contract/decide.ts b/src/contract/decide.ts index e3d98dc..6cc4c8f 100644 --- a/src/contract/decide.ts +++ b/src/contract/decide.ts @@ -27,12 +27,12 @@ import { dataRecord, validationFailure } from "./input"; export const REVIEW_CONFIDENCE_THRESHOLD = 0.8; /** The answer that speaks for the proposal, per question. */ -export const FAVORABLE: Record = { +export const FAVORABLE: Readonly> = Object.freeze({ addresses_task: "yes", evidence_supports: "yes", unrelated_changes: "no", needs_clarification: "no", -}; +}); export interface Decision { verdict: ReviewVerdict; diff --git a/src/contract/types.ts b/src/contract/types.ts index 3266ae4..2ef169f 100644 --- a/src/contract/types.ts +++ b/src/contract/types.ts @@ -11,7 +11,7 @@ * the playground's API module. */ -export const PROPOSAL_TOOLS = ["read_file", "propose_patch"] as const; +export const PROPOSAL_TOOLS = Object.freeze(["read_file", "propose_patch"] as const); export type ProposalTool = (typeof PROPOSAL_TOOLS)[number]; export interface Proposal { @@ -25,12 +25,12 @@ export interface Proposal { evidence: string[]; } -export const REVIEW_QUESTION_IDS = [ +export const REVIEW_QUESTION_IDS = Object.freeze([ "addresses_task", "evidence_supports", "unrelated_changes", "needs_clarification", -] as const; +] as const); export type ReviewQuestionId = (typeof REVIEW_QUESTION_IDS)[number]; /** Question set v1. Ids are stable; wording changes bump the version. */ diff --git a/tests/policy-metadata.test.ts b/tests/policy-metadata.test.ts new file mode 100644 index 0000000..64d57bf --- /dev/null +++ b/tests/policy-metadata.test.ts @@ -0,0 +1,29 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { decide, FAVORABLE, PROPOSAL_TOOLS, REVIEW_QUESTION_IDS, JEV_MODEL } from "../src"; + +test("exported policy metadata is frozen at runtime", () => { + for (const value of [FAVORABLE, REVIEW_QUESTION_IDS, PROPOSAL_TOOLS]) + assert.equal(Object.isFrozen(value), true); + assert.equal(Reflect.set(FAVORABLE, "addresses_task", "no"), false); + assert.equal(Reflect.set(REVIEW_QUESTION_IDS, "length", 0), false); + assert.equal(Reflect.set(PROPOSAL_TOOLS, "0", "execute"), false); + assert.equal(FAVORABLE.addresses_task, "yes"); + assert.equal(REVIEW_QUESTION_IDS.length, 4); + assert.equal(PROPOSAL_TOOLS[0], "read_file"); +}); + +test("attempting metadata changes cannot bypass answer evaluation", () => { + const ok = { ok: true, errors: [] }; + const empty = { model: JEV_MODEL, answers: {}, error: null, source: "mock", latencyMs: 0 }; + assert.equal(decide(ok, empty).verdict, "proposal_only"); + assert.equal(Reflect.set(REVIEW_QUESTION_IDS, "length", 0), false); + assert.equal(decide(ok, empty).verdict, "proposal_only"); + const compileTimeReadonly = () => { + // @ts-expect-error FAVORABLE is part of the immutable decision policy. + FAVORABLE.addresses_task = "no"; + // @ts-expect-error Question IDs are a readonly tuple. + REVIEW_QUESTION_IDS.length = 0; + }; + assert.equal(typeof compileTimeReadonly, "function"); +}); From a7a2356fc3e959f607b5f042ef4766e7bef84f2d Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 22 Sep 2026 06:18:38 -0400 Subject: [PATCH 5/7] docs: document canonical answer policy with boundary coverage --- README.md | 9 +++++++-- docs/architecture.md | 6 +++++- tests/answer-invariants.test.ts | 13 +++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 65dd759..59f2e55 100644 --- a/README.md +++ b/README.md @@ -111,11 +111,16 @@ A `noul` answer is one probability of "yes". The harness reads `answer = p ≥ 0 ```text validation.ok = false → reject Jev never consulted; withheld jev = null or jev.answers = null → unavailable withheld; NEVER treated as safe -any answer unfavorable, - or favorable but < threshold → proposal_only recorded pending; a human sees it +any answer missing, non-finite, out of range, + inconsistent, unfavorable, or < threshold → proposal_only recorded pending; a human sees it all four favorable, each ≥ 0.8 → permit recorded pending; evidence, not authorization ``` +Each answer must have finite probability in `[0,1]`, confidence in `[0.5,1]`, +and exactly match `answer = p >= 0.5 ? "yes" : "no"` and +`confidence = Math.max(p, 1 - p)`. Every failing question is named in the reason; +canonical fixture verdicts are unchanged. + `REVIEW_CONFIDENCE_THRESHOLD = 0.8` is exported and uncalibrated; hosts can pass their own. A threshold outside `[0.5, 1]` is refused. ### Receipt diff --git a/docs/architecture.md b/docs/architecture.md index 7d032cd..cace6ba 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -47,9 +47,13 @@ A `noul` answer is a single probability of "yes". `answer = p ≥ 0.5 ? yes : no | --- | --- | --- | | Validation failed (schema, tool, path, diff) | `reject` | withheld; Jev not consulted | | No review ran, or `answers` is `null` | `unavailable` | withheld; never treated as safe | -| Any answer unfavorable, or favorable but below threshold | `proposal_only` | recorded pending; a human sees it | +| Any answer missing, non-finite, out of range, inconsistent with its probability, unfavorable, or below threshold | `proposal_only` | recorded pending; a human sees it | | All four favorable and each `confidence ≥ REVIEW_CONFIDENCE_THRESHOLD` (0.8) | `permit` | recorded pending; evidence, not authorization | +Canonical triples have probability in `[0,1]`, confidence in `[0.5,1]`, and +exactly match `p >= 0.5 ? "yes" : "no"` and `Math.max(p, 1 - p)`. Every +failing question is named; canonical fixture verdicts are unchanged. + `base` mode (bench only) is validate-only: anything that validates is `permit`, with a reason that says no reviewer checked whether the proposal is on task. It exists to show the gap Jev closes. ## Receipt v1 diff --git a/tests/answer-invariants.test.ts b/tests/answer-invariants.test.ts index 8038b3c..5af01c9 100644 --- a/tests/answer-invariants.test.ts +++ b/tests/answer-invariants.test.ts @@ -40,3 +40,16 @@ test("canonical probability grid preserves the v1 decision table", () => { assert.equal(decide(ok, review(answers)).verdict, expected); } }); + +test("canonical JSON answers preserve threshold endpoints and name every miss", () => { + const answers = favorable(); + for (const id of REVIEW_QUESTION_IDS) answers[id] = answer(FAVORABLE[id] === "yes" ? 1 : 0); + const parsed = JSON.parse(JSON.stringify(review(answers))); + assert.equal(decide(ok, parsed, 1).verdict, "permit"); + for (const id of REVIEW_QUESTION_IDS) answers[id] = answer(0.5); + const result = decide(ok, review(answers), 0.5); + assert.equal(result.verdict, "proposal_only"); + assert.match(result.reason, /unrelated_changes/); + assert.match(result.reason, /needs_clarification/); + assert.doesNotMatch(result.reason, /addresses_task|evidence_supports/); +}); From 497e90b6c7100d47a44e0e0aaccd316e4d872b47 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 22 Sep 2026 06:18:41 -0400 Subject: [PATCH 6/7] docs: describe provider error precedence in decision tables --- README.md | 2 +- docs/architecture.md | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 59f2e55..44f9506 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ A `noul` answer is one probability of "yes". The harness reads `answer = p ≥ 0 ```text validation.ok = false → reject Jev never consulted; withheld -jev = null or jev.answers = null → unavailable withheld; NEVER treated as safe +jev = null, answers = null, or error != null → unavailable withheld; NEVER treated as safe any answer missing, non-finite, out of range, inconsistent, unfavorable, or < threshold → proposal_only recorded pending; a human sees it all four favorable, each ≥ 0.8 → permit recorded pending; evidence, not authorization diff --git a/docs/architecture.md b/docs/architecture.md index cace6ba..1b2d8e2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -46,10 +46,13 @@ A `noul` answer is a single probability of "yes". `answer = p ≥ 0.5 ? yes : no | Condition | Verdict | Execution | | --- | --- | --- | | Validation failed (schema, tool, path, diff) | `reject` | withheld; Jev not consulted | -| No review ran, or `answers` is `null` | `unavailable` | withheld; never treated as safe | +| No review ran, `answers` is `null`, or `error` is non-null (even with retained answers) | `unavailable` | withheld; never treated as safe | | Any answer missing, non-finite, out of range, inconsistent with its probability, unfavorable, or below threshold | `proposal_only` | recorded pending; a human sees it | | All four favorable and each `confidence ≥ REVIEW_CONFIDENCE_THRESHOLD` (0.8) | `permit` | recorded pending; evidence, not authorization | +A successful review pairs answers with `error: null`; a failed review pairs +`answers: null` with a diagnostic string. Validation failure retains precedence. + Canonical triples have probability in `[0,1]`, confidence in `[0.5,1]`, and exactly match `p >= 0.5 ? "yes" : "no"` and `Math.max(p, 1 - p)`. Every failing question is named; canonical fixture verdicts are unchanged. From 0b1df9e64858212c16455f0911d338cda56d9032 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Tue, 22 Sep 2026 06:19:30 -0400 Subject: [PATCH 7/7] fix: read validation arrays without invoking custom behavior --- README.md | 4 ++-- docs/architecture.md | 10 ++++++++-- docs/hardening/03-runtime-boundary.md | 6 ++++-- src/contract/input.ts | 19 ++++++++++++++++--- tests/runtime-inputs.test.ts | 14 ++++++++++++++ 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 44f9506..6f8ea72 100644 --- a/README.md +++ b/README.md @@ -109,8 +109,8 @@ A `noul` answer is one probability of "yes". The harness reads `answer = p ≥ 0 ### Decision table ```text -validation.ok = false → reject Jev never consulted; withheld -jev = null, answers = null, or error != null → unavailable withheld; NEVER treated as safe +malformed validation, ok != true, or nonempty errors → reject Jev never consulted; withheld +missing/malformed review, null answers, or error != null → unavailable withheld; NEVER treated as safe any answer missing, non-finite, out of range, inconsistent, unfavorable, or < threshold → proposal_only recorded pending; a human sees it all four favorable, each ≥ 0.8 → permit recorded pending; evidence, not authorization diff --git a/docs/architecture.md b/docs/architecture.md index 1b2d8e2..ecfc71c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -45,11 +45,17 @@ A `noul` answer is a single probability of "yes". `answer = p ≥ 0.5 ? yes : no | Condition | Verdict | Execution | | --- | --- | --- | -| Validation failed (schema, tool, path, diff) | `reject` | withheld; Jev not consulted | -| No review ran, `answers` is `null`, or `error` is non-null (even with retained answers) | `unavailable` | withheld; never treated as safe | +| Malformed validation, `ok !== true`, or nonempty errors | `reject` | withheld; Jev not consulted | +| Missing/malformed review envelope, null answers, or non-null error (even with retained answers) | `unavailable` | withheld; never treated as safe | | Any answer missing, non-finite, out of range, inconsistent with its probability, unfavorable, or below threshold | `proposal_only` | recorded pending; a human sees it | | All four favorable and each `confidence ≥ REVIEW_CONFIDENCE_THRESHOLD` (0.8) | `permit` | recorded pending; evidence, not authorization | +Decision inputs are plain data. Success validation requires literal `true` and +an empty dense array of errors. Accessors, inherited fields, and decorated error +arrays are rejected without running their iterators or getters. A malformed +answer entry remains `proposal_only`; an absent answer object is `unavailable`. +These checks are not a sandbox for proxies or code running in the host process. + A successful review pairs answers with `error: null`; a failed review pairs `answers: null` with a diagnostic string. Validation failure retains precedence. diff --git a/docs/hardening/03-runtime-boundary.md b/docs/hardening/03-runtime-boundary.md index b3a091a..2a8112f 100644 --- a/docs/hardening/03-runtime-boundary.md +++ b/docs/hardening/03-runtime-boundary.md @@ -13,8 +13,10 @@ throws as a deliberate configuration error; it is never clamped. Plain records are copied through own data-property descriptors. Inherited fields, class instances, and accessors are rejected. This boundary is for -JSON/plain data, not adversarial same-process JavaScript: proxies and malicious -array accessors require process isolation and are not a supported trust domain. +JSON/plain data, not adversarial same-process JavaScript. Error arrays must be +dense own data entries; custom iterators and accessors are rejected without +invocation. Proxies still require process isolation and are not a supported +trust domain. Review metadata is shape-checked, not authenticated. The host still pins the expected model/source, verifies provenance, binds snapshots, and authorizes diff --git a/src/contract/input.ts b/src/contract/input.ts index 9f03b38..f5b9779 100644 --- a/src/contract/input.ts +++ b/src/contract/input.ts @@ -13,13 +13,26 @@ export function dataRecord(value: unknown): Record | null { return out; } +/** Copy dense plain arrays without invoking custom iterators or index getters. */ +export function dataArray(value: unknown): unknown[] | null { + if (!Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype) return null; + if (Reflect.ownKeys(value).length !== value.length + 1) return null; + const out: unknown[] = []; + for (let index = 0; index < value.length; index++) { + const descriptor = Object.getOwnPropertyDescriptor(value, String(index)); + if (!descriptor || !("value" in descriptor)) return null; + out.push(descriptor.value); + } + return out; +} + /** A success requires a real boolean and an empty, well-formed error list. */ export function validationFailure(value: unknown): string | null { const validation = dataRecord(value); - if (!validation || typeof validation.ok !== "boolean" || !Array.isArray(validation.errors)) + if (!validation || typeof validation.ok !== "boolean") return "Validation failed: malformed validation result."; - const errors: unknown[] = Array.from(validation.errors); - if (!errors.every((error): error is string => typeof error === "string")) + const errors = dataArray(validation.errors); + if (!errors || !errors.every((error): error is string => typeof error === "string")) return "Validation failed: malformed validation errors."; if (!validation.ok || errors.length > 0) return `Validation failed: ${errors.join("; ") || "validator did not succeed"}`; diff --git a/tests/runtime-inputs.test.ts b/tests/runtime-inputs.test.ts index 2b5b208..f18bc16 100644 --- a/tests/runtime-inputs.test.ts +++ b/tests/runtime-inputs.test.ts @@ -38,3 +38,17 @@ test("JSON round-trips remain usable and invalid configuration still throws", () assert.equal(decide(JSON.parse(JSON.stringify(ok)), JSON.parse(JSON.stringify(review()))).verdict, "permit"); assert.throws(() => decide(undefined, undefined, 0.49), /between 0.5 and 1/); }); + +test("validation error arrays cannot hide failures through iterators or getters", () => { + let reads = 0; + const iterator = ["validation failed"]; + Object.defineProperty(iterator, Symbol.iterator, { value: function* () { reads++; } }); + const getter = Object.defineProperty([], "0", { get() { reads++; return "failed"; } }); + const inherited = Object.setPrototypeOf(Array(1), { 0: "failed" }); + const decorated = Object.assign([], { ignored: "failed" }); + for (const errors of [iterator, getter, inherited, decorated]) { + assert.equal(decide({ ok: true, errors }, review()).verdict, "reject"); + assert.equal(decideBase({ ok: true, errors }).verdict, "reject"); + } + assert.equal(reads, 0); +});