From 383032abf0cee8150bd1bc0d83ff5d1dcc54398b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Hern=C3=A1ndez?= Date: Tue, 4 Aug 2026 16:53:33 -0400 Subject: [PATCH 1/2] Stop counting rules that measure the genre instead of the machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ninety texts published before generative models existed produced 888 flagged tells — a median of seven per human academic paper, with only two of the ninety coming back clean. `rhet.rule-of-three` fired on half of them. The scores were fine, because those rules weigh one or two, so the published false-positive rate was never the problem. The evidence panel was, and it costs this project more than it would cost anyone else: showing the evidence instead of a percentage is the whole argument, and a teacher who gets seven confident tells on a colleague's paper learns to disbelieve the eighth. Each rule now carries the rate at which humans use it — hits per thousand words at the ninetieth percentile of the corpus, per language, and only for rules with at least eight texts of evidence. Below that rate a finding is marked AtHumanRate: still shown, still offered to the rewriter, worth nothing to the score. Rules the corpus never sees keep counting on a single occurrence, which is why delve and tapestry are untouched. Marking rather than deleting is the second attempt. Deleting made evidence vanish as a document grew, which also handed anyone a way to bury a tell by padding; it cut the rewriter off from words it knows how to replace; and it answered "our evidence panel is noisy" by hiding evidence. All three are worse than the disease. The numbers are derived by `dotnet run --project tools/SignsOfAI.Calibration -- thresholds` and written back into the packs, so they can be regenerated when the corpus grows rather than taken on trust. The first version of this change pasted in numbers from a throwaway script while the code claimed the tool produced them, which is exactly the sort of unverifiable "measured" figure this project criticises elsewhere. Reported honestly: thresholds are fitted on the same corpus the project measures itself against, so the improvement is quoted leave-one-out — each text judged against rates derived from the other eighty-nine. That gives findings per text 9.9 -> 7.0 (-30%) and texts with nothing flagged 2 -> 9, against -32% and 10 in-sample. The recommended threshold is unchanged at 25/100 and nothing in the corpus reaches it. What this does not fix, and the code says so where it happens: rates are per rule, and language models are tuned away from repeating any one tell. The recognisable shape of machine prose is fifteen different tells at one occurrence each, every one individually at a human rate. Marking keeps that shape visible and countable; nothing here scores it. Breadth needs its own evidence, not a constant chosen today. Refs #31 Co-Authored-By: Claude Opus 5 --- Docs/CALIBRATION.md | 68 +++---- src/SignsOfAI.Cli/Program.cs | 22 ++- src/SignsOfAI.Core/AiWritingAnalyzer.cs | 7 +- .../Calibration/CalibrationModel.cs | 16 +- src/SignsOfAI.Core/Model/Finding.cs | 14 ++ src/SignsOfAI.Core/Rules/GenreGate.cs | 68 +++++++ src/SignsOfAI.Core/Rules/Packs/rules.en.json | 20 ++ src/SignsOfAI.Core/Rules/Packs/rules.es.json | 2 + src/SignsOfAI.Core/Rules/RulePack.cs | 77 +++++++- src/SignsOfAI.Core/Scoring/Scorer.cs | 5 + tests/SignsOfAI.Core.Tests/GenreGateTests.cs | 171 +++++++++++++++++ tools/SignsOfAI.Calibration/Program.cs | 37 +++- tools/SignsOfAI.Calibration/Thresholds.cs | 176 ++++++++++++++++++ 13 files changed, 639 insertions(+), 44 deletions(-) create mode 100644 src/SignsOfAI.Core/Rules/GenreGate.cs create mode 100644 tests/SignsOfAI.Core.Tests/GenreGateTests.cs create mode 100644 tools/SignsOfAI.Calibration/Thresholds.cs diff --git a/Docs/CALIBRATION.md b/Docs/CALIBRATION.md index b04ec94..0d29c0b 100644 --- a/Docs/CALIBRATION.md +++ b/Docs/CALIBRATION.md @@ -10,8 +10,8 @@ It is **not an accuracy figure**. Accuracy needs machine-written text to measure - **Corpus** `signsofai-human-baseline`, fingerprint `123fa5b9ebca3f29` - **Texts** 90 (280,221 words) -- **Engine** SignsOfAI.Core 0.2.1 -- **Run** 2026-08-03 +- **Engine** SignsOfAI.Core 0.3.0 +- **Run** 2026-08-04 - **Target false-positive rate** 5% Every text here was published before generative models could have written it. That is the whole basis for calling it human, and it is a stronger guarantee than any classifier offers about anything. The manifest names each source, its licence and its year, so the claim can be traced rather than trusted. @@ -28,8 +28,8 @@ A rate that holds in English and fails in Spanish is not one number, and reporti | Group | Texts | Median | 90th pct | Highest | Threshold for 5% | Best bound it can support | |---|---|---|---|---|---|---| -| **en** | 65 | 6.4 | 12.3 | 23.4 | — | 5.6% | -| **es** | 25 | 7.7 | 15.4 | 18.9 | — | 13.3% | +| **en** | 65 | 5.8 | 11.8 | 23.4 | — | 5.6% | +| **es** | 25 | 7.2 | 15.1 | 18.4 | — | 13.3% | A dash means this group has too few texts to bound that rate at all — with nothing flagged it still takes roughly seventy-five before the interval alone gets under 5%. That is a statement about the corpus, not the tool. @@ -39,22 +39,22 @@ The reason the whole exercise exists. If this project cannot show a rate for sec | Group | Texts | Median | 90th pct | Highest | Threshold for 5% | Best bound it can support | |---|---|---|---|---|---|---| -| **en-anglophone-affiliation** | 21 | 7.0 | 9.8 | 15.4 | — | 15.5% | -| **en-other-affiliation** | 19 | 6.4 | 14.1 | 18.3 | — | 16.8% | -| **en-wikipedia** | 25 | 5.2 | 10.4 | 23.4 | — | 13.3% | -| **es-wikipedia** | 25 | 7.7 | 15.4 | 18.9 | — | 13.3% | +| **en-anglophone-affiliation** | 21 | 5.9 | 9.0 | 14.2 | — | 15.5% | +| **en-other-affiliation** | 19 | 6.2 | 13.6 | 18.0 | — | 16.8% | +| **en-wikipedia** | 25 | 4.9 | 10.4 | 23.4 | — | 13.3% | +| **es-wikipedia** | 25 | 7.2 | 15.1 | 18.4 | — | 13.3% | A dash means this group has too few texts to bound that rate at all — with nothing flagged it still takes roughly seventy-five before the interval alone gets under 5%. That is a statement about the corpus, not the tool. -Across these groups the median score runs from 7.7 (**es-wikipedia**) down to 5.2 (**en-wikipedia**), a spread of 2.5 points on a scale of a hundred. The longest tail belongs to **es-wikipedia** at 15.4 for the ninetieth percentile. A tool with the defect this project criticises would show one group sitting well above the rest; on this corpus none does. It is a first indication rather than a finding — these are tens of texts, not hundreds — and the numbers move as the corpus grows, in whichever direction they move. +Across these groups the median score runs from 7.2 (**es-wikipedia**) down to 4.9 (**en-wikipedia**), a spread of 2.3 points on a scale of a hundred. The longest tail belongs to **es-wikipedia** at 15.1 for the ninetieth percentile. A tool with the defect this project criticises would show one group sitting well above the rest; on this corpus none does. It is a first indication rather than a finding — these are tens of texts, not hundreds — and the numbers move as the corpus grows, in whichever direction they move. ## Every threshold | Score at or above | Human texts flagged | Rate | 95% interval | |---|---|---|---| -| 5 | 75 / 90 | 83.3% | 74.3% – 89.6% | -| 10 | 18 / 90 | 20% | 13% – 29.4% | -| 15 | 7 / 90 | 7.8% | 3.8% – 15.2% | +| 5 | 61 / 90 | 67.8% | 57.6% – 76.5% | +| 10 | 16 / 90 | 17.8% | 11.2% – 26.9% | +| 15 | 6 / 90 | 6.7% | 3.1% – 13.8% | | 20 | 1 / 90 | 1.1% | 0.2% – 6% | | 25 | 0 / 90 | 0% | 0% – 4.1% | | 30 | 0 / 90 | 0% | 0% – 4.1% | @@ -79,31 +79,31 @@ Every rule below fired on text no machine wrote, so each hit is a false positive | Rule | Texts it fired on | Share | Total hits | |---|---|---|---| -| `rhet.rule-of-three` | 45 | 50% | 100 | | `stat.burstiness` | 25 | 27.8% | 25 | -| `lex.moreover` | 23 | 25.6% | 67 | -| `lex.furthermore` | 23 | 25.6% | 55 | -| `rhet.in-order-to` | 22 | 24.4% | 60 | -| `lex.just` | 19 | 21.1% | 30 | -| `lex.facilitate` | 17 | 18.9% | 29 | -| `rhet.in-terms-of` | 16 | 17.8% | 30 | -| `lex.comprehensive` | 14 | 15.6% | 36 | -| `lex.simply` | 14 | 15.6% | 24 | -| `lex.ademas` | 12 | 13.3% | 28 | -| `lex.utilize` | 11 | 12.2% | 28 | -| `rhet.not-only-but` | 11 | 12.2% | 19 | -| `lex.robust` | 10 | 11.1% | 22 | -| `lex.notably` | 10 | 11.1% | 17 | -| `rhet.regla-de-tres` | 9 | 10% | 14 | -| `lex.actually` | 9 | 10% | 11 | -| `rhet.in-conclusion` | 9 | 10% | 11 | -| `syn.serves-as` | 9 | 10% | 10 | -| `lex.importantly` | 8 | 8.9% | 11 | -| `rhet.weasel-attribution` | 8 | 8.9% | 11 | -| `rhet.with-regard-to` | 8 | 8.9% | 11 | -| `lex.crucial` | 8 | 8.9% | 9 | +| `rhet.in-terms-of` | 9 | 10% | 22 | +| `rhet.not-only-but` | 8 | 8.9% | 16 | +| `rhet.in-order-to` | 7 | 7.8% | 33 | +| `lex.furthermore` | 7 | 7.8% | 24 | +| `lex.robust` | 7 | 7.8% | 19 | +| `lex.just` | 7 | 7.8% | 16 | +| `lex.simply` | 7 | 7.8% | 16 | | `lex.utilizar` | 7 | 7.8% | 15 | | `rhet.in-this-article` | 7 | 7.8% | 15 | +| `lex.notably` | 7 | 7.8% | 14 | +| `syn.superficial-ing` | 7 | 7.8% | 12 | +| `rhet.with-regard-to` | 7 | 7.8% | 10 | +| `lex.crucial` | 7 | 7.8% | 8 | +| `syn.serves-as` | 7 | 7.8% | 8 | +| `lex.moreover` | 6 | 6.7% | 35 | +| `lex.comprehensive` | 6 | 6.7% | 27 | +| `lex.utilize` | 6 | 6.7% | 22 | +| `lex.facilitate` | 6 | 6.7% | 16 | +| `rhet.rule-of-three` | 6 | 6.7% | 16 | +| `lex.importantly` | 6 | 6.7% | 9 | +| `rhet.weasel-attribution` | 6 | 6.7% | 9 | +| `lex.actually` | 6 | 6.7% | 8 | +| `rhet.in-conclusion` | 6 | 6.7% | 8 | +| `rhet.important-note` | 6 | 6.7% | 7 | A rule near the top is not automatically wrong. Some tells genuinely appear in human academic prose and the catalog says so. But a rule firing on most human texts is measuring the genre rather than the machine, and should be reweighted or retired. diff --git a/src/SignsOfAI.Cli/Program.cs b/src/SignsOfAI.Cli/Program.cs index 7723847..6def91c 100644 --- a/src/SignsOfAI.Cli/Program.cs +++ b/src/SignsOfAI.Cli/Program.cs @@ -183,7 +183,11 @@ findings = result.Findings.Select(f => new { f.RuleId, category = f.Category.ToString(), severity = f.Severity.ToString(), - f.MatchedText, f.Message, f.Suggestion, f.Evidence + f.MatchedText, f.Message, f.Suggestion, f.Evidence, + // Reported so a consumer can tell the two apart: this one matched, and it matched at a + // rate people write at, so it counts for nothing. Leaving it out would make the findings + // list and the score disagree with no way to see why. + f.AtHumanRate }), // Kept in its own object rather than folded in with the findings: these are characters at // offsets, not judgements about prose, and a consumer should not have to tell them apart. @@ -324,6 +328,17 @@ static void PrintBaseline(string path, BaselineReport r, bool useColor) Console.WriteLine(); } +// The headline counts evidence, so it agrees with the category tallies and with the score. What +// matched at a human rate is named beside it rather than folded in: a reader who sees eleven +// highlights under a headline of three deserves to know why, on the same line. +static int Counted(AnalysisResult r) => r.Findings.Count(f => !f.AtHumanRate); + +static string AtRate(AnalysisResult r) +{ + var n = r.Findings.Count(f => f.AtHumanRate); + return n == 0 ? "" : $" + {n} at a human rate"; +} + static void PrintReport(string path, AnalysisResult r, int top, bool useColor) { string Col(string s, int code) => useColor ? $"[{code}m{s}" : s; @@ -333,7 +348,7 @@ static void PrintReport(string path, AnalysisResult r, int top, bool useColor) Console.WriteLine(); Console.WriteLine(Bold($" ✍ Signs of AI Writing — {Path.GetFileName(path)}")); Console.WriteLine($" {Col($"{r.OverallScore:0}/100", scoreColor)} {Bold(r.Verdict)} " + - $"({r.Findings.Count} signal{(r.Findings.Count == 1 ? "" : "s")}, {(r.Language == "es" ? "Español" : "English")})"); + $"({Counted(r)} signal{(Counted(r) == 1 ? "" : "s")}{AtRate(r)}, {(r.Language == "es" ? "Español" : "English")})"); Console.WriteLine($" words {r.Statistics.WordCount} · sentences {r.Statistics.SentenceCount} · " + $"burstiness {r.Statistics.Burstiness:0.00} · lexical diversity {r.Statistics.LexicalDiversity:0.00}"); @@ -349,9 +364,12 @@ static void PrintReport(string path, AnalysisResult r, int top, bool useColor) foreach (var f in shown) { int sev = f.Severity switch { Severity.High => 31, Severity.Medium => 33, Severity.Low => 36, _ => 90 }; + if (f.AtHumanRate) sev = 90; var head = $" {Col("●", sev)} [{f.Category}] " + (string.IsNullOrEmpty(f.MatchedText) ? "" : Bold(f.MatchedText)); Console.WriteLine(head.TrimEnd()); Console.WriteLine($" {f.Message}"); + if (f.AtHumanRate) + Console.WriteLine(Col(" used here at a rate people write at — shown, not counted", 90)); Console.WriteLine(Col($" → {f.Suggestion}", 90)); } if (r.Findings.Count > shown.Count) diff --git a/src/SignsOfAI.Core/AiWritingAnalyzer.cs b/src/SignsOfAI.Core/AiWritingAnalyzer.cs index d01bf90..ed330cd 100644 --- a/src/SignsOfAI.Core/AiWritingAnalyzer.cs +++ b/src/SignsOfAI.Core/AiWritingAnalyzer.cs @@ -72,13 +72,18 @@ public AnalysisResult Analyze(string text, string? language = null, IReadOnlyLis Statistics = statistics, }; - var findings = _analyzers + var matched = _analyzers .SelectMany(a => a.Analyze(context)) .Select(f => normalized.Changed ? ToSource(f, normalized, text) : f) .OrderBy(f => f.Span.Start) .ThenBy(f => f.Span.Length) .ToList(); + // Rules that measured the genre rather than the machine are silenced here, against rates taken + // from writing that predates generative models. Nothing is re-decided per finding: a rule is + // either used at a human rate in this text, and says nothing, or it is not. + var findings = GenreGate.Apply(matched, rulePack, statistics.WordCount); + var (overall, byCategory) = Scorer.Score(findings, statistics); return new AnalysisResult diff --git a/src/SignsOfAI.Core/Calibration/CalibrationModel.cs b/src/SignsOfAI.Core/Calibration/CalibrationModel.cs index 6c43913..42b43c4 100644 --- a/src/SignsOfAI.Core/Calibration/CalibrationModel.cs +++ b/src/SignsOfAI.Core/Calibration/CalibrationModel.cs @@ -25,8 +25,22 @@ public sealed record CalibrationSample public required int WordCount { get; init; } - /// Every rule that fired on this human text — each one a false positive by construction. + /// + /// The rules that produced evidence on this human text — each one a false positive by + /// construction, since no machine wrote any of it. + /// + /// This excludes rules the text used at a rate people write at, which are shown to a reader but + /// score nothing. Reporting those here would make the published misfire table look unchanged + /// while the scores moved, which is the opposite of informative. + /// public required IReadOnlyList RuleIds { get; init; } + + /// + /// Everything that matched, including what was found at a human rate. Two things need it: the + /// derivation of the rates themselves, which must see all usage or it would measure the effect of + /// its own previous output, and the published count of how much the rates are absorbing. + /// + public IReadOnlyList MatchedRuleIds { get; init; } = []; } /// diff --git a/src/SignsOfAI.Core/Model/Finding.cs b/src/SignsOfAI.Core/Model/Finding.cs index c8a8ff7..01065b2 100644 --- a/src/SignsOfAI.Core/Model/Finding.cs +++ b/src/SignsOfAI.Core/Model/Finding.cs @@ -30,4 +30,18 @@ public sealed record Finding /// Contribution of this finding to the overall score (higher = stronger AI signal). public double Weight { get; init; } + + /// + /// True when this rule is being used in this text at a rate people write at, measured against + /// writing published before generative models existed. The finding still describes something real + /// and is still shown, but it is not evidence of a machine and contributes nothing to the score. + /// + /// It is marked rather than removed on purpose. Ninety human academic papers produced a median of + /// seven flagged tells each, which is the sort of thing that teaches a reader to disbelieve the + /// eighth — but a tool whose whole argument is *showing the evidence* cannot answer that by hiding + /// evidence. "Furthermore appears once in three thousand words, which is how people write" is a + /// more useful thing to tell someone than silence, and it keeps the finding available to the + /// rewriter for a writer whose goal is removing the word rather than proving anything. + /// + public bool AtHumanRate { get; init; } } diff --git a/src/SignsOfAI.Core/Rules/GenreGate.cs b/src/SignsOfAI.Core/Rules/GenreGate.cs new file mode 100644 index 0000000..15d5c64 --- /dev/null +++ b/src/SignsOfAI.Core/Rules/GenreGate.cs @@ -0,0 +1,68 @@ +using SignsOfAI.Core.Model; + +namespace SignsOfAI.Core.Rules; + +/// +/// Marks the findings of rules that are describing the genre rather than the machine. +/// +/// Measuring the analyzer against ninety texts published before generative models existed produced a +/// number worth staring at: a median of seven flagged tells per human academic paper, 888 +/// across the corpus, with only two of the ninety coming back clean. The score survived that — the +/// median was 6.8 out of 100 and nothing reached the recommended threshold — because the rules +/// involved carry weights of one and two. So the false-positive *rate* was never the problem. +/// +/// The problem is the evidence panel, and it costs this project more than it would cost anyone else, +/// because showing the evidence instead of a percentage is the entire argument. A teacher who pastes +/// a colleague's paper and gets seven confident-looking tells learns not to believe the eighth, which +/// is the one that mattered. +/// +/// Why this marks rather than deletes. Deleting was the first attempt and it was wrong in +/// three ways that are worth keeping written down. It made evidence vanish as a document grew, which +/// also handed anyone a way to bury a tell by padding. It cut the live rewriter off from words it +/// knows how to replace, for a writer whose goal is removing "utilize", not proving anything about it. +/// And it answered "our evidence panel is noisy" by hiding evidence, from a tool that exists to show +/// it. Marking keeps every finding visible and honest about what it is: present, and present at a rate +/// people write at. +/// +/// What this does not fix. The rate is measured per rule, and language models are tuned +/// away from repeating any single tell — the recognisable shape of machine prose is fifteen different +/// tells appearing once each, every one of them individually at a human rate. Marking rather than +/// deleting means that shape stays visible and countable, but nothing here scores it. Measuring +/// breadth rather than density is a separate question and it needs its own evidence, not a constant +/// chosen today. +/// +public static class GenreGate +{ + /// + /// with the genre-rate ones flagged . + /// Findings whose rule carries no measured human rate are returned untouched. + /// + /// must be the word count of the same text the findings came from, + /// as counted by StatisticsCalculator — the thresholds are derived against that counter, so + /// a different one silently rescales every comparison. + /// + public static IReadOnlyList Apply( + IReadOnlyList findings, RulePack pack, int wordCount) + { + if (findings.Count == 0 || wordCount <= 0) return findings; + + var thresholds = pack.HumanRates; + if (thresholds.Count == 0) return findings; + + var counts = new Dictionary(StringComparer.Ordinal); + foreach (var f in findings) + if (thresholds.ContainsKey(f.RuleId)) + counts[f.RuleId] = counts.GetValueOrDefault(f.RuleId) + 1; + + if (counts.Count == 0) return findings; + + var atHumanRate = new HashSet(StringComparer.Ordinal); + foreach (var (ruleId, hits) in counts) + if (hits / (double)wordCount * 1000.0 <= thresholds[ruleId]) + atHumanRate.Add(ruleId); + + return atHumanRate.Count == 0 + ? findings + : [.. findings.Select(f => atHumanRate.Contains(f.RuleId) ? f with { AtHumanRate = true } : f)]; + } +} diff --git a/src/SignsOfAI.Core/Rules/Packs/rules.en.json b/src/SignsOfAI.Core/Rules/Packs/rules.en.json index fc199c8..e480482 100644 --- a/src/SignsOfAI.Core/Rules/Packs/rules.en.json +++ b/src/SignsOfAI.Core/Rules/Packs/rules.en.json @@ -160,6 +160,7 @@ }, { "id": "lex.crucial", + "humanRatePer1000": 0.16, "terms": [ "crucial" ], @@ -221,6 +222,7 @@ }, { "id": "lex.robust", + "humanRatePer1000": 0.27, "terms": [ "robust" ], @@ -474,6 +476,7 @@ }, { "id": "lex.comprehensive", + "humanRatePer1000": 0.41, "terms": [ "comprehensive" ], @@ -527,6 +530,7 @@ }, { "id": "lex.moreover", + "humanRatePer1000": 0.98, "terms": [ "moreover" ], @@ -541,6 +545,7 @@ }, { "id": "lex.furthermore", + "humanRatePer1000": 0.77, "terms": [ "furthermore" ], @@ -555,6 +560,7 @@ }, { "id": "lex.notably", + "humanRatePer1000": 0.27, "terms": [ "notably" ], @@ -565,6 +571,7 @@ }, { "id": "lex.utilize", + "humanRatePer1000": 0.35, "terms": [ "utilize", "utilizes", @@ -582,6 +589,7 @@ }, { "id": "lex.facilitate", + "humanRatePer1000": 0.3, "terms": [ "facilitate", "facilitates", @@ -661,6 +669,7 @@ }, { "id": "lex.just", + "humanRatePer1000": 0.38, "terms": [ "just" ], @@ -671,6 +680,7 @@ }, { "id": "lex.simply", + "humanRatePer1000": 0.27, "terms": [ "simply" ], @@ -681,6 +691,7 @@ }, { "id": "lex.actually", + "humanRatePer1000": 0.14, "terms": [ "actually" ], @@ -721,6 +732,7 @@ }, { "id": "lex.importantly", + "humanRatePer1000": 0.13, "terms": [ "importantly" ], @@ -782,6 +794,7 @@ }, { "id": "rhet.not-only-but", + "humanRatePer1000": 0.16, "category": "Rhetorical", "regex": "\\bnot only\\b[^.?!\\n]{1,80}?\\bbut (also|it also)\\b", "weight": 5.0, @@ -854,6 +867,7 @@ }, { "id": "rhet.in-conclusion", + "humanRatePer1000": 0.22, "category": "Rhetorical", "regex": "\\bin conclusion\\b|\\bto sum up\\b|\\bin summary\\b", "weight": 2.5, @@ -872,6 +886,7 @@ }, { "id": "rhet.rule-of-three", + "humanRatePer1000": 1.09, "category": "Rhetorical", "regex": "\\b(\\w+), (\\w+),? and (\\w+)[.,;]", "weight": 1.2, @@ -881,6 +896,7 @@ }, { "id": "syn.serves-as", + "humanRatePer1000": 0.16, "category": "Syntactic", "regex": "\\b(serves|stands|acts|functions) as (a|an|the)\\b", "weight": 3.5, @@ -962,6 +978,7 @@ }, { "id": "rhet.in-terms-of", + "humanRatePer1000": 0.27, "category": "Rhetorical", "regex": "\\bin terms of\\b", "weight": 2.0, @@ -971,6 +988,7 @@ }, { "id": "rhet.with-regard-to", + "humanRatePer1000": 0.2, "category": "Rhetorical", "regex": "\\b(with|in) regards? to\\b", "weight": 2.0, @@ -980,6 +998,7 @@ }, { "id": "rhet.in-order-to", + "humanRatePer1000": 0.8, "category": "Rhetorical", "regex": "\\bin order to\\b", "weight": 1.0, @@ -1016,6 +1035,7 @@ }, { "id": "rhet.weasel-attribution", + "humanRatePer1000": 0.15, "category": "Rhetorical", "regex": "\\b(experts?|scientists?|researchers?|studies|research|critics|analysts) (agree|say|claim|believe|show|shows|suggest|argue|note|found)\\b|\\bit is widely (regarded|considered|believed|accepted|known)\\b|\\bwidely (regarded|considered|seen) as\\b|\\bit(('| i)s| is) (said|believed|thought) that\\b|\\bmany (people )?(believe|argue|say)\\b", "weight": 4.0, diff --git a/src/SignsOfAI.Core/Rules/Packs/rules.es.json b/src/SignsOfAI.Core/Rules/Packs/rules.es.json index 5444830..1fde181 100644 --- a/src/SignsOfAI.Core/Rules/Packs/rules.es.json +++ b/src/SignsOfAI.Core/Rules/Packs/rules.es.json @@ -487,6 +487,7 @@ }, { "id": "lex.ademas", + "humanRatePer1000": 1.65, "terms": [ "además" ], @@ -755,6 +756,7 @@ }, { "id": "rhet.regla-de-tres", + "humanRatePer1000": 1.29, "category": "Rhetorical", "regex": "\\b(\\w+), (\\w+) y (\\w+)[.,;]", "weight": 1.2, diff --git a/src/SignsOfAI.Core/Rules/RulePack.cs b/src/SignsOfAI.Core/Rules/RulePack.cs index 4d27e30..55bb85f 100644 --- a/src/SignsOfAI.Core/Rules/RulePack.cs +++ b/src/SignsOfAI.Core/Rules/RulePack.cs @@ -5,7 +5,7 @@ namespace SignsOfAI.Core.Rules; /// A vocabulary rule: one concept with all its surface forms. -public sealed class LexicalRule +public sealed record LexicalRule { public required string Id { get; init; } @@ -39,6 +39,9 @@ public sealed class LexicalRule /// Optional supporting evidence shown to the user. public string? Evidence { get; init; } + /// + public double? HumanRatePer1000 { get; init; } + /// /// What the live rewriter can substitute for a match, best first. Empty when this rule has no /// mechanical fix (the writer has to make a judgement call), which the rewriter treats as @@ -96,7 +99,7 @@ public static IReadOnlyList LeadingTerms(string? suggestion) } /// A regex rule for rhetorical/syntactic patterns spanning multiple words. -public sealed class PatternRule +public sealed record PatternRule { public required string Id { get; init; } @@ -114,6 +117,35 @@ public sealed class PatternRule public required string Suggestion { get; init; } public string? Evidence { get; init; } + + /// + /// How often this rule fires on writing no machine wrote, in hits per thousand words, at the + /// ninetieth percentile of the human calibration corpus. Absent — the default — means "never + /// measured", and the rule counts on presence as it always did. Present and zero means a catalog + /// turned the rate off deliberately, which is a different statement and has to stay distinguishable. + /// + /// When it is set, the rule still reports what it finds, but below that rate the findings are + /// marked and score nothing. That is the difference between + /// "this word appears" and "this word appears at a rate people don't write at", and it matters + /// because measuring on the corpus showed a median of seven flagged tells in every human academic + /// paper. "Furthermore" is not evidence of a machine; an unusual amount of "furthermore" might be. + /// + /// These numbers are measured, not chosen, and the repository can regenerate them: + /// + /// dotnet run --project tools/SignsOfAI.Calibration -- thresholds + /// + /// derives each one from the calibration corpus and writes it back here. A rule only gets a rate + /// when it fired on at least eight texts in that language — below that there is not enough evidence + /// to set a threshold, and inventing one is the failure this project exists to avoid. Rules with no + /// measured rate are untouched, which is why the strong tells (delve, tapestry) still count on a + /// single occurrence: they essentially never appear in the human corpus. + /// + /// A caveat that belongs next to the numbers rather than in a commit message: these are fitted on + /// the same ninety texts the project reports its false-positive rate against. The improvement is + /// therefore quoted leave-one-out — each text judged against thresholds derived from the other + /// eighty-nine — which is a smaller number than the in-sample one and the only honest version. + /// + public double? HumanRatePer1000 { get; init; } } /// A full rule-pack (a "catalog") — built-in or supplied by the user. @@ -126,6 +158,29 @@ public sealed class RulePack public PatternRule[] Patterns { get; init; } = []; + private Dictionary? _humanRates; + + /// + /// Rule id to measured human rate, for the rules in this pack that carry one. Built once and kept: + /// the alternative is rebuilding a dictionary over every rule in the pack on each analysis, and + /// analysis runs on every keystroke in the live editor. + /// + public IReadOnlyDictionary HumanRates + { + get + { + if (_humanRates is not null) return _humanRates; + + var map = new Dictionary(StringComparer.Ordinal); + foreach (var rule in Lexical ?? []) + if (rule.HumanRatePer1000 is { } lexRate and > 0) map[rule.Id] = lexRate; + foreach (var rule in Patterns ?? []) + if (rule.HumanRatePer1000 is { } patRate and > 0) map[rule.Id] = patRate; + + return _humanRates = map; + } + } + /// /// Wording for the analyzers that compute their findings instead of matching a rule — the /// overused-word message, the rhythm one, the em-dash one. Optional: anything missing falls back @@ -187,8 +242,22 @@ public static RulePack Merge(string language, IEnumerable packs) // and a pack that supplies one means to use it instead of, not on top of, the built-in. if (pack.FunctionWords is { Length: > 0 }) functionWords = pack.FunctionWords; // A custom pack parsed from JSON may omit a section, leaving the array null under source-gen. - foreach (var rule in pack.Lexical ?? []) lexical[rule.Id] = rule; - foreach (var rule in pack.Patterns ?? []) patterns[rule.Id] = rule; + // A measured human rate survives an override that does not restate it. Contributing a + // reworded suggestion for `lex.moreover` is the documented way to help; silently switching + // that rule's calibration off because the contributor had no reason to know the field + // existed is not a trade this project should make. Setting it explicitly still works — + // including to zero, which turns the rate off on purpose. + foreach (var rule in pack.Lexical ?? []) + lexical[rule.Id] = rule.HumanRatePer1000 is null + && lexical.TryGetValue(rule.Id, out var priorLex) + ? rule with { HumanRatePer1000 = priorLex.HumanRatePer1000 } + : rule; + + foreach (var rule in pack.Patterns ?? []) + patterns[rule.Id] = rule.HumanRatePer1000 is null + && patterns.TryGetValue(rule.Id, out var priorPat) + ? rule with { HumanRatePer1000 = priorPat.HumanRatePer1000 } + : rule; // Merged key by key, so a custom catalog can reword one message without restating them all. foreach (var (key, text) in pack.Messages ?? []) messages[key] = text; } diff --git a/src/SignsOfAI.Core/Scoring/Scorer.cs b/src/SignsOfAI.Core/Scoring/Scorer.cs index 260a75c..3d5c06d 100644 --- a/src/SignsOfAI.Core/Scoring/Scorer.cs +++ b/src/SignsOfAI.Core/Scoring/Scorer.cs @@ -43,6 +43,11 @@ public static (double Overall, IReadOnlyList ByCategory) Score( int words = Math.Max(stats.WordCount, WordFloor); double burstiness = BurstinessScore(stats); + // Findings whose rule is being used at a rate people write at are shown to the reader but + // score nothing. They are not evidence of a machine, and counting them is how a tool ends up + // reporting that ordinary academic prose looks generated. + findings = [.. findings.Where(f => !f.AtHumanRate)]; + var byCategory = new List(); foreach (SignCategory category in Enum.GetValues()) { diff --git a/tests/SignsOfAI.Core.Tests/GenreGateTests.cs b/tests/SignsOfAI.Core.Tests/GenreGateTests.cs new file mode 100644 index 0000000..5a97c5e --- /dev/null +++ b/tests/SignsOfAI.Core.Tests/GenreGateTests.cs @@ -0,0 +1,171 @@ +using SignsOfAI.Core; +using SignsOfAI.Core.Model; +using SignsOfAI.Core.Rules; + +namespace SignsOfAI.Core.Tests; + +/// +/// The gate decides which findings count as evidence of a machine, so the tests that matter are the +/// ones about what it must never do: hide a finding, silence a rule that has no measured rate, or +/// turn a contributed catalog's calibration off by accident. +/// +public class GenreGateTests +{ + private static RulePack Pack(params (string Id, double Rate)[] rules) => new() + { + Language = "en", + Lexical = [.. rules.Select(r => new LexicalRule + { + Id = r.Id, + Terms = [r.Id], + Suggestion = "something, else", + HumanRatePer1000 = r.Rate, + })], + }; + + private static Finding Hit(string ruleId) => new() + { + RuleId = ruleId, + Category = SignCategory.Lexical, + Severity = Severity.Low, + Span = new TextSpan(0, 1), + Message = "m", + Suggestion = "s", + Weight = 2.0, + }; + + [Fact] + public void Marks_but_never_removes() + { + var findings = new[] { Hit("lex.a"), Hit("lex.a") }; + + var result = GenreGate.Apply(findings, Pack(("lex.a", 5.0)), wordCount: 1000); + + // Two hits in a thousand words is 2.0, well under the threshold. + Assert.Equal(2, result.Count); + Assert.All(result, f => Assert.True(f.AtHumanRate)); + } + + [Fact] + public void Leaves_a_rule_above_its_human_rate_alone() + { + var findings = new[] { Hit("lex.a"), Hit("lex.a"), Hit("lex.a") }; + + var result = GenreGate.Apply(findings, Pack(("lex.a", 2.0)), wordCount: 1000); + + Assert.All(result, f => Assert.False(f.AtHumanRate)); + } + + [Fact] + public void A_rule_with_no_measured_rate_is_untouched() + { + // The strong tells — delve, tapestry — never appear in the human corpus, get no threshold, + // and must keep counting on a single occurrence however long the document is. + var findings = new[] { Hit("lex.delve") }; + + var result = GenreGate.Apply(findings, Pack(("lex.other", 1.0)), wordCount: 100_000); + + Assert.Single(result); + Assert.False(result[0].AtHumanRate); + } + + [Fact] + public void The_boundary_is_inclusive_so_exactly_the_human_rate_is_human() + { + var findings = new[] { Hit("lex.a") }; + + var atExactly = GenreGate.Apply(findings, Pack(("lex.a", 1.0)), wordCount: 1000); + var justOver = GenreGate.Apply(findings, Pack(("lex.a", 1.0)), wordCount: 999); + + Assert.True(atExactly[0].AtHumanRate); + Assert.False(justOver[0].AtHumanRate); + } + + [Fact] + public void Rates_are_counted_per_rule_not_across_the_document() + { + var findings = new[] { Hit("lex.a"), Hit("lex.b"), Hit("lex.b"), Hit("lex.b") }; + + var result = GenreGate.Apply(findings, Pack(("lex.a", 2.0), ("lex.b", 2.0)), wordCount: 1000); + + Assert.True(result.Single(f => f.RuleId == "lex.a").AtHumanRate); + Assert.All(result.Where(f => f.RuleId == "lex.b"), f => Assert.False(f.AtHumanRate)); + } + + [Fact] + public void Findings_at_a_human_rate_do_not_move_the_score() + { + // The whole point: an academic paper using "furthermore" the way academics use it should read + // exactly as human as one that never uses it. + var analyzer = new AiWritingAnalyzer(); + var body = string.Join(" ", Enumerable.Repeat("The study examined the data carefully and reported what it found.", 160)); + + var withOne = analyzer.Analyze(body + " Furthermore, the result held.", "en"); + + // Reported, so the reader still sees it… + Assert.Contains(withOne.Findings, f => f.RuleId == "lex.furthermore"); + Assert.True(withOne.Findings.Single(f => f.RuleId == "lex.furthermore").AtHumanRate); + + // …and worth nothing, so it cannot push anybody toward an accusation. The lexical category is + // asserted rather than the overall score on purpose: adding any sentence moves burstiness, and + // a test that watched the overall number would be measuring sentence rhythm, not this gate. + // The category tallies count evidence, not highlights: they exist to explain the score, so a + // finding that scores nothing must not appear in them even though it is still in the panel. + var lexical = withOne.CategoryScores.Single(c => c.Category == SignCategory.Lexical); + Assert.Equal(0, lexical.FindingCount); + Assert.Equal(0, lexical.Score); + } + + [Fact] + public void A_custom_catalog_that_reworks_a_rule_keeps_its_measured_rate() + { + // Rewording a suggestion is the documented way to contribute. Losing the rule's calibration + // because the contributor had no reason to restate a number they never saw would be a trap. + var contributed = new RulePack + { + Language = "en", + Lexical = [new LexicalRule + { + Id = "lex.furthermore", + Terms = ["furthermore"], + Suggestion = "besides, also", + }], + }; + + var merged = AiWritingAnalyzer.ResolvePack("en", [contributed]); + + Assert.True(merged.HumanRates.TryGetValue("lex.furthermore", out var rate)); + Assert.True(rate > 0); + } + + [Fact] + public void A_catalog_may_still_turn_a_rate_off_deliberately() + { + // Inheriting the built-in rate must not become impossible to override — only impossible to + // lose by accident. A pack that means to disable it states zero, which is different from stating nothing. + var contributed = new RulePack + { + Language = "en", + Lexical = [new LexicalRule + { + Id = "lex.furthermore", + Terms = ["furthermore"], + Suggestion = "besides, also", + HumanRatePer1000 = 0, + }], + }; + + var merged = AiWritingAnalyzer.ResolvePack("en", [contributed]); + + Assert.False(merged.HumanRates.ContainsKey("lex.furthermore")); + } + + [Fact] + public void The_built_in_packs_carry_measured_rates() + { + // A regression guard for the packaging, not the analysis: these numbers live in JSON that is + // rewritten by a tool, and losing them would quietly restore the seven-false-tells behaviour. + Assert.NotEmpty(AiWritingAnalyzer.ResolvePack("en").HumanRates); + Assert.NotEmpty(AiWritingAnalyzer.ResolvePack("es").HumanRates); + } +} diff --git a/tools/SignsOfAI.Calibration/Program.cs b/tools/SignsOfAI.Calibration/Program.cs index fb9a211..0710d79 100644 --- a/tools/SignsOfAI.Calibration/Program.cs +++ b/tools/SignsOfAI.Calibration/Program.cs @@ -29,6 +29,7 @@ string source = ""; string fetchLanguage = "en"; int count = 40, fromYear = 2018, toYear = 2020; +string packsDir = "src/SignsOfAI.Core/Rules/Packs"; for (int i = 1; i < argv.Count; i++) { @@ -38,6 +39,7 @@ case "--texts": textsDir = Next(); break; case "--out": outPath = Next(); break; case "--record-hashes": recordHashes = true; break; + case "--packs": packsDir = Next(); break; case "--source": source = Next(); break; case "--lang": fetchLanguage = Next(); break; case "--count": count = int.Parse(Next()); break; @@ -50,7 +52,7 @@ string Next() => ++i < argv.Count ? argv[i] : throw new ArgumentException($"Missing value for {argv[i - 1]}"); } -if (argv[0] is not ("run" or "fetch")) +if (argv[0] is not ("run" or "fetch" or "thresholds")) { Console.Error.WriteLine($"Unknown command '{argv[0]}'. Run --help."); return 2; @@ -144,7 +146,8 @@ Stratum = entry.Stratum, Score = result.OverallScore, WordCount = result.Statistics.WordCount, - RuleIds = [.. result.Findings.Select(f => f.RuleId)], + RuleIds = [.. result.Findings.Where(f => !f.AtHumanRate).Select(f => f.RuleId)], + MatchedRuleIds = [.. result.Findings.Select(f => f.RuleId)], }); } @@ -161,6 +164,36 @@ return 1; } +// ── `thresholds` ───────────────────────────────────────────────────────────── +// Derives each rule's human usage rate and writes it into the packs, so the numbers the analyzer +// runs on can be regenerated by anyone holding the corpus rather than taken on trust. +if (argv[0] == "thresholds") +{ + var derived = Thresholds.Derive(samples); + foreach (var (language, rates) in derived.OrderBy(p => p.Key, StringComparer.Ordinal)) + { + var pack = Path.Combine(packsDir, $"rules.{language}.json"); + if (!File.Exists(pack)) + { + Console.Error.WriteLine($" no pack for '{language}' at {pack}"); + continue; + } + + var written = Thresholds.WriteInto(pack, rates); + Console.WriteLine($" {language}: {written} rules given a measured rate → {pack}"); + foreach (var (id, rate) in rates.OrderByDescending(r => r.Value)) + Console.WriteLine($" {id,-28} {rate,5:0.00} per 1,000 words"); + } + + var (before, after, cleanBefore, cleanAfter) = Thresholds.LeaveOneOut(samples); + Console.WriteLine(); + Console.WriteLine(" Held out of its own thresholds, each text keeps:"); + Console.WriteLine($" findings per text {before:0.0} → {after:0.0} ({(after - before) / before:P0})"); + Console.WriteLine($" texts with none {cleanBefore} → {cleanAfter} of {samples.Count}"); + Console.WriteLine(); + return 0; +} + var calibration = Calibrator.Compute( samples, manifest.Id, manifest.Fingerprint(), manifest.TargetFalsePositiveRate); diff --git a/tools/SignsOfAI.Calibration/Thresholds.cs b/tools/SignsOfAI.Calibration/Thresholds.cs new file mode 100644 index 0000000..8a38ebe --- /dev/null +++ b/tools/SignsOfAI.Calibration/Thresholds.cs @@ -0,0 +1,176 @@ +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using SignsOfAI.Core.Calibration; + +namespace SignsOfAI.Calibration; + +/// +/// Derives each rule's human usage rate from the corpus and writes it into the rule packs. +/// +/// This exists because the first version of those numbers was computed in a throwaway script and +/// pasted in, which made the packs claim something the repository could not back up. A measured +/// threshold nobody can regenerate is a chosen threshold with better manners, and choosing thresholds +/// is the practice this project criticises in every article it has published. +/// +public static class Thresholds +{ + /// + /// A rule needs this many texts before its rate means anything. The ninetieth percentile of eight + /// samples is already close to the largest of them; below that it is one author's habit. + /// + public const int MinimumTexts = 8; + + /// The percentile of human usage a text must exceed before the rule counts as evidence. + public const double Percentile = 0.90; + + /// Language → rule id → hits per thousand words at . + public static Dictionary> Derive( + IReadOnlyList samples) + { + var result = new Dictionary>(StringComparer.Ordinal); + + foreach (var language in samples.Select(s => s.Language).Distinct()) + { + var texts = samples.Where(s => s.Language == language).ToList(); + var rates = new Dictionary(StringComparer.Ordinal); + + foreach (var ruleId in texts.SelectMany(t => t.MatchedRuleIds).Distinct()) + { + // Statistical findings are one per document by construction, so a rate per thousand + // words measures the document's length rather than the rule's behaviour. + if (ruleId.StartsWith("stat.", StringComparison.Ordinal)) continue; + + if (texts.Count(t => t.MatchedRuleIds.Contains(ruleId)) < MinimumTexts) continue; + + var rate = RateAt(texts, ruleId, Percentile); + if (rate > 0) rates[ruleId] = Math.Round(rate, 2); + } + + if (rates.Count > 0) result[language] = rates; + } + + return result; + } + + /// + /// The out-of-sample answer to "how much noise did this remove", and the only honest one: each + /// text is judged against thresholds derived from the other texts, never from itself. Fitting on + /// the corpus and then reporting the improvement on the same corpus is how a tool reports a + /// number it cannot reproduce on anybody else's writing. + /// + public static (double Before, double After, int TextsCleanBefore, int TextsCleanAfter) LeaveOneOut( + IReadOnlyList samples) + { + double before = 0, after = 0; + int cleanBefore = 0, cleanAfter = 0; + + foreach (var held in samples) + { + var others = samples.Where(s => s.Language == held.Language && !ReferenceEquals(s, held)).ToList(); + var rates = new Dictionary(StringComparer.Ordinal); + + foreach (var ruleId in others.SelectMany(t => t.MatchedRuleIds).Distinct()) + { + if (ruleId.StartsWith("stat.", StringComparison.Ordinal)) continue; + if (others.Count(t => t.MatchedRuleIds.Contains(ruleId)) < MinimumTexts) continue; + var rate = RateAt(others, ruleId, Percentile); + if (rate > 0) rates[ruleId] = Math.Round(rate, 2); + } + + int kept = 0; + foreach (var group in held.MatchedRuleIds.GroupBy(id => id, StringComparer.Ordinal)) + { + var hits = group.Count(); + var rate = held.WordCount > 0 ? hits / (double)held.WordCount * 1000.0 : 0; + if (!rates.TryGetValue(group.Key, out var threshold) || rate > threshold) kept += hits; + } + + before += held.MatchedRuleIds.Count; + after += kept; + if (held.MatchedRuleIds.Count == 0) cleanBefore++; + if (kept == 0) cleanAfter++; + } + + return (before / samples.Count, after / samples.Count, cleanBefore, cleanAfter); + } + + /// + /// The rate at which is used, at the given percentile, across every text + /// in — including the ones where it never fires, which count as zero. A + /// percentile over only the texts that fired would answer "how heavily do the people who use this + /// word use it", and the question here is "how often does this appear in writing at all". + /// + private static double RateAt(IReadOnlyList texts, string ruleId, double percentile) + { + var rates = texts + .Select(t => t.WordCount > 0 + ? t.MatchedRuleIds.Count(id => id == ruleId) / (double)t.WordCount * 1000.0 + : 0) + .OrderBy(r => r) + .ToList(); + + var index = Math.Min(rates.Count - 1, (int)(rates.Count * percentile)); + return rates[index]; + } + + /// + /// Writes the derived rates into a rule pack, adding humanRatePer1000 to the rules that have + /// one and removing it from those that no longer do — a rule that drops below + /// as the corpus changes must lose its threshold rather than keep a + /// stale one. Returns how many rules were written. + /// + /// The file is rewritten from its parsed form, so formatting is normalised once and owned by this + /// tool from then on. Non-ASCII is left as itself: the Spanish pack is full of it and escaping it + /// would make every future diff unreadable. + /// + public static int WriteInto(string packPath, IReadOnlyDictionary rates) + { + var root = JsonNode.Parse(File.ReadAllText(packPath))!.AsObject(); + int written = 0; + + foreach (var section in new[] { "lexical", "patterns" }) + { + if (root[section] is not JsonArray rules) continue; + + foreach (var node in rules) + { + if (node is not JsonObject rule || rule["id"]?.GetValue() is not { } id) continue; + + rule.Remove("humanRatePer1000"); + if (!rates.TryGetValue(id, out var rate)) continue; + + // Placed straight after the id so it reads as a property of the rule rather than an + // afterthought appended to whatever the last field happened to be. + var rebuilt = new JsonObject(); + foreach (var (key, value) in rule.ToList()) + { + rule.Remove(key); + rebuilt[key] = value; + if (key == "id") rebuilt["humanRatePer1000"] = rate; + } + + foreach (var (key, value) in rebuilt.ToList()) + { + rebuilt.Remove(key); + rule[key] = value; + } + + written++; + } + } + + var options = new JsonSerializerOptions + { + WriteIndented = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + // Newlines are written LF explicitly. The packs are LF in the repository, and letting this run + // on Windows rewrite them as CRLF would mark every line of a 900-line file as changed the + // first time anyone regenerates the thresholds. + var json = root.ToJsonString(options).ReplaceLineEndings("\n") + "\n"; + File.WriteAllText(packPath, json); + return written; + } +} From dd4ea07a9c778c61feb98bfd2d3b547af189a324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Hern=C3=A1ndez?= Date: Tue, 4 Aug 2026 17:31:00 -0400 Subject: [PATCH 2/2] Let the result say which findings are evidence, instead of each host deciding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An architecture review caught this within a day of the flag landing, and it was already producing contradictory reports. The web headline counted every match while its own category chips counted only the scoring ones, so a page could say eleven signals above chips summing to three — and the share card published the inflated number. The MCP server handed an agent, under "why this reads like AI", matches the engine had already ruled out, with no field in its payload that could have revealed the difference. The CLI happened to be right because it was written last. A semantic every consumer has to remember is one some consumer will forget. AnalysisResult now partitions its own findings: Signals is the evidence and what "N signals" means everywhere, Observations is what the text uses at a rate people write at, and Findings remains everything for highlighting and for the rewriter, which should offer to replace a word whether or not its rate proves anything. MCP findings also carry RuleId and AtHumanRate now. Without them an agent had no way to tell a tell from a habit, which is precisely the judgement this project exists to hand over rather than make. A test asserts the partition and that the headline agrees with the category tallies, because that is the disagreement that actually happened. Refs #31 Co-Authored-By: Claude Opus 5 --- src/SignsOfAI.Cli/Program.cs | 4 +-- src/SignsOfAI.Core/Model/AnalysisResult.cs | 33 ++++++++++++++++++-- src/SignsOfAI.Mcp/Tools/WritingTools.cs | 18 +++++++++-- src/SignsOfAI.UI/Pages/Home.razor | 8 ++--- tests/SignsOfAI.Core.Tests/GenreGateTests.cs | 19 +++++++++++ 5 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/SignsOfAI.Cli/Program.cs b/src/SignsOfAI.Cli/Program.cs index 6def91c..7332af3 100644 --- a/src/SignsOfAI.Cli/Program.cs +++ b/src/SignsOfAI.Cli/Program.cs @@ -331,11 +331,11 @@ static void PrintBaseline(string path, BaselineReport r, bool useColor) // The headline counts evidence, so it agrees with the category tallies and with the score. What // matched at a human rate is named beside it rather than folded in: a reader who sees eleven // highlights under a headline of three deserves to know why, on the same line. -static int Counted(AnalysisResult r) => r.Findings.Count(f => !f.AtHumanRate); +static int Counted(AnalysisResult r) => r.Signals.Count; static string AtRate(AnalysisResult r) { - var n = r.Findings.Count(f => f.AtHumanRate); + var n = r.Observations.Count; return n == 0 ? "" : $" + {n} at a human rate"; } diff --git a/src/SignsOfAI.Core/Model/AnalysisResult.cs b/src/SignsOfAI.Core/Model/AnalysisResult.cs index 84f2e1c..76e0995 100644 --- a/src/SignsOfAI.Core/Model/AnalysisResult.cs +++ b/src/SignsOfAI.Core/Model/AnalysisResult.cs @@ -12,10 +12,39 @@ public sealed record AnalysisResult /// Language code actually used for analysis ("en" or "es"). public required string Language { get; init; } - /// All findings, ordered by position in the text. + /// + /// Everything that matched, ordered by position in the text — both the findings that count as + /// evidence and the ones the writer is using at a rate people write at. Highlighting works from + /// this list, and so does the rewriter, which should offer to replace a word regardless of whether + /// its rate proves anything. + /// + /// Do not count this to report how many signals were found. Use . The + /// two differ exactly when the genre gate has marked something, and a host that counts this list + /// will contradict its own category tallies and its own score. + /// public required IReadOnlyList Findings { get; init; } - /// Score contribution per category (0–100). + /// + /// The findings that are evidence: everything except what the text uses at a human rate. This is + /// what "N signals" means, it is what counts, and it is what moved + /// . + /// + /// Derived here rather than left to each host on purpose. The distinction arrived as a flag on + /// , and within a day the five hosts disagreed about it: one counted + /// correctly, one reported a headline that its own category chips contradicted, and the MCP server + /// handed an agent, as "why this reads like AI", matches the engine had already ruled out. A + /// semantic that every consumer has to remember is a semantic that some consumer will forget. + /// + public IReadOnlyList Signals => field ??= [.. Findings.Where(f => !f.AtHumanRate)]; + + /// + /// What matched but is not evidence: rules this text uses at a rate measured on writing published + /// before generative models existed. Worth showing — "you use 'furthermore' about as often as + /// other people do" is a useful thing to be told — and worth nothing to the score. + /// + public IReadOnlyList Observations => field ??= [.. Findings.Where(f => f.AtHumanRate)]; + + /// Score contribution per category (0–100), counting only. public required IReadOnlyList CategoryScores { get; init; } /// Overall "reads like AI" score, 0 (human) – 100 (unmistakably AI). diff --git a/src/SignsOfAI.Mcp/Tools/WritingTools.cs b/src/SignsOfAI.Mcp/Tools/WritingTools.cs index f70c069..685ff30 100644 --- a/src/SignsOfAI.Mcp/Tools/WritingTools.cs +++ b/src/SignsOfAI.Mcp/Tools/WritingTools.cs @@ -28,7 +28,7 @@ public static AnalysisReport AnalyzeAiWriting( Math.Round(r.OverallScore, 1), r.Verdict, r.Language, - r.Findings.Count, + r.Signals.Count, r.CategoryScores .Where(c => c.FindingCount > 0) .Select(c => new CategoryCount(c.Category.ToString(), c.FindingCount, Math.Round(c.Score, 1))) @@ -41,7 +41,8 @@ public static AnalysisReport AnalyzeAiWriting( Math.Round(r.Statistics.LexicalDiversity, 3)), r.Findings .Select(f => new FindingItem( - f.Category.ToString(), f.Severity.ToString(), f.MatchedText, f.Message, f.Suggestion, f.Evidence)) + f.RuleId, f.Category.ToString(), f.Severity.ToString(), f.MatchedText, f.Message, + f.Suggestion, f.Evidence, f.AtHumanRate)) .ToList(), // A one-line pointer, not the report. Anything found here deserves the dedicated tool, // which returns coordinates; folding those into the score's payload would invite an @@ -88,4 +89,15 @@ public sealed record CategoryCount(string Category, int Count, double Score); public sealed record DocStats(int Words, int Sentences, double MeanSentenceLength, double Burstiness, double LexicalDiversity); -public sealed record FindingItem(string Category, string Severity, string MatchedText, string Message, string Suggestion, string? Evidence); +public sealed record FindingItem( + string RuleId, + string Category, + string Severity, + string MatchedText, + string Message, + string Suggestion, + string? Evidence, + // An agent reading this list has no other way to tell the two apart, and the difference is the + // whole point: true here means the engine found the phrase and decided it is not evidence, + // because the text uses it about as often as people do. + bool AtHumanRate); diff --git a/src/SignsOfAI.UI/Pages/Home.razor b/src/SignsOfAI.UI/Pages/Home.razor index 183ab89..c3f878c 100644 --- a/src/SignsOfAI.UI/Pages/Home.razor +++ b/src/SignsOfAI.UI/Pages/Home.razor @@ -210,7 +210,7 @@

@L.Verdict(r.OverallScore)

-

@L.P(r.Findings.Count, "home.signals") · @L["home.analyzedas"] +

@L.P(r.Signals.Count, "home.signals") · @L["home.analyzedas"] @L.TextLanguageName(r.Language)

@foreach (var c in r.CategoryScores.Where(c => c.FindingCount > 0)) @@ -354,8 +354,8 @@
-

@L.F("home.recommendations", r.Findings.Count)

- @if (r.Findings.Count == 0) +

@L.F("home.recommendations", r.Signals.Count)

+ @if (r.Signals.Count == 0) {

@L["home.nofindings"]

} @@ -669,7 +669,7 @@ else Score: r.OverallScore, ScoreColor: ScoreHex(r.OverallScore), Verdict: L.Verdict(r.OverallScore), - Signals: r.Findings.Count, + Signals: r.Signals.Count, Language: L.TextLanguageName(r.Language), Categories: r.CategoryScores.Where(c => c.FindingCount > 0) .Select(c => new CardCategory(L.Category(c.Category), c.FindingCount, Highlighter.CategoryHex(c.Category))) diff --git a/tests/SignsOfAI.Core.Tests/GenreGateTests.cs b/tests/SignsOfAI.Core.Tests/GenreGateTests.cs index 5a97c5e..2caea77 100644 --- a/tests/SignsOfAI.Core.Tests/GenreGateTests.cs +++ b/tests/SignsOfAI.Core.Tests/GenreGateTests.cs @@ -168,4 +168,23 @@ public void The_built_in_packs_carry_measured_rates() Assert.NotEmpty(AiWritingAnalyzer.ResolvePack("en").HumanRates); Assert.NotEmpty(AiWritingAnalyzer.ResolvePack("es").HumanRates); } + + [Fact] + public void The_result_partitions_its_own_findings_so_no_host_has_to() + { + // The regression this guards against actually happened: the flag went onto Finding and within + // a day the web headline counted every match while its category chips counted only the scoring + // ones, and the MCP server handed an agent, as evidence, matches the engine had ruled out. + var analyzer = new AiWritingAnalyzer(); + var body = string.Join(" ", Enumerable.Repeat("The study examined the data carefully and reported what it found.", 160)); + + var r = analyzer.Analyze(body + " Furthermore, the result held.", "en"); + + Assert.Equal(r.Findings.Count, r.Signals.Count + r.Observations.Count); + Assert.All(r.Signals, f => Assert.False(f.AtHumanRate)); + Assert.All(r.Observations, f => Assert.True(f.AtHumanRate)); + + // The headline number and the category tallies have to be the same number. + Assert.Equal(r.Signals.Count, r.CategoryScores.Sum(c => c.FindingCount)); + } }