@L.P(r.Signals.Count, "home.signals") · @L["home.analyzedas"]
@L.TextLanguageName(r.Language)
@foreach (var c in r.CategoryScores.Where(c => c.FindingCount > 0))
@@ -354,8 +354,8 @@
-
@L.F("home.recommendations", r.Findings.Count)
- @if (r.Findings.Count == 0)
+
@L.F("home.recommendations", r.Signals.Count)
+ @if (r.Signals.Count == 0)
{
@L["home.nofindings"]
}
@@ -669,7 +669,7 @@ else
Score: r.OverallScore,
ScoreColor: ScoreHex(r.OverallScore),
Verdict: L.Verdict(r.OverallScore),
- Signals: r.Findings.Count,
+ Signals: r.Signals.Count,
Language: L.TextLanguageName(r.Language),
Categories: r.CategoryScores.Where(c => c.FindingCount > 0)
.Select(c => new CardCategory(L.Category(c.Category), c.FindingCount, Highlighter.CategoryHex(c.Category)))
diff --git a/tests/SignsOfAI.Core.Tests/GenreGateTests.cs b/tests/SignsOfAI.Core.Tests/GenreGateTests.cs
new file mode 100644
index 0000000..2caea77
--- /dev/null
+++ b/tests/SignsOfAI.Core.Tests/GenreGateTests.cs
@@ -0,0 +1,190 @@
+using SignsOfAI.Core;
+using SignsOfAI.Core.Model;
+using SignsOfAI.Core.Rules;
+
+namespace SignsOfAI.Core.Tests;
+
+///
+/// The gate decides which findings count as evidence of a machine, so the tests that matter are the
+/// ones about what it must never do: hide a finding, silence a rule that has no measured rate, or
+/// turn a contributed catalog's calibration off by accident.
+///
+public class GenreGateTests
+{
+ private static RulePack Pack(params (string Id, double Rate)[] rules) => new()
+ {
+ Language = "en",
+ Lexical = [.. rules.Select(r => new LexicalRule
+ {
+ Id = r.Id,
+ Terms = [r.Id],
+ Suggestion = "something, else",
+ HumanRatePer1000 = r.Rate,
+ })],
+ };
+
+ private static Finding Hit(string ruleId) => new()
+ {
+ RuleId = ruleId,
+ Category = SignCategory.Lexical,
+ Severity = Severity.Low,
+ Span = new TextSpan(0, 1),
+ Message = "m",
+ Suggestion = "s",
+ Weight = 2.0,
+ };
+
+ [Fact]
+ public void Marks_but_never_removes()
+ {
+ var findings = new[] { Hit("lex.a"), Hit("lex.a") };
+
+ var result = GenreGate.Apply(findings, Pack(("lex.a", 5.0)), wordCount: 1000);
+
+ // Two hits in a thousand words is 2.0, well under the threshold.
+ Assert.Equal(2, result.Count);
+ Assert.All(result, f => Assert.True(f.AtHumanRate));
+ }
+
+ [Fact]
+ public void Leaves_a_rule_above_its_human_rate_alone()
+ {
+ var findings = new[] { Hit("lex.a"), Hit("lex.a"), Hit("lex.a") };
+
+ var result = GenreGate.Apply(findings, Pack(("lex.a", 2.0)), wordCount: 1000);
+
+ Assert.All(result, f => Assert.False(f.AtHumanRate));
+ }
+
+ [Fact]
+ public void A_rule_with_no_measured_rate_is_untouched()
+ {
+ // The strong tells — delve, tapestry — never appear in the human corpus, get no threshold,
+ // and must keep counting on a single occurrence however long the document is.
+ var findings = new[] { Hit("lex.delve") };
+
+ var result = GenreGate.Apply(findings, Pack(("lex.other", 1.0)), wordCount: 100_000);
+
+ Assert.Single(result);
+ Assert.False(result[0].AtHumanRate);
+ }
+
+ [Fact]
+ public void The_boundary_is_inclusive_so_exactly_the_human_rate_is_human()
+ {
+ var findings = new[] { Hit("lex.a") };
+
+ var atExactly = GenreGate.Apply(findings, Pack(("lex.a", 1.0)), wordCount: 1000);
+ var justOver = GenreGate.Apply(findings, Pack(("lex.a", 1.0)), wordCount: 999);
+
+ Assert.True(atExactly[0].AtHumanRate);
+ Assert.False(justOver[0].AtHumanRate);
+ }
+
+ [Fact]
+ public void Rates_are_counted_per_rule_not_across_the_document()
+ {
+ var findings = new[] { Hit("lex.a"), Hit("lex.b"), Hit("lex.b"), Hit("lex.b") };
+
+ var result = GenreGate.Apply(findings, Pack(("lex.a", 2.0), ("lex.b", 2.0)), wordCount: 1000);
+
+ Assert.True(result.Single(f => f.RuleId == "lex.a").AtHumanRate);
+ Assert.All(result.Where(f => f.RuleId == "lex.b"), f => Assert.False(f.AtHumanRate));
+ }
+
+ [Fact]
+ public void Findings_at_a_human_rate_do_not_move_the_score()
+ {
+ // The whole point: an academic paper using "furthermore" the way academics use it should read
+ // exactly as human as one that never uses it.
+ var analyzer = new AiWritingAnalyzer();
+ var body = string.Join(" ", Enumerable.Repeat("The study examined the data carefully and reported what it found.", 160));
+
+ var withOne = analyzer.Analyze(body + " Furthermore, the result held.", "en");
+
+ // Reported, so the reader still sees it…
+ Assert.Contains(withOne.Findings, f => f.RuleId == "lex.furthermore");
+ Assert.True(withOne.Findings.Single(f => f.RuleId == "lex.furthermore").AtHumanRate);
+
+ // …and worth nothing, so it cannot push anybody toward an accusation. The lexical category is
+ // asserted rather than the overall score on purpose: adding any sentence moves burstiness, and
+ // a test that watched the overall number would be measuring sentence rhythm, not this gate.
+ // The category tallies count evidence, not highlights: they exist to explain the score, so a
+ // finding that scores nothing must not appear in them even though it is still in the panel.
+ var lexical = withOne.CategoryScores.Single(c => c.Category == SignCategory.Lexical);
+ Assert.Equal(0, lexical.FindingCount);
+ Assert.Equal(0, lexical.Score);
+ }
+
+ [Fact]
+ public void A_custom_catalog_that_reworks_a_rule_keeps_its_measured_rate()
+ {
+ // Rewording a suggestion is the documented way to contribute. Losing the rule's calibration
+ // because the contributor had no reason to restate a number they never saw would be a trap.
+ var contributed = new RulePack
+ {
+ Language = "en",
+ Lexical = [new LexicalRule
+ {
+ Id = "lex.furthermore",
+ Terms = ["furthermore"],
+ Suggestion = "besides, also",
+ }],
+ };
+
+ var merged = AiWritingAnalyzer.ResolvePack("en", [contributed]);
+
+ Assert.True(merged.HumanRates.TryGetValue("lex.furthermore", out var rate));
+ Assert.True(rate > 0);
+ }
+
+ [Fact]
+ public void A_catalog_may_still_turn_a_rate_off_deliberately()
+ {
+ // Inheriting the built-in rate must not become impossible to override — only impossible to
+ // lose by accident. A pack that means to disable it states zero, which is different from stating nothing.
+ var contributed = new RulePack
+ {
+ Language = "en",
+ Lexical = [new LexicalRule
+ {
+ Id = "lex.furthermore",
+ Terms = ["furthermore"],
+ Suggestion = "besides, also",
+ HumanRatePer1000 = 0,
+ }],
+ };
+
+ var merged = AiWritingAnalyzer.ResolvePack("en", [contributed]);
+
+ Assert.False(merged.HumanRates.ContainsKey("lex.furthermore"));
+ }
+
+ [Fact]
+ public void The_built_in_packs_carry_measured_rates()
+ {
+ // A regression guard for the packaging, not the analysis: these numbers live in JSON that is
+ // rewritten by a tool, and losing them would quietly restore the seven-false-tells behaviour.
+ Assert.NotEmpty(AiWritingAnalyzer.ResolvePack("en").HumanRates);
+ Assert.NotEmpty(AiWritingAnalyzer.ResolvePack("es").HumanRates);
+ }
+
+ [Fact]
+ public void The_result_partitions_its_own_findings_so_no_host_has_to()
+ {
+ // The regression this guards against actually happened: the flag went onto Finding and within
+ // a day the web headline counted every match while its category chips counted only the scoring
+ // ones, and the MCP server handed an agent, as evidence, matches the engine had ruled out.
+ var analyzer = new AiWritingAnalyzer();
+ var body = string.Join(" ", Enumerable.Repeat("The study examined the data carefully and reported what it found.", 160));
+
+ var r = analyzer.Analyze(body + " Furthermore, the result held.", "en");
+
+ Assert.Equal(r.Findings.Count, r.Signals.Count + r.Observations.Count);
+ Assert.All(r.Signals, f => Assert.False(f.AtHumanRate));
+ Assert.All(r.Observations, f => Assert.True(f.AtHumanRate));
+
+ // The headline number and the category tallies have to be the same number.
+ Assert.Equal(r.Signals.Count, r.CategoryScores.Sum(c => c.FindingCount));
+ }
+}
diff --git a/tools/SignsOfAI.Calibration/Program.cs b/tools/SignsOfAI.Calibration/Program.cs
index fb9a211..0710d79 100644
--- a/tools/SignsOfAI.Calibration/Program.cs
+++ b/tools/SignsOfAI.Calibration/Program.cs
@@ -29,6 +29,7 @@
string source = "";
string fetchLanguage = "en";
int count = 40, fromYear = 2018, toYear = 2020;
+string packsDir = "src/SignsOfAI.Core/Rules/Packs";
for (int i = 1; i < argv.Count; i++)
{
@@ -38,6 +39,7 @@
case "--texts": textsDir = Next(); break;
case "--out": outPath = Next(); break;
case "--record-hashes": recordHashes = true; break;
+ case "--packs": packsDir = Next(); break;
case "--source": source = Next(); break;
case "--lang": fetchLanguage = Next(); break;
case "--count": count = int.Parse(Next()); break;
@@ -50,7 +52,7 @@
string Next() => ++i < argv.Count ? argv[i] : throw new ArgumentException($"Missing value for {argv[i - 1]}");
}
-if (argv[0] is not ("run" or "fetch"))
+if (argv[0] is not ("run" or "fetch" or "thresholds"))
{
Console.Error.WriteLine($"Unknown command '{argv[0]}'. Run --help.");
return 2;
@@ -144,7 +146,8 @@
Stratum = entry.Stratum,
Score = result.OverallScore,
WordCount = result.Statistics.WordCount,
- RuleIds = [.. result.Findings.Select(f => f.RuleId)],
+ RuleIds = [.. result.Findings.Where(f => !f.AtHumanRate).Select(f => f.RuleId)],
+ MatchedRuleIds = [.. result.Findings.Select(f => f.RuleId)],
});
}
@@ -161,6 +164,36 @@
return 1;
}
+// ── `thresholds` ─────────────────────────────────────────────────────────────
+// Derives each rule's human usage rate and writes it into the packs, so the numbers the analyzer
+// runs on can be regenerated by anyone holding the corpus rather than taken on trust.
+if (argv[0] == "thresholds")
+{
+ var derived = Thresholds.Derive(samples);
+ foreach (var (language, rates) in derived.OrderBy(p => p.Key, StringComparer.Ordinal))
+ {
+ var pack = Path.Combine(packsDir, $"rules.{language}.json");
+ if (!File.Exists(pack))
+ {
+ Console.Error.WriteLine($" no pack for '{language}' at {pack}");
+ continue;
+ }
+
+ var written = Thresholds.WriteInto(pack, rates);
+ Console.WriteLine($" {language}: {written} rules given a measured rate → {pack}");
+ foreach (var (id, rate) in rates.OrderByDescending(r => r.Value))
+ Console.WriteLine($" {id,-28} {rate,5:0.00} per 1,000 words");
+ }
+
+ var (before, after, cleanBefore, cleanAfter) = Thresholds.LeaveOneOut(samples);
+ Console.WriteLine();
+ Console.WriteLine(" Held out of its own thresholds, each text keeps:");
+ Console.WriteLine($" findings per text {before:0.0} → {after:0.0} ({(after - before) / before:P0})");
+ Console.WriteLine($" texts with none {cleanBefore} → {cleanAfter} of {samples.Count}");
+ Console.WriteLine();
+ return 0;
+}
+
var calibration = Calibrator.Compute(
samples, manifest.Id, manifest.Fingerprint(), manifest.TargetFalsePositiveRate);
diff --git a/tools/SignsOfAI.Calibration/Thresholds.cs b/tools/SignsOfAI.Calibration/Thresholds.cs
new file mode 100644
index 0000000..8a38ebe
--- /dev/null
+++ b/tools/SignsOfAI.Calibration/Thresholds.cs
@@ -0,0 +1,176 @@
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using SignsOfAI.Core.Calibration;
+
+namespace SignsOfAI.Calibration;
+
+///
+/// Derives each rule's human usage rate from the corpus and writes it into the rule packs.
+///
+/// This exists because the first version of those numbers was computed in a throwaway script and
+/// pasted in, which made the packs claim something the repository could not back up. A measured
+/// threshold nobody can regenerate is a chosen threshold with better manners, and choosing thresholds
+/// is the practice this project criticises in every article it has published.
+///
+public static class Thresholds
+{
+ ///
+ /// A rule needs this many texts before its rate means anything. The ninetieth percentile of eight
+ /// samples is already close to the largest of them; below that it is one author's habit.
+ ///
+ public const int MinimumTexts = 8;
+
+ ///
The percentile of human usage a text must exceed before the rule counts as evidence.
+ public const double Percentile = 0.90;
+
+ ///
Language → rule id → hits per thousand words at .
+ public static Dictionary
> Derive(
+ IReadOnlyList samples)
+ {
+ var result = new Dictionary>(StringComparer.Ordinal);
+
+ foreach (var language in samples.Select(s => s.Language).Distinct())
+ {
+ var texts = samples.Where(s => s.Language == language).ToList();
+ var rates = new Dictionary(StringComparer.Ordinal);
+
+ foreach (var ruleId in texts.SelectMany(t => t.MatchedRuleIds).Distinct())
+ {
+ // Statistical findings are one per document by construction, so a rate per thousand
+ // words measures the document's length rather than the rule's behaviour.
+ if (ruleId.StartsWith("stat.", StringComparison.Ordinal)) continue;
+
+ if (texts.Count(t => t.MatchedRuleIds.Contains(ruleId)) < MinimumTexts) continue;
+
+ var rate = RateAt(texts, ruleId, Percentile);
+ if (rate > 0) rates[ruleId] = Math.Round(rate, 2);
+ }
+
+ if (rates.Count > 0) result[language] = rates;
+ }
+
+ return result;
+ }
+
+ ///
+ /// The out-of-sample answer to "how much noise did this remove", and the only honest one: each
+ /// text is judged against thresholds derived from the other texts, never from itself. Fitting on
+ /// the corpus and then reporting the improvement on the same corpus is how a tool reports a
+ /// number it cannot reproduce on anybody else's writing.
+ ///
+ public static (double Before, double After, int TextsCleanBefore, int TextsCleanAfter) LeaveOneOut(
+ IReadOnlyList samples)
+ {
+ double before = 0, after = 0;
+ int cleanBefore = 0, cleanAfter = 0;
+
+ foreach (var held in samples)
+ {
+ var others = samples.Where(s => s.Language == held.Language && !ReferenceEquals(s, held)).ToList();
+ var rates = new Dictionary(StringComparer.Ordinal);
+
+ foreach (var ruleId in others.SelectMany(t => t.MatchedRuleIds).Distinct())
+ {
+ if (ruleId.StartsWith("stat.", StringComparison.Ordinal)) continue;
+ if (others.Count(t => t.MatchedRuleIds.Contains(ruleId)) < MinimumTexts) continue;
+ var rate = RateAt(others, ruleId, Percentile);
+ if (rate > 0) rates[ruleId] = Math.Round(rate, 2);
+ }
+
+ int kept = 0;
+ foreach (var group in held.MatchedRuleIds.GroupBy(id => id, StringComparer.Ordinal))
+ {
+ var hits = group.Count();
+ var rate = held.WordCount > 0 ? hits / (double)held.WordCount * 1000.0 : 0;
+ if (!rates.TryGetValue(group.Key, out var threshold) || rate > threshold) kept += hits;
+ }
+
+ before += held.MatchedRuleIds.Count;
+ after += kept;
+ if (held.MatchedRuleIds.Count == 0) cleanBefore++;
+ if (kept == 0) cleanAfter++;
+ }
+
+ return (before / samples.Count, after / samples.Count, cleanBefore, cleanAfter);
+ }
+
+ ///
+ /// The rate at which is used, at the given percentile, across every text
+ /// in — including the ones where it never fires, which count as zero. A
+ /// percentile over only the texts that fired would answer "how heavily do the people who use this
+ /// word use it", and the question here is "how often does this appear in writing at all".
+ ///
+ private static double RateAt(IReadOnlyList texts, string ruleId, double percentile)
+ {
+ var rates = texts
+ .Select(t => t.WordCount > 0
+ ? t.MatchedRuleIds.Count(id => id == ruleId) / (double)t.WordCount * 1000.0
+ : 0)
+ .OrderBy(r => r)
+ .ToList();
+
+ var index = Math.Min(rates.Count - 1, (int)(rates.Count * percentile));
+ return rates[index];
+ }
+
+ ///
+ /// Writes the derived rates into a rule pack, adding humanRatePer1000 to the rules that have
+ /// one and removing it from those that no longer do — a rule that drops below
+ /// as the corpus changes must lose its threshold rather than keep a
+ /// stale one. Returns how many rules were written.
+ ///
+ /// The file is rewritten from its parsed form, so formatting is normalised once and owned by this
+ /// tool from then on. Non-ASCII is left as itself: the Spanish pack is full of it and escaping it
+ /// would make every future diff unreadable.
+ ///
+ public static int WriteInto(string packPath, IReadOnlyDictionary rates)
+ {
+ var root = JsonNode.Parse(File.ReadAllText(packPath))!.AsObject();
+ int written = 0;
+
+ foreach (var section in new[] { "lexical", "patterns" })
+ {
+ if (root[section] is not JsonArray rules) continue;
+
+ foreach (var node in rules)
+ {
+ if (node is not JsonObject rule || rule["id"]?.GetValue() is not { } id) continue;
+
+ rule.Remove("humanRatePer1000");
+ if (!rates.TryGetValue(id, out var rate)) continue;
+
+ // Placed straight after the id so it reads as a property of the rule rather than an
+ // afterthought appended to whatever the last field happened to be.
+ var rebuilt = new JsonObject();
+ foreach (var (key, value) in rule.ToList())
+ {
+ rule.Remove(key);
+ rebuilt[key] = value;
+ if (key == "id") rebuilt["humanRatePer1000"] = rate;
+ }
+
+ foreach (var (key, value) in rebuilt.ToList())
+ {
+ rebuilt.Remove(key);
+ rule[key] = value;
+ }
+
+ written++;
+ }
+ }
+
+ var options = new JsonSerializerOptions
+ {
+ WriteIndented = true,
+ Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+ };
+
+ // Newlines are written LF explicitly. The packs are LF in the repository, and letting this run
+ // on Windows rewrite them as CRLF would mark every line of a 900-line file as changed the
+ // first time anyone regenerates the thresholds.
+ var json = root.ToJsonString(options).ReplaceLineEndings("\n") + "\n";
+ File.WriteAllText(packPath, json);
+ return written;
+ }
+}