From 1301f304667bd38e8de278a8e7ea16c91266c479 Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Thu, 30 Apr 2026 10:19:07 -0600 Subject: [PATCH] Add ability to Note tests with known Issues (flakiness) --- docs/api.md | 1 + docs/examples.md | 20 ++++++++++++++++++++ trial.go | 30 +++++++++++++++++++++++++----- trial_test.go | 23 +++++++++++++++++++++++ 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/docs/api.md b/docs/api.md index cc9a2b7..57a8fb5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -43,6 +43,7 @@ type CompareFunc func(actual, expected interface{}) (equal bool, differences str | `.Comparer(fn)` | Set custom comparison function | | `.Timeout(d)` | Set max duration per case | | `.Parallel()` | Enable parallel execution for subtests (use with `SubTest()`) | +| `.KnownIssue(reason)` | Add message for known flaky tests | ## Case Fields diff --git a/docs/examples.md b/docs/examples.md index 63687cc..905aa12 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -364,6 +364,26 @@ trial.New(fn, cases). --- +## Known Issue Visibility + +Use `KnownIssue()` to annotate failures with a warning reason when a test is known to be flaky. This is visibility only: failures still fail. + +```go +trial.New(fn, cases). + KnownIssue("tracked flaky behavior - TODO fix #123"). + SubTest(t) +``` + +When a case fails, output includes a suffix like: + +```text +FAIL: "case name" ... (known issue: tracked flaky behavior - TODO fix #123) +``` + +Passing cases are unchanged. + +--- + ## Complete Example Copy-paste-ready test file: diff --git a/trial.go b/trial.go index d7aa30f..0ec4211 100644 --- a/trial.go +++ b/trial.go @@ -48,6 +48,14 @@ func colorGreen(s string) string { return s } +// colorYellow wraps text in yellow ANSI color codes if colors are enabled. +func colorYellow(s string) string { + if colorEnabled { + return "\033[33m" + s + "\033[0m" + } + return s +} + type ( // TestFunc a wrapper function used to setup the method being tested. TestFunc func(in Input) (result interface{}, err error) @@ -70,11 +78,12 @@ type Comparer interface { // Trial framework used to test different logical states type Trial[In any, Out any] struct { - cases map[string]Case[In, Out] - testFn testFunc[In, Out] - equalFn CompareFunc - timeout time.Duration - parallel bool + cases map[string]Case[In, Out] + testFn testFunc[In, Out] + equalFn CompareFunc + timeout time.Duration + parallel bool + knownIssueReason string } // Cases made during the trial @@ -126,6 +135,14 @@ func (t *Trial[In, Out]) Parallel() *Trial[In, Out] { return t } +// KnownIssue marks this trial run with a known-issue reason. +// If a case fails, the reason is appended to improve visibility. +// This does not change pass/fail behavior. +func (t *Trial[In, Out]) KnownIssue(reason string) *Trial[In, Out] { + t.knownIssueReason = reason + return t +} + // SubTest runs all cases as individual subtests func (t *Trial[In, Out]) SubTest(tst testing.TB) { if h, ok := tst.(tHelper); ok { @@ -220,6 +237,9 @@ func (t *Trial[In, Out]) testCase(msg string, test Case[In, Out]) result { result.pass("PASS: %q", msg) } } + if !result.Success && t.knownIssueReason != "" { + result.Message += colorYellow(fmt.Sprintf(" (known issue: %s)", t.knownIssueReason)) + } return *result } diff --git a/trial_test.go b/trial_test.go index ebbb35a..78d6e83 100644 --- a/trial_test.go +++ b/trial_test.go @@ -335,6 +335,29 @@ func TestParallel(t *testing.T) { New(fn, cases).Parallel().SubTest(t) } +func TestKnownIssue(t *testing.T) { + + fn := func(in int) (int, error) { + return in * 2, nil + } + cases := map[string]Case[int, int]{ + "mismatch": {Input: 2, Expected: 999}, + "match": {Input: 2, Expected: 4}, + } + testRunner := New(fn, cases).KnownIssue("Issue #1234") + + for testName, test := range cases { + result := testRunner.testCase(testName, test) + if result.Success && strings.Contains(result.Message, "Issue #1234") { + t.Logf("FAIL: Unexpecte Known issue %v", result.Message) + t.Fail() + } else if !result.Success && !strings.Contains(result.Message, "Issue #1234") { + t.Logf("FAIL: Expected Known Issue in %v", result.Message) + t.Fail() + } + } +} + func TestColorDiagnostics(t *testing.T) { if colorEnabled { t.Log("Color Enabled " + colorGreen("GREEN") + " " + colorRed("RED"))