diff --git a/Cargo.lock b/Cargo.lock index 7432f0ca..f629419a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1115,6 +1115,7 @@ dependencies = [ "karva_metadata", "karva_version", "ruff_db", + "serde_json", ] [[package]] @@ -1286,6 +1287,7 @@ dependencies = [ "karva_project", "karva_static", "karva_version", + "serde_json", "tracing", "uuid", "which", diff --git a/crates/karva/tests/it/configuration/mod.rs b/crates/karva/tests/it/configuration/mod.rs index 9f553a30..62b2f2b6 100644 --- a/crates/karva/tests/it/configuration/mod.rs +++ b/crates/karva/tests/it/configuration/mod.rs @@ -1,3 +1,4 @@ +mod overrides; mod profile; use insta_cmd::assert_cmd_snapshot; diff --git a/crates/karva/tests/it/configuration/overrides.rs b/crates/karva/tests/it/configuration/overrides.rs new file mode 100644 index 00000000..c1a6d1fa --- /dev/null +++ b/crates/karva/tests/it/configuration/overrides.rs @@ -0,0 +1,327 @@ +use insta_cmd::assert_cmd_snapshot; + +use crate::common::TestContext; + +#[test] +fn override_retries_for_tagged_test() { + let context = TestContext::with_files([ + ( + "karva.toml", + r#" +[[profile.default.overrides]] +filter = "tag(network)" +retries = 2 +"#, + ), + ( + "test.py", + r" +import karva + +counter = 0 + +@karva.tags.network +def test_flaky(): + global counter + counter += 1 + assert counter >= 2 +", + ), + ]); + + assert_cmd_snapshot!(context.command_no_parallel(), @r" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + TRY 1 FAIL [TIME] test::test_flaky + TRY 2 PASS [TIME] test::test_flaky + ──────────── + Summary [TIME] 1 test run: 1 passed (1 flaky), 0 skipped + FLAKY 2/3 [TIME] test::test_flaky + + ----- stderr ----- + "); +} + +/// A failing test that does not match any override should fall through to +/// the profile-level `retry` value. +#[test] +fn override_retries_does_not_apply_to_non_matching_test() { + let context = TestContext::with_files([ + ( + "karva.toml", + r#" +[profile.default.test] +retry = 0 + +[[profile.default.overrides]] +filter = "tag(network)" +retries = 5 +"#, + ), + ( + "test.py", + r" +def test_flaky(): + assert False +", + ), + ]); + + assert_cmd_snapshot!(context.command_no_parallel(), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 1 test across 1 worker + FAIL [TIME] test::test_flaky + + diagnostics: + + error[test-failure]: Test `test_flaky` failed + --> test.py:2:5 + | + 2 | def test_flaky(): + | ^^^^^^^^^^ + | + info: Test failed here + --> test.py:3:5 + | + 3 | assert False + | ^^^^^^^^^^^^ + | + + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + "); +} + +/// `retries = 0` on a matching override defeats a higher profile-level +/// `retry` value. +#[test] +fn override_retries_zero_disables_retries_for_matching_test() { + let context = TestContext::with_files([ + ( + "karva.toml", + r#" +[profile.default.test] +retry = 5 + +[[profile.default.overrides]] +filter = "tag(unit)" +retries = 0 +"#, + ), + ( + "test.py", + r" +import karva + +@karva.tags.unit +def test_unit(): + assert False +", + ), + ]); + + assert_cmd_snapshot!(context.command_no_parallel(), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 1 test across 1 worker + FAIL [TIME] test::test_unit + + diagnostics: + + error[test-failure]: Test `test_unit` failed + --> test.py:5:5 + | + 5 | def test_unit(): + | ^^^^^^^^^ + | + info: Test failed here + --> test.py:6:5 + | + 6 | assert False + | ^^^^^^^^^^^^ + | + + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + "); +} + +#[test] +fn override_retries_invalid_filter_errors_at_load() { + let context = TestContext::with_files([ + ( + "karva.toml", + r#" +[[profile.default.overrides]] +filter = "tag(" +retries = 1 +"#, + ), + ("test.py", "def test_a(): pass"), + ]); + + assert_cmd_snapshot!(context.command_no_parallel(), @r#" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Karva failed + Cause: /karva.toml is not a valid `karva.toml`: TOML parse error at line 3, column 10 + | + 3 | filter = "tag(" + | ^^^^^^ + expected a matcher body in filter expression `tag(` + + Cause: TOML parse error at line 3, column 10 + | + 3 | filter = "tag(" + | ^^^^^^ + expected a matcher body in filter expression `tag(` + "#); +} + +/// A matching override's `timeout` overrides the profile-level value. +#[test] +fn override_timeout_kills_matching_test() { + let context = TestContext::with_files([ + ( + "karva.toml", + r#" +[[profile.default.overrides]] +filter = "tag(slow)" +timeout = 0.1 +"#, + ), + ( + "test.py", + r" +import time +import karva + +@karva.tags.slow +def test_slow(): + time.sleep(2) +", + ), + ]); + + assert_cmd_snapshot!(context.command_no_parallel(), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 1 test across 1 worker + FAIL [TIME] test::test_slow + + diagnostics: + + error[test-failure]: Test `test_slow` failed + --> test.py:6:5 + | + 6 | def test_slow(): + | ^^^^^^^^^ + | + info: Test exceeded timeout of 0.1 seconds + + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + "); +} + +/// `timeout = 0` on a matching override disables the hard limit even when +/// the profile sets one. +#[test] +fn override_timeout_zero_disables_profile_timeout() { + let context = TestContext::with_files([ + ( + "karva.toml", + r#" +[profile.default.test] +timeout = 0.1 + +[[profile.default.overrides]] +filter = "tag(integration)" +timeout = 0 +"#, + ), + ( + "test.py", + r" +import time +import karva + +@karva.tags.integration +def test_long_lived(): + time.sleep(0.3) +", + ), + ]); + + assert_cmd_snapshot!(context.command_no_parallel(), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test::test_long_lived + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + "); +} + +/// A matching override's `slow-timeout` flags only the matched test as +/// slow. +#[test] +fn override_slow_timeout_flags_matching_test() { + let context = TestContext::with_files([ + ( + "karva.toml", + r#" +[[profile.default.overrides]] +filter = "tag(integration)" +slow-timeout = 0.001 +"#, + ), + ( + "test.py", + r" +import time +import karva + +@karva.tags.integration +def test_integration(): + time.sleep(0.05) + +def test_unit(): + pass +", + ), + ]); + + assert_cmd_snapshot!( + context.command_no_parallel().arg("--status-level=slow"), + @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 2 tests across 1 worker + SLOW [TIME] test::test_integration + ──────────── + Summary [TIME] 2 tests run: 2 passed, 0 skipped, 1 slow + + ----- stderr ----- + " + ); +} diff --git a/crates/karva_cli/Cargo.toml b/crates/karva_cli/Cargo.toml index 14f76bd5..93b0ebac 100644 --- a/crates/karva_cli/Cargo.toml +++ b/crates/karva_cli/Cargo.toml @@ -17,6 +17,7 @@ karva_version = { workspace = true } camino = { workspace = true } clap = { workspace = true, features = ["wrap_help", "string", "env"] } ruff_db = { workspace = true } +serde_json = { workspace = true } [lints] workspace = true diff --git a/crates/karva_cli/src/test.rs b/crates/karva_cli/src/test.rs index e5952ddd..d9b0c0e0 100644 --- a/crates/karva_cli/src/test.rs +++ b/crates/karva_cli/src/test.rs @@ -4,8 +4,8 @@ use camino::Utf8PathBuf; use clap::Parser; use karva_logging::{FinalStatusLevel, StatusLevel, TerminalColor}; use karva_metadata::{ - CovFailUnder, CoverageOptions, MaxFail, Options, SlowTimeoutSecs, SrcOptions, TerminalOptions, - TestOptions, TestTimeoutSecs, + CovFailUnder, CoverageOptions, MaxFail, Options, OverrideOptions, SlowTimeoutSecs, SrcOptions, + TerminalOptions, TestOptions, TestTimeoutSecs, }; use crate::enums::{CovReport, NoTests, OutputFormat, RunIgnored}; @@ -219,6 +219,21 @@ pub struct SubTestCommand { /// for direct use. #[clap(long, hide = true, value_name = "PATH")] pub cov_data_file: Option, + + /// Internal: a single per-test override entry, encoded as JSON. + /// + /// Workers receive overrides from the main process via this flag, one + /// entry per occurrence. Users configure overrides via + /// `[[profile..overrides]]` in `karva.toml` rather than this + /// flag. + #[clap( + long = "override-json", + hide = true, + value_name = "JSON", + action = clap::ArgAction::Append, + value_parser = parse_override_json, + )] + pub override_json: Vec, } #[derive(Debug, Parser)] @@ -355,6 +370,7 @@ impl SubTestCommand { fail_under: self.cov_fail_under.map(CovFailUnder), disabled: self.no_cov.then_some(true), }), + overrides: self.override_json, } } } @@ -369,6 +385,15 @@ impl TestCommand { } } +/// Parse a `--override-json` argument from its JSON encoding. +/// +/// The main process forwards each `[[profile..overrides]]` entry to +/// the worker via this flag, so the JSON must round-trip the +/// [`OverrideOptions`] schema (including filter validation). +fn parse_override_json(raw: &str) -> Result { + serde_json::from_str(raw).map_err(|err| err.to_string()) +} + /// Parse and validate a `--cov-fail-under=N` argument. /// /// Accepts any finite percentage in `0..=100`. diff --git a/crates/karva_metadata/src/filter.rs b/crates/karva_metadata/src/filter.rs index 0354a676..6694c758 100644 --- a/crates/karva_metadata/src/filter.rs +++ b/crates/karva_metadata/src/filter.rs @@ -2,6 +2,7 @@ use std::fmt; use globset::{Glob, GlobMatcher}; use regex::Regex; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; /// How the body of a predicate should be compared against the value it's @@ -92,6 +93,64 @@ impl Filterset { } } +/// A filter expression that has been validated at construction time. +/// +/// Wraps the raw string and a compiled [`Filterset`] so callers can +/// evaluate the filter without re-parsing or risking a panic. Equality +/// is defined on the raw string so structural comparisons (used by the +/// `Options` derive) remain meaningful. +#[derive(Debug, Clone)] +pub struct ValidatedFilter { + raw: String, + compiled: Filterset, +} + +impl ValidatedFilter { + pub fn new(raw: String) -> Result { + let compiled = Filterset::new(&raw)?; + Ok(Self { raw, compiled }) + } + + pub fn as_str(&self) -> &str { + &self.raw + } + + pub fn matches(&self, ctx: &EvalContext<'_>) -> bool { + self.compiled.matches(ctx) + } + + pub fn filterset(&self) -> &Filterset { + &self.compiled + } +} + +impl PartialEq for ValidatedFilter { + fn eq(&self, other: &Self) -> bool { + self.raw == other.raw + } +} + +impl Eq for ValidatedFilter {} + +impl Serialize for ValidatedFilter { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.raw.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for ValidatedFilter { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let raw = String::deserialize(deserializer)?; + Self::new(raw).map_err(serde::de::Error::custom) + } +} + /// A set of filterset expressions combined with OR semantics (matches if any /// filter matches). An empty set matches everything. #[derive(Debug, Clone, Default)] diff --git a/crates/karva_metadata/src/lib.rs b/crates/karva_metadata/src/lib.rs index a14fd64b..9d66e84b 100644 --- a/crates/karva_metadata/src/lib.rs +++ b/crates/karva_metadata/src/lib.rs @@ -11,13 +11,13 @@ mod settings; pub use max_fail::MaxFail; pub use options::{ - Config, CovReport, CoverageOptions, DEFAULT_PROFILE, Options, OutputFormat, + Config, CovReport, CoverageOptions, DEFAULT_PROFILE, Options, OutputFormat, OverrideOptions, ProjectOptionsOverrides, SrcOptions, TerminalOptions, TestOptions, UnknownProfile, }; pub use pyproject::{PyProject, PyProjectError}; pub use settings::{ - CovFailUnder, CoverageSettings, NoTestsMode, ProjectSettings, RunIgnoredMode, SlowTimeoutSecs, - TestTimeoutSecs, + CovFailUnder, CoverageSettings, NoTestsMode, OverrideSettings, ProjectSettings, RunIgnoredMode, + SlowTimeoutSecs, TestTimeoutSecs, }; use crate::options::KarvaTomlError; diff --git a/crates/karva_metadata/src/options/mod.rs b/crates/karva_metadata/src/options/mod.rs index 0e27f4df..b45a3cfe 100644 --- a/crates/karva_metadata/src/options/mod.rs +++ b/crates/karva_metadata/src/options/mod.rs @@ -10,16 +10,14 @@ use serde::{Deserialize, Serialize}; pub use config::{Config, DEFAULT_PROFILE, KarvaTomlError, UnknownProfile}; pub use overrides::ProjectOptionsOverrides; -use crate::filter::FiltersetSet; +use crate::filter::{FiltersetSet, ValidatedFilter}; use crate::max_fail::MaxFail; use crate::settings::{ - CovFailUnder, CoverageSettings, NoTestsMode, ProjectSettings, RunIgnoredMode, SlowTimeoutSecs, - SrcSettings, TerminalSettings, TestSettings, TestTimeoutSecs, + CovFailUnder, CoverageSettings, NoTestsMode, OverrideSettings, ProjectSettings, RunIgnoredMode, + SlowTimeoutSecs, SrcSettings, TerminalSettings, TestSettings, TestTimeoutSecs, }; -#[derive( - Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, OptionsMetadata, Combine, -)] +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, OptionsMetadata)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] pub struct Options { #[serde(skip_serializing_if = "Option::is_none")] @@ -34,6 +32,29 @@ pub struct Options { #[serde(skip_serializing_if = "Option::is_none")] #[option_group] pub coverage: Option, + + /// Per-test configuration overrides. + /// + /// Each entry pairs a [filter expression](#filter) with one or more + /// option overrides. The first override whose filter matches the + /// running test wins for any given option. Fields left unset on a + /// matching override fall through to the next match (or the + /// profile-level default). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub overrides: Vec, +} + +impl Combine for Options { + fn combine_with(&mut self, other: Self) { + Combine::combine_with(&mut self.src, other.src); + Combine::combine_with(&mut self.terminal, other.terminal); + Combine::combine_with(&mut self.test, other.test); + Combine::combine_with(&mut self.coverage, other.coverage); + // Overrides obey "first match wins"; higher-precedence entries + // (i.e. those from `self`) must come first, so prepend rather + // than using the default `Vec::combine_with` which appends. + self.overrides.extend(other.overrides); + } } impl Options { @@ -43,6 +64,54 @@ impl Options { src: self.src.clone().unwrap_or_default().to_settings(), test: self.test.clone().unwrap_or_default().to_settings(), coverage: self.coverage.clone().unwrap_or_default().to_settings(), + overrides: self + .overrides + .iter() + .map(OverrideOptions::to_settings) + .collect(), + } + } +} + +/// A single per-test override entry. +/// +/// Mirrors `[[profile..overrides]]` in `karva.toml`. Each override +/// pairs a filter expression with one or more option fields to apply when +/// the filter matches a given test. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct OverrideOptions { + /// A filter expression evaluated against each test. Tests whose name + /// or tags match the expression pick up this override's settings. + pub filter: ValidatedFilter, + + /// Number of times to retry a matching test before giving up. Mirrors + /// the profile-level [`retry`](#retry) field. + #[serde(skip_serializing_if = "Option::is_none")] + pub retries: Option, + + /// Hard per-test timeout, in seconds, applied to matching tests. + /// Mirrors the profile-level [`timeout`](#timeout) field. A value of + /// `0` (or any non-positive value) disables the hard timeout for the + /// matching test. + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + + /// Threshold (in seconds) above which a matching test is flagged as + /// slow. Mirrors the profile-level + /// [`slow-timeout`](#slow-timeout) field. A non-positive value + /// disables slow tracking for the matching test. + #[serde(skip_serializing_if = "Option::is_none")] + pub slow_timeout: Option, +} + +impl OverrideOptions { + pub(crate) fn to_settings(&self) -> OverrideSettings { + OverrideSettings { + filter: self.filter.clone(), + retries: self.retries, + timeout: self.timeout, + slow_timeout: self.slow_timeout, } } } @@ -1012,4 +1081,205 @@ nonsense = 1 " ); } + + #[test] + fn parse_overrides_section() { + let toml = r#" +[[profile.default.overrides]] +filter = "tag(network)" +retries = 5 + +[[profile.default.overrides]] +filter = "tag(unit)" +retries = 0 +"#; + let resolved = Config::from_toml_str(toml) + .expect("parse") + .resolve_profile(None) + .expect("resolves"); + let overrides = resolved.overrides; + assert_eq!(overrides.len(), 2); + assert_eq!(overrides[0].filter.as_str(), "tag(network)"); + assert_eq!(overrides[0].retries, Some(5)); + assert_eq!(overrides[1].filter.as_str(), "tag(unit)"); + assert_eq!(overrides[1].retries, Some(0)); + } + + /// Named profile entries layer on top of the default profile's + /// overrides — both lists end up in the resolved options. + #[test] + fn resolve_profile_appends_named_overrides_on_top_of_default() { + let toml = r#" +[[profile.default.overrides]] +filter = "tag(network)" +retries = 3 + +[[profile.ci.overrides]] +filter = "tag(slow)" +retries = 1 +"#; + let resolved = Config::from_toml_str(toml) + .expect("parse") + .resolve_profile(Some("ci")) + .expect("resolves"); + let raw: Vec<&str> = resolved + .overrides + .iter() + .map(|o| o.filter.as_str()) + .collect(); + assert_eq!(raw, vec!["tag(slow)", "tag(network)"]); + } + + #[test] + fn from_toml_str_rejects_invalid_override_filter() { + let toml = r#" +[[profile.default.overrides]] +filter = "tag(" +retries = 1 +"#; + let err = Config::from_toml_str(toml).expect_err("invalid filter"); + assert!( + err.to_string().contains("expected a matcher body"), + "expected filter parse error in: {err}" + ); + } + + #[test] + fn to_settings_compiles_overrides() { + let toml = r#" +[[profile.default.overrides]] +filter = "tag(network)" +retries = 5 +"#; + let resolved = Config::from_toml_str(toml) + .expect("parse") + .resolve_profile(None) + .expect("resolves"); + let settings = resolved.to_settings(); + let overrides = settings.overrides(); + assert_eq!(overrides.len(), 1); + assert_eq!(overrides[0].retries, Some(5)); + let ctx = crate::filter::EvalContext { + test_name: "test::foo", + tags: &["network"], + }; + assert!(overrides[0].matches(&ctx)); + let other = crate::filter::EvalContext { + test_name: "test::bar", + tags: &["unit"], + }; + assert!(!overrides[0].matches(&other)); + } + + #[test] + fn retry_for_picks_first_matching_override() { + let toml = r#" +[profile.default.test] +retry = 1 + +[[profile.default.overrides]] +filter = "tag(network)" +retries = 5 + +[[profile.default.overrides]] +filter = "tag(unit)" +retries = 0 +"#; + let resolved = Config::from_toml_str(toml) + .expect("parse") + .resolve_profile(None) + .expect("resolves"); + let settings = resolved.to_settings(); + let net = crate::filter::EvalContext { + test_name: "test::a", + tags: &["network"], + }; + let unit = crate::filter::EvalContext { + test_name: "test::b", + tags: &["unit"], + }; + let other = crate::filter::EvalContext { + test_name: "test::c", + tags: &[], + }; + assert_eq!(settings.retry_for(&net), 5); + assert_eq!(settings.retry_for(&unit), 0); + assert_eq!(settings.retry_for(&other), 1); + } + + #[test] + fn timeout_for_picks_first_matching_override() { + let toml = r#" +[profile.default.test] +timeout = 30.0 + +[[profile.default.overrides]] +filter = "tag(slow)" +timeout = 300.0 + +[[profile.default.overrides]] +filter = "tag(unit)" +timeout = 0 +"#; + let resolved = Config::from_toml_str(toml) + .expect("parse") + .resolve_profile(None) + .expect("resolves"); + let settings = resolved.to_settings(); + let slow = crate::filter::EvalContext { + test_name: "test::a", + tags: &["slow"], + }; + let unit = crate::filter::EvalContext { + test_name: "test::b", + tags: &["unit"], + }; + let other = crate::filter::EvalContext { + test_name: "test::c", + tags: &[], + }; + assert_eq!( + settings.timeout_for(&slow), + Some(std::time::Duration::from_secs(300)) + ); + // `timeout = 0` on a matching override disables the hard limit. + assert_eq!(settings.timeout_for(&unit), None); + assert_eq!( + settings.timeout_for(&other), + Some(std::time::Duration::from_secs(30)) + ); + } + + #[test] + fn slow_timeout_for_picks_first_matching_override() { + let toml = r#" +[profile.default.test] +slow-timeout = 1.0 + +[[profile.default.overrides]] +filter = "tag(integration)" +slow-timeout = 30.0 +"#; + let resolved = Config::from_toml_str(toml) + .expect("parse") + .resolve_profile(None) + .expect("resolves"); + let settings = resolved.to_settings(); + let integration = crate::filter::EvalContext { + test_name: "test::a", + tags: &["integration"], + }; + let other = crate::filter::EvalContext { + test_name: "test::b", + tags: &[], + }; + assert_eq!( + settings.slow_timeout_for(&integration), + Some(std::time::Duration::from_secs(30)) + ); + assert_eq!( + settings.slow_timeout_for(&other), + Some(std::time::Duration::from_secs(1)) + ); + } } diff --git a/crates/karva_metadata/src/settings.rs b/crates/karva_metadata/src/settings.rs index 94babf39..f1d4fd8f 100644 --- a/crates/karva_metadata/src/settings.rs +++ b/crates/karva_metadata/src/settings.rs @@ -4,7 +4,7 @@ use karva_combine::Combine; use karva_logging::{FinalStatusLevel, StatusLevel}; use serde::{Deserialize, Serialize}; -use crate::filter::FiltersetSet; +use crate::filter::{EvalContext, FiltersetSet, ValidatedFilter}; use crate::max_fail::MaxFail; use crate::options::{CovReport, OutputFormat}; @@ -127,6 +127,23 @@ pub struct ProjectSettings { pub(crate) src: SrcSettings, pub(crate) test: TestSettings, pub(crate) coverage: CoverageSettings, + pub(crate) overrides: Vec, +} + +/// A compiled per-test override applied when its [filter](Self::filter) +/// matches the running test. +#[derive(Debug, Clone)] +pub struct OverrideSettings { + pub filter: ValidatedFilter, + pub retries: Option, + pub timeout: Option, + pub slow_timeout: Option, +} + +impl OverrideSettings { + pub fn matches(&self, ctx: &EvalContext<'_>) -> bool { + self.filter.matches(ctx) + } } impl ProjectSettings { @@ -150,6 +167,55 @@ impl ProjectSettings { self.test.max_fail } + pub fn overrides(&self) -> &[OverrideSettings] { + &self.overrides + } + + /// Find the first matching override that sets a value for `field`. + fn first_matching_override( + &self, + ctx: &EvalContext<'_>, + field: impl Fn(&OverrideSettings) -> Option, + ) -> Option { + self.overrides + .iter() + .find_map(|ovr| ovr.matches(ctx).then(|| field(ovr)).flatten()) + } + + /// Resolve the retry budget for a single test. + /// + /// Walks through the configured overrides in order; the first match + /// with `retries` set wins. Falls back to the profile-level + /// [`TestSettings::retry`] when no override matches. + pub fn retry_for(&self, ctx: &EvalContext<'_>) -> u32 { + self.first_matching_override(ctx, |ovr| ovr.retries) + .unwrap_or(self.test.retry) + } + + /// Resolve the hard per-test timeout for a single test. + /// + /// First match wins. A matching override with a non-positive + /// `timeout` disables the hard limit for that test even when the + /// profile sets one. + pub fn timeout_for(&self, ctx: &EvalContext<'_>) -> Option { + if let Some(secs) = self.first_matching_override(ctx, |ovr| ovr.timeout) { + return secs.as_duration(); + } + self.test.timeout + } + + /// Resolve the slow-test threshold for a single test. + /// + /// First match wins. A matching override with a non-positive value + /// disables slow tracking for that test even when the profile sets a + /// threshold. + pub fn slow_timeout_for(&self, ctx: &EvalContext<'_>) -> Option { + if let Some(secs) = self.first_matching_override(ctx, |ovr| ovr.slow_timeout) { + return secs.as_duration(); + } + self.test.slow_timeout + } + pub fn set_filter(&mut self, filter: FiltersetSet) { self.test.filter = filter; } diff --git a/crates/karva_runner/Cargo.toml b/crates/karva_runner/Cargo.toml index 322e73bb..c360d7d6 100644 --- a/crates/karva_runner/Cargo.toml +++ b/crates/karva_runner/Cargo.toml @@ -28,6 +28,7 @@ crossbeam-channel = { workspace = true } ctrlc = { workspace = true } fastrand = { workspace = true } ignore = { workspace = true } +serde_json = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } which = { workspace = true } diff --git a/crates/karva_runner/src/worker_args.rs b/crates/karva_runner/src/worker_args.rs index 41c0b6d3..a6739607 100644 --- a/crates/karva_runner/src/worker_args.rs +++ b/crates/karva_runner/src/worker_args.rs @@ -133,5 +133,16 @@ fn inner_cli_args(settings: &ProjectSettings, args: &SubTestCommand) -> Vec PackageRunner<'ctx, 'a> { &self, test_name: &QualifiedTestName, total_duration: std::time::Duration, + threshold: Option, ) { - if let Some(threshold) = self.context.settings().test().slow_timeout + if let Some(threshold) = threshold && total_duration > threshold { self.context.register_slow_test(test_name, total_duration); @@ -515,13 +516,19 @@ impl<'ctx, 'a> PackageRunner<'ctx, 'a> { snapshot_test_name, ); + let custom_tag_names = tags.custom_tag_names(); + let qualified_name_str = qualified_test_name.to_string(); + let eval_ctx = karva_metadata::filter::EvalContext { + test_name: &qualified_name_str, + tags: &custom_tag_names, + }; + let is_async = stmt_function_def.is_async && !crate::utils::patch_async_test_function(py, &function).unwrap_or(false); let timeout_seconds = tags.timeout_tag().map(TimeoutTag::seconds).or_else(|| { self.context .settings() - .test() - .timeout + .timeout_for(&eval_ctx) .map(|d| d.as_secs_f64()) }); let run_test = || { @@ -550,7 +557,7 @@ impl<'ctx, 'a> PackageRunner<'ctx, 'a> { } }; - let configured_retries = self.context.settings().test().retry; + let configured_retries = self.context.settings().retry_for(&eval_ctx); let RetryOutcome { test_result, attempt, @@ -567,7 +574,11 @@ impl<'ctx, 'a> PackageRunner<'ctx, 'a> { }; let total_duration = start_time.elapsed(); - self.maybe_register_slow(&qualified_test_name, total_duration); + self.maybe_register_slow( + &qualified_test_name, + total_duration, + self.context.settings().slow_timeout_for(&eval_ctx), + ); let passed = if was_retried { let passed_on = attempt; diff --git a/docs/usage/failure-handling/retries.md b/docs/usage/failure-handling/retries.md index 52a0af18..9d3d6bc6 100644 --- a/docs/usage/failure-handling/retries.md +++ b/docs/usage/failure-handling/retries.md @@ -25,6 +25,37 @@ karva test --retry 3 --status-level=retry --final-status-level=retry The summary line then includes a `N retried` counter so flake patterns are visible at a glance. +## Per-test retry overrides + +Profile-level `retry` applies to every test. To grant a flakier subset more attempts (or fewer) without changing the global default, define one or more `[[profile..overrides]]` entries. Each entry pairs a [filter expression](../../configuration/configuration.md) with one or more option fields; the first matching override wins. + +```toml +[profile.default.test] +retry = 1 + +[[profile.default.overrides]] +filter = "tag(network)" +retries = 5 + +[[profile.default.overrides]] +filter = "tag(unit)" +retries = 0 +``` + +In this example tests tagged `network` retry up to five times, tests tagged `unit` never retry, and everything else falls back to `retry = 1`. Overrides defined in a named profile (`[[profile.ci.overrides]]`) take precedence over those defined under `default`. + +The same `[[profile..overrides]]` block also supports `timeout` and `slow-timeout` fields, mirroring the [profile-level timeout](../../configuration/configuration.md) and slow-test threshold. A matching override with a non-positive value disables the corresponding limit for that test, even when the profile sets one. + +```toml +[profile.default.test] +timeout = 30.0 + +[[profile.default.overrides]] +filter = "tag(integration)" +timeout = 300.0 +slow-timeout = 30.0 +``` + ## Detecting attempts from inside a test Tests can read `KARVA_ATTEMPT` (1-indexed) and `KARVA_TOTAL_ATTEMPTS` (`retries + 1`) from the environment: