diff --git a/crates/karva/tests/it/extensions/tags/timeout.rs b/crates/karva/tests/it/extensions/tags/timeout.rs index a4e4b14a..5ba93769 100644 --- a/crates/karva/tests/it/extensions/tags/timeout.rs +++ b/crates/karva/tests/it/extensions/tags/timeout.rs @@ -282,3 +282,137 @@ def test_always_slow(): ----- stderr ----- "); } + +/// `--timeout` applies to every test that does not already carry an +/// `@karva.tags.timeout` decorator. +#[test] +fn test_cli_timeout_kills_slow_test() { + let context = TestContext::with_file( + "test.py", + r" +import time + +def test_slow(): + time.sleep(2) + ", + ); + + assert_cmd_snapshot!(context.command().arg("--timeout=0.1"), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 1 test across 1 worker + FAIL [TIME] test::test_slow + + diagnostics: + + error[test-failure]: Test `test_slow` failed + --> test.py:4:5 + | + 4 | def test_slow(): + | ^^^^^^^^^ + | + info: Test exceeded timeout of 0.1 seconds + + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + "); +} + +#[test] +fn test_cli_timeout_does_not_flag_fast_tests() { + let context = TestContext::with_file( + "test.py", + r" +def test_fast(): + assert True + ", + ); + + assert_cmd_snapshot!(context.command().arg("--timeout=60"), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test::test_fast + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + "); +} + +/// A test-level `@karva.tags.timeout` overrides the configured default. +#[test] +fn test_cli_timeout_tag_overrides_default() { + let context = TestContext::with_file( + "test.py", + r" +import time +import karva + +@karva.tags.timeout(2.0) +def test_under_tag_limit(): + time.sleep(0.3) + ", + ); + + assert_cmd_snapshot!(context.command().arg("--timeout=0.1"), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test::test_under_tag_limit + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + "); +} + +#[test] +fn test_config_timeout_kills_slow_test() { + let context = TestContext::with_files([ + ( + "pyproject.toml", + r" +[tool.karva.profile.default.test] +timeout = 0.1 + ", + ), + ( + "test.py", + r" +import time + +def test_slow(): + time.sleep(2) + ", + ), + ]); + + assert_cmd_snapshot!(context.command(), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 1 test across 1 worker + FAIL [TIME] test::test_slow + + diagnostics: + + error[test-failure]: Test `test_slow` failed + --> test.py:4:5 + | + 4 | def test_slow(): + | ^^^^^^^^^ + | + info: Test exceeded timeout of 0.1 seconds + + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + "); +} diff --git a/crates/karva_cli/src/test.rs b/crates/karva_cli/src/test.rs index 188a58da..dc1ef9b3 100644 --- a/crates/karva_cli/src/test.rs +++ b/crates/karva_cli/src/test.rs @@ -5,7 +5,7 @@ use clap::Parser; use karva_logging::{FinalStatusLevel, StatusLevel, TerminalColor}; use karva_metadata::{ CovFailUnder, CoverageOptions, MaxFail, Options, SlowTimeoutSecs, SrcOptions, TerminalOptions, - TestOptions, + TestOptions, TestTimeoutSecs, }; use crate::enums::{CovReport, NoTests, OutputFormat, RunIgnored}; @@ -114,6 +114,16 @@ pub struct SubTestCommand { #[clap(long, value_name = "SECONDS", help_heading = "Runner options")] pub slow_timeout: Option, + /// Hard per-test timeout, in seconds. + /// + /// Tests that run longer than this duration are killed and reported + /// as failures. A test-level [`@karva.tags.timeout`] decorator + /// overrides the default for that specific test. + /// + /// Accepts fractional seconds such as `--timeout=120` or `--timeout=0.5`. + #[clap(long, value_name = "SECONDS", help_heading = "Runner options")] + pub timeout: Option, + /// Update snapshots directly instead of creating pending `.snap.new` files. /// /// When set, `karva.assert_snapshot()` will write directly to `.snap` files, @@ -321,6 +331,7 @@ impl SubTestCommand { retry: self.retry, no_tests: self.no_tests.map(Into::into), slow_timeout: self.slow_timeout.map(SlowTimeoutSecs), + timeout: self.timeout.map(TestTimeoutSecs), }), coverage: Some(CoverageOptions { sources: (!self.cov.is_empty()).then(|| self.cov.clone()), diff --git a/crates/karva_metadata/src/lib.rs b/crates/karva_metadata/src/lib.rs index 2a806428..a14fd64b 100644 --- a/crates/karva_metadata/src/lib.rs +++ b/crates/karva_metadata/src/lib.rs @@ -17,6 +17,7 @@ pub use options::{ pub use pyproject::{PyProject, PyProjectError}; pub use settings::{ CovFailUnder, CoverageSettings, NoTestsMode, ProjectSettings, RunIgnoredMode, SlowTimeoutSecs, + TestTimeoutSecs, }; use crate::options::KarvaTomlError; diff --git a/crates/karva_metadata/src/options/mod.rs b/crates/karva_metadata/src/options/mod.rs index fc61ac42..0e27f4df 100644 --- a/crates/karva_metadata/src/options/mod.rs +++ b/crates/karva_metadata/src/options/mod.rs @@ -14,7 +14,7 @@ use crate::filter::FiltersetSet; use crate::max_fail::MaxFail; use crate::settings::{ CovFailUnder, CoverageSettings, NoTestsMode, ProjectSettings, RunIgnoredMode, SlowTimeoutSecs, - SrcSettings, TerminalSettings, TestSettings, + SrcSettings, TerminalSettings, TestSettings, TestTimeoutSecs, }; #[derive( @@ -277,6 +277,25 @@ pub struct TestOptions { "# )] pub slow_timeout: Option, + + /// Hard per-test timeout (in seconds). + /// + /// When set, every test that runs longer than this duration is killed + /// and reported as a failure. Tests can override the limit individually + /// with [`@karva.tags.timeout`](https://docs.karva.dev/usage/tags/timeout/), + /// which takes precedence over the configured default. + /// + /// Defaults to unset, which disables hard timeouts unless a tag is + /// applied to the test. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"null"#, + value_type = "float (seconds)", + example = r#" + timeout = 120.0 + "# + )] + pub timeout: Option, } impl TestOptions { @@ -298,6 +317,7 @@ impl TestOptions { run_ignored: RunIgnoredMode::default(), no_tests: self.no_tests.unwrap_or_default(), slow_timeout: self.slow_timeout.and_then(SlowTimeoutSecs::as_duration), + timeout: self.timeout.and_then(TestTimeoutSecs::as_duration), } } } @@ -513,7 +533,7 @@ nonsense = 42 | 4 | nonsense = 42 | ^^^^^^^^ - unknown field `nonsense`, expected one of `test-function-prefix`, `fail-fast`, `max-fail`, `try-import-fixtures`, `retry`, `no-tests`, `slow-timeout` + unknown field `nonsense`, expected one of `test-function-prefix`, `fail-fast`, `max-fail`, `try-import-fixtures`, `retry`, `no-tests`, `slow-timeout`, `timeout` " ); } @@ -611,6 +631,7 @@ max-fail = 0 ), no_tests: None, slow_timeout: None, + timeout: None, } "#); } @@ -639,6 +660,7 @@ max-fail = 0 ), no_tests: None, slow_timeout: None, + timeout: None, } "#); } @@ -700,6 +722,7 @@ retry = 2 ), no_tests: None, slow_timeout: None, + timeout: None, }, ) "#); @@ -755,6 +778,7 @@ retry = 5 ), no_tests: None, slow_timeout: None, + timeout: None, }, ) "#); diff --git a/crates/karva_metadata/src/settings.rs b/crates/karva_metadata/src/settings.rs index 2b1c2282..94babf39 100644 --- a/crates/karva_metadata/src/settings.rs +++ b/crates/karva_metadata/src/settings.rs @@ -68,6 +68,37 @@ impl Combine for SlowTimeoutSecs { } } +/// A per-test timeout expressed in seconds. +/// +/// Wraps `f64` for the same reason as [`SlowTimeoutSecs`]. Tests exceeding +/// this duration are killed and reported as failures (see +/// [`crate::settings::TestSettings::timeout`]). +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct TestTimeoutSecs(pub f64); + +impl Eq for TestTimeoutSecs {} + +impl TestTimeoutSecs { + pub fn as_duration(self) -> Option { + if self.0.is_finite() && self.0 > 0.0 { + Some(Duration::from_secs_f64(self.0)) + } else { + None + } + } +} + +impl Combine for TestTimeoutSecs { + #[inline(always)] + fn combine_with(&mut self, _other: Self) {} + + #[inline] + fn combine(self, _other: Self) -> Self { + self + } +} + /// A coverage threshold expressed as a percentage (`0..=100`). /// /// Wraps `f64` for the same reason as [`SlowTimeoutSecs`]: keeps the @@ -164,4 +195,8 @@ pub struct TestSettings { /// Threshold after which a test is flagged as slow. `None` disables /// slow-test detection entirely. pub slow_timeout: Option, + /// Hard per-test timeout. Tests that run longer than this duration are + /// killed and reported as failures. `None` disables the hard timeout + /// (tests may still set their own limit via `@karva.tags.timeout`). + pub timeout: Option, } diff --git a/crates/karva_runner/src/worker_args.rs b/crates/karva_runner/src/worker_args.rs index 315cbbc1..41c0b6d3 100644 --- a/crates/karva_runner/src/worker_args.rs +++ b/crates/karva_runner/src/worker_args.rs @@ -114,6 +114,11 @@ fn inner_cli_args(settings: &ProjectSettings, args: &SubTestCommand) -> Vec PackageRunner<'ctx, 'a> { let is_async = stmt_function_def.is_async && !crate::utils::patch_async_test_function(py, &function).unwrap_or(false); - let timeout_seconds = tags.timeout_tag().map(TimeoutTag::seconds); + let timeout_seconds = tags.timeout_tag().map(TimeoutTag::seconds).or_else(|| { + self.context + .settings() + .test() + .timeout + .map(|d| d.as_secs_f64()) + }); let run_test = || { if let Some(seconds) = timeout_seconds { return run_test_with_timeout( diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 0a4cd474..5b37aa7c 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -331,6 +331,31 @@ test-function-prefix = "test" --- +### `timeout` + +Hard per-test timeout (in seconds). + +When set, every test that runs longer than this duration is killed +and reported as a failure. Tests can override the limit individually +with [`@karva.tags.timeout`](https://docs.karva.dev/usage/tags/timeout/), +which takes precedence over the configured default. + +Defaults to unset, which disables hard timeouts unless a tag is +applied to the test. + +**Default value**: `null` + +**Type**: `float (seconds)` + +**Example usage** (`pyproject.toml`): + +```toml +[tool.karva.profile.default.test] +timeout = 120.0 +``` + +--- + ### `try-import-fixtures` When set, we will try to import functions in each test file as well as parsing the ast to find them. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 6253b571..d58a796f 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -126,6 +126,9 @@ karva test [OPTIONS] [PATH]...
  • skip: Additionally display skipped test results
  • all: Display all test result statuses
    • --test-prefix test-prefix

    The prefix of the test functions

    +
    --timeout seconds

    Hard per-test timeout, in seconds.

    +

    Tests that run longer than this duration are killed and reported as failures. A test-level [@karva.tags.timeout] decorator overrides the default for that specific test.

    +

    Accepts fractional seconds such as --timeout=120 or --timeout=0.5.

    --try-import-fixtures

    When set, we will try to import functions in each test file as well as parsing the ast to find them.

    This is often slower, so it is not recommended for most projects.

    --verbose, -v

    Use verbose output (or -vv and -vvv for more verbose output)

    diff --git a/docs/usage/tags/timeout.md b/docs/usage/tags/timeout.md index 6d0e2c28..f65d70ff 100644 --- a/docs/usage/tags/timeout.md +++ b/docs/usage/tags/timeout.md @@ -13,6 +13,21 @@ def test_function(): The threshold accepts fractional seconds (`@karva.tags.timeout(0.5)`). +## Configuring a default timeout + +Use the `timeout` setting (or `--timeout=SECONDS` on the CLI) to apply the same hard limit to every test in the project: + +```bash +karva test --timeout=120 +``` + +```toml +[tool.karva.profile.default.test] +timeout = 120 +``` + +A test-level `@karva.tags.timeout` always wins over the configured default, so individual tests can opt into a longer or shorter window. + ## Sync vs async tests Sync tests are submitted to a single-worker `concurrent.futures.ThreadPoolExecutor`. When the limit elapses, a `TimeoutError` is raised against the test and the worker thread is abandoned — Python has no safe way to interrupt arbitrary code, so any side effects already started will continue. If a test repeatedly times out and leaks resources, fix the test rather than the timeout.