Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 134 additions & 0 deletions crates/karva/tests/it/extensions/tags/timeout.rs
Original file line number Diff line number Diff line change
Expand Up @@ -282,3 +282,137 @@ def test_always_slow():
----- stderr -----
");
}

/// `--timeout` applies to every test that does not already carry an
/// `@karva.tags.timeout` decorator.
#[test]
fn test_cli_timeout_kills_slow_test() {
let context = TestContext::with_file(
"test.py",
r"
import time

def test_slow():
time.sleep(2)
",
);

assert_cmd_snapshot!(context.command().arg("--timeout=0.1"), @"
success: false
exit_code: 1
----- stdout -----
Starting 1 test across 1 worker
FAIL [TIME] test::test_slow

diagnostics:

error[test-failure]: Test `test_slow` failed
--> test.py:4:5
|
4 | def test_slow():
| ^^^^^^^^^
|
info: Test exceeded timeout of 0.1 seconds

────────────
Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped

----- stderr -----
");
}

#[test]
fn test_cli_timeout_does_not_flag_fast_tests() {
let context = TestContext::with_file(
"test.py",
r"
def test_fast():
assert True
",
);

assert_cmd_snapshot!(context.command().arg("--timeout=60"), @"
success: true
exit_code: 0
----- stdout -----
Starting 1 test across 1 worker
PASS [TIME] test::test_fast
────────────
Summary [TIME] 1 test run: 1 passed, 0 skipped

----- stderr -----
");
}

/// A test-level `@karva.tags.timeout` overrides the configured default.
#[test]
fn test_cli_timeout_tag_overrides_default() {
let context = TestContext::with_file(
"test.py",
r"
import time
import karva

@karva.tags.timeout(2.0)
def test_under_tag_limit():
time.sleep(0.3)
",
);

assert_cmd_snapshot!(context.command().arg("--timeout=0.1"), @"
success: true
exit_code: 0
----- stdout -----
Starting 1 test across 1 worker
PASS [TIME] test::test_under_tag_limit
────────────
Summary [TIME] 1 test run: 1 passed, 0 skipped

----- stderr -----
");
}

#[test]
fn test_config_timeout_kills_slow_test() {
let context = TestContext::with_files([
(
"pyproject.toml",
r"
[tool.karva.profile.default.test]
timeout = 0.1
",
),
(
"test.py",
r"
import time

def test_slow():
time.sleep(2)
",
),
]);

assert_cmd_snapshot!(context.command(), @"
success: false
exit_code: 1
----- stdout -----
Starting 1 test across 1 worker
FAIL [TIME] test::test_slow

diagnostics:

error[test-failure]: Test `test_slow` failed
--> test.py:4:5
|
4 | def test_slow():
| ^^^^^^^^^
|
info: Test exceeded timeout of 0.1 seconds

────────────
Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped

----- stderr -----
");
}
13 changes: 12 additions & 1 deletion crates/karva_cli/src/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use clap::Parser;
use karva_logging::{FinalStatusLevel, StatusLevel, TerminalColor};
use karva_metadata::{
CovFailUnder, CoverageOptions, MaxFail, Options, SlowTimeoutSecs, SrcOptions, TerminalOptions,
TestOptions,
TestOptions, TestTimeoutSecs,
};

use crate::enums::{CovReport, NoTests, OutputFormat, RunIgnored};
Expand Down Expand Up @@ -114,6 +114,16 @@ pub struct SubTestCommand {
#[clap(long, value_name = "SECONDS", help_heading = "Runner options")]
pub slow_timeout: Option<f64>,

/// Hard per-test timeout, in seconds.
///
/// Tests that run longer than this duration are killed and reported
/// as failures. A test-level [`@karva.tags.timeout`] decorator
/// overrides the default for that specific test.
///
/// Accepts fractional seconds such as `--timeout=120` or `--timeout=0.5`.
#[clap(long, value_name = "SECONDS", help_heading = "Runner options")]
pub timeout: Option<f64>,

/// Update snapshots directly instead of creating pending `.snap.new` files.
///
/// When set, `karva.assert_snapshot()` will write directly to `.snap` files,
Expand Down Expand Up @@ -321,6 +331,7 @@ impl SubTestCommand {
retry: self.retry,
no_tests: self.no_tests.map(Into::into),
slow_timeout: self.slow_timeout.map(SlowTimeoutSecs),
timeout: self.timeout.map(TestTimeoutSecs),
}),
coverage: Some(CoverageOptions {
sources: (!self.cov.is_empty()).then(|| self.cov.clone()),
Expand Down
1 change: 1 addition & 0 deletions crates/karva_metadata/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ pub use options::{
pub use pyproject::{PyProject, PyProjectError};
pub use settings::{
CovFailUnder, CoverageSettings, NoTestsMode, ProjectSettings, RunIgnoredMode, SlowTimeoutSecs,
TestTimeoutSecs,
};

use crate::options::KarvaTomlError;
Expand Down
28 changes: 26 additions & 2 deletions crates/karva_metadata/src/options/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use crate::filter::FiltersetSet;
use crate::max_fail::MaxFail;
use crate::settings::{
CovFailUnder, CoverageSettings, NoTestsMode, ProjectSettings, RunIgnoredMode, SlowTimeoutSecs,
SrcSettings, TerminalSettings, TestSettings,
SrcSettings, TerminalSettings, TestSettings, TestTimeoutSecs,
};

#[derive(
Expand Down Expand Up @@ -277,6 +277,25 @@ pub struct TestOptions {
"#
)]
pub slow_timeout: Option<SlowTimeoutSecs>,

/// Hard per-test timeout (in seconds).
///
/// When set, every test that runs longer than this duration is killed
/// and reported as a failure. Tests can override the limit individually
/// with [`@karva.tags.timeout`](https://docs.karva.dev/usage/tags/timeout/),
/// which takes precedence over the configured default.
///
/// Defaults to unset, which disables hard timeouts unless a tag is
/// applied to the test.
#[serde(skip_serializing_if = "Option::is_none")]
#[option(
default = r#"null"#,
value_type = "float (seconds)",
example = r#"
timeout = 120.0
"#
)]
pub timeout: Option<TestTimeoutSecs>,
}

impl TestOptions {
Expand All @@ -298,6 +317,7 @@ impl TestOptions {
run_ignored: RunIgnoredMode::default(),
no_tests: self.no_tests.unwrap_or_default(),
slow_timeout: self.slow_timeout.and_then(SlowTimeoutSecs::as_duration),
timeout: self.timeout.and_then(TestTimeoutSecs::as_duration),
}
}
}
Expand Down Expand Up @@ -513,7 +533,7 @@ nonsense = 42
|
4 | nonsense = 42
| ^^^^^^^^
unknown field `nonsense`, expected one of `test-function-prefix`, `fail-fast`, `max-fail`, `try-import-fixtures`, `retry`, `no-tests`, `slow-timeout`
unknown field `nonsense`, expected one of `test-function-prefix`, `fail-fast`, `max-fail`, `try-import-fixtures`, `retry`, `no-tests`, `slow-timeout`, `timeout`
"
);
}
Expand Down Expand Up @@ -611,6 +631,7 @@ max-fail = 0
),
no_tests: None,
slow_timeout: None,
timeout: None,
}
"#);
}
Expand Down Expand Up @@ -639,6 +660,7 @@ max-fail = 0
),
no_tests: None,
slow_timeout: None,
timeout: None,
}
"#);
}
Expand Down Expand Up @@ -700,6 +722,7 @@ retry = 2
),
no_tests: None,
slow_timeout: None,
timeout: None,
},
)
"#);
Expand Down Expand Up @@ -755,6 +778,7 @@ retry = 5
),
no_tests: None,
slow_timeout: None,
timeout: None,
},
)
"#);
Expand Down
35 changes: 35 additions & 0 deletions crates/karva_metadata/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,37 @@ impl Combine for SlowTimeoutSecs {
}
}

/// A per-test timeout expressed in seconds.
///
/// Wraps `f64` for the same reason as [`SlowTimeoutSecs`]. Tests exceeding
/// this duration are killed and reported as failures (see
/// [`crate::settings::TestSettings::timeout`]).
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(transparent)]
pub struct TestTimeoutSecs(pub f64);

impl Eq for TestTimeoutSecs {}

impl TestTimeoutSecs {
pub fn as_duration(self) -> Option<Duration> {
if self.0.is_finite() && self.0 > 0.0 {
Some(Duration::from_secs_f64(self.0))
} else {
None
}
}
}

impl Combine for TestTimeoutSecs {
#[inline(always)]
fn combine_with(&mut self, _other: Self) {}

#[inline]
fn combine(self, _other: Self) -> Self {
self
}
}

/// A coverage threshold expressed as a percentage (`0..=100`).
///
/// Wraps `f64` for the same reason as [`SlowTimeoutSecs`]: keeps the
Expand Down Expand Up @@ -164,4 +195,8 @@ pub struct TestSettings {
/// Threshold after which a test is flagged as slow. `None` disables
/// slow-test detection entirely.
pub slow_timeout: Option<Duration>,
/// Hard per-test timeout. Tests that run longer than this duration are
/// killed and reported as failures. `None` disables the hard timeout
/// (tests may still set their own limit via `@karva.tags.timeout`).
pub timeout: Option<Duration>,
}
5 changes: 5 additions & 0 deletions crates/karva_runner/src/worker_args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,11 @@ fn inner_cli_args(settings: &ProjectSettings, args: &SubTestCommand) -> Vec<Stri
cli_args.push(format!("{}", threshold.as_secs_f64()));
}

if let Some(timeout) = settings.test().timeout {
cli_args.push("--timeout".to_string());
cli_args.push(format!("{}", timeout.as_secs_f64()));
}

for expr in &args.filter_expressions {
cli_args.push("--filter".to_string());
cli_args.push(expr.clone());
Expand Down
8 changes: 7 additions & 1 deletion crates/karva_test_semantic/src/runner/package_runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,13 @@ impl<'ctx, 'a> PackageRunner<'ctx, 'a> {

let is_async = stmt_function_def.is_async
&& !crate::utils::patch_async_test_function(py, &function).unwrap_or(false);
let timeout_seconds = tags.timeout_tag().map(TimeoutTag::seconds);
let timeout_seconds = tags.timeout_tag().map(TimeoutTag::seconds).or_else(|| {
self.context
.settings()
.test()
.timeout
.map(|d| d.as_secs_f64())
});
let run_test = || {
if let Some(seconds) = timeout_seconds {
return run_test_with_timeout(
Expand Down
25 changes: 25 additions & 0 deletions docs/configuration/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,31 @@ test-function-prefix = "test"

---

### `timeout`

Hard per-test timeout (in seconds).

When set, every test that runs longer than this duration is killed
and reported as a failure. Tests can override the limit individually
with [`@karva.tags.timeout`](https://docs.karva.dev/usage/tags/timeout/),
which takes precedence over the configured default.

Defaults to unset, which disables hard timeouts unless a tag is
applied to the test.

**Default value**: `null`

**Type**: `float (seconds)`

**Example usage** (`pyproject.toml`):

```toml
[tool.karva.profile.default.test]
timeout = 120.0
```

---

### `try-import-fixtures`

When set, we will try to import functions in each test file as well as parsing the ast to find them.
Expand Down
3 changes: 3 additions & 0 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,9 @@ karva test [OPTIONS] [PATH]...
<li><code>skip</code>: Additionally display skipped test results</li>
<li><code>all</code>: Display all test result statuses</li>
</ul></dd><dt id="karva-test--test-prefix"><a href="#karva-test--test-prefix"><code>--test-prefix</code></a> <i>test-prefix</i></dt><dd><p>The prefix of the test functions</p>
</dd><dt id="karva-test--timeout"><a href="#karva-test--timeout"><code>--timeout</code></a> <i>seconds</i></dt><dd><p>Hard per-test timeout, in seconds.</p>
<p>Tests that run longer than this duration are killed and reported as failures. A test-level &#91;<code>@karva.tags.timeout</code>&#93; decorator overrides the default for that specific test.</p>
<p>Accepts fractional seconds such as <code>--timeout=120</code> or <code>--timeout=0.5</code>.</p>
</dd><dt id="karva-test--try-import-fixtures"><a href="#karva-test--try-import-fixtures"><code>--try-import-fixtures</code></a></dt><dd><p>When set, we will try to import functions in each test file as well as parsing the ast to find them.</p>
<p>This is often slower, so it is not recommended for most projects.</p>
</dd><dt id="karva-test--verbose"><a href="#karva-test--verbose"><code>--verbose</code></a>, <code>-v</code></dt><dd><p>Use verbose output (or <code>-vv</code> and <code>-vvv</code> for more verbose output)</p>
Expand Down
Loading
Loading