From 46042e16c271d551b15ced4647f0c02cfa0fae43 Mon Sep 17 00:00:00 2001 From: Fredrik Jansson Date: Thu, 19 Dec 2024 22:55:32 +0100 Subject: [PATCH 1/6] wip --- defaults/default-bacon.toml | 1 + src/analysis/analyzer.rs | 18 +-- src/analysis/mod.rs | 1 + src/analysis/nextest_json/mod.rs | 213 +++++++++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 14 deletions(-) create mode 100644 src/analysis/nextest_json/mod.rs diff --git a/defaults/default-bacon.toml b/defaults/default-bacon.toml index 917449a0..922c9dba 100644 --- a/defaults/default-bacon.toml +++ b/defaults/default-bacon.toml @@ -48,6 +48,7 @@ need_stdout = true [jobs.nextest] command = [ "cargo", "nextest", "run", + "--message-format", "libtest-json", "--hide-progress-bar", "--failure-output", "final" ] need_stdout = true diff --git a/src/analysis/analyzer.rs b/src/analysis/analyzer.rs index f62e71e7..9af4dabe 100644 --- a/src/analysis/analyzer.rs +++ b/src/analysis/analyzer.rs @@ -1,19 +1,8 @@ use { - super::{ - biome, - cargo_json, - cpp, - eslint, - nextest, - python, - standard, - }, + super::{biome, cargo_json, cpp, eslint, nextest_json, python, standard}, crate::*, anyhow::Result, - serde::{ - Deserialize, - Serialize, - }, + serde::{Deserialize, Serialize}, }; /// A stateless operator building a report from a list of command output lines. @@ -39,7 +28,8 @@ impl AnalyzerRef { pub fn create_analyzer(self) -> Box { match self { Self::Standard => Box::new(standard::StandardAnalyzer::default()), - Self::Nextest => Box::new(nextest::NextestAnalyzer::default()), + //Self::Nextest => Box::new(nextest::NextestAnalyzer::default()), + Self::Nextest => Box::new(nextest_json::NextestJSONAnalyzer::default()), Self::Eslint => Box::new(eslint::EslintAnalyzer::default()), Self::Biome => Box::new(biome::BiomeAnalyzer::default()), Self::PythonUnittest => Box::new(python::unittest::UnittestAnalyzer::default()), diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs index 03254719..87ec4cf0 100644 --- a/src/analysis/mod.rs +++ b/src/analysis/mod.rs @@ -9,6 +9,7 @@ mod line_analyzer; mod line_pattern; mod line_type; mod nextest; +mod nextest_json; mod python; mod standard; mod stats; diff --git a/src/analysis/nextest_json/mod.rs b/src/analysis/nextest_json/mod.rs new file mode 100644 index 00000000..3fa9a424 --- /dev/null +++ b/src/analysis/nextest_json/mod.rs @@ -0,0 +1,213 @@ +use crate::{CommandOutput, CommandOutputLine, CommandStream, Report, TLine}; + +use super::{Analyzer, LineType, Stats}; + +#[derive(Debug, Default)] +pub struct NextestJSONAnalyzer { + lines: Vec, +} + +impl Analyzer for NextestJSONAnalyzer { + fn start( + &mut self, + _mission: &crate::Mission, + ) { + self.lines.clear(); + } + + fn receive_line( + &mut self, + line: CommandOutputLine, + command_output: &mut crate::CommandOutput, + ) { + match line.origin { + CommandStream::StdErr => command_output.push(line), + CommandStream::StdOut => self.lines.push(line), + } + } + + fn build_report(&mut self) -> anyhow::Result { + let mut report = Report { + lines: Vec::with_capacity(self.lines.len()), + stats: Stats::default(), + suggest_backtrace: false, + output: CommandOutput::default(), + failure_keys: Vec::default(), + }; + let mut fail_idx = 0; + for line in &self.lines { + let line: OutputLine = serde_json::from_str(&line.content.to_raw())?; + match line { + OutputLine::Suite { event } => match event { + SuiteEvent::Started { test_count: _ } => (), + SuiteEvent::Ok {..} => (), + SuiteEvent::Failed { + passed, + failed, + ignored: _, + measured: _, + filtered_out: _, + exec_time: _, + } => { + report.stats.test_fails = failed; + report.stats.passed_tests = passed; + } + }, + OutputLine::Test { event } => match event { + TestEvent::Started => (), + TestEvent::Ok { name: _ } => (), + TestEvent::Failed { name, stdout } => { + report.lines.push(crate::Line { + item_idx: fail_idx, + line_type: LineType::TestResult(false), + content: TLine::failed(&name), + }); + report.lines.push(crate::Line { + item_idx: fail_idx, + line_type: LineType::Normal, + content: TLine::from_raw(stdout), + }); + report.failure_keys.push(name); + fail_idx += 1; + } + }, + } + } + report.lines.push(crate::Line { + item_idx: fail_idx, + line_type: LineType::Normal, + content: TLine::default(), + }); + Ok(report) + } +} + +#[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq)] +#[serde(rename_all = "lowercase")] +#[serde(tag = "event")] +enum TestEvent { + Started, + Ok { name: String }, + Failed { name: String, stdout: String }, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq)] +#[serde(rename_all = "lowercase")] +#[serde(tag = "event")] +enum SuiteEvent { + Started { + test_count: usize, + }, + Ok { + passed: usize, + failed: usize, + ignored: usize, + measured: usize, + filtered_out: usize, + exec_time: f32, + }, + Failed { + passed: usize, + failed: usize, + ignored: usize, + measured: usize, + filtered_out: usize, + exec_time: f32, + }, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq)] +#[serde(rename_all = "lowercase")] +#[serde(tag = "type")] +enum OutputLine { + Suite { + #[serde(flatten)] + event: SuiteEvent, + }, + Test { + #[serde(flatten)] + event: TestEvent, + }, +} + +#[test] +fn test_parse() { + let suite_started = r#"{"type":"suite","event":"started","test_count":7}"#; + assert_eq!( + serde_json::from_str::(suite_started).unwrap(), + OutputLine::Suite { + event: SuiteEvent::Started { test_count: 7 } + } + ); + + let suite_failed = r#"{"type":"suite","event":"failed","passed":6,"failed":1,"ignored":0,"measured":0,"filtered_out":0,"exec_time":0.015213355}"#; + assert_eq!( + serde_json::from_str::(suite_failed).unwrap(), + OutputLine::Suite { + event: SuiteEvent::Failed { + passed: 6, + failed: 1, + ignored: 0, + measured: 0, + filtered_out: 0, + exec_time: 0.015213355 + } + } + ); + + let suite_ok = r#"{"type":"suite","event":"ok","passed":13,"failed":0,"ignored":0,"measured":0,"filtered_out":0,"exec_time":0.140588}"#; + + assert_eq!( + serde_json::from_str::(suite_ok).unwrap(), + OutputLine::Suite { + event: SuiteEvent::Ok { + passed: 13, + failed: 0, + ignored: 0, + measured: 0, + filtered_out: 0, + exec_time: 0.140588 + } + } + ); + + let test_started = + r#"{"type":"test","event":"started","name":"llm::llm$parser::test::number"}"#; + assert_eq!( + serde_json::from_str::(test_started).unwrap(), + OutputLine::Test { + event: TestEvent::Started {} + } + ); + + let test_ok = r#"{"type":"test","event":"ok","name":"llm::llm$parser::test::identifier","exec_time":0.002138244}"#; + + assert_eq!( + serde_json::from_str::(test_ok).unwrap(), + OutputLine::Test { + event: TestEvent::Ok { + name: "llm::llm$parser::test::identifier".to_owned(), + } + } + ); + + let test_fail = r#" {"type":"test","event":"failed","name":"llm::llm$parser::test::var","exec_time":0.002140747,"stdout":"thread 'parser::test::var' panicked at src"}"#; + assert_eq!( + serde_json::from_str::(test_fail).unwrap(), + OutputLine::Test { + event: TestEvent::Failed { + name: "llm::llm$parser::test::var".to_owned(), + stdout: "thread 'parser::test::var' panicked at src".to_string(), + } + } + ); +} + +#[test] +fn test_fail() { + assert!(false); +} + +/* + * NEXTEST_EXPERIMENTAL_LIBTEST_JSON=1 cargo nextest run --message-format libtest-json --failure-output final --hide-progress-bar 2>/dev/null +*/ From 44aa2c851698d355e5e48f1d5e8329991aaebd30 Mon Sep 17 00:00:00 2001 From: Fredrik Jansson Date: Fri, 20 Dec 2024 07:23:15 +0100 Subject: [PATCH 2/6] Reformatted. --- defaults/default-bacon.toml | 1 + src/analysis/analyzer.rs | 15 +++++++++++++-- src/analysis/nextest_json/mod.rs | 18 ++++++++++++++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/defaults/default-bacon.toml b/defaults/default-bacon.toml index 922c9dba..bca44b9d 100644 --- a/defaults/default-bacon.toml +++ b/defaults/default-bacon.toml @@ -46,6 +46,7 @@ command = ["cargo", "test"] need_stdout = true [jobs.nextest] +env.NEXTEST_EXPERIMENTAL_LIBTEST_JSON="1" command = [ "cargo", "nextest", "run", "--message-format", "libtest-json", diff --git a/src/analysis/analyzer.rs b/src/analysis/analyzer.rs index 9af4dabe..5ed3b754 100644 --- a/src/analysis/analyzer.rs +++ b/src/analysis/analyzer.rs @@ -1,8 +1,19 @@ use { - super::{biome, cargo_json, cpp, eslint, nextest_json, python, standard}, + super::{ + biome, + cargo_json, + cpp, + eslint, + nextest_json, + python, + standard, + }, crate::*, anyhow::Result, - serde::{Deserialize, Serialize}, + serde::{ + Deserialize, + Serialize, + }, }; /// A stateless operator building a report from a list of command output lines. diff --git a/src/analysis/nextest_json/mod.rs b/src/analysis/nextest_json/mod.rs index 3fa9a424..9a81c9c2 100644 --- a/src/analysis/nextest_json/mod.rs +++ b/src/analysis/nextest_json/mod.rs @@ -1,6 +1,16 @@ -use crate::{CommandOutput, CommandOutputLine, CommandStream, Report, TLine}; - -use super::{Analyzer, LineType, Stats}; +use crate::{ + CommandOutput, + CommandOutputLine, + CommandStream, + Report, + TLine, +}; + +use super::{ + Analyzer, + LineType, + Stats, +}; #[derive(Debug, Default)] pub struct NextestJSONAnalyzer { @@ -40,7 +50,7 @@ impl Analyzer for NextestJSONAnalyzer { match line { OutputLine::Suite { event } => match event { SuiteEvent::Started { test_count: _ } => (), - SuiteEvent::Ok {..} => (), + SuiteEvent::Ok { .. } => (), SuiteEvent::Failed { passed, failed, From dc20bf3b03db7ceccce3e63cba2b6e074f1cf905 Mon Sep 17 00:00:00 2001 From: Fredrik Jansson Date: Fri, 20 Dec 2024 07:23:41 +0100 Subject: [PATCH 3/6] Set nextest experimental env. --- defaults/default-bacon.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/defaults/default-bacon.toml b/defaults/default-bacon.toml index bca44b9d..922c9dba 100644 --- a/defaults/default-bacon.toml +++ b/defaults/default-bacon.toml @@ -46,7 +46,6 @@ command = ["cargo", "test"] need_stdout = true [jobs.nextest] -env.NEXTEST_EXPERIMENTAL_LIBTEST_JSON="1" command = [ "cargo", "nextest", "run", "--message-format", "libtest-json", From da7a6b8d460542e3db7f59d24c6ccdb9d1f5a898 Mon Sep 17 00:00:00 2001 From: Fredrik Jansson Date: Fri, 20 Dec 2024 08:16:42 +0100 Subject: [PATCH 4/6] Set nextest env --- defaults/default-bacon.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/defaults/default-bacon.toml b/defaults/default-bacon.toml index 922c9dba..d6a046ae 100644 --- a/defaults/default-bacon.toml +++ b/defaults/default-bacon.toml @@ -46,10 +46,13 @@ command = ["cargo", "test"] need_stdout = true [jobs.nextest] +env.NEXTEST_EXPERIMENTAL_LIBTEST_JSON="1" command = [ "cargo", "nextest", "run", "--message-format", "libtest-json", - "--hide-progress-bar", "--failure-output", "final" + "--hide-progress-bar", + "--failure-output", "final", + "--no-input-handler" ] need_stdout = true analyzer = "nextest" From 76ddc944118b5bee63e51504816338e992872df4 Mon Sep 17 00:00:00 2001 From: Fredrik Jansson Date: Fri, 20 Dec 2024 08:16:56 +0100 Subject: [PATCH 5/6] Use standard analyzer for build errors --- src/analysis/nextest_json/mod.rs | 197 +++++++++++++++++-------------- 1 file changed, 109 insertions(+), 88 deletions(-) diff --git a/src/analysis/nextest_json/mod.rs b/src/analysis/nextest_json/mod.rs index 9a81c9c2..577bd454 100644 --- a/src/analysis/nextest_json/mod.rs +++ b/src/analysis/nextest_json/mod.rs @@ -10,11 +10,13 @@ use super::{ Analyzer, LineType, Stats, + standard::StandardAnalyzer, }; #[derive(Debug, Default)] pub struct NextestJSONAnalyzer { lines: Vec, + standard_analyzer: StandardAnalyzer, } impl Analyzer for NextestJSONAnalyzer { @@ -31,7 +33,10 @@ impl Analyzer for NextestJSONAnalyzer { command_output: &mut crate::CommandOutput, ) { match line.origin { - CommandStream::StdErr => command_output.push(line), + // In JSON mode, stderr output is human readable + CommandStream::StdErr => self.standard_analyzer.receive_line(line, command_output), + + // The JSON payloads come on stdout CommandStream::StdOut => self.lines.push(line), } } @@ -44,7 +49,8 @@ impl Analyzer for NextestJSONAnalyzer { output: CommandOutput::default(), failure_keys: Vec::default(), }; - let mut fail_idx = 0; + + let mut item_idx = 0; for line in &self.lines { let line: OutputLine = serde_json::from_str(&line.content.to_raw())?; match line { @@ -67,31 +73,37 @@ impl Analyzer for NextestJSONAnalyzer { TestEvent::Started => (), TestEvent::Ok { name: _ } => (), TestEvent::Failed { name, stdout } => { + let name = cleanup_name(&name); report.lines.push(crate::Line { - item_idx: fail_idx, - line_type: LineType::TestResult(false), + item_idx, + line_type: LineType::Title(super::Kind::Error), content: TLine::failed(&name), }); - report.lines.push(crate::Line { - item_idx: fail_idx, - line_type: LineType::Normal, - content: TLine::from_raw(stdout), - }); + for outline in stdout.lines() { + report.lines.push(crate::Line { + item_idx, + line_type: LineType::Normal, + content: TLine::from_raw(outline.to_owned()), + }); + } report.failure_keys.push(name); - fail_idx += 1; + item_idx += 1; } }, } } - report.lines.push(crate::Line { - item_idx: fail_idx, - line_type: LineType::Normal, - content: TLine::default(), - }); Ok(report) } } +fn cleanup_name(name: &str) -> String { + if let Some(idx) = name.chars().position(|ch| ch == '$') { + name.chars().skip(idx + 1).collect() + } else { + name.to_owned() + } +} + #[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq)] #[serde(rename_all = "lowercase")] #[serde(tag = "event")] @@ -140,84 +152,93 @@ enum OutputLine { }, } -#[test] -fn test_parse() { - let suite_started = r#"{"type":"suite","event":"started","test_count":7}"#; - assert_eq!( - serde_json::from_str::(suite_started).unwrap(), - OutputLine::Suite { - event: SuiteEvent::Started { test_count: 7 } - } - ); - - let suite_failed = r#"{"type":"suite","event":"failed","passed":6,"failed":1,"ignored":0,"measured":0,"filtered_out":0,"exec_time":0.015213355}"#; - assert_eq!( - serde_json::from_str::(suite_failed).unwrap(), - OutputLine::Suite { - event: SuiteEvent::Failed { - passed: 6, - failed: 1, - ignored: 0, - measured: 0, - filtered_out: 0, - exec_time: 0.015213355 +#[cfg(test)] +mod test { + use super::{ + OutputLine, + SuiteEvent, + TestEvent, + }; + + #[test] + fn parse() { + let suite_started = r#"{"type":"suite","event":"started","test_count":7}"#; + assert_eq!( + serde_json::from_str::(suite_started).unwrap(), + OutputLine::Suite { + event: SuiteEvent::Started { test_count: 7 } } - } - ); - - let suite_ok = r#"{"type":"suite","event":"ok","passed":13,"failed":0,"ignored":0,"measured":0,"filtered_out":0,"exec_time":0.140588}"#; - - assert_eq!( - serde_json::from_str::(suite_ok).unwrap(), - OutputLine::Suite { - event: SuiteEvent::Ok { - passed: 13, - failed: 0, - ignored: 0, - measured: 0, - filtered_out: 0, - exec_time: 0.140588 + ); + + let suite_failed = r#"{"type":"suite","event":"failed","passed":6,"failed":1,"ignored":0,"measured":0,"filtered_out":0,"exec_time":0.015213355}"#; + assert_eq!( + serde_json::from_str::(suite_failed).unwrap(), + OutputLine::Suite { + event: SuiteEvent::Failed { + passed: 6, + failed: 1, + ignored: 0, + measured: 0, + filtered_out: 0, + exec_time: 0.015213355 + } } - } - ); - - let test_started = - r#"{"type":"test","event":"started","name":"llm::llm$parser::test::number"}"#; - assert_eq!( - serde_json::from_str::(test_started).unwrap(), - OutputLine::Test { - event: TestEvent::Started {} - } - ); + ); + + let suite_ok = r#"{"type":"suite","event":"ok","passed":13,"failed":0,"ignored":0,"measured":0,"filtered_out":0,"exec_time":0.140588}"#; + + assert_eq!( + serde_json::from_str::(suite_ok).unwrap(), + OutputLine::Suite { + event: SuiteEvent::Ok { + passed: 13, + failed: 0, + ignored: 0, + measured: 0, + filtered_out: 0, + exec_time: 0.140588 + } + } + ); + + let test_started = + r#"{"type":"test","event":"started","name":"llm::llm$parser::test::number"}"#; + assert_eq!( + serde_json::from_str::(test_started).unwrap(), + OutputLine::Test { + event: TestEvent::Started {} + } + ); - let test_ok = r#"{"type":"test","event":"ok","name":"llm::llm$parser::test::identifier","exec_time":0.002138244}"#; + let test_ok = r#"{"type":"test","event":"ok","name":"llm::llm$parser::test::identifier","exec_time":0.002138244}"#; - assert_eq!( - serde_json::from_str::(test_ok).unwrap(), - OutputLine::Test { - event: TestEvent::Ok { - name: "llm::llm$parser::test::identifier".to_owned(), + assert_eq!( + serde_json::from_str::(test_ok).unwrap(), + OutputLine::Test { + event: TestEvent::Ok { + name: "llm::llm$parser::test::identifier".to_owned(), + } } - } - ); - - let test_fail = r#" {"type":"test","event":"failed","name":"llm::llm$parser::test::var","exec_time":0.002140747,"stdout":"thread 'parser::test::var' panicked at src"}"#; - assert_eq!( - serde_json::from_str::(test_fail).unwrap(), - OutputLine::Test { - event: TestEvent::Failed { - name: "llm::llm$parser::test::var".to_owned(), - stdout: "thread 'parser::test::var' panicked at src".to_string(), + ); + + let test_fail = r#" {"type":"test","event":"failed","name":"llm::llm$parser::test::var","exec_time":0.002140747,"stdout":"thread 'parser::test::var' panicked at src"}"#; + assert_eq!( + serde_json::from_str::(test_fail).unwrap(), + OutputLine::Test { + event: TestEvent::Failed { + name: "llm::llm$parser::test::var".to_owned(), + stdout: "thread 'parser::test::var' panicked at src".to_string(), + } } - } - ); -} + ); + } -#[test] -fn test_fail() { - assert!(false); + #[test] + fn cleanup_name() { + let name = "bacon::bacon$analysis::nextest_json::test_fail"; + assert_eq!( + super::cleanup_name(name), + "analysis::nextest_json::test_fail".to_owned() + ); + } } - -/* - * NEXTEST_EXPERIMENTAL_LIBTEST_JSON=1 cargo nextest run --message-format libtest-json --failure-output final --hide-progress-bar 2>/dev/null -*/ From 72982d62dc29b808cb6a2d8b00237c673070a9c1 Mon Sep 17 00:00:00 2001 From: Fredrik Jansson Date: Fri, 20 Dec 2024 08:22:06 +0100 Subject: [PATCH 6/6] Added ignored test parsing. --- src/analysis/nextest_json/mod.rs | 36 +++++++++++++++----------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/analysis/nextest_json/mod.rs b/src/analysis/nextest_json/mod.rs index 577bd454..693decc4 100644 --- a/src/analysis/nextest_json/mod.rs +++ b/src/analysis/nextest_json/mod.rs @@ -1,17 +1,6 @@ -use crate::{ - CommandOutput, - CommandOutputLine, - CommandStream, - Report, - TLine, -}; - -use super::{ - Analyzer, - LineType, - Stats, - standard::StandardAnalyzer, -}; +use crate::{CommandOutput, CommandOutputLine, CommandStream, Report, TLine}; + +use super::{Analyzer, LineType, Stats, standard::StandardAnalyzer}; #[derive(Debug, Default)] pub struct NextestJSONAnalyzer { @@ -72,6 +61,7 @@ impl Analyzer for NextestJSONAnalyzer { OutputLine::Test { event } => match event { TestEvent::Started => (), TestEvent::Ok { name: _ } => (), + TestEvent::Ignored { name: _ } => (), TestEvent::Failed { name, stdout } => { let name = cleanup_name(&name); report.lines.push(crate::Line { @@ -111,6 +101,7 @@ enum TestEvent { Started, Ok { name: String }, Failed { name: String, stdout: String }, + Ignored { name: String }, } #[derive(Debug, serde::Deserialize, serde::Serialize, PartialEq)] @@ -154,11 +145,7 @@ enum OutputLine { #[cfg(test)] mod test { - use super::{ - OutputLine, - SuiteEvent, - TestEvent, - }; + use super::{OutputLine, SuiteEvent, TestEvent}; #[test] fn parse() { @@ -231,6 +218,17 @@ mod test { } } ); + + let test_ignored = + r#"{"type":"test","event":"ignored","name":"llvm::llvm$parser::test::var"}"#; + assert_eq!( + serde_json::from_str::(test_ignored).unwrap(), + OutputLine::Test { + event: TestEvent::Ignored { + name: "llm::llm$parser::test::var".to_owned(), + } + } + ); } #[test]