From aacebc17bd710ad83349901b284275b990c0b520 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Wed, 20 Aug 2025 14:20:34 -0500 Subject: [PATCH 1/6] chore: Silence warning --- crates/libtest-json/src/event.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/libtest-json/src/event.rs b/crates/libtest-json/src/event.rs index 5eb9c9c..abc42ca 100644 --- a/crates/libtest-json/src/event.rs +++ b/crates/libtest-json/src/event.rs @@ -397,10 +397,12 @@ impl RunComplete { } } +#[cfg(feature = "serde")] fn true_default() -> bool { true } +#[cfg(feature = "serde")] fn is_true(yes: &bool) -> bool { *yes } @@ -423,6 +425,7 @@ impl RunMode { } } + #[cfg(any(feature = "serde", feature = "json"))] fn is_default(&self) -> bool { *self == Default::default() } From 38cbc23d5bb6d530adc5fdf5a62b1112e4fc24ca Mon Sep 17 00:00:00 2001 From: Ed Page Date: Wed, 20 Aug 2025 14:17:45 -0500 Subject: [PATCH 2/6] feat(json): Specify MessageKind to be in precedence order --- crates/libtest-json/event.schema.json | 4 ++-- crates/libtest-json/src/event.rs | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/libtest-json/event.schema.json b/crates/libtest-json/event.schema.json index 6e1a40e..10d5331 100644 --- a/crates/libtest-json/event.schema.json +++ b/crates/libtest-json/event.schema.json @@ -203,8 +203,8 @@ "RunStatus": { "type": "string", "enum": [ - "ignored", - "failed" + "failed", + "ignored" ] }, "CaseComplete": { diff --git a/crates/libtest-json/src/event.rs b/crates/libtest-json/src/event.rs index abc42ca..5137157 100644 --- a/crates/libtest-json/src/event.rs +++ b/crates/libtest-json/src/event.rs @@ -431,20 +431,21 @@ impl RunMode { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] #[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] pub enum RunStatus { - Ignored, + // Highest precedent items for determining test status last Failed, + Ignored, } impl RunStatus { pub fn as_str(&self) -> &str { match self { - Self::Ignored => "ignored", Self::Failed => "failed", + Self::Ignored => "ignored", } } } From 4200d4f97e9019a2009d559ed39c540eb1e81cb4 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 15 Aug 2025 16:07:00 -0500 Subject: [PATCH 3/6] feat(json)!: Add CaseMessage The aim is to allow multiple failures --- DESIGN.md | 1 + crates/libtest-json/event.schema.json | 44 ++++++++++++ crates/libtest-json/src/event.rs | 69 +++++++++++++++++++ crates/libtest-json/tests/roundtrip.rs | 25 +++++++ crates/libtest2-harness/src/notify/pretty.rs | 1 + crates/libtest2-harness/src/notify/summary.rs | 69 ++++++++++++++++--- crates/libtest2-harness/src/notify/terse.rs | 2 + 7 files changed, 202 insertions(+), 9 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index e34c55c..66eb7d9 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -37,6 +37,7 @@ Decisions - Terse and pretty progress indicators are too nebulous to render (see their notifiers) - There is likely not enough value add in the failure message - This puts more of a burden on custom test harnesses for their implementation than is strictly needed +- Report failures separate from test-complete so we can have multiple ### Prior Art diff --git a/crates/libtest-json/event.schema.json b/crates/libtest-json/event.schema.json index 10d5331..2461ea6 100644 --- a/crates/libtest-json/event.schema.json +++ b/crates/libtest-json/event.schema.json @@ -67,6 +67,19 @@ "event" ] }, + { + "type": "object", + "properties": { + "event": { + "type": "string", + "const": "case_message" + } + }, + "$ref": "#/$defs/CaseMessage", + "required": [ + "event" + ] + }, { "type": "object", "properties": { @@ -207,6 +220,37 @@ "ignored" ] }, + "CaseMessage": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "$ref": "#/$defs/RunStatus" + }, + "message": { + "type": [ + "string", + "null" + ] + }, + "elapsed_s": { + "anyOf": [ + { + "$ref": "#/$defs/Elapsed" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "name", + "status" + ] + }, "CaseComplete": { "type": "object", "properties": { diff --git a/crates/libtest-json/src/event.rs b/crates/libtest-json/src/event.rs index 5137157..c0a69a6 100644 --- a/crates/libtest-json/src/event.rs +++ b/crates/libtest-json/src/event.rs @@ -9,6 +9,7 @@ pub enum Event { DiscoverComplete(DiscoverComplete), RunStart(RunStart), CaseStart(CaseStart), + CaseMessage(CaseMessage), CaseComplete(CaseComplete), RunComplete(RunComplete), } @@ -22,6 +23,7 @@ impl Event { Self::DiscoverComplete(event) => event.to_jsonline(), Self::RunStart(event) => event.to_jsonline(), Self::CaseStart(event) => event.to_jsonline(), + Self::CaseMessage(event) => event.to_jsonline(), Self::CaseComplete(event) => event.to_jsonline(), Self::RunComplete(event) => event.to_jsonline(), } @@ -58,6 +60,12 @@ impl From for Event { } } +impl From for Event { + fn from(inner: CaseMessage) -> Self { + Self::CaseMessage(inner) + } +} + impl From for Event { fn from(inner: CaseComplete) -> Self { Self::CaseComplete(inner) @@ -292,6 +300,67 @@ impl CaseStart { } } +#[derive(Clone, Debug)] +#[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub struct CaseMessage { + pub name: String, + pub status: RunStatus, + #[cfg_attr( + feature = "serde", + serde(default, skip_serializing_if = "Option::is_none") + )] + pub message: Option, + #[cfg_attr( + feature = "serde", + serde(default, skip_serializing_if = "Option::is_none") + )] + pub elapsed_s: Option, +} + +impl CaseMessage { + #[cfg(feature = "json")] + pub fn to_jsonline(&self) -> String { + use json_write::JsonWrite as _; + + let mut buffer = String::new(); + buffer.open_object().unwrap(); + + buffer.key("event").unwrap(); + buffer.keyval_sep().unwrap(); + buffer.value("case_message").unwrap(); + + buffer.val_sep().unwrap(); + buffer.key("name").unwrap(); + buffer.keyval_sep().unwrap(); + buffer.value(&self.name).unwrap(); + + buffer.val_sep().unwrap(); + buffer.key("status").unwrap(); + buffer.keyval_sep().unwrap(); + buffer.value(self.status.as_str()).unwrap(); + + if let Some(message) = &self.message { + buffer.val_sep().unwrap(); + buffer.key("message").unwrap(); + buffer.keyval_sep().unwrap(); + buffer.value(message).unwrap(); + } + + if let Some(elapsed_s) = self.elapsed_s { + buffer.val_sep().unwrap(); + buffer.key("elapsed_s").unwrap(); + buffer.keyval_sep().unwrap(); + buffer.value(String::from(elapsed_s)).unwrap(); + } + + buffer.close_object().unwrap(); + + buffer + } +} + #[derive(Clone, Debug)] #[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] diff --git a/crates/libtest-json/tests/roundtrip.rs b/crates/libtest-json/tests/roundtrip.rs index 4f33bbc..3528ec8 100644 --- a/crates/libtest-json/tests/roundtrip.rs +++ b/crates/libtest-json/tests/roundtrip.rs @@ -101,6 +101,31 @@ fn case_start() { ); } +#[test] +fn case_message() { + t( + libtest_json::event::CaseMessage { + name: "Hello\tworld!".to_owned(), + status: libtest_json::RunStatus::Failed, + message: None, + elapsed_s: None, + }, + str![[r#"{"event":"case_message","name":"Hello\tworld!","status":"failed"}"#]], + ); + + t( + libtest_json::event::CaseMessage { + name: "Hello\tworld!".to_owned(), + status: libtest_json::RunStatus::Ignored, + message: Some("This\tfailed".to_owned()), + elapsed_s: Some(libtest_json::Elapsed(Default::default())), + }, + str![[ + r#"{"event":"case_message","name":"Hello\tworld!","status":"ignored","message":"This\tfailed","elapsed_s":"0"}"# + ]], + ); +} + #[test] fn case_complete() { t( diff --git a/crates/libtest2-harness/src/notify/pretty.rs b/crates/libtest2-harness/src/notify/pretty.rs index 2a4583a..a665931 100644 --- a/crates/libtest2-harness/src/notify/pretty.rs +++ b/crates/libtest2-harness/src/notify/pretty.rs @@ -51,6 +51,7 @@ impl super::Notifier for PrettyRunNotifier { self.writer.flush()?; } } + Event::CaseMessage(_) => {} Event::CaseComplete(inner) => { let status = self.summary.get_status(&inner.name); let (s, style) = match status { diff --git a/crates/libtest2-harness/src/notify/summary.rs b/crates/libtest2-harness/src/notify/summary.rs index 0c4bf09..87a1692 100644 --- a/crates/libtest2-harness/src/notify/summary.rs +++ b/crates/libtest2-harness/src/notify/summary.rs @@ -1,4 +1,4 @@ -use super::event::CaseComplete; +use super::event::CaseMessage; use super::Event; use super::RunStatus; use super::FAILED; @@ -11,14 +11,14 @@ pub(crate) struct Summary { /// filter-in pattern or by `--skip` arguments). num_filtered_out: usize, - status: std::collections::HashMap, + status: std::collections::HashMap, elapsed_s: Option, } impl Summary { pub(crate) fn get_status(&self, name: &str) -> Option { - let event = self.status.get(name)?; - event.status + let status = self.status.get(name)?; + find_run_status(status) } pub(crate) fn write_start(&self, writer: &mut dyn std::io::Write) -> std::io::Result<()> { @@ -34,12 +34,27 @@ impl Summary { let mut num_failed = 0; let mut num_ignored = 0; let mut failures = std::collections::BTreeMap::new(); - for event in self.status.values() { - match event.status { + for (name, case_status) in &self.status { + let mut status = find_run_status(case_status); + if !case_status.started { + // Even override `Ignored` + status = Some(RunStatus::Failed); + failures.insert(name, Some("test found that never started")); + } + if !case_status.completed { + // Even override `Ignored` + status = Some(RunStatus::Failed); + failures.insert(name, Some("test never completed")); + } + match status { Some(RunStatus::Ignored) => num_ignored += 1, Some(RunStatus::Failed) => { num_failed += 1; - failures.insert(&event.name, &event.message); + for event in &case_status.messages { + if Some(event.status) == status { + failures.insert(name, event.message.as_deref()); + } + } } None => num_passed += 1, } @@ -106,9 +121,30 @@ impl super::Notifier for Summary { } Event::DiscoverComplete(_) => {} Event::RunStart(_) => {} - Event::CaseStart(_) => {} + Event::CaseStart(inner) => { + self.status.entry(inner.name).or_default().started = true; + } + Event::CaseMessage(inner) => { + self.status + .entry(inner.name.clone()) + .or_default() + .messages + .push(inner); + } Event::CaseComplete(inner) => { - self.status.insert(inner.name.clone(), inner); + if let Some(status) = inner.status { + self.status + .entry(inner.name.clone()) + .or_default() + .messages + .push(CaseMessage { + name: inner.name.clone(), + status, + message: inner.message.clone(), + elapsed_s: inner.elapsed_s, + }); + } + self.status.entry(inner.name).or_default().completed = true; } Event::RunComplete(inner) => { self.elapsed_s = inner.elapsed_s; @@ -117,3 +153,18 @@ impl super::Notifier for Summary { Ok(()) } } + +fn find_run_status(case_status: &CaseStatus) -> Option { + let mut status = None; + for event in &case_status.messages { + status = status.max(Some(event.status)); + } + status +} + +#[derive(Default, Clone, Debug)] +struct CaseStatus { + messages: Vec, + started: bool, + completed: bool, +} diff --git a/crates/libtest2-harness/src/notify/terse.rs b/crates/libtest2-harness/src/notify/terse.rs index a901131..7ceada5 100644 --- a/crates/libtest2-harness/src/notify/terse.rs +++ b/crates/libtest2-harness/src/notify/terse.rs @@ -35,6 +35,7 @@ impl super::Notifier for TerseListNotifier { } Event::RunStart(_) => {} Event::CaseStart(_) => {} + Event::CaseMessage(_) => {} Event::CaseComplete(_) => {} Event::RunComplete(_) => {} } @@ -68,6 +69,7 @@ impl super::Notifier for TerseRunNotifier { self.summary.write_start(&mut self.writer)?; } Event::CaseStart(_) => {} + Event::CaseMessage(_) => {} Event::CaseComplete(inner) => { let status = self.summary.get_status(&inner.name); let (c, style) = match status { From 47ffcfd7990f72bdd623bf57a981bed62ac2df0b Mon Sep 17 00:00:00 2001 From: Ed Page Date: Wed, 20 Aug 2025 14:06:16 -0500 Subject: [PATCH 4/6] fix(json)!: Remove CaseMessage content from CaseComplete --- crates/libtest-json/event.schema.json | 17 ------ crates/libtest-json/src/event.rs | 25 --------- crates/libtest-json/tests/roundtrip.rs | 8 +-- crates/libtest2-harness/src/harness.rs | 52 ++++++++++++++----- crates/libtest2-harness/src/notify/summary.rs | 12 ----- .../tests/testsuite/mixed_bag.rs | 47 +++++++++++++---- crates/libtest2/tests/testsuite/mixed_bag.rs | 47 +++++++++++++---- 7 files changed, 112 insertions(+), 96 deletions(-) diff --git a/crates/libtest-json/event.schema.json b/crates/libtest-json/event.schema.json index 2461ea6..598dd2c 100644 --- a/crates/libtest-json/event.schema.json +++ b/crates/libtest-json/event.schema.json @@ -257,23 +257,6 @@ "name": { "type": "string" }, - "status": { - "description": "`None` means success", - "anyOf": [ - { - "$ref": "#/$defs/RunStatus" - }, - { - "type": "null" - } - ] - }, - "message": { - "type": [ - "string", - "null" - ] - }, "elapsed_s": { "anyOf": [ { diff --git a/crates/libtest-json/src/event.rs b/crates/libtest-json/src/event.rs index c0a69a6..e511255 100644 --- a/crates/libtest-json/src/event.rs +++ b/crates/libtest-json/src/event.rs @@ -367,17 +367,6 @@ impl CaseMessage { #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] pub struct CaseComplete { pub name: String, - /// `None` means success - #[cfg_attr( - feature = "serde", - serde(default, skip_serializing_if = "Option::is_none") - )] - pub status: Option, - #[cfg_attr( - feature = "serde", - serde(default, skip_serializing_if = "Option::is_none") - )] - pub message: Option, #[cfg_attr( feature = "serde", serde(default, skip_serializing_if = "Option::is_none") @@ -402,20 +391,6 @@ impl CaseComplete { buffer.keyval_sep().unwrap(); buffer.value(&self.name).unwrap(); - if let Some(status) = self.status { - buffer.val_sep().unwrap(); - buffer.key("status").unwrap(); - buffer.keyval_sep().unwrap(); - buffer.value(status.as_str()).unwrap(); - } - - if let Some(message) = &self.message { - buffer.val_sep().unwrap(); - buffer.key("message").unwrap(); - buffer.keyval_sep().unwrap(); - buffer.value(message).unwrap(); - } - if let Some(elapsed_s) = self.elapsed_s { buffer.val_sep().unwrap(); buffer.key("elapsed_s").unwrap(); diff --git a/crates/libtest-json/tests/roundtrip.rs b/crates/libtest-json/tests/roundtrip.rs index 3528ec8..a23ab14 100644 --- a/crates/libtest-json/tests/roundtrip.rs +++ b/crates/libtest-json/tests/roundtrip.rs @@ -131,8 +131,6 @@ fn case_complete() { t( libtest_json::event::CaseComplete { name: "Hello\tworld!".to_owned(), - status: None, - message: None, elapsed_s: None, }, str![[r#"{"event":"case_complete","name":"Hello\tworld!"}"#]], @@ -141,13 +139,9 @@ fn case_complete() { t( libtest_json::event::CaseComplete { name: "Hello\tworld!".to_owned(), - status: Some(libtest_json::RunStatus::Ignored), - message: Some("This\tfailed".to_owned()), elapsed_s: Some(libtest_json::Elapsed(Default::default())), }, - str![[ - r#"{"event":"case_complete","name":"Hello\tworld!","status":"ignored","message":"This\tfailed","elapsed_s":"0"}"# - ]], + str![[r#"{"event":"case_complete","name":"Hello\tworld!","elapsed_s":"0"}"#]], ); } diff --git a/crates/libtest2-harness/src/harness.rs b/crates/libtest2-harness/src/harness.rs index 49a4917..cd36f28 100644 --- a/crates/libtest2-harness/src/harness.rs +++ b/crates/libtest2-harness/src/harness.rs @@ -310,11 +310,26 @@ fn run( } impl RunningTest { - fn join(self, event: &mut notify::event::CaseComplete) { - if self.join_handle.join().is_err() && event.status.is_none() { - event.status = Some(notify::RunStatus::Failed); - event.message = Some("panicked after reporting success".to_owned()); + fn join( + self, + start: &std::time::Instant, + event: ¬ify::event::CaseComplete, + notifier: &mut dyn notify::Notifier, + ) -> std::io::Result<()> { + if self.join_handle.join().is_err() { + let status = notify::RunStatus::Failed; + let message = Some("panicked after reporting success".to_owned()); + notifier.notify( + notify::event::CaseMessage { + name: event.name.clone(), + status, + message, + elapsed_s: Some(notify::Elapsed(start.elapsed())), + } + .into(), + )?; } + Ok(()) } } @@ -379,10 +394,10 @@ fn run( } } - let mut event = rx.recv().unwrap(); - if let notify::Event::CaseComplete(event) = &mut event { + let event = rx.recv().unwrap(); + if let notify::Event::CaseComplete(event) = &event { let running_test = running_tests.remove(&event.name).unwrap(); - running_test.join(event); + running_test.join(start, event, notifier)?; pending -= 1; } notifier.notify(event)?; @@ -446,20 +461,31 @@ fn run_case( Err(RunError::fail(msg)) }); - let err = outcome.as_ref().err(); - let status = err.map(|e| e.status()); - let message = err.and_then(|e| e.cause().map(|c| c.to_string())); + let mut case_status = None; + if let Some(err) = outcome.as_ref().err() { + let status = err.status(); + case_status = Some(status); + let message = err.cause().map(|c| c.to_string()); + notifier.notify( + notify::event::CaseMessage { + name: case.name().to_owned(), + status, + message, + elapsed_s: Some(notify::Elapsed(start.elapsed())), + } + .into(), + )?; + } + notifier.notify( notify::event::CaseComplete { name: case.name().to_owned(), - status, - message, elapsed_s: Some(notify::Elapsed(start.elapsed())), } .into(), )?; - Ok(status != Some(notify::RunStatus::Failed)) + Ok(case_status != Some(notify::RunStatus::Failed)) } /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`. diff --git a/crates/libtest2-harness/src/notify/summary.rs b/crates/libtest2-harness/src/notify/summary.rs index 87a1692..7622c91 100644 --- a/crates/libtest2-harness/src/notify/summary.rs +++ b/crates/libtest2-harness/src/notify/summary.rs @@ -132,18 +132,6 @@ impl super::Notifier for Summary { .push(inner); } Event::CaseComplete(inner) => { - if let Some(status) = inner.status { - self.status - .entry(inner.name.clone()) - .or_default() - .messages - .push(CaseMessage { - name: inner.name.clone(), - status, - message: inner.message.clone(), - elapsed_s: inner.elapsed_s, - }); - } self.status.entry(inner.name).or_default().completed = true; } Event::RunComplete(inner) => { diff --git a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs index 0c26e07..7f5779b 100644 --- a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs @@ -898,12 +898,17 @@ fn test_json() { "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "bear", "status": "ignored", "message": "fails", "elapsed_s": "[..]" }, + { + "event": "case_complete", + "name": "bear", + "elapsed_s": "[..]" + }, { "event": "case_start", "name": "cat", @@ -928,13 +933,6 @@ fn test_json() { "elapsed_s": "[..]", "event": "discover_complete" }, - { - "elapsed_s": "[..]", - "event": "case_complete", - "message": "fails", - "name": "bear", - "status": "ignored" - }, { "elapsed_s": "[..]", "event": "case_complete", @@ -1007,6 +1005,18 @@ fn test_json() { "name": "owl", "selected": false, "elapsed_s": "[..]" + }, + { + "event": "case_message", + "name": "bear", + "status": "ignored", + "message": "fails", + "elapsed_s": "[..]" + }, + { + "event": "case_complete", + "name": "bear", + "elapsed_s": "[..]" } ] "#]] @@ -1157,24 +1167,34 @@ fn fail_fast_json() { "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "bear", "status": "ignored", "message": "fails", "elapsed_s": "[..]" }, + { + "event": "case_complete", + "name": "bear", + "elapsed_s": "[..]" + }, { "event": "case_start", "name": "bunny", "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "bunny", "status": "ignored", "message": "fails", "elapsed_s": "[..]" }, + { + "event": "case_complete", + "name": "bunny", + "elapsed_s": "[..]" + }, { "event": "case_start", "name": "cat", @@ -1191,12 +1211,17 @@ fn fail_fast_json() { "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "dog", "status": "failed", "message": "was not a good boy", "elapsed_s": "[..]" }, + { + "event": "case_complete", + "name": "dog", + "elapsed_s": "[..]" + }, { "event": "run_complete", "elapsed_s": "[..]" diff --git a/crates/libtest2/tests/testsuite/mixed_bag.rs b/crates/libtest2/tests/testsuite/mixed_bag.rs index 89fc58d..5b92871 100644 --- a/crates/libtest2/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2/tests/testsuite/mixed_bag.rs @@ -905,12 +905,17 @@ fn test_json() { "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "bear", "status": "ignored", "message": "fails", "elapsed_s": "[..]" }, + { + "event": "case_complete", + "name": "bear", + "elapsed_s": "[..]" + }, { "event": "case_start", "name": "cat", @@ -935,13 +940,6 @@ fn test_json() { "elapsed_s": "[..]", "event": "discover_complete" }, - { - "elapsed_s": "[..]", - "event": "case_complete", - "message": "fails", - "name": "bear", - "status": "ignored" - }, { "elapsed_s": "[..]", "event": "case_complete", @@ -1014,6 +1012,18 @@ fn test_json() { "name": "owl", "selected": false, "elapsed_s": "[..]" + }, + { + "event": "case_message", + "name": "bear", + "status": "ignored", + "message": "fails", + "elapsed_s": "[..]" + }, + { + "event": "case_complete", + "name": "bear", + "elapsed_s": "[..]" } ] "#]] @@ -1164,24 +1174,34 @@ fn fail_fast_json() { "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "bear", "status": "ignored", "message": "fails", "elapsed_s": "[..]" }, + { + "event": "case_complete", + "name": "bear", + "elapsed_s": "[..]" + }, { "event": "case_start", "name": "bunny", "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "bunny", "status": "ignored", "message": "fails", "elapsed_s": "[..]" }, + { + "event": "case_complete", + "name": "bunny", + "elapsed_s": "[..]" + }, { "event": "case_start", "name": "cat", @@ -1198,12 +1218,17 @@ fn fail_fast_json() { "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "dog", "status": "failed", "message": "was not a good boy", "elapsed_s": "[..]" }, + { + "event": "case_complete", + "name": "dog", + "elapsed_s": "[..]" + }, { "event": "run_complete", "elapsed_s": "[..]" From e22e8af31d4b963523f530453ebe9ad6b5f5cdd0 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Wed, 20 Aug 2025 14:15:26 -0500 Subject: [PATCH 5/6] fix(json)!: Rename RunStatus to MessageKind --- crates/libtest-json/event.schema.json | 8 ++++---- crates/libtest-json/src/event.rs | 10 +++++----- crates/libtest-json/src/lib.rs | 2 +- crates/libtest-json/tests/roundtrip.rs | 10 ++++------ crates/libtest2-harness/src/case.rs | 10 +++++----- crates/libtest2-harness/src/harness.rs | 12 ++++++------ crates/libtest2-harness/src/notify/pretty.rs | 8 ++++---- crates/libtest2-harness/src/notify/summary.rs | 18 +++++++++--------- crates/libtest2-harness/src/notify/terse.rs | 8 ++++---- .../tests/testsuite/mixed_bag.rs | 16 ++++++++-------- crates/libtest2/tests/testsuite/mixed_bag.rs | 16 ++++++++-------- 11 files changed, 58 insertions(+), 60 deletions(-) diff --git a/crates/libtest-json/event.schema.json b/crates/libtest-json/event.schema.json index 598dd2c..af23661 100644 --- a/crates/libtest-json/event.schema.json +++ b/crates/libtest-json/event.schema.json @@ -213,7 +213,7 @@ "name" ] }, - "RunStatus": { + "MessageKind": { "type": "string", "enum": [ "failed", @@ -226,8 +226,8 @@ "name": { "type": "string" }, - "status": { - "$ref": "#/$defs/RunStatus" + "kind": { + "$ref": "#/$defs/MessageKind" }, "message": { "type": [ @@ -248,7 +248,7 @@ }, "required": [ "name", - "status" + "kind" ] }, "CaseComplete": { diff --git a/crates/libtest-json/src/event.rs b/crates/libtest-json/src/event.rs index e511255..10c4376 100644 --- a/crates/libtest-json/src/event.rs +++ b/crates/libtest-json/src/event.rs @@ -306,7 +306,7 @@ impl CaseStart { #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] pub struct CaseMessage { pub name: String, - pub status: RunStatus, + pub kind: MessageKind, #[cfg_attr( feature = "serde", serde(default, skip_serializing_if = "Option::is_none") @@ -337,9 +337,9 @@ impl CaseMessage { buffer.value(&self.name).unwrap(); buffer.val_sep().unwrap(); - buffer.key("status").unwrap(); + buffer.key("kind").unwrap(); buffer.keyval_sep().unwrap(); - buffer.value(self.status.as_str()).unwrap(); + buffer.value(self.kind.as_str()).unwrap(); if let Some(message) = &self.message { buffer.val_sep().unwrap(); @@ -479,13 +479,13 @@ impl RunMode { #[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] -pub enum RunStatus { +pub enum MessageKind { // Highest precedent items for determining test status last Failed, Ignored, } -impl RunStatus { +impl MessageKind { pub fn as_str(&self) -> &str { match self { Self::Failed => "failed", diff --git a/crates/libtest-json/src/lib.rs b/crates/libtest-json/src/lib.rs index 2a20c9a..0075c61 100644 --- a/crates/libtest-json/src/lib.rs +++ b/crates/libtest-json/src/lib.rs @@ -9,8 +9,8 @@ pub mod event; pub use event::Elapsed; pub use event::Event; +pub use event::MessageKind; pub use event::RunMode; -pub use event::RunStatus; #[doc = include_str!("../README.md")] #[cfg(doctest)] diff --git a/crates/libtest-json/tests/roundtrip.rs b/crates/libtest-json/tests/roundtrip.rs index a23ab14..160fa20 100644 --- a/crates/libtest-json/tests/roundtrip.rs +++ b/crates/libtest-json/tests/roundtrip.rs @@ -106,23 +106,21 @@ fn case_message() { t( libtest_json::event::CaseMessage { name: "Hello\tworld!".to_owned(), - status: libtest_json::RunStatus::Failed, + kind: libtest_json::MessageKind::Failed, message: None, elapsed_s: None, }, - str![[r#"{"event":"case_message","name":"Hello\tworld!","status":"failed"}"#]], + str![[r#"{"event":"case_message","name":"Hello\tworld!","kind":"failed"}"#]], ); t( libtest_json::event::CaseMessage { name: "Hello\tworld!".to_owned(), - status: libtest_json::RunStatus::Ignored, + kind: libtest_json::MessageKind::Ignored, message: Some("This\tfailed".to_owned()), elapsed_s: Some(libtest_json::Elapsed(Default::default())), }, - str![[ - r#"{"event":"case_message","name":"Hello\tworld!","status":"ignored","message":"This\tfailed","elapsed_s":"0"}"# - ]], + str![[r#"{"event":"case_message","name":"Hello\tworld!","kind":"ignored","message":"This\tfailed","elapsed_s":"0"}"#]], ); } diff --git a/crates/libtest2-harness/src/case.rs b/crates/libtest2-harness/src/case.rs index 7dc9e94..d1c0ee0 100644 --- a/crates/libtest2-harness/src/case.rs +++ b/crates/libtest2-harness/src/case.rs @@ -53,14 +53,14 @@ pub type RunResult = Result<(), RunError>; #[derive(Debug)] pub struct RunError { - status: notify::RunStatus, + status: notify::MessageKind, cause: Option>, } impl RunError { pub fn with_cause(cause: impl std::error::Error + Send + Sync + 'static) -> Self { Self { - status: notify::RunStatus::Failed, + status: notify::MessageKind::Failed, cause: Some(Box::new(cause)), } } @@ -71,19 +71,19 @@ impl RunError { pub(crate) fn ignore() -> Self { Self { - status: notify::RunStatus::Ignored, + status: notify::MessageKind::Ignored, cause: None, } } pub(crate) fn ignore_for(reason: String) -> Self { Self { - status: notify::RunStatus::Ignored, + status: notify::MessageKind::Ignored, cause: Some(Box::new(Message(reason))), } } - pub(crate) fn status(&self) -> notify::RunStatus { + pub(crate) fn status(&self) -> notify::MessageKind { self.status } diff --git a/crates/libtest2-harness/src/harness.rs b/crates/libtest2-harness/src/harness.rs index cd36f28..3a3aab1 100644 --- a/crates/libtest2-harness/src/harness.rs +++ b/crates/libtest2-harness/src/harness.rs @@ -317,12 +317,12 @@ fn run( notifier: &mut dyn notify::Notifier, ) -> std::io::Result<()> { if self.join_handle.join().is_err() { - let status = notify::RunStatus::Failed; + let kind = notify::MessageKind::Failed; let message = Some("panicked after reporting success".to_owned()); notifier.notify( notify::event::CaseMessage { name: event.name.clone(), - status, + kind, message, elapsed_s: Some(notify::Elapsed(start.elapsed())), } @@ -463,13 +463,13 @@ fn run_case( let mut case_status = None; if let Some(err) = outcome.as_ref().err() { - let status = err.status(); - case_status = Some(status); + let kind = err.status(); + case_status = Some(kind); let message = err.cause().map(|c| c.to_string()); notifier.notify( notify::event::CaseMessage { name: case.name().to_owned(), - status, + kind, message, elapsed_s: Some(notify::Elapsed(start.elapsed())), } @@ -485,7 +485,7 @@ fn run_case( .into(), )?; - Ok(case_status != Some(notify::RunStatus::Failed)) + Ok(case_status != Some(notify::MessageKind::Failed)) } /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`. diff --git a/crates/libtest2-harness/src/notify/pretty.rs b/crates/libtest2-harness/src/notify/pretty.rs index a665931..9d59e96 100644 --- a/crates/libtest2-harness/src/notify/pretty.rs +++ b/crates/libtest2-harness/src/notify/pretty.rs @@ -1,5 +1,5 @@ use super::Event; -use super::RunStatus; +use super::MessageKind; use super::FAILED; use super::IGNORED; use super::OK; @@ -53,10 +53,10 @@ impl super::Notifier for PrettyRunNotifier { } Event::CaseMessage(_) => {} Event::CaseComplete(inner) => { - let status = self.summary.get_status(&inner.name); + let status = self.summary.get_kind(&inner.name); let (s, style) = match status { - Some(RunStatus::Ignored) => ("ignored", IGNORED), - Some(RunStatus::Failed) => ("FAILED", FAILED), + Some(MessageKind::Ignored) => ("ignored", IGNORED), + Some(MessageKind::Failed) => ("FAILED", FAILED), None => ("ok", OK), }; diff --git a/crates/libtest2-harness/src/notify/summary.rs b/crates/libtest2-harness/src/notify/summary.rs index 7622c91..f1b5e6d 100644 --- a/crates/libtest2-harness/src/notify/summary.rs +++ b/crates/libtest2-harness/src/notify/summary.rs @@ -1,6 +1,6 @@ use super::event::CaseMessage; use super::Event; -use super::RunStatus; +use super::MessageKind; use super::FAILED; use super::OK; @@ -16,7 +16,7 @@ pub(crate) struct Summary { } impl Summary { - pub(crate) fn get_status(&self, name: &str) -> Option { + pub(crate) fn get_kind(&self, name: &str) -> Option { let status = self.status.get(name)?; find_run_status(status) } @@ -38,20 +38,20 @@ impl Summary { let mut status = find_run_status(case_status); if !case_status.started { // Even override `Ignored` - status = Some(RunStatus::Failed); + status = Some(MessageKind::Failed); failures.insert(name, Some("test found that never started")); } if !case_status.completed { // Even override `Ignored` - status = Some(RunStatus::Failed); + status = Some(MessageKind::Failed); failures.insert(name, Some("test never completed")); } match status { - Some(RunStatus::Ignored) => num_ignored += 1, - Some(RunStatus::Failed) => { + Some(MessageKind::Ignored) => num_ignored += 1, + Some(MessageKind::Failed) => { num_failed += 1; for event in &case_status.messages { - if Some(event.status) == status { + if Some(event.kind) == status { failures.insert(name, event.message.as_deref()); } } @@ -142,10 +142,10 @@ impl super::Notifier for Summary { } } -fn find_run_status(case_status: &CaseStatus) -> Option { +fn find_run_status(case_status: &CaseStatus) -> Option { let mut status = None; for event in &case_status.messages { - status = status.max(Some(event.status)); + status = status.max(Some(event.kind)); } status } diff --git a/crates/libtest2-harness/src/notify/terse.rs b/crates/libtest2-harness/src/notify/terse.rs index 7ceada5..c9917c3 100644 --- a/crates/libtest2-harness/src/notify/terse.rs +++ b/crates/libtest2-harness/src/notify/terse.rs @@ -1,5 +1,5 @@ use super::Event; -use super::RunStatus; +use super::MessageKind; use super::FAILED; use super::IGNORED; use super::OK; @@ -71,10 +71,10 @@ impl super::Notifier for TerseRunNotifier { Event::CaseStart(_) => {} Event::CaseMessage(_) => {} Event::CaseComplete(inner) => { - let status = self.summary.get_status(&inner.name); + let status = self.summary.get_kind(&inner.name); let (c, style) = match status { - Some(RunStatus::Ignored) => ('i', IGNORED), - Some(RunStatus::Failed) => ('F', FAILED), + Some(MessageKind::Ignored) => ('i', IGNORED), + Some(MessageKind::Failed) => ('F', FAILED), None => ('.', OK), }; write!(self.writer, "{style}{c}{style:#}")?; diff --git a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs index 7f5779b..3702818 100644 --- a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs @@ -900,7 +900,7 @@ fn test_json() { { "event": "case_message", "name": "bear", - "status": "ignored", + "kind": "ignored", "message": "fails", "elapsed_s": "[..]" }, @@ -1007,15 +1007,15 @@ fn test_json() { "elapsed_s": "[..]" }, { - "event": "case_message", + "event": "case_complete", "name": "bear", - "status": "ignored", - "message": "fails", "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "bear", + "kind": "ignored", + "message": "fails", "elapsed_s": "[..]" } ] @@ -1169,7 +1169,7 @@ fn fail_fast_json() { { "event": "case_message", "name": "bear", - "status": "ignored", + "kind": "ignored", "message": "fails", "elapsed_s": "[..]" }, @@ -1186,7 +1186,7 @@ fn fail_fast_json() { { "event": "case_message", "name": "bunny", - "status": "ignored", + "kind": "ignored", "message": "fails", "elapsed_s": "[..]" }, @@ -1213,7 +1213,7 @@ fn fail_fast_json() { { "event": "case_message", "name": "dog", - "status": "failed", + "kind": "failed", "message": "was not a good boy", "elapsed_s": "[..]" }, diff --git a/crates/libtest2/tests/testsuite/mixed_bag.rs b/crates/libtest2/tests/testsuite/mixed_bag.rs index 5b92871..84b6896 100644 --- a/crates/libtest2/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2/tests/testsuite/mixed_bag.rs @@ -907,7 +907,7 @@ fn test_json() { { "event": "case_message", "name": "bear", - "status": "ignored", + "kind": "ignored", "message": "fails", "elapsed_s": "[..]" }, @@ -1014,15 +1014,15 @@ fn test_json() { "elapsed_s": "[..]" }, { - "event": "case_message", + "event": "case_complete", "name": "bear", - "status": "ignored", - "message": "fails", "elapsed_s": "[..]" }, { - "event": "case_complete", + "event": "case_message", "name": "bear", + "kind": "ignored", + "message": "fails", "elapsed_s": "[..]" } ] @@ -1176,7 +1176,7 @@ fn fail_fast_json() { { "event": "case_message", "name": "bear", - "status": "ignored", + "kind": "ignored", "message": "fails", "elapsed_s": "[..]" }, @@ -1193,7 +1193,7 @@ fn fail_fast_json() { { "event": "case_message", "name": "bunny", - "status": "ignored", + "kind": "ignored", "message": "fails", "elapsed_s": "[..]" }, @@ -1220,7 +1220,7 @@ fn fail_fast_json() { { "event": "case_message", "name": "dog", - "status": "failed", + "kind": "failed", "message": "was not a good boy", "elapsed_s": "[..]" }, From 10674e43c4b273e7e2922dac2dbfd2aa329a3e49 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Wed, 20 Aug 2025 14:27:51 -0500 Subject: [PATCH 6/6] fix(json)!: Rename MessageKind::Failed to Error --- crates/libtest-json/event.schema.json | 2 +- crates/libtest-json/src/event.rs | 4 ++-- crates/libtest-json/tests/roundtrip.rs | 8 +++++--- crates/libtest2-harness/src/case.rs | 2 +- crates/libtest2-harness/src/harness.rs | 4 ++-- crates/libtest2-harness/src/notify/pretty.rs | 2 +- crates/libtest2-harness/src/notify/summary.rs | 6 +++--- crates/libtest2-harness/src/notify/terse.rs | 2 +- crates/libtest2-mimic/tests/testsuite/mixed_bag.rs | 2 +- crates/libtest2/tests/testsuite/mixed_bag.rs | 2 +- 10 files changed, 18 insertions(+), 16 deletions(-) diff --git a/crates/libtest-json/event.schema.json b/crates/libtest-json/event.schema.json index af23661..efff322 100644 --- a/crates/libtest-json/event.schema.json +++ b/crates/libtest-json/event.schema.json @@ -216,7 +216,7 @@ "MessageKind": { "type": "string", "enum": [ - "failed", + "error", "ignored" ] }, diff --git a/crates/libtest-json/src/event.rs b/crates/libtest-json/src/event.rs index 10c4376..68132f5 100644 --- a/crates/libtest-json/src/event.rs +++ b/crates/libtest-json/src/event.rs @@ -481,14 +481,14 @@ impl RunMode { #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] pub enum MessageKind { // Highest precedent items for determining test status last - Failed, + Error, Ignored, } impl MessageKind { pub fn as_str(&self) -> &str { match self { - Self::Failed => "failed", + Self::Error => "error", Self::Ignored => "ignored", } } diff --git a/crates/libtest-json/tests/roundtrip.rs b/crates/libtest-json/tests/roundtrip.rs index 160fa20..13381d6 100644 --- a/crates/libtest-json/tests/roundtrip.rs +++ b/crates/libtest-json/tests/roundtrip.rs @@ -106,11 +106,11 @@ fn case_message() { t( libtest_json::event::CaseMessage { name: "Hello\tworld!".to_owned(), - kind: libtest_json::MessageKind::Failed, + kind: libtest_json::MessageKind::Error, message: None, elapsed_s: None, }, - str![[r#"{"event":"case_message","name":"Hello\tworld!","kind":"failed"}"#]], + str![[r#"{"event":"case_message","name":"Hello\tworld!","kind":"error"}"#]], ); t( @@ -120,7 +120,9 @@ fn case_message() { message: Some("This\tfailed".to_owned()), elapsed_s: Some(libtest_json::Elapsed(Default::default())), }, - str![[r#"{"event":"case_message","name":"Hello\tworld!","kind":"ignored","message":"This\tfailed","elapsed_s":"0"}"#]], + str![[ + r#"{"event":"case_message","name":"Hello\tworld!","kind":"ignored","message":"This\tfailed","elapsed_s":"0"}"# + ]], ); } diff --git a/crates/libtest2-harness/src/case.rs b/crates/libtest2-harness/src/case.rs index d1c0ee0..d7639f5 100644 --- a/crates/libtest2-harness/src/case.rs +++ b/crates/libtest2-harness/src/case.rs @@ -60,7 +60,7 @@ pub struct RunError { impl RunError { pub fn with_cause(cause: impl std::error::Error + Send + Sync + 'static) -> Self { Self { - status: notify::MessageKind::Failed, + status: notify::MessageKind::Error, cause: Some(Box::new(cause)), } } diff --git a/crates/libtest2-harness/src/harness.rs b/crates/libtest2-harness/src/harness.rs index 3a3aab1..c823114 100644 --- a/crates/libtest2-harness/src/harness.rs +++ b/crates/libtest2-harness/src/harness.rs @@ -317,7 +317,7 @@ fn run( notifier: &mut dyn notify::Notifier, ) -> std::io::Result<()> { if self.join_handle.join().is_err() { - let kind = notify::MessageKind::Failed; + let kind = notify::MessageKind::Error; let message = Some("panicked after reporting success".to_owned()); notifier.notify( notify::event::CaseMessage { @@ -485,7 +485,7 @@ fn run_case( .into(), )?; - Ok(case_status != Some(notify::MessageKind::Failed)) + Ok(case_status != Some(notify::MessageKind::Error)) } /// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`. diff --git a/crates/libtest2-harness/src/notify/pretty.rs b/crates/libtest2-harness/src/notify/pretty.rs index 9d59e96..515397b 100644 --- a/crates/libtest2-harness/src/notify/pretty.rs +++ b/crates/libtest2-harness/src/notify/pretty.rs @@ -56,7 +56,7 @@ impl super::Notifier for PrettyRunNotifier { let status = self.summary.get_kind(&inner.name); let (s, style) = match status { Some(MessageKind::Ignored) => ("ignored", IGNORED), - Some(MessageKind::Failed) => ("FAILED", FAILED), + Some(MessageKind::Error) => ("FAILED", FAILED), None => ("ok", OK), }; diff --git a/crates/libtest2-harness/src/notify/summary.rs b/crates/libtest2-harness/src/notify/summary.rs index f1b5e6d..4f0b956 100644 --- a/crates/libtest2-harness/src/notify/summary.rs +++ b/crates/libtest2-harness/src/notify/summary.rs @@ -38,17 +38,17 @@ impl Summary { let mut status = find_run_status(case_status); if !case_status.started { // Even override `Ignored` - status = Some(MessageKind::Failed); + status = Some(MessageKind::Error); failures.insert(name, Some("test found that never started")); } if !case_status.completed { // Even override `Ignored` - status = Some(MessageKind::Failed); + status = Some(MessageKind::Error); failures.insert(name, Some("test never completed")); } match status { Some(MessageKind::Ignored) => num_ignored += 1, - Some(MessageKind::Failed) => { + Some(MessageKind::Error) => { num_failed += 1; for event in &case_status.messages { if Some(event.kind) == status { diff --git a/crates/libtest2-harness/src/notify/terse.rs b/crates/libtest2-harness/src/notify/terse.rs index c9917c3..ac212bb 100644 --- a/crates/libtest2-harness/src/notify/terse.rs +++ b/crates/libtest2-harness/src/notify/terse.rs @@ -74,7 +74,7 @@ impl super::Notifier for TerseRunNotifier { let status = self.summary.get_kind(&inner.name); let (c, style) = match status { Some(MessageKind::Ignored) => ('i', IGNORED), - Some(MessageKind::Failed) => ('F', FAILED), + Some(MessageKind::Error) => ('F', FAILED), None => ('.', OK), }; write!(self.writer, "{style}{c}{style:#}")?; diff --git a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs index 3702818..7e36001 100644 --- a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs @@ -1213,7 +1213,7 @@ fn fail_fast_json() { { "event": "case_message", "name": "dog", - "kind": "failed", + "kind": "error", "message": "was not a good boy", "elapsed_s": "[..]" }, diff --git a/crates/libtest2/tests/testsuite/mixed_bag.rs b/crates/libtest2/tests/testsuite/mixed_bag.rs index 84b6896..fc88276 100644 --- a/crates/libtest2/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2/tests/testsuite/mixed_bag.rs @@ -1220,7 +1220,7 @@ fn fail_fast_json() { { "event": "case_message", "name": "dog", - "kind": "failed", + "kind": "error", "message": "was not a good boy", "elapsed_s": "[..]" },