feat(json)!: Multi-error test support

DESIGN.md

@@ -37,6 +37,7 @@ Decisions
   - Terse and pretty progress indicators are too nebulous to render (see their notifiers)
   - There is likely not enough value add in the failure message
   - This puts more of a burden on custom test harnesses for their implementation than is strictly needed
+- Report failures separate from test-complete so we can have multiple
 
 ### Prior Art
 

crates/libtest-json/event.schema.json

@@ -67,6 +67,19 @@
         "event"
       ]
     },
+    {
+      "type": "object",
+      "properties": {
+        "event": {
+          "type": "string",
+          "const": "case_message"
+        }
+      },
+      "$ref": "#/$defs/CaseMessage",
+      "required": [
+        "event"
+      ]
+    },
     {
       "type": "object",
       "properties": {
@@ -200,35 +213,49 @@
         "name"
       ]
     },
-    "RunStatus": {
+    "MessageKind": {
       "type": "string",
       "enum": [
-        "ignored",
-        "failed"
+        "error",
+        "ignored"
       ]
     },
-    "CaseComplete": {
+    "CaseMessage": {
       "type": "object",
       "properties": {
         "name": {
           "type": "string"
         },
-        "status": {
-          "description": "`None` means success",
+        "kind": {
+          "$ref": "#/$defs/MessageKind"
+        },
+        "message": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "elapsed_s": {
           "anyOf": [
             {
-              "$ref": "#/$defs/RunStatus"
+              "$ref": "#/$defs/Elapsed"
             },
             {
               "type": "null"
             }
           ]
-        },
-        "message": {
-          "type": [
-            "string",
-            "null"
-          ]
+        }
+      },
+      "required": [
+        "name",
+        "kind"
+      ]
+    },
+    "CaseComplete": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
         },
         "elapsed_s": {
           "anyOf": [

crates/libtest-json/src/event.rs

@@ -9,6 +9,7 @@ pub enum Event {
     DiscoverComplete(DiscoverComplete),
     RunStart(RunStart),
     CaseStart(CaseStart),
+    CaseMessage(CaseMessage),
     CaseComplete(CaseComplete),
     RunComplete(RunComplete),
 }
@@ -22,6 +23,7 @@ impl Event {
             Self::DiscoverComplete(event) => event.to_jsonline(),
             Self::RunStart(event) => event.to_jsonline(),
             Self::CaseStart(event) => event.to_jsonline(),
+            Self::CaseMessage(event) => event.to_jsonline(),
             Self::CaseComplete(event) => event.to_jsonline(),
             Self::RunComplete(event) => event.to_jsonline(),
         }
@@ -58,6 +60,12 @@ impl From<CaseStart> for Event {
     }
 }
 
+impl From<CaseMessage> for Event {
+    fn from(inner: CaseMessage) -> Self {
+        Self::CaseMessage(inner)
+    }
+}
+
 impl From<CaseComplete> for Event {
     fn from(inner: CaseComplete) -> Self {
         Self::CaseComplete(inner)
@@ -296,14 +304,9 @@ impl CaseStart {
 #[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
-pub struct CaseComplete {
+pub struct CaseMessage {
     pub name: String,
-    /// `None` means success
-    #[cfg_attr(
-        feature = "serde",
-        serde(default, skip_serializing_if = "Option::is_none")
-    )]
-    pub status: Option<RunStatus>,
+    pub kind: MessageKind,
     #[cfg_attr(
         feature = "serde",
         serde(default, skip_serializing_if = "Option::is_none")
@@ -316,7 +319,7 @@ pub struct CaseComplete {
     pub elapsed_s: Option<Elapsed>,
 }
 
-impl CaseComplete {
+impl CaseMessage {
     #[cfg(feature = "json")]
     pub fn to_jsonline(&self) -> String {
         use json_write::JsonWrite as _;
@@ -326,19 +329,17 @@ impl CaseComplete {
 
         buffer.key("event").unwrap();
         buffer.keyval_sep().unwrap();
-        buffer.value("case_complete").unwrap();
+        buffer.value("case_message").unwrap();
 
         buffer.val_sep().unwrap();
         buffer.key("name").unwrap();
         buffer.keyval_sep().unwrap();
         buffer.value(&self.name).unwrap();
 
-        if let Some(status) = self.status {
-            buffer.val_sep().unwrap();
-            buffer.key("status").unwrap();
-            buffer.keyval_sep().unwrap();
-            buffer.value(status.as_str()).unwrap();
-        }
+        buffer.val_sep().unwrap();
+        buffer.key("kind").unwrap();
+        buffer.keyval_sep().unwrap();
+        buffer.value(self.kind.as_str()).unwrap();
 
         if let Some(message) = &self.message {
             buffer.val_sep().unwrap();
@@ -360,6 +361,49 @@ impl CaseComplete {
     }
 }
 
+#[derive(Clone, Debug)]
+#[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
+pub struct CaseComplete {
+    pub name: String,
+    #[cfg_attr(
+        feature = "serde",
+        serde(default, skip_serializing_if = "Option::is_none")
+    )]
+    pub elapsed_s: Option<Elapsed>,
+}
+
+impl CaseComplete {
+    #[cfg(feature = "json")]
+    pub fn to_jsonline(&self) -> String {
+        use json_write::JsonWrite as _;
+
+        let mut buffer = String::new();
+        buffer.open_object().unwrap();
+
+        buffer.key("event").unwrap();
+        buffer.keyval_sep().unwrap();
+        buffer.value("case_complete").unwrap();
+
+        buffer.val_sep().unwrap();
+        buffer.key("name").unwrap();
+        buffer.keyval_sep().unwrap();
+        buffer.value(&self.name).unwrap();
+
+        if let Some(elapsed_s) = self.elapsed_s {
+            buffer.val_sep().unwrap();
+            buffer.key("elapsed_s").unwrap();
+            buffer.keyval_sep().unwrap();
+            buffer.value(String::from(elapsed_s)).unwrap();
+        }
+
+        buffer.close_object().unwrap();
+
+        buffer
+    }
+}
+
 #[derive(Clone, Debug)]
 #[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
@@ -397,10 +441,12 @@ impl RunComplete {
     }
 }
 
+#[cfg(feature = "serde")]
 fn true_default() -> bool {
     true
 }
 
+#[cfg(feature = "serde")]
 fn is_true(yes: &bool) -> bool {
     *yes
 }
@@ -423,25 +469,27 @@ impl RunMode {
         }
     }
 
+    #[cfg(any(feature = "serde", feature = "json"))]
     fn is_default(&self) -> bool {
         *self == Default::default()
     }
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
 #[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
-pub enum RunStatus {
+pub enum MessageKind {
+    // Highest precedent items for determining test status last
+    Error,
     Ignored,
-    Failed,
 }
 
-impl RunStatus {
+impl MessageKind {
     pub fn as_str(&self) -> &str {
         match self {
+            Self::Error => "error",
             Self::Ignored => "ignored",
-            Self::Failed => "failed",
         }
     }
 }

crates/libtest-json/src/lib.rs

@@ -9,8 +9,8 @@ pub mod event;
 
 pub use event::Elapsed;
 pub use event::Event;
+pub use event::MessageKind;
 pub use event::RunMode;
-pub use event::RunStatus;
 
 #[doc = include_str!("../README.md")]
 #[cfg(doctest)]

crates/libtest-json/tests/roundtrip.rs

@@ -102,30 +102,49 @@ fn case_start() {
 }
 
 #[test]
-fn case_complete() {
+fn case_message() {
     t(
-        libtest_json::event::CaseComplete {
+        libtest_json::event::CaseMessage {
             name: "Hello\tworld!".to_owned(),
-            status: None,
+            kind: libtest_json::MessageKind::Error,
             message: None,
             elapsed_s: None,
         },
-        str![[r#"{"event":"case_complete","name":"Hello\tworld!"}"#]],
+        str![[r#"{"event":"case_message","name":"Hello\tworld!","kind":"error"}"#]],
     );
 
     t(
-        libtest_json::event::CaseComplete {
+        libtest_json::event::CaseMessage {
             name: "Hello\tworld!".to_owned(),
-            status: Some(libtest_json::RunStatus::Ignored),
+            kind: libtest_json::MessageKind::Ignored,
             message: Some("This\tfailed".to_owned()),
             elapsed_s: Some(libtest_json::Elapsed(Default::default())),
         },
         str![[
-            r#"{"event":"case_complete","name":"Hello\tworld!","status":"ignored","message":"This\tfailed","elapsed_s":"0"}"#
+            r#"{"event":"case_message","name":"Hello\tworld!","kind":"ignored","message":"This\tfailed","elapsed_s":"0"}"#
         ]],
     );
 }
 
+#[test]
+fn case_complete() {
+    t(
+        libtest_json::event::CaseComplete {
+            name: "Hello\tworld!".to_owned(),
+            elapsed_s: None,
+        },
+        str![[r#"{"event":"case_complete","name":"Hello\tworld!"}"#]],
+    );
+
+    t(
+        libtest_json::event::CaseComplete {
+            name: "Hello\tworld!".to_owned(),
+            elapsed_s: Some(libtest_json::Elapsed(Default::default())),
+        },
+        str![[r#"{"event":"case_complete","name":"Hello\tworld!","elapsed_s":"0"}"#]],
+    );
+}
+
 #[test]
 fn suite_complete() {
     t(

crates/libtest2-harness/src/case.rs

@@ -53,14 +53,14 @@ pub type RunResult = Result<(), RunError>;
 
 #[derive(Debug)]
 pub struct RunError {
-    status: notify::RunStatus,
+    status: notify::MessageKind,
     cause: Option<Box<dyn std::error::Error + Send + Sync + 'static>>,
 }
 
 impl RunError {
     pub fn with_cause(cause: impl std::error::Error + Send + Sync + 'static) -> Self {
         Self {
-            status: notify::RunStatus::Failed,
+            status: notify::MessageKind::Error,
             cause: Some(Box::new(cause)),
         }
     }
@@ -71,19 +71,19 @@ impl RunError {
 
     pub(crate) fn ignore() -> Self {
         Self {
-            status: notify::RunStatus::Ignored,
+            status: notify::MessageKind::Ignored,
             cause: None,
         }
     }
 
     pub(crate) fn ignore_for(reason: String) -> Self {
         Self {
-            status: notify::RunStatus::Ignored,
+            status: notify::MessageKind::Ignored,
             cause: Some(Box::new(Message(reason))),
         }
     }
 
-    pub(crate) fn status(&self) -> notify::RunStatus {
+    pub(crate) fn status(&self) -> notify::MessageKind {
         self.status
     }