diff --git a/Cargo.lock b/Cargo.lock index bb307ce..2f55a02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -363,6 +363,8 @@ dependencies = [ "anstream", "anstyle", "normalize-line-endings", + "serde", + "serde_json", "similar", "snapbox-macros", ] diff --git a/DESIGN.md b/DESIGN.md index 88a78f3..6bfb3b0 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -149,3 +149,10 @@ support for multi-character shorts is something people may want to opt-in to (it Performance isn't the top priority, so remoing `&str` -> `char` conversions isn't necessarily viewed as a benefit. This also makes `match` need to work off of `&str` instead of `char`. Unsure which of those would be slower and how the different characteristics match up. + +## Harness + +### Decision: report and run tests in filter order + +Rather than build into every harness shuffle, sharding, and any other specific logic like that, +we can instead give the user direct control over the test order by the order they are specified on the command line. diff --git a/crates/libtest2-harness/src/harness.rs b/crates/libtest2-harness/src/harness.rs index 6b4cc0e..3abc016 100644 --- a/crates/libtest2-harness/src/harness.rs +++ b/crates/libtest2-harness/src/harness.rs @@ -161,9 +161,6 @@ fn discover( notifier.notify(notify::Event::DiscoverStart)?; let timer = std::time::Instant::now(); - // Do this first so it applies to both discover and running - cases.sort_unstable_by_key(|case| case.name().to_owned()); - let matches_filter = |case: &dyn Case, filter: &str| { let test_name = case.name(); @@ -172,6 +169,20 @@ fn discover( false => test_name.contains(filter), } }; + + // Do this first so it applies to both discover and running + cases.sort_unstable_by_key(|case| { + let priority = if opts.filters.is_empty() { + Some(0) + } else { + opts.filters + .iter() + .position(|filter| matches_filter(case.as_ref(), filter)) + }; + let name = case.name().to_owned(); + (priority, name) + }); + let mut retain_cases = Vec::with_capacity(cases.len()); for case in cases.iter() { let filtered_in = opts.filters.is_empty() diff --git a/crates/libtest2-mimic/Cargo.toml b/crates/libtest2-mimic/Cargo.toml index 6e03675..a9abd2b 100644 --- a/crates/libtest2-mimic/Cargo.toml +++ b/crates/libtest2-mimic/Cargo.toml @@ -37,7 +37,7 @@ dunce = "1.0.4" escargot = "0.5.8" once_cell_polyfill = "1.56.0" pathdiff = "0.2.1" -snapbox = "0.6.0" +snapbox = { version = "0.6.0", features = ["json"] } [lints] workspace = true diff --git a/crates/libtest2-mimic/tests/testsuite/all_passing.rs b/crates/libtest2-mimic/tests/testsuite/all_passing.rs index 4a9f681..6438fe5 100644 --- a/crates/libtest2-mimic/tests/testsuite/all_passing.rs +++ b/crates/libtest2-mimic/tests/testsuite/all_passing.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -23,7 +26,7 @@ fn main() { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], single: &str, parallel: &str) { +fn check(args: &[&str], single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -37,7 +40,8 @@ fn check(args: &[&str], single: &str, parallel: &str) { fn normal() { check( &[], - r#" + str![[r#" + running 3 tests test bar ... ok test barro ... ok @@ -45,14 +49,17 @@ test foo ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 3 tests ... test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -60,20 +67,24 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..] fn filter_one() { check( &["foo"], - r#" + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -81,21 +92,25 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn filter_two() { check( &["bar"], - r#" + str![[r#" + running 2 tests test bar ... ok test barro ... ok test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -103,20 +118,24 @@ test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..] fn filter_exact() { check( &["bar", "--exact"], - r#" + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -124,20 +143,24 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn filter_two_and_skip() { check( &["--skip", "barro", "bar"], - r#" + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -145,7 +168,8 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn skip_nothing() { check( &["--skip", "peter"], - r#" + str![[r#" + running 3 tests test bar ... ok test barro ... ok @@ -153,14 +177,17 @@ test foo ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 3 tests ... test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -168,20 +195,24 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..] fn skip_two() { check( &["--skip", "bar"], - r#" + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -189,21 +220,25 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn skip_exact() { check( &["--exact", "--skip", "bar"], - r#" + str![[r#" + running 2 tests test barro ... ok test foo ... ok test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -211,17 +246,21 @@ test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..] fn terse_output() { check( &["--quiet", "--skip", "foo"], - r#" + str![[r#" + running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2-mimic/tests/testsuite/main_thread.rs b/crates/libtest2-mimic/tests/testsuite/main_thread.rs index f303e87..48cf9ed 100644 --- a/crates/libtest2-mimic/tests/testsuite/main_thread.rs +++ b/crates/libtest2-mimic/tests/testsuite/main_thread.rs @@ -1,3 +1,5 @@ +use snapbox::str; + #[test] fn check_test_on_main_thread() { let package_root = crate::util::new_test( @@ -22,9 +24,10 @@ fn main() { .current_dir(package_root) .assert() .success() - .stdout_eq( - " + .stdout_eq(str![[r#" + running 1 test -...", - ); +... + +"#]]); } diff --git a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs index 9f90376..b95d7eb 100644 --- a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -44,7 +47,7 @@ fn main() { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], code: i32, single: &str, parallel: &str) { +fn check(args: &[&str], code: i32, single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -63,7 +66,8 @@ fn normal() { check( &[], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -85,8 +89,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -101,7 +107,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -110,7 +117,8 @@ fn test_mode() { check( &["--test"], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -132,8 +140,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -148,7 +158,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -157,7 +168,8 @@ fn bench_mode() { check( &["--bench"], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -179,8 +191,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -195,7 +209,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -204,7 +219,8 @@ fn list() { check( &["--list"], 0, - r#"bear: test + str![[r#" +bear: test bunny: test cat: test dog: test @@ -215,8 +231,10 @@ owl: test 8 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test bunny: test cat: test dog: test @@ -227,7 +245,8 @@ owl: test 8 tests -"#, + +"#]], ); } @@ -236,7 +255,8 @@ fn list_ignored() { check( &["--list", "--ignored"], 0, - r#"bear: test + str![[r#" +bear: test bunny: test cat: test dog: test @@ -247,8 +267,10 @@ owl: test 8 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test bunny: test cat: test dog: test @@ -259,7 +281,8 @@ owl: test 8 tests -"#, + +"#]], ); } @@ -268,18 +291,50 @@ fn list_with_filter() { check( &["--list", "a"], 0, - r#"bear: test + str![[r#" +bear: test cat: test 2 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test cat: test 2 tests -"#, + +"#]], + ); +} + +#[test] +fn list_with_specified_order() { + check( + &["--list", "--exact", "owl", "fox", "bunny", "frog"], + 0, + str![[r#" +owl: test +fox: test +bunny: test +frog: test + +4 tests + + +"#]], + str![[r#" +owl: test +fox: test +bunny: test +frog: test + +4 tests + + +"#]], ); } @@ -288,21 +343,25 @@ fn filter_c() { check( &["a"], 0, - r#" + str![[r#" + running 2 tests test bear ... ignored test cat ... ok test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -311,21 +370,25 @@ fn filter_o_test() { check( &["--test", "a"], 0, - r#" + str![[r#" + running 2 tests test bear ... ignored test cat ... ok test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -334,7 +397,8 @@ fn filter_o_test_include_ignored() { check( &["--test", "--include-ignored", "o"], 101, - r#" + str![[r#" + running 4 tests test dog ... FAILED test fox ... ok @@ -356,8 +420,10 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 4 tests ... @@ -376,7 +442,8 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -385,7 +452,8 @@ fn filter_o_test_ignored() { check( &["--test", "--ignored", "o"], 101, - r#" + str![[r#" + running 4 tests test dog ... FAILED test fox ... ok @@ -407,8 +475,10 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 4 tests ... @@ -427,7 +497,8 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -436,7 +507,8 @@ fn normal_include_ignored() { check( &["--include-ignored"], 101, - r#" + str![[r#" + running 8 tests test bear ... FAILED test bunny ... FAILED @@ -470,8 +542,10 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -498,7 +572,8 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -507,7 +582,8 @@ fn normal_ignored() { check( &["--ignored"], 101, - r#" + str![[r#" + running 8 tests test bear ... FAILED test bunny ... FAILED @@ -541,8 +617,10 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -569,7 +647,8 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -578,7 +657,8 @@ fn lots_of_flags() { check( &["--ignored", "--skip", "g", "--test", "o"], 101, - r#" + str![[r#" + running 2 tests test fox ... ok test owl ... FAILED @@ -594,8 +674,10 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... @@ -610,7 +692,8 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -620,28 +703,128 @@ fn list_json() { check( &["-Zunstable-options", "--format=json", "--list", "a"], 0, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -"#, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -"#, + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), ); } @@ -651,40 +834,191 @@ fn test_json() { check( &["-Zunstable-options", "--format=json", "a"], 0, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -{"event":"suite-start"} -{"event":"case-start","name":"bear"} -{"event":"case-complete","name":"bear","mode":"test","status":"ignored","message":"fails","elapsed_s":"[..]"} -{"event":"case-start","name":"cat"} -{"event":"case-complete","name":"cat","mode":"test","status":null,"message":null,"elapsed_s":"[..]"} -{"event":"suite-complete","elapsed_s":"[..]"} -"#, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -{"event":"suite-start"} -[..] -[..] -[..] -[..] -{"event":"suite-complete","elapsed_s":"[..]"} -"#, + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + }, + { + "event": "suite-start" + }, + { + "event": "case-start", + "name": "bear" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": "fails", + "mode": "test", + "name": "bear", + "status": "ignored" + }, + { + "event": "case-start", + "name": "cat" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": null, + "mode": "test", + "name": "cat", + "status": null + }, + { + "elapsed_s": "[..]", + "event": "suite-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + }, + { + "event": "suite-start" + }, + { + "event": "case-start", + "name": "bear" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": "fails", + "mode": "test", + "name": "bear", + "status": "ignored" + }, + { + "event": "case-start", + "name": "cat" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": null, + "mode": "test", + "name": "cat", + "status": null + }, + { + "elapsed_s": "[..]", + "event": "suite-complete" + } +] +"#]] + .unordered() + .is_json() + .against_jsonlines(), ); } @@ -693,7 +1027,8 @@ fn terse_output() { check( &["--quiet"], 101, - r#" + str![[r#" + running 8 tests ii.Fi.ii failures: @@ -707,8 +1042,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... failures: @@ -722,6 +1059,7 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2-mimic/tests/testsuite/panic.rs b/crates/libtest2-mimic/tests/testsuite/panic.rs index e4006bf..9d2db79 100644 --- a/crates/libtest2-mimic/tests/testsuite/panic.rs +++ b/crates/libtest2-mimic/tests/testsuite/panic.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -22,7 +25,7 @@ fn main() { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], code: i32, single: &str, parallel: &str) { +fn check(args: &[&str], code: i32, single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -41,7 +44,8 @@ fn normal() { check( &[], 101, - r#" + str![[r#" + running 2 tests test panics ... FAILED test passes ... ok @@ -57,8 +61,10 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... @@ -73,6 +79,7 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2/Cargo.toml b/crates/libtest2/Cargo.toml index ca33579..cac52b8 100644 --- a/crates/libtest2/Cargo.toml +++ b/crates/libtest2/Cargo.toml @@ -37,7 +37,7 @@ dunce = "1.0.4" escargot = "0.5.8" once_cell_polyfill = "1.56.0" pathdiff = "0.2.1" -snapbox = "0.6.0" +snapbox = { version = "0.6.0", features = ["json"] } [lints] workspace = true diff --git a/crates/libtest2/tests/testsuite/all_passing.rs b/crates/libtest2/tests/testsuite/all_passing.rs index 912b529..38586a4 100644 --- a/crates/libtest2/tests/testsuite/all_passing.rs +++ b/crates/libtest2/tests/testsuite/all_passing.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -26,7 +29,7 @@ fn barro(_state: &libtest2::State) -> libtest2::RunResult { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], single: &str, parallel: &str) { +fn check(args: &[&str], single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -40,7 +43,8 @@ fn check(args: &[&str], single: &str, parallel: &str) { fn normal() { check( &[], - r#" + str![[r#" + running 3 tests test bar ... ok test barro ... ok @@ -48,14 +52,17 @@ test foo ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 3 tests ... test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -63,20 +70,24 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..] fn filter_one() { check( &["foo"], - r#" + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -84,21 +95,25 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn filter_two() { check( &["bar"], - r#" + str![[r#" + running 2 tests test bar ... ok test barro ... ok test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -106,20 +121,24 @@ test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..] fn filter_exact() { check( &["bar", "--exact"], - r#" + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -127,20 +146,24 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn filter_two_and_skip() { check( &["--skip", "barro", "bar"], - r#" + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -148,7 +171,8 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn skip_nothing() { check( &["--skip", "peter"], - r#" + str![[r#" + running 3 tests test bar ... ok test barro ... ok @@ -156,14 +180,17 @@ test foo ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 3 tests ... test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -171,20 +198,24 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..] fn skip_two() { check( &["--skip", "bar"], - r#" + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -192,21 +223,25 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn skip_exact() { check( &["--exact", "--skip", "bar"], - r#" + str![[r#" + running 2 tests test barro ... ok test foo ... ok test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -214,17 +249,21 @@ test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..] fn terse_output() { check( &["--quiet", "--skip", "foo"], - r#" + str![[r#" + running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2/tests/testsuite/mixed_bag.rs b/crates/libtest2/tests/testsuite/mixed_bag.rs index c845b76..c46391a 100644 --- a/crates/libtest2/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2/tests/testsuite/mixed_bag.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -51,7 +54,7 @@ fn bear(state: &libtest2::State) -> libtest2::RunResult { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], code: i32, single: &str, parallel: &str) { +fn check(args: &[&str], code: i32, single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -70,7 +73,8 @@ fn normal() { check( &[], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -92,8 +96,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -108,7 +114,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -117,7 +124,8 @@ fn test_mode() { check( &["--test"], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -139,8 +147,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -155,7 +165,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -164,7 +175,8 @@ fn bench_mode() { check( &["--bench"], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -186,8 +198,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -202,7 +216,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -211,7 +226,8 @@ fn list() { check( &["--list"], 0, - r#"bear: test + str![[r#" +bear: test bunny: test cat: test dog: test @@ -222,8 +238,10 @@ owl: test 8 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test bunny: test cat: test dog: test @@ -234,7 +252,8 @@ owl: test 8 tests -"#, + +"#]], ); } @@ -243,7 +262,8 @@ fn list_ignored() { check( &["--list", "--ignored"], 0, - r#"bear: test + str![[r#" +bear: test bunny: test cat: test dog: test @@ -254,8 +274,10 @@ owl: test 8 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test bunny: test cat: test dog: test @@ -266,7 +288,8 @@ owl: test 8 tests -"#, + +"#]], ); } @@ -275,18 +298,50 @@ fn list_with_filter() { check( &["--list", "a"], 0, - r#"bear: test + str![[r#" +bear: test cat: test 2 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test cat: test 2 tests -"#, + +"#]], + ); +} + +#[test] +fn list_with_specified_order() { + check( + &["--list", "--exact", "owl", "fox", "bunny", "frog"], + 0, + str![[r#" +owl: test +fox: test +bunny: test +frog: test + +4 tests + + +"#]], + str![[r#" +owl: test +fox: test +bunny: test +frog: test + +4 tests + + +"#]], ); } @@ -295,21 +350,25 @@ fn filter_c() { check( &["a"], 0, - r#" + str![[r#" + running 2 tests test bear ... ignored test cat ... ok test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -318,21 +377,25 @@ fn filter_o_test() { check( &["--test", "a"], 0, - r#" + str![[r#" + running 2 tests test bear ... ignored test cat ... ok test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -341,7 +404,8 @@ fn filter_o_test_include_ignored() { check( &["--test", "--include-ignored", "o"], 101, - r#" + str![[r#" + running 4 tests test dog ... FAILED test fox ... ok @@ -363,8 +427,10 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 4 tests ... @@ -383,7 +449,8 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -392,7 +459,8 @@ fn filter_o_test_ignored() { check( &["--test", "--ignored", "o"], 101, - r#" + str![[r#" + running 4 tests test dog ... FAILED test fox ... ok @@ -414,8 +482,10 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 4 tests ... @@ -434,7 +504,8 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -443,7 +514,8 @@ fn normal_include_ignored() { check( &["--include-ignored"], 101, - r#" + str![[r#" + running 8 tests test bear ... FAILED test bunny ... FAILED @@ -477,8 +549,10 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -505,7 +579,8 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -514,7 +589,8 @@ fn normal_ignored() { check( &["--ignored"], 101, - r#" + str![[r#" + running 8 tests test bear ... FAILED test bunny ... FAILED @@ -548,8 +624,10 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -576,7 +654,8 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -585,7 +664,8 @@ fn lots_of_flags() { check( &["--ignored", "--skip", "g", "--test", "o"], 101, - r#" + str![[r#" + running 2 tests test fox ... ok test owl ... FAILED @@ -601,8 +681,10 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... @@ -617,7 +699,8 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -627,28 +710,128 @@ fn list_json() { check( &["-Zunstable-options", "--format=json", "--list", "a"], 0, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -"#, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -"#, + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), ); } @@ -658,40 +841,191 @@ fn test_json() { check( &["-Zunstable-options", "--format=json", "a"], 0, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -{"event":"suite-start"} -{"event":"case-start","name":"bear"} -{"event":"case-complete","name":"bear","mode":"test","status":"ignored","message":"fails","elapsed_s":"[..]"} -{"event":"case-start","name":"cat"} -{"event":"case-complete","name":"cat","mode":"test","status":null,"message":null,"elapsed_s":"[..]"} -{"event":"suite-complete","elapsed_s":"[..]"} -"#, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -{"event":"suite-start"} -[..] -[..] -[..] -[..] -{"event":"suite-complete","elapsed_s":"[..]"} -"#, + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + }, + { + "event": "suite-start" + }, + { + "event": "case-start", + "name": "bear" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": "fails", + "mode": "test", + "name": "bear", + "status": "ignored" + }, + { + "event": "case-start", + "name": "cat" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": null, + "mode": "test", + "name": "cat", + "status": null + }, + { + "elapsed_s": "[..]", + "event": "suite-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + }, + { + "event": "suite-start" + }, + { + "event": "case-start", + "name": "bear" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": "fails", + "mode": "test", + "name": "bear", + "status": "ignored" + }, + { + "event": "case-start", + "name": "cat" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": null, + "mode": "test", + "name": "cat", + "status": null + }, + { + "elapsed_s": "[..]", + "event": "suite-complete" + } +] +"#]] + .unordered() + .is_json() + .against_jsonlines(), ); } @@ -700,7 +1034,8 @@ fn terse_output() { check( &["--quiet"], 101, - r#" + str![[r#" + running 8 tests ii.Fi.ii failures: @@ -714,8 +1049,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... failures: @@ -729,6 +1066,7 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2/tests/testsuite/panic.rs b/crates/libtest2/tests/testsuite/panic.rs index a9af8bf..0c786a3 100644 --- a/crates/libtest2/tests/testsuite/panic.rs +++ b/crates/libtest2/tests/testsuite/panic.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -22,7 +25,7 @@ fn panics(_state: &libtest2::State) -> libtest2::RunResult { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], code: i32, single: &str, parallel: &str) { +fn check(args: &[&str], code: i32, single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -41,7 +44,8 @@ fn normal() { check( &[], 101, - r#" + str![[r#" + running 2 tests test panics ... FAILED test passes ... ok @@ -57,8 +61,10 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... @@ -73,6 +79,7 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); }