From b0397998cf282ea40ce340eb8bb90499a6ad71df Mon Sep 17 00:00:00 2001 From: Ed Page Date: Thu, 10 Jul 2025 16:43:42 -0500 Subject: [PATCH 1/3] test: Switch to snapshot testing --- Cargo.lock | 2 + crates/libtest2-mimic/Cargo.toml | 2 +- .../tests/testsuite/all_passing.rs | 113 ++-- .../tests/testsuite/main_thread.rs | 11 +- .../tests/testsuite/mixed_bag.rs | 536 ++++++++++++++---- .../libtest2-mimic/tests/testsuite/panic.rs | 17 +- crates/libtest2/Cargo.toml | 2 +- .../libtest2/tests/testsuite/all_passing.rs | 113 ++-- crates/libtest2/tests/testsuite/mixed_bag.rs | 536 ++++++++++++++---- crates/libtest2/tests/testsuite/panic.rs | 17 +- 10 files changed, 1033 insertions(+), 316 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb307ce..2f55a02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -363,6 +363,8 @@ dependencies = [ "anstream", "anstyle", "normalize-line-endings", + "serde", + "serde_json", "similar", "snapbox-macros", ] diff --git a/crates/libtest2-mimic/Cargo.toml b/crates/libtest2-mimic/Cargo.toml index 6e03675..a9abd2b 100644 --- a/crates/libtest2-mimic/Cargo.toml +++ b/crates/libtest2-mimic/Cargo.toml @@ -37,7 +37,7 @@ dunce = "1.0.4" escargot = "0.5.8" once_cell_polyfill = "1.56.0" pathdiff = "0.2.1" -snapbox = "0.6.0" +snapbox = { version = "0.6.0", features = ["json"] } [lints] workspace = true diff --git a/crates/libtest2-mimic/tests/testsuite/all_passing.rs b/crates/libtest2-mimic/tests/testsuite/all_passing.rs index 4a9f681..6438fe5 100644 --- a/crates/libtest2-mimic/tests/testsuite/all_passing.rs +++ b/crates/libtest2-mimic/tests/testsuite/all_passing.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -23,7 +26,7 @@ fn main() { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], single: &str, parallel: &str) { +fn check(args: &[&str], single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -37,7 +40,8 @@ fn check(args: &[&str], single: &str, parallel: &str) { fn normal() { check( &[], - r#" + str![[r#" + running 3 tests test bar ... ok test barro ... ok @@ -45,14 +49,17 @@ test foo ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 3 tests ... test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -60,20 +67,24 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..] fn filter_one() { check( &["foo"], - r#" + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -81,21 +92,25 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn filter_two() { check( &["bar"], - r#" + str![[r#" + running 2 tests test bar ... ok test barro ... ok test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -103,20 +118,24 @@ test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..] fn filter_exact() { check( &["bar", "--exact"], - r#" + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -124,20 +143,24 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn filter_two_and_skip() { check( &["--skip", "barro", "bar"], - r#" + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -145,7 +168,8 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn skip_nothing() { check( &["--skip", "peter"], - r#" + str![[r#" + running 3 tests test bar ... ok test barro ... ok @@ -153,14 +177,17 @@ test foo ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 3 tests ... test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -168,20 +195,24 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..] fn skip_two() { check( &["--skip", "bar"], - r#" + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -189,21 +220,25 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn skip_exact() { check( &["--exact", "--skip", "bar"], - r#" + str![[r#" + running 2 tests test barro ... ok test foo ... ok test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -211,17 +246,21 @@ test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..] fn terse_output() { check( &["--quiet", "--skip", "foo"], - r#" + str![[r#" + running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2-mimic/tests/testsuite/main_thread.rs b/crates/libtest2-mimic/tests/testsuite/main_thread.rs index f303e87..48cf9ed 100644 --- a/crates/libtest2-mimic/tests/testsuite/main_thread.rs +++ b/crates/libtest2-mimic/tests/testsuite/main_thread.rs @@ -1,3 +1,5 @@ +use snapbox::str; + #[test] fn check_test_on_main_thread() { let package_root = crate::util::new_test( @@ -22,9 +24,10 @@ fn main() { .current_dir(package_root) .assert() .success() - .stdout_eq( - " + .stdout_eq(str![[r#" + running 1 test -...", - ); +... + +"#]]); } diff --git a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs index 9f90376..49c16d2 100644 --- a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -44,7 +47,7 @@ fn main() { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], code: i32, single: &str, parallel: &str) { +fn check(args: &[&str], code: i32, single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -63,7 +66,8 @@ fn normal() { check( &[], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -85,8 +89,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -101,7 +107,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -110,7 +117,8 @@ fn test_mode() { check( &["--test"], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -132,8 +140,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -148,7 +158,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -157,7 +168,8 @@ fn bench_mode() { check( &["--bench"], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -179,8 +191,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -195,7 +209,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -204,7 +219,8 @@ fn list() { check( &["--list"], 0, - r#"bear: test + str![[r#" +bear: test bunny: test cat: test dog: test @@ -215,8 +231,10 @@ owl: test 8 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test bunny: test cat: test dog: test @@ -227,7 +245,8 @@ owl: test 8 tests -"#, + +"#]], ); } @@ -236,7 +255,8 @@ fn list_ignored() { check( &["--list", "--ignored"], 0, - r#"bear: test + str![[r#" +bear: test bunny: test cat: test dog: test @@ -247,8 +267,10 @@ owl: test 8 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test bunny: test cat: test dog: test @@ -259,7 +281,8 @@ owl: test 8 tests -"#, + +"#]], ); } @@ -268,18 +291,22 @@ fn list_with_filter() { check( &["--list", "a"], 0, - r#"bear: test + str![[r#" +bear: test cat: test 2 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test cat: test 2 tests -"#, + +"#]], ); } @@ -288,21 +315,25 @@ fn filter_c() { check( &["a"], 0, - r#" + str![[r#" + running 2 tests test bear ... ignored test cat ... ok test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -311,21 +342,25 @@ fn filter_o_test() { check( &["--test", "a"], 0, - r#" + str![[r#" + running 2 tests test bear ... ignored test cat ... ok test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -334,7 +369,8 @@ fn filter_o_test_include_ignored() { check( &["--test", "--include-ignored", "o"], 101, - r#" + str![[r#" + running 4 tests test dog ... FAILED test fox ... ok @@ -356,8 +392,10 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 4 tests ... @@ -376,7 +414,8 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -385,7 +424,8 @@ fn filter_o_test_ignored() { check( &["--test", "--ignored", "o"], 101, - r#" + str![[r#" + running 4 tests test dog ... FAILED test fox ... ok @@ -407,8 +447,10 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 4 tests ... @@ -427,7 +469,8 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -436,7 +479,8 @@ fn normal_include_ignored() { check( &["--include-ignored"], 101, - r#" + str![[r#" + running 8 tests test bear ... FAILED test bunny ... FAILED @@ -470,8 +514,10 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -498,7 +544,8 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -507,7 +554,8 @@ fn normal_ignored() { check( &["--ignored"], 101, - r#" + str![[r#" + running 8 tests test bear ... FAILED test bunny ... FAILED @@ -541,8 +589,10 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -569,7 +619,8 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -578,7 +629,8 @@ fn lots_of_flags() { check( &["--ignored", "--skip", "g", "--test", "o"], 101, - r#" + str![[r#" + running 2 tests test fox ... ok test owl ... FAILED @@ -594,8 +646,10 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... @@ -610,7 +664,8 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -620,28 +675,128 @@ fn list_json() { check( &["-Zunstable-options", "--format=json", "--list", "a"], 0, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -"#, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -"#, + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), ); } @@ -651,40 +806,191 @@ fn test_json() { check( &["-Zunstable-options", "--format=json", "a"], 0, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -{"event":"suite-start"} -{"event":"case-start","name":"bear"} -{"event":"case-complete","name":"bear","mode":"test","status":"ignored","message":"fails","elapsed_s":"[..]"} -{"event":"case-start","name":"cat"} -{"event":"case-complete","name":"cat","mode":"test","status":null,"message":null,"elapsed_s":"[..]"} -{"event":"suite-complete","elapsed_s":"[..]"} -"#, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -{"event":"suite-start"} -[..] -[..] -[..] -[..] -{"event":"suite-complete","elapsed_s":"[..]"} -"#, + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + }, + { + "event": "suite-start" + }, + { + "event": "case-start", + "name": "bear" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": "fails", + "mode": "test", + "name": "bear", + "status": "ignored" + }, + { + "event": "case-start", + "name": "cat" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": null, + "mode": "test", + "name": "cat", + "status": null + }, + { + "elapsed_s": "[..]", + "event": "suite-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + }, + { + "event": "suite-start" + }, + { + "event": "case-start", + "name": "bear" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": "fails", + "mode": "test", + "name": "bear", + "status": "ignored" + }, + { + "event": "case-start", + "name": "cat" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": null, + "mode": "test", + "name": "cat", + "status": null + }, + { + "elapsed_s": "[..]", + "event": "suite-complete" + } +] +"#]] + .unordered() + .is_json() + .against_jsonlines(), ); } @@ -693,7 +999,8 @@ fn terse_output() { check( &["--quiet"], 101, - r#" + str![[r#" + running 8 tests ii.Fi.ii failures: @@ -707,8 +1014,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... failures: @@ -722,6 +1031,7 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2-mimic/tests/testsuite/panic.rs b/crates/libtest2-mimic/tests/testsuite/panic.rs index e4006bf..9d2db79 100644 --- a/crates/libtest2-mimic/tests/testsuite/panic.rs +++ b/crates/libtest2-mimic/tests/testsuite/panic.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -22,7 +25,7 @@ fn main() { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], code: i32, single: &str, parallel: &str) { +fn check(args: &[&str], code: i32, single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -41,7 +44,8 @@ fn normal() { check( &[], 101, - r#" + str![[r#" + running 2 tests test panics ... FAILED test passes ... ok @@ -57,8 +61,10 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... @@ -73,6 +79,7 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2/Cargo.toml b/crates/libtest2/Cargo.toml index ca33579..cac52b8 100644 --- a/crates/libtest2/Cargo.toml +++ b/crates/libtest2/Cargo.toml @@ -37,7 +37,7 @@ dunce = "1.0.4" escargot = "0.5.8" once_cell_polyfill = "1.56.0" pathdiff = "0.2.1" -snapbox = "0.6.0" +snapbox = { version = "0.6.0", features = ["json"] } [lints] workspace = true diff --git a/crates/libtest2/tests/testsuite/all_passing.rs b/crates/libtest2/tests/testsuite/all_passing.rs index 912b529..38586a4 100644 --- a/crates/libtest2/tests/testsuite/all_passing.rs +++ b/crates/libtest2/tests/testsuite/all_passing.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -26,7 +29,7 @@ fn barro(_state: &libtest2::State) -> libtest2::RunResult { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], single: &str, parallel: &str) { +fn check(args: &[&str], single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -40,7 +43,8 @@ fn check(args: &[&str], single: &str, parallel: &str) { fn normal() { check( &[], - r#" + str![[r#" + running 3 tests test bar ... ok test barro ... ok @@ -48,14 +52,17 @@ test foo ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 3 tests ... test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -63,20 +70,24 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..] fn filter_one() { check( &["foo"], - r#" + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -84,21 +95,25 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn filter_two() { check( &["bar"], - r#" + str![[r#" + running 2 tests test bar ... ok test barro ... ok test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -106,20 +121,24 @@ test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..] fn filter_exact() { check( &["bar", "--exact"], - r#" + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -127,20 +146,24 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn filter_two_and_skip() { check( &["--skip", "barro", "bar"], - r#" + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test bar ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -148,7 +171,8 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn skip_nothing() { check( &["--skip", "peter"], - r#" + str![[r#" + running 3 tests test bar ... ok test barro ... ok @@ -156,14 +180,17 @@ test foo ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 3 tests ... test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -171,20 +198,24 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..] fn skip_two() { check( &["--skip", "bar"], - r#" + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 1 test test foo ... ok test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -192,21 +223,25 @@ test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..] fn skip_exact() { check( &["--exact", "--skip", "bar"], - r#" + str![[r#" + running 2 tests test barro ... ok test foo ... ok test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -214,17 +249,21 @@ test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..] fn terse_output() { check( &["--quiet", "--skip", "foo"], - r#" + str![[r#" + running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests .. test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2/tests/testsuite/mixed_bag.rs b/crates/libtest2/tests/testsuite/mixed_bag.rs index c845b76..18346c8 100644 --- a/crates/libtest2/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2/tests/testsuite/mixed_bag.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -51,7 +54,7 @@ fn bear(state: &libtest2::State) -> libtest2::RunResult { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], code: i32, single: &str, parallel: &str) { +fn check(args: &[&str], code: i32, single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -70,7 +73,8 @@ fn normal() { check( &[], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -92,8 +96,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -108,7 +114,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -117,7 +124,8 @@ fn test_mode() { check( &["--test"], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -139,8 +147,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -155,7 +165,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -164,7 +175,8 @@ fn bench_mode() { check( &["--bench"], 101, - r#" + str![[r#" + running 8 tests test bear ... ignored test bunny ... ignored @@ -186,8 +198,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -202,7 +216,8 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -211,7 +226,8 @@ fn list() { check( &["--list"], 0, - r#"bear: test + str![[r#" +bear: test bunny: test cat: test dog: test @@ -222,8 +238,10 @@ owl: test 8 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test bunny: test cat: test dog: test @@ -234,7 +252,8 @@ owl: test 8 tests -"#, + +"#]], ); } @@ -243,7 +262,8 @@ fn list_ignored() { check( &["--list", "--ignored"], 0, - r#"bear: test + str![[r#" +bear: test bunny: test cat: test dog: test @@ -254,8 +274,10 @@ owl: test 8 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test bunny: test cat: test dog: test @@ -266,7 +288,8 @@ owl: test 8 tests -"#, + +"#]], ); } @@ -275,18 +298,22 @@ fn list_with_filter() { check( &["--list", "a"], 0, - r#"bear: test + str![[r#" +bear: test cat: test 2 tests -"#, - r#"bear: test + +"#]], + str![[r#" +bear: test cat: test 2 tests -"#, + +"#]], ); } @@ -295,21 +322,25 @@ fn filter_c() { check( &["a"], 0, - r#" + str![[r#" + running 2 tests test bear ... ignored test cat ... ok test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -318,21 +349,25 @@ fn filter_o_test() { check( &["--test", "a"], 0, - r#" + str![[r#" + running 2 tests test bear ... ignored test cat ... ok test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... test result: ok. 1 passed; 0 failed; 1 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -341,7 +376,8 @@ fn filter_o_test_include_ignored() { check( &["--test", "--include-ignored", "o"], 101, - r#" + str![[r#" + running 4 tests test dog ... FAILED test fox ... ok @@ -363,8 +399,10 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 4 tests ... @@ -383,7 +421,8 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -392,7 +431,8 @@ fn filter_o_test_ignored() { check( &["--test", "--ignored", "o"], 101, - r#" + str![[r#" + running 4 tests test dog ... FAILED test fox ... ok @@ -414,8 +454,10 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 4 tests ... @@ -434,7 +476,8 @@ failures: test result: FAILED. 2 passed; 2 failed; 0 ignored; 4 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -443,7 +486,8 @@ fn normal_include_ignored() { check( &["--include-ignored"], 101, - r#" + str![[r#" + running 8 tests test bear ... FAILED test bunny ... FAILED @@ -477,8 +521,10 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -505,7 +551,8 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -514,7 +561,8 @@ fn normal_ignored() { check( &["--ignored"], 101, - r#" + str![[r#" + running 8 tests test bear ... FAILED test bunny ... FAILED @@ -548,8 +596,10 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... @@ -576,7 +626,8 @@ failures: test result: FAILED. 4 passed; 4 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -585,7 +636,8 @@ fn lots_of_flags() { check( &["--ignored", "--skip", "g", "--test", "o"], 101, - r#" + str![[r#" + running 2 tests test fox ... ok test owl ... FAILED @@ -601,8 +653,10 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 6 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... @@ -617,7 +671,8 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 6 filtered out; finished in [..]s -"#, + +"#]], ); } @@ -627,28 +682,128 @@ fn list_json() { check( &["-Zunstable-options", "--format=json", "--list", "a"], 0, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -"#, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -"#, + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), ); } @@ -658,40 +813,191 @@ fn test_json() { check( &["-Zunstable-options", "--format=json", "a"], 0, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -{"event":"suite-start"} -{"event":"case-start","name":"bear"} -{"event":"case-complete","name":"bear","mode":"test","status":"ignored","message":"fails","elapsed_s":"[..]"} -{"event":"case-start","name":"cat"} -{"event":"case-complete","name":"cat","mode":"test","status":null,"message":null,"elapsed_s":"[..]"} -{"event":"suite-complete","elapsed_s":"[..]"} -"#, - r#"{"event":"discover-start"} -{"event":"discover-case","name":"bear","mode":"test","run":true} -{"event":"discover-case","name":"bunny","mode":"test","run":false} -{"event":"discover-case","name":"cat","mode":"test","run":true} -{"event":"discover-case","name":"dog","mode":"test","run":false} -{"event":"discover-case","name":"fly","mode":"test","run":false} -{"event":"discover-case","name":"fox","mode":"test","run":false} -{"event":"discover-case","name":"frog","mode":"test","run":false} -{"event":"discover-case","name":"owl","mode":"test","run":false} -{"event":"discover-complete","elapsed_s":"[..]"} -{"event":"suite-start"} -[..] -[..] -[..] -[..] -{"event":"suite-complete","elapsed_s":"[..]"} -"#, + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + }, + { + "event": "suite-start" + }, + { + "event": "case-start", + "name": "bear" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": "fails", + "mode": "test", + "name": "bear", + "status": "ignored" + }, + { + "event": "case-start", + "name": "cat" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": null, + "mode": "test", + "name": "cat", + "status": null + }, + { + "elapsed_s": "[..]", + "event": "suite-complete" + } +] +"#]] + .is_json() + .against_jsonlines(), + str![[r#" +[ + { + "event": "discover-start" + }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "bunny", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "dog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fly", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "fox", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "frog", + "run": false + }, + { + "event": "discover-case", + "mode": "test", + "name": "owl", + "run": false + }, + { + "elapsed_s": "[..]", + "event": "discover-complete" + }, + { + "event": "suite-start" + }, + { + "event": "case-start", + "name": "bear" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": "fails", + "mode": "test", + "name": "bear", + "status": "ignored" + }, + { + "event": "case-start", + "name": "cat" + }, + { + "elapsed_s": "[..]", + "event": "case-complete", + "message": null, + "mode": "test", + "name": "cat", + "status": null + }, + { + "elapsed_s": "[..]", + "event": "suite-complete" + } +] +"#]] + .unordered() + .is_json() + .against_jsonlines(), ); } @@ -700,7 +1006,8 @@ fn terse_output() { check( &["--quiet"], 101, - r#" + str![[r#" + running 8 tests ii.Fi.ii failures: @@ -714,8 +1021,10 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 8 tests ... failures: @@ -729,6 +1038,7 @@ failures: test result: FAILED. 2 passed; 1 failed; 5 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } diff --git a/crates/libtest2/tests/testsuite/panic.rs b/crates/libtest2/tests/testsuite/panic.rs index a9af8bf..0c786a3 100644 --- a/crates/libtest2/tests/testsuite/panic.rs +++ b/crates/libtest2/tests/testsuite/panic.rs @@ -1,3 +1,6 @@ +use snapbox::prelude::*; +use snapbox::str; + fn test_cmd() -> snapbox::cmd::Command { static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = once_cell_polyfill::sync::OnceLock::new(); @@ -22,7 +25,7 @@ fn panics(_state: &libtest2::State) -> libtest2::RunResult { snapbox::cmd::Command::new(bin).current_dir(current_dir) } -fn check(args: &[&str], code: i32, single: &str, parallel: &str) { +fn check(args: &[&str], code: i32, single: impl IntoData, parallel: impl IntoData) { test_cmd() .args(args) .args(["--test-threads", "1"]) @@ -41,7 +44,8 @@ fn normal() { check( &[], 101, - r#" + str![[r#" + running 2 tests test panics ... FAILED test passes ... ok @@ -57,8 +61,10 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, - r#" + +"#]], + str![[r#" + running 2 tests ... @@ -73,6 +79,7 @@ failures: test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 filtered out; finished in [..]s -"#, + +"#]], ); } From f285ae40a76b966079f7b834bfa5f3ae8fb46d99 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Thu, 10 Jul 2025 16:45:10 -0500 Subject: [PATCH 2/3] test: Show list ordering behavior --- .../tests/testsuite/mixed_bag.rs | 28 +++++++++++++++++++ crates/libtest2/tests/testsuite/mixed_bag.rs | 28 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs index 49c16d2..03fc1c5 100644 --- a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs @@ -306,6 +306,34 @@ cat: test 2 tests +"#]], + ); +} + +#[test] +fn list_with_specified_order() { + check( + &["--list", "--exact", "owl", "fox", "bunny", "frog"], + 0, + str![[r#" +bunny: test +fox: test +frog: test +owl: test + +4 tests + + +"#]], + str![[r#" +bunny: test +fox: test +frog: test +owl: test + +4 tests + + "#]], ); } diff --git a/crates/libtest2/tests/testsuite/mixed_bag.rs b/crates/libtest2/tests/testsuite/mixed_bag.rs index 18346c8..2e7694d 100644 --- a/crates/libtest2/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2/tests/testsuite/mixed_bag.rs @@ -313,6 +313,34 @@ cat: test 2 tests +"#]], + ); +} + +#[test] +fn list_with_specified_order() { + check( + &["--list", "--exact", "owl", "fox", "bunny", "frog"], + 0, + str![[r#" +bunny: test +fox: test +frog: test +owl: test + +4 tests + + +"#]], + str![[r#" +bunny: test +fox: test +frog: test +owl: test + +4 tests + + "#]], ); } From b9036a323fa53b1cea3b63700d7d0452b0827553 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Thu, 10 Jul 2025 17:02:29 -0500 Subject: [PATCH 3/3] feat(harness): Give users control over sort order --- DESIGN.md | 7 ++ crates/libtest2-harness/src/harness.rs | 17 +++- .../tests/testsuite/mixed_bag.rs | 80 +++++++++---------- crates/libtest2/tests/testsuite/mixed_bag.rs | 80 +++++++++---------- 4 files changed, 101 insertions(+), 83 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 88a78f3..6bfb3b0 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -149,3 +149,10 @@ support for multi-character shorts is something people may want to opt-in to (it Performance isn't the top priority, so remoing `&str` -> `char` conversions isn't necessarily viewed as a benefit. This also makes `match` need to work off of `&str` instead of `char`. Unsure which of those would be slower and how the different characteristics match up. + +## Harness + +### Decision: report and run tests in filter order + +Rather than build into every harness shuffle, sharding, and any other specific logic like that, +we can instead give the user direct control over the test order by the order they are specified on the command line. diff --git a/crates/libtest2-harness/src/harness.rs b/crates/libtest2-harness/src/harness.rs index 6b4cc0e..3abc016 100644 --- a/crates/libtest2-harness/src/harness.rs +++ b/crates/libtest2-harness/src/harness.rs @@ -161,9 +161,6 @@ fn discover( notifier.notify(notify::Event::DiscoverStart)?; let timer = std::time::Instant::now(); - // Do this first so it applies to both discover and running - cases.sort_unstable_by_key(|case| case.name().to_owned()); - let matches_filter = |case: &dyn Case, filter: &str| { let test_name = case.name(); @@ -172,6 +169,20 @@ fn discover( false => test_name.contains(filter), } }; + + // Do this first so it applies to both discover and running + cases.sort_unstable_by_key(|case| { + let priority = if opts.filters.is_empty() { + Some(0) + } else { + opts.filters + .iter() + .position(|filter| matches_filter(case.as_ref(), filter)) + }; + let name = case.name().to_owned(); + (priority, name) + }); + let mut retain_cases = Vec::with_capacity(cases.len()); for case in cases.iter() { let filtered_in = opts.filters.is_empty() diff --git a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs index 03fc1c5..b95d7eb 100644 --- a/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2-mimic/tests/testsuite/mixed_bag.rs @@ -316,20 +316,20 @@ fn list_with_specified_order() { &["--list", "--exact", "owl", "fox", "bunny", "frog"], 0, str![[r#" -bunny: test +owl: test fox: test +bunny: test frog: test -owl: test 4 tests "#]], str![[r#" -bunny: test +owl: test fox: test +bunny: test frog: test -owl: test 4 tests @@ -708,24 +708,12 @@ fn list_json() { { "event": "discover-start" }, - { - "event": "discover-case", - "mode": "test", - "name": "bear", - "run": true - }, { "event": "discover-case", "mode": "test", "name": "bunny", "run": false }, - { - "event": "discover-case", - "mode": "test", - "name": "cat", - "run": true - }, { "event": "discover-case", "mode": "test", @@ -756,6 +744,18 @@ fn list_json() { "name": "owl", "run": false }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, { "elapsed_s": "[..]", "event": "discover-complete" @@ -769,24 +769,12 @@ fn list_json() { { "event": "discover-start" }, - { - "event": "discover-case", - "mode": "test", - "name": "bear", - "run": true - }, { "event": "discover-case", "mode": "test", "name": "bunny", "run": false }, - { - "event": "discover-case", - "mode": "test", - "name": "cat", - "run": true - }, { "event": "discover-case", "mode": "test", @@ -817,6 +805,18 @@ fn list_json() { "name": "owl", "run": false }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, { "elapsed_s": "[..]", "event": "discover-complete" @@ -839,24 +839,12 @@ fn test_json() { { "event": "discover-start" }, - { - "event": "discover-case", - "mode": "test", - "name": "bear", - "run": true - }, { "event": "discover-case", "mode": "test", "name": "bunny", "run": false }, - { - "event": "discover-case", - "mode": "test", - "name": "cat", - "run": true - }, { "event": "discover-case", "mode": "test", @@ -887,6 +875,18 @@ fn test_json() { "name": "owl", "run": false }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, { "elapsed_s": "[..]", "event": "discover-complete" diff --git a/crates/libtest2/tests/testsuite/mixed_bag.rs b/crates/libtest2/tests/testsuite/mixed_bag.rs index 2e7694d..c46391a 100644 --- a/crates/libtest2/tests/testsuite/mixed_bag.rs +++ b/crates/libtest2/tests/testsuite/mixed_bag.rs @@ -323,20 +323,20 @@ fn list_with_specified_order() { &["--list", "--exact", "owl", "fox", "bunny", "frog"], 0, str![[r#" -bunny: test +owl: test fox: test +bunny: test frog: test -owl: test 4 tests "#]], str![[r#" -bunny: test +owl: test fox: test +bunny: test frog: test -owl: test 4 tests @@ -715,24 +715,12 @@ fn list_json() { { "event": "discover-start" }, - { - "event": "discover-case", - "mode": "test", - "name": "bear", - "run": true - }, { "event": "discover-case", "mode": "test", "name": "bunny", "run": false }, - { - "event": "discover-case", - "mode": "test", - "name": "cat", - "run": true - }, { "event": "discover-case", "mode": "test", @@ -763,6 +751,18 @@ fn list_json() { "name": "owl", "run": false }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, { "elapsed_s": "[..]", "event": "discover-complete" @@ -776,24 +776,12 @@ fn list_json() { { "event": "discover-start" }, - { - "event": "discover-case", - "mode": "test", - "name": "bear", - "run": true - }, { "event": "discover-case", "mode": "test", "name": "bunny", "run": false }, - { - "event": "discover-case", - "mode": "test", - "name": "cat", - "run": true - }, { "event": "discover-case", "mode": "test", @@ -824,6 +812,18 @@ fn list_json() { "name": "owl", "run": false }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, { "elapsed_s": "[..]", "event": "discover-complete" @@ -846,24 +846,12 @@ fn test_json() { { "event": "discover-start" }, - { - "event": "discover-case", - "mode": "test", - "name": "bear", - "run": true - }, { "event": "discover-case", "mode": "test", "name": "bunny", "run": false }, - { - "event": "discover-case", - "mode": "test", - "name": "cat", - "run": true - }, { "event": "discover-case", "mode": "test", @@ -894,6 +882,18 @@ fn test_json() { "name": "owl", "run": false }, + { + "event": "discover-case", + "mode": "test", + "name": "bear", + "run": true + }, + { + "event": "discover-case", + "mode": "test", + "name": "cat", + "run": true + }, { "elapsed_s": "[..]", "event": "discover-complete"