Skip to content

Commit 31becb4

Browse files
authored
Merge pull request #175 from kellpossible/luke/m4-rewrite-parser
m4
2 parents 453b0d0 + ffce090 commit 31becb4

File tree

196 files changed

+7955
-7
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

196 files changed

+7955
-7
lines changed

Cargo.lock

Lines changed: 291 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ members = [
88
"display",
99
"file",
1010
"fs",
11+
"m4",
12+
"m4/test-manager",
1113
"gettext-rs",
1214
"misc",
1315
"pathnames",
@@ -24,7 +26,7 @@ members = [
2426

2527
[workspace.dependencies]
2628
atty = "0.2"
27-
clap = { version = "4", default-features = false, features = ["std", "derive", "help", "usage"] }
29+
clap = { version = "4", default-features = false, features = ["std", "derive", "help", "usage", "error-context", "cargo"] }
2830
chrono = { version = "0.4", default-features = false, features = ["clock"] }
2931
libc = "0.2"
3032
regex = "1.10"

datetime/tests/time/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,5 +89,5 @@ fn parse_error_test() {
8989

9090
#[test]
9191
fn command_error_test() {
92-
run_test_time(&["-s", "ls", "-l"], "", "unexpected argument found", 0);
92+
run_test_time(&["-s", "ls", "-l"], "", "unexpected argument '-s' found", 0);
9393
}

m4/Cargo.toml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
[package]
2+
name = "m4"
3+
version = "0.1.0"
4+
edition = "2021"
5+
6+
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
7+
8+
[dependencies]
9+
clap.workspace = true
10+
env_logger = "0.11"
11+
errno = "0.3"
12+
libc.workspace = true
13+
log = "0.4"
14+
nom = "7.1"
15+
once_cell = "1.19"
16+
thiserror = "1.0"
17+
18+
[dev-dependencies]
19+
env_logger = "0.11"
20+
m4-test-manager = { path = "./test-manager" }
21+
regex-lite = "0.1"
22+
similar-asserts = "1.5"
23+
test-log = { version = "0.2", default-features=false, features=["log"]}
24+
25+
[build-dependencies]
26+
m4-test-manager = { path = "./test-manager" }
27+

m4/README.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# `m4`
2+
3+
Implementation of `m4` according to the specification <https://pubs.opengroup.org/onlinepubs/9699919799/utilities/m4.html>.
4+
5+
6+
## Other Implementations
7+
8+
Other known implementations of `m4`:
9+
10+
* [illumos m4](https://github.com/illumos/illumos-gate/blob/master/usr/src/cmd/sgs/m4/common/m4.c)
11+
* BSD
12+
* [FreeBSD m4](https://github.com/freebsd/freebsd-src/tree/main/usr.bin/m4)
13+
* [OpenBSD m4](https://github.com/openbsd/src/tree/master/usr.bin/m4)
14+
* Portable BSD m4 - [here](https://github.com/ibara/m4) and [here](https://github.com/chimera-linux/bsdm4)
15+
* [GNU m4](https://www.gnu.org/software/m4/)

m4/build.rs

Lines changed: 327 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
use std::{
2+
collections::BTreeMap,
3+
fs::read_dir,
4+
os::unix::ffi::OsStrExt,
5+
path::{Path, PathBuf},
6+
};
7+
8+
use m4_test_manager::TestSnapshot;
9+
10+
/// A candididate for an integration test [`Test`], can be converted into one if both
11+
/// [`TestCandidate::input`] and [`TestCandidate::output`] are `Some`. This is created during the
12+
/// process of analyzing the available integration test files in order to pair up the input and
13+
/// output for the same test.
14+
#[derive(Default)]
15+
struct TestCandidate {
16+
/// The name of the test.
17+
name: String,
18+
/// Input `.m4` or `.args` file. Will be `None` if has not yet been found.
19+
input: Option<PathBuf>,
20+
/// Output `.out` file. Will be `None` if has not yet been found.
21+
output: Option<PathBuf>,
22+
/// See [`TestSnapshot::ignore`].
23+
ignore: bool,
24+
/// See [`TestSnapshot::expect_error`].
25+
expect_error: bool,
26+
/// See [`TestSnapshot::stdout_regex`].
27+
stdout_regex: Option<String>,
28+
}
29+
30+
impl TryFrom<TestCandidate> for Test {
31+
type Error = &'static str;
32+
33+
fn try_from(value: TestCandidate) -> Result<Self, Self::Error> {
34+
Ok(Test {
35+
name: value.name,
36+
input: value
37+
.input
38+
.ok_or("No input provided")?
39+
.to_str()
40+
.ok_or("Error converting input path to string")?
41+
.to_owned(),
42+
output: value
43+
.output
44+
.ok_or("No output file provided, please run m4-test-manager update-snapshots")?
45+
.to_str()
46+
.ok_or("Error converting output path to string")?
47+
.to_owned(),
48+
ignore: value.ignore,
49+
expect_error: value.expect_error,
50+
stdout_regex: value.stdout_regex,
51+
})
52+
}
53+
}
54+
55+
struct Test {
56+
/// The name of the test.
57+
name: String,
58+
/// Input `.m4` file.
59+
input: String,
60+
/// Output `.out` file
61+
output: String,
62+
/// See [`TestSnapshot::ignore`].
63+
ignore: bool,
64+
/// See [`TestSnapshot::expect_error`].
65+
expect_error: bool,
66+
/// See [`TestSnapshot::stdout_regex`].
67+
stdout_regex: Option<String>,
68+
}
69+
impl Test {
70+
fn as_code(&self) -> String {
71+
let Self {
72+
name,
73+
input,
74+
output,
75+
ignore,
76+
expect_error,
77+
stdout_regex,
78+
} = self;
79+
let mut s = String::new();
80+
81+
if *ignore {
82+
s.push_str("#[ignore]");
83+
}
84+
85+
s.push_str(&format!(
86+
r##"#[test]
87+
fn test_{name}() {{
88+
init();
89+
let output = run_command(&Path::new("{input}"));
90+
91+
let test: TestSnapshot = read_test("{output}");
92+
assert_eq!(output.status, std::process::ExitStatus::from_raw(test.status), "status (\x1b[31mcurrent\x1b[0m|\x1b[32mexpected\x1b[0m)");
93+
94+
"##
95+
));
96+
97+
if let Some(stdout_regex) = stdout_regex {
98+
s.push_str(&format!(
99+
r##"
100+
let r = regex_lite::Regex::new(r"{stdout_regex}").unwrap();
101+
assert!(r.is_match(&String::from_utf8(output.stdout).unwrap()), "stdout doesn't match regex: r\"{{}}\"", "{stdout_regex}");
102+
"##
103+
));
104+
} else {
105+
s.push_str(r##"
106+
assert_eq!(String::from_utf8(output.stdout).unwrap(), test.stdout, "stdout (\x1b[31mcurrent\x1b[0m|\x1b[32mexpected\x1b[0m)");
107+
"##);
108+
}
109+
110+
if *expect_error {
111+
s.push_str(
112+
r##"
113+
if !test.stderr.is_empty() {
114+
assert!(!output.stderr.is_empty());
115+
}"##,
116+
);
117+
} else {
118+
s.push_str(r##"
119+
assert_eq!(String::from_utf8(output.stderr).unwrap(), test.stderr, "stderr (\x1b[31mcurrent\x1b[0m|\x1b[32mexpected\x1b[0m)");"##);
120+
}
121+
122+
s.push('}');
123+
124+
s
125+
}
126+
}
127+
128+
fn name_from_path(path: &Path) -> Option<String> {
129+
Some(path.file_name()?.to_str()?.split('.').next()?.to_owned())
130+
}
131+
132+
fn main() {
133+
println!("cargo::rerun-if-changed=fixtures/");
134+
let mut test_candidates: BTreeMap<String, TestCandidate> = BTreeMap::new();
135+
let fixtures_directory = Path::new("fixtures/integration_tests");
136+
for entry in read_dir(fixtures_directory).unwrap() {
137+
let entry = entry.unwrap();
138+
let path = entry.path();
139+
140+
match path.extension().map(|e| e.as_bytes()) {
141+
Some(b"m4") | Some(b"args") => {
142+
let name = name_from_path(&path).unwrap();
143+
let snapshot_file_name = format!("{name}.out");
144+
let snapshot_file = fixtures_directory.join(snapshot_file_name);
145+
let (ignore, expect_error, stdout_regex) = if snapshot_file.exists() {
146+
let mut f = std::fs::OpenOptions::new()
147+
.read(true)
148+
.open(&snapshot_file)
149+
.unwrap();
150+
let snapshot = TestSnapshot::deserialize(&mut f);
151+
(
152+
snapshot.ignore,
153+
snapshot.expect_error,
154+
snapshot.stdout_regex,
155+
)
156+
} else {
157+
(false, false, None)
158+
};
159+
let candidate = test_candidates
160+
.entry(name.clone())
161+
.or_insert(TestCandidate {
162+
name,
163+
ignore,
164+
expect_error,
165+
stdout_regex: stdout_regex.clone(),
166+
..TestCandidate::default()
167+
});
168+
candidate.input = Some(path);
169+
candidate.ignore = ignore;
170+
candidate.expect_error = expect_error;
171+
candidate.stdout_regex = stdout_regex;
172+
}
173+
Some(b"out") => {
174+
let name = name_from_path(&path).unwrap();
175+
let candidate = test_candidates
176+
.entry(name.clone())
177+
.or_insert(TestCandidate {
178+
name,
179+
..TestCandidate::default()
180+
});
181+
candidate.output = Some(path);
182+
}
183+
_ => eprintln!("Ignoring file {path:?}"),
184+
}
185+
}
186+
let mut integration_test: String =
187+
r#"//! NOTE: This file has been auto generated using build.rs, don't edit by hand!
188+
//! You can regenerate the tests (which are based on the fixtures in `fixtures/integration_tests/`)
189+
//! using the following command:
190+
//! `cargo run -p m4-test-manager update-snapshots`
191+
use similar_asserts::assert_eq;
192+
use std::process::ExitStatus;
193+
use std::os::unix::ffi::OsStrExt;
194+
use std::os::unix::process::ExitStatusExt;
195+
use std::fs::read_to_string;
196+
use std::path::Path;
197+
use m4::error::GetExitCode;
198+
use m4_test_manager::TestSnapshot;
199+
200+
fn init() {
201+
let _ = env_logger::builder()
202+
.is_test(true)
203+
// No timestamp to make it easier to diff output
204+
.format_timestamp(None)
205+
.try_init();
206+
}
207+
208+
fn read_test(path: impl AsRef<std::path::Path>) -> TestSnapshot {
209+
let mut f = std::fs::File::open(path).unwrap();
210+
let snapshot = TestSnapshot::deserialize(&mut f);
211+
log::info!(
212+
"Expecting stdout:\n\x1b[34m{}\x1b[0m",
213+
snapshot.stdout,
214+
);
215+
log::info!(
216+
"Expecting stderr:\n\x1b[34m{}\x1b[0m",
217+
snapshot.stderr,
218+
);
219+
log::info!(
220+
"Expecting status:\n\x1b[34m{}\x1b[0m",
221+
snapshot.status,
222+
);
223+
snapshot
224+
}
225+
226+
fn run_command(input: &Path) -> std::process::Output {
227+
let input_string = read_to_string(input).unwrap();
228+
log::info!(
229+
"Running command with input {input:?}:\n\x1b[34m{}\x1b[0m",
230+
input_string,
231+
);
232+
#[derive(Default, Clone)]
233+
struct StdoutRef(std::rc::Rc<std::cell::RefCell<Vec<u8>>>);
234+
impl std::io::Write for StdoutRef {
235+
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
236+
self.0.borrow_mut().write(buf)
237+
}
238+
239+
fn flush(&mut self) -> std::io::Result<()> {
240+
self.0.borrow_mut().flush()
241+
}
242+
}
243+
impl StdoutRef {
244+
fn into_inner(self) -> Vec<u8> {
245+
std::rc::Rc::into_inner(self.0).unwrap().into_inner()
246+
}
247+
}
248+
let (stdout, stderr, status) = match input
249+
.extension()
250+
.expect("Input file should have extension")
251+
.as_bytes()
252+
{
253+
b"m4" => {
254+
// The reason why we run the command using this as a library is so we can run with it built in
255+
// test configuration, with all the associated conditionally compiled test log instrumentation.
256+
257+
let stdout = StdoutRef::default();
258+
let mut stderr: Vec<u8> = Vec::new();
259+
let args = m4::Args {
260+
files: vec![input.into()],
261+
..m4::Args::default()
262+
};
263+
let result = m4::run(stdout.clone(), &mut stderr, args);
264+
let status = ExitStatus::from_raw(result.get_exit_code() as i32);
265+
(stdout.into_inner(), stderr, status)
266+
}
267+
b"args" => {
268+
let args = input_string;
269+
let _cargo_build_output = std::process::Command::new("cargo")
270+
.arg("build")
271+
.output()
272+
.unwrap();
273+
274+
log::info!("RUST_LOG is ignored for this test because it interferes with output");
275+
let output = std::process::Command::new("sh")
276+
.env("RUST_LOG", "") // Disable rust log output because it interferes with the test.
277+
.arg("-c")
278+
.arg(format!("../target/debug/m4 {args}"))
279+
.output()
280+
.unwrap();
281+
282+
(output.stdout, output.stderr, output.status)
283+
}
284+
_ => panic!("Unsupported input extension {input:?}"),
285+
};
286+
287+
log::info!("Received status: {status}");
288+
log::info!(
289+
"Received stdout:\n\x1b[34m{}\x1b[0m",
290+
String::from_utf8_lossy(&stdout)
291+
);
292+
log::info!(
293+
"Received stderr:\n\x1b[34m{}\x1b[0m",
294+
String::from_utf8_lossy(&stderr)
295+
);
296+
std::process::Output {
297+
stdout,
298+
stderr,
299+
status,
300+
}
301+
}
302+
"#
303+
.to_owned();
304+
for (name, candidate) in test_candidates {
305+
let test: Test = candidate.try_into().unwrap_or_else(|error| {
306+
panic!("Error creating test from candidate for {name}: {error}")
307+
});
308+
309+
integration_test.push('\n');
310+
integration_test.push_str(&test.as_code());
311+
integration_test.push('\n');
312+
}
313+
314+
std::fs::write("tests/integration_test.rs", integration_test).unwrap();
315+
let output = std::process::Command::new("cargo")
316+
.arg("fmt")
317+
.arg("--")
318+
.arg("tests/integration_test.rs")
319+
.output()
320+
.unwrap();
321+
if !output.status.success() {
322+
panic!(
323+
"Error executing cargo fmt: {}",
324+
String::from_utf8_lossy(&output.stderr)
325+
);
326+
}
327+
}

0 commit comments

Comments
 (0)