rickub
nandi/rustnimpublic⑂ Fork 0
⑂ 428f3741e2904549bd0157081162006f3952b324
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/rustnim.git
git clone ssh://git@rickub.com/nandi/rustnim.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

differential.rs · 420 lines · 14.1 KBRust Blame HistoryRaw
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 17h ago1//! Differential test runner: rustc is the oracle.
2//!
3//! For each `tests/cases/*.rs`:
4//!
5//! ```text
6//! rustc case.rs -o a && ./a -> (stdout_rs, status_rs)
7//! rustnim case.rs -o case.nim
8//! nim c case.nim -o b && ./b -> (stdout_nim, status_nim)
9//! assert stdout_rs == stdout_nim && status_rs == status_nim
10//! ```
11//!
12//! A case passes only when *both* binaries build and produce identical output.
13//! Every intermediate stage is checked explicitly, because the failure mode
14//! this project exists to avoid is a transpiler reporting success while
15//! emitting nothing: `rustnim` exiting 0 with an empty or Nim-unparseable
16//! output file is a hard failure here, not a silent pass.
17//!
18//! Profile: rustc is invoked *without* `-O`, so debug-profile integer overflow
19//! checks are on. Nim's default `nim c` also has overflow checks on. That is
20//! the matching pair, and it is the profile this project models.
21//!
22//! Directives, recognised in `//@ ...` comments at the top of a case:
23//!
24//! //@ reject: <substring> rustnim must fail, with <substring> in stderr.
25//! (The "fail loudly" rule, tested.)
26//! //@ skip: <reason> Not run; reported as skipped.
27//! //@ args: <argv> Passed to both binaries.
28//! //@ stdin: <line> Fed to both binaries on stdin.
29
30use std::collections::BTreeMap;
31use std::fmt::Write as _;
32use std::fs;
33use std::io::Write as _;
34use std::path::{Path, PathBuf};
35use std::process::{Command, Stdio};
36
37const RUSTNIM: &str = env!("CARGO_BIN_EXE_rustnim");
38
39// ---------------------------------------------------------------- outcomes
40
41#[derive(Debug)]
42enum Outcome {
43 Pass,
44 Skip(String),
45 /// A stage failed. `stage` is where, `detail` is the evidence.
46 Fail { stage: &'static str, detail: String },
47}
48
49impl Outcome {
50 fn fail(stage: &'static str, detail: impl Into<String>) -> Self {
51 Outcome::Fail { stage, detail: detail.into() }
52 }
53}
54
55struct Run {
56 status: Option<i32>,
57 stdout: Vec<u8>,
58 stderr: String,
59}
60
61impl Run {
62 fn ok(&self) -> bool {
63 self.status == Some(0)
64 }
65}
66
67// ------------------------------------------------------------------ helpers
68
69fn run(cmd: &mut Command, stdin: Option<&str>) -> Result<Run, String> {
70 cmd.stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() })
71 .stdout(Stdio::piped())
72 .stderr(Stdio::piped());
73 let mut child = cmd.spawn().map_err(|e| format!("spawn {:?}: {e}", cmd.get_program()))?;
74 if let Some(s) = stdin {
75 child.stdin.as_mut().unwrap().write_all(s.as_bytes()).map_err(|e| e.to_string())?;
76 }
77 let out = child.wait_with_output().map_err(|e| e.to_string())?;
78 Ok(Run {
79 status: out.status.code(),
80 stdout: out.stdout,
81 stderr: String::from_utf8_lossy(&out.stderr).into_owned(),
82 })
83}
84
85/// Locate the vendored Nim. It lives at the *repository* root, which is not
86/// the manifest dir when running inside a git worktree, so walk upwards.
87fn find_nim() -> Result<PathBuf, String> {
88 if let Ok(p) = std::env::var("RUSTNIM_NIM") {
89 let p = PathBuf::from(p);
90 if p.is_file() {
91 return Ok(p);
92 }
93 return Err(format!("RUSTNIM_NIM={} is not a file", p.display()));
94 }
95 let mut dir: Option<&Path> = Some(Path::new(env!("CARGO_MANIFEST_DIR")));
96 while let Some(d) = dir {
97 let cand = d.join(".nim-toolchain/bin/nim");
98 if cand.is_file() {
99 return Ok(cand);
100 }
101 dir = d.parent();
102 }
103 Err("no .nim-toolchain/bin/nim found in this directory or any parent; \
104 set RUSTNIM_NIM to the nim binary"
105 .into())
106}
107
108#[derive(Default)]
109struct Directives {
110 reject: Option<String>,
111 skip: Option<String>,
112 args: Vec<String>,
113 stdin: Option<String>,
114}
115
116fn directives(src: &str) -> Directives {
117 let mut d = Directives::default();
118 for line in src.lines() {
119 let line = line.trim();
120 let Some(rest) = line.strip_prefix("//@") else {
121 // Directives must be in the leading comment block; stop at the
122 // first line of real code so a `//@` inside a string can't count.
123 if line.is_empty() || line.starts_with("//") || line.starts_with("#!") {
124 continue;
125 }
126 break;
127 };
128 let rest = rest.trim();
129 let (key, val) = match rest.split_once(':') {
130 Some((k, v)) => (k.trim(), v.trim().to_string()),
131 None => (rest, String::new()),
132 };
133 match key {
134 "reject" => d.reject = Some(val),
135 "skip" => d.skip = Some(val),
136 "args" => d.args = val.split_whitespace().map(str::to_string).collect(),
137 "stdin" => d.stdin = Some(format!("{val}\n")),
138 _ => {}
139 }
140 }
141 d
142}
143
144/// Byte-for-byte diff, rendered readably: show the first differing line with
145/// escapes, so a trailing-newline or whitespace difference is visible.
146fn diff_report(want: &[u8], got: &[u8]) -> String {
147 let w = String::from_utf8_lossy(want);
148 let g = String::from_utf8_lossy(got);
149 let (wl, gl): (Vec<_>, Vec<_>) = (w.lines().collect(), g.lines().collect());
150 let mut out = String::new();
151 for i in 0..wl.len().max(gl.len()) {
152 let (a, b) = (wl.get(i), gl.get(i));
153 if a != b {
154 let _ = writeln!(out, " first difference at line {}:", i + 1);
155 let _ = writeln!(out, " rustc: {}", a.map(|s| format!("{s:?}")).unwrap_or("<eof>".into()));
156 let _ = writeln!(out, " nim : {}", b.map(|s| format!("{s:?}")).unwrap_or("<eof>".into()));
157 break;
158 }
159 }
160 if out.is_empty() {
161 // Lines all matched, so the difference is in trailing bytes.
162 let _ = writeln!(out, " lines match; raw bytes differ (trailing newline?)");
163 let _ = writeln!(out, " rustc: {:?}", w);
164 let _ = writeln!(out, " nim : {:?}", g);
165 }
166 let _ = writeln!(out, " ({} bytes from rustc, {} from nim)", want.len(), got.len());
167 out
168}
169
170fn tail(s: &str, lines: usize) -> String {
171 let v: Vec<&str> = s.trim_end().lines().collect();
172 let start = v.len().saturating_sub(lines);
173 v[start..]
174 .iter()
175 .map(|l| format!(" {l}"))
176 .collect::<Vec<_>>()
177 .join("\n")
178}
179
180// --------------------------------------------------------------- one case
181
182fn run_case(case: &Path, work: &Path, nim: &Path) -> Outcome {
183 let name = case.file_stem().unwrap().to_string_lossy().into_owned();
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 16h ago184 // For a directory case, `main.rs` is the crate root and carries the
185 // directives; every `.rs` beside it is passed to rustnim as well.
186 let (root, extra) = if case.is_dir() {
187 let mut others: Vec<PathBuf> = fs::read_dir(case)
188 .map(|d| {
189 d.filter_map(|e| e.ok().map(|e| e.path()))
190 .filter(|p| {
191 p.extension().is_some_and(|x| x == "rs")
192 && p.file_name().is_some_and(|f| f != "main.rs")
193 })
194 .collect()
195 })
196 .unwrap_or_default();
197 others.sort();
198 (case.join("main.rs"), others)
199 } else {
200 (case.to_path_buf(), Vec::new())
201 };
202 let src = match fs::read_to_string(&root) {
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 17h ago203 Ok(s) => s,
204 Err(e) => return Outcome::fail("read", e.to_string()),
205 };
206 let d = directives(&src);
207 if let Some(why) = d.skip {
208 return Outcome::Skip(why);
209 }
210
211 let dir = work.join(&name);
212 let _ = fs::remove_dir_all(&dir);
213 if let Err(e) = fs::create_dir_all(&dir) {
214 return Outcome::fail("setup", e.to_string());
215 }
216
217 // -- stage: transpile. Checked first, and checked strictly: exit status,
218 // stderr on failure, and that the file actually has content in it.
219 // Nim module names must be valid Nim identifiers, which case names
220 // (`005-base16ct-decode-core`) are not.
221 let mod_name: String = format!(
222 "c_{}",
223 name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect::<String>()
224 );
225 let nim_src = dir.join(format!("{mod_name}.nim"));
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 16h ago226 let mut cmd = Command::new(RUSTNIM);
227 cmd.arg(&root);
228 for e in &extra {
229 cmd.arg(e);
230 }
231 cmd.arg("-o").arg(&nim_src).env("TMPDIR", &dir);
232 let transpile = match run(&mut cmd, None) {
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 17h ago233 Ok(r) => r,
234 Err(e) => return Outcome::fail("rustnim", e),
235 };
236
237 if let Some(want) = &d.reject {
238 // A `reject` case asserts the "fail loudly" rule.
239 if transpile.ok() {
240 return Outcome::fail(
241 "rustnim",
242 format!("expected rejection containing {want:?}, but transpile succeeded"),
243 );
244 }
245 if !transpile.stderr.contains(want.as_str()) {
246 return Outcome::fail(
247 "rustnim",
248 format!("rejected, but message lacks {want:?}:\n{}", tail(&transpile.stderr, 10)),
249 );
250 }
251 return Outcome::Pass;
252 }
253
254 if !transpile.ok() {
255 return Outcome::fail(
256 "rustnim",
257 format!("exit {:?}\n{}", transpile.status, tail(&transpile.stderr, 20)),
258 );
259 }
260 match fs::metadata(&nim_src) {
261 Err(_) => {
262 return Outcome::fail("rustnim", "exited 0 but wrote no output file".to_string());
263 }
264 Ok(m) if m.len() == 0 => {
265 return Outcome::fail("rustnim", "exited 0 but wrote an empty output file".to_string());
266 }
267 Ok(_) => {}
268 }
269
270 // -- stage: rustc. No -O: debug profile, overflow checks on.
271 let rs_bin = dir.join("rs.bin");
272 let rc = match run(
273 Command::new("rustc")
274 .arg("--edition=2021")
275 .arg("-A").arg("warnings")
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 16h ago276 .arg(&root)
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 17h ago277 .arg("-o").arg(&rs_bin)
278 .env("TMPDIR", &dir),
279 None,
280 ) {
281 Ok(r) => r,
282 Err(e) => return Outcome::fail("rustc", e),
283 };
284 if !rc.ok() {
285 return Outcome::fail("rustc", format!("case does not compile as Rust:\n{}", tail(&rc.stderr, 20)));
286 }
287
288 // -- stage: nim c
289 let nim_bin = dir.join("nim.bin");
290 let nc = match run(
291 Command::new(nim)
292 .arg("c")
293 .arg("--hints:off")
294 .arg("--warnings:off")
295 .arg("--colors:off")
296 .arg(format!("--nimcache:{}", dir.join("nimcache").display()))
297 .arg(format!("-o:{}", nim_bin.display()))
298 .arg(&nim_src)
299 .env("TMPDIR", &dir),
300 None,
301 ) {
302 Ok(r) => r,
303 Err(e) => return Outcome::fail("nim", e),
304 };
305 if !nc.ok() {
306 // Nim reports compile errors on stdout.
307 let msg = if nc.stderr.trim().is_empty() {
308 String::from_utf8_lossy(&nc.stdout).into_owned()
309 } else {
310 nc.stderr.clone()
311 };
312 return Outcome::fail(
313 "nim",
314 format!("generated Nim does not compile:\n{}\n --- generated ---\n{}", tail(&msg, 20), numbered(&nim_src)),
315 );
316 }
317
318 // -- stage: execute both
319 let exec = |bin: &Path| {
320 let mut c = Command::new(bin);
321 c.args(&d.args).env("TMPDIR", &dir);
322 run(&mut c, d.stdin.as_deref())
323 };
324 let (a, b) = match (exec(&rs_bin), exec(&nim_bin)) {
325 (Ok(a), Ok(b)) => (a, b),
326 (Err(e), _) | (_, Err(e)) => return Outcome::fail("run", e),
327 };
328
329 if a.stdout != b.stdout {
330 return Outcome::fail("diff", diff_report(&a.stdout, &b.stdout));
331 }
332 if a.status != b.status {
333 return Outcome::fail(
334 "diff",
335 format!(
336 "stdout matches but exit status differs: rustc {:?}, nim {:?}\n nim stderr:\n{}",
337 a.status,
338 b.status,
339 tail(&b.stderr, 10)
340 ),
341 );
342 }
343 Outcome::Pass
344}
345
346fn numbered(p: &Path) -> String {
347 fs::read_to_string(p)
348 .unwrap_or_default()
349 .lines()
350 .enumerate()
351 .map(|(i, l)| format!(" {:>3} | {l}", i + 1))
352 .collect::<Vec<_>>()
353 .join("\n")
354}
355
356// ------------------------------------------------------------------ driver
357
358#[test]
359fn differential() {
360 let root = Path::new(env!("CARGO_MANIFEST_DIR"));
361 let cases_dir = root.join("tests/cases");
362 let work = root.join("tests/.work");
363 let _ = fs::create_dir_all(&work);
364
365 let nim = match find_nim() {
366 Ok(n) => n,
367 Err(e) => panic!("cannot locate Nim: {e}"),
368 };
369
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 16h ago370 // A case is either a single `.rs` file or a directory of them whose
371 // crate root is `main.rs` -- which is how a multi-file crate is tested.
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 17h ago372 let mut cases: Vec<PathBuf> = fs::read_dir(&cases_dir)
373 .unwrap_or_else(|e| panic!("no {}: {e}", cases_dir.display()))
374 .filter_map(|e| e.ok().map(|e| e.path()))
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 16h ago375 .filter(|p| {
376 p.extension().is_some_and(|x| x == "rs") || p.join("main.rs").is_file()
377 })
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 17h ago378 .collect();
379 cases.sort();
380
381 // An empty corpus must not read as success. That is the exact failure this
382 // runner exists to catch, and it applies to the runner itself.
383 assert!(!cases.is_empty(), "no cases in {}", cases_dir.display());
384
385 let filter = std::env::var("RUSTNIM_CASE").ok();
386 let mut results: BTreeMap<String, Outcome> = BTreeMap::new();
387 for case in &cases {
388 let name = case.file_stem().unwrap().to_string_lossy().into_owned();
389 if let Some(f) = &filter {
390 if !name.contains(f.as_str()) {
391 continue;
392 }
393 }
394 let outcome = run_case(case, &work, &nim);
395 match &outcome {
396 Outcome::Pass => eprintln!("ok {name}"),
397 Outcome::Skip(why) => eprintln!("skip {name} ({why})"),
398 Outcome::Fail { .. } => eprintln!("FAIL {name}"),
399 }
400 results.insert(name, outcome);
401 }
402
403 let mut failed = Vec::new();
404 let (mut pass, mut skip) = (0, 0);
405 for (name, o) in &results {
406 match o {
407 Outcome::Pass => pass += 1,
408 Outcome::Skip(_) => skip += 1,
409 Outcome::Fail { stage, detail } => failed.push(format!(
410 "\n--- {name}: failed at stage `{stage}`\n{}",
411 detail.trim_end()
412 )),
413 }
414 }
415
416 eprintln!("\n{pass} passed, {} failed, {skip} skipped", failed.len());
417 if !failed.is_empty() {
418 panic!("{}", failed.join("\n"));
419 }
420}