//! Differential test runner: rustc is the oracle. //! //! For each `tests/cases/*.rs`: //! //! ```text //! rustc case.rs -o a && ./a -> (stdout_rs, status_rs) //! rustnim case.rs -o case.nim //! nim c case.nim -o b && ./b -> (stdout_nim, status_nim) //! assert stdout_rs == stdout_nim && status_rs == status_nim //! ``` //! //! A case passes only when *both* binaries build and produce identical output. //! Every intermediate stage is checked explicitly, because the failure mode //! this project exists to avoid is a transpiler reporting success while //! emitting nothing: `rustnim` exiting 0 with an empty or Nim-unparseable //! output file is a hard failure here, not a silent pass. //! //! Profile: rustc is invoked *without* `-O`, so debug-profile integer overflow //! checks are on. Nim's default `nim c` also has overflow checks on. That is //! the matching pair, and it is the profile this project models. //! //! Directives, recognised in `//@ ...` comments at the top of a case: //! //! //@ reject: rustnim must fail, with in stderr. //! (The "fail loudly" rule, tested.) //! //@ skip: Not run; reported as skipped. //! //@ args: Passed to both binaries. //! //@ stdin: Fed to both binaries on stdin. use std::collections::BTreeMap; use std::fmt::Write as _; use std::fs; use std::io::Write as _; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; const RUSTNIM: &str = env!("CARGO_BIN_EXE_rustnim"); // ---------------------------------------------------------------- outcomes #[derive(Debug)] enum Outcome { Pass, Skip(String), /// A stage failed. `stage` is where, `detail` is the evidence. Fail { stage: &'static str, detail: String }, } impl Outcome { fn fail(stage: &'static str, detail: impl Into) -> Self { Outcome::Fail { stage, detail: detail.into() } } } struct Run { status: Option, stdout: Vec, stderr: String, } impl Run { fn ok(&self) -> bool { self.status == Some(0) } } // ------------------------------------------------------------------ helpers fn run(cmd: &mut Command, stdin: Option<&str>) -> Result { cmd.stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() }) .stdout(Stdio::piped()) .stderr(Stdio::piped()); let mut child = cmd.spawn().map_err(|e| format!("spawn {:?}: {e}", cmd.get_program()))?; if let Some(s) = stdin { child.stdin.as_mut().unwrap().write_all(s.as_bytes()).map_err(|e| e.to_string())?; } let out = child.wait_with_output().map_err(|e| e.to_string())?; Ok(Run { status: out.status.code(), stdout: out.stdout, stderr: String::from_utf8_lossy(&out.stderr).into_owned(), }) } /// Locate the vendored Nim. It lives at the *repository* root, which is not /// the manifest dir when running inside a git worktree, so walk upwards. fn find_nim() -> Result { if let Ok(p) = std::env::var("RUSTNIM_NIM") { let p = PathBuf::from(p); if p.is_file() { return Ok(p); } return Err(format!("RUSTNIM_NIM={} is not a file", p.display())); } let mut dir: Option<&Path> = Some(Path::new(env!("CARGO_MANIFEST_DIR"))); while let Some(d) = dir { let cand = d.join(".nim-toolchain/bin/nim"); if cand.is_file() { return Ok(cand); } dir = d.parent(); } Err("no .nim-toolchain/bin/nim found in this directory or any parent; \ set RUSTNIM_NIM to the nim binary" .into()) } #[derive(Default)] struct Directives { reject: Option, skip: Option, args: Vec, stdin: Option, } fn directives(src: &str) -> Directives { let mut d = Directives::default(); for line in src.lines() { let line = line.trim(); let Some(rest) = line.strip_prefix("//@") else { // Directives must be in the leading comment block; stop at the // first line of real code so a `//@` inside a string can't count. if line.is_empty() || line.starts_with("//") || line.starts_with("#!") { continue; } break; }; let rest = rest.trim(); let (key, val) = match rest.split_once(':') { Some((k, v)) => (k.trim(), v.trim().to_string()), None => (rest, String::new()), }; match key { "reject" => d.reject = Some(val), "skip" => d.skip = Some(val), "args" => d.args = val.split_whitespace().map(str::to_string).collect(), "stdin" => d.stdin = Some(format!("{val}\n")), _ => {} } } d } /// Byte-for-byte diff, rendered readably: show the first differing line with /// escapes, so a trailing-newline or whitespace difference is visible. fn diff_report(want: &[u8], got: &[u8]) -> String { let w = String::from_utf8_lossy(want); let g = String::from_utf8_lossy(got); let (wl, gl): (Vec<_>, Vec<_>) = (w.lines().collect(), g.lines().collect()); let mut out = String::new(); for i in 0..wl.len().max(gl.len()) { let (a, b) = (wl.get(i), gl.get(i)); if a != b { let _ = writeln!(out, " first difference at line {}:", i + 1); let _ = writeln!(out, " rustc: {}", a.map(|s| format!("{s:?}")).unwrap_or("".into())); let _ = writeln!(out, " nim : {}", b.map(|s| format!("{s:?}")).unwrap_or("".into())); break; } } if out.is_empty() { // Lines all matched, so the difference is in trailing bytes. let _ = writeln!(out, " lines match; raw bytes differ (trailing newline?)"); let _ = writeln!(out, " rustc: {:?}", w); let _ = writeln!(out, " nim : {:?}", g); } let _ = writeln!(out, " ({} bytes from rustc, {} from nim)", want.len(), got.len()); out } fn tail(s: &str, lines: usize) -> String { let v: Vec<&str> = s.trim_end().lines().collect(); let start = v.len().saturating_sub(lines); v[start..] .iter() .map(|l| format!(" {l}")) .collect::>() .join("\n") } // --------------------------------------------------------------- one case fn run_case(case: &Path, work: &Path, nim: &Path) -> Outcome { let name = case.file_stem().unwrap().to_string_lossy().into_owned(); let src = match fs::read_to_string(case) { Ok(s) => s, Err(e) => return Outcome::fail("read", e.to_string()), }; let d = directives(&src); if let Some(why) = d.skip { return Outcome::Skip(why); } let dir = work.join(&name); let _ = fs::remove_dir_all(&dir); if let Err(e) = fs::create_dir_all(&dir) { return Outcome::fail("setup", e.to_string()); } // -- stage: transpile. Checked first, and checked strictly: exit status, // stderr on failure, and that the file actually has content in it. // Nim module names must be valid Nim identifiers, which case names // (`005-base16ct-decode-core`) are not. let mod_name: String = format!( "c_{}", name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect::() ); let nim_src = dir.join(format!("{mod_name}.nim")); let transpile = match run( Command::new(RUSTNIM).arg(case).arg("-o").arg(&nim_src).env("TMPDIR", &dir), None, ) { Ok(r) => r, Err(e) => return Outcome::fail("rustnim", e), }; if let Some(want) = &d.reject { // A `reject` case asserts the "fail loudly" rule. if transpile.ok() { return Outcome::fail( "rustnim", format!("expected rejection containing {want:?}, but transpile succeeded"), ); } if !transpile.stderr.contains(want.as_str()) { return Outcome::fail( "rustnim", format!("rejected, but message lacks {want:?}:\n{}", tail(&transpile.stderr, 10)), ); } return Outcome::Pass; } if !transpile.ok() { return Outcome::fail( "rustnim", format!("exit {:?}\n{}", transpile.status, tail(&transpile.stderr, 20)), ); } match fs::metadata(&nim_src) { Err(_) => { return Outcome::fail("rustnim", "exited 0 but wrote no output file".to_string()); } Ok(m) if m.len() == 0 => { return Outcome::fail("rustnim", "exited 0 but wrote an empty output file".to_string()); } Ok(_) => {} } // -- stage: rustc. No -O: debug profile, overflow checks on. let rs_bin = dir.join("rs.bin"); let rc = match run( Command::new("rustc") .arg("--edition=2021") .arg("-A").arg("warnings") .arg(case) .arg("-o").arg(&rs_bin) .env("TMPDIR", &dir), None, ) { Ok(r) => r, Err(e) => return Outcome::fail("rustc", e), }; if !rc.ok() { return Outcome::fail("rustc", format!("case does not compile as Rust:\n{}", tail(&rc.stderr, 20))); } // -- stage: nim c let nim_bin = dir.join("nim.bin"); let nc = match run( Command::new(nim) .arg("c") .arg("--hints:off") .arg("--warnings:off") .arg("--colors:off") .arg(format!("--nimcache:{}", dir.join("nimcache").display())) .arg(format!("-o:{}", nim_bin.display())) .arg(&nim_src) .env("TMPDIR", &dir), None, ) { Ok(r) => r, Err(e) => return Outcome::fail("nim", e), }; if !nc.ok() { // Nim reports compile errors on stdout. let msg = if nc.stderr.trim().is_empty() { String::from_utf8_lossy(&nc.stdout).into_owned() } else { nc.stderr.clone() }; return Outcome::fail( "nim", format!("generated Nim does not compile:\n{}\n --- generated ---\n{}", tail(&msg, 20), numbered(&nim_src)), ); } // -- stage: execute both let exec = |bin: &Path| { let mut c = Command::new(bin); c.args(&d.args).env("TMPDIR", &dir); run(&mut c, d.stdin.as_deref()) }; let (a, b) = match (exec(&rs_bin), exec(&nim_bin)) { (Ok(a), Ok(b)) => (a, b), (Err(e), _) | (_, Err(e)) => return Outcome::fail("run", e), }; if a.stdout != b.stdout { return Outcome::fail("diff", diff_report(&a.stdout, &b.stdout)); } if a.status != b.status { return Outcome::fail( "diff", format!( "stdout matches but exit status differs: rustc {:?}, nim {:?}\n nim stderr:\n{}", a.status, b.status, tail(&b.stderr, 10) ), ); } Outcome::Pass } fn numbered(p: &Path) -> String { fs::read_to_string(p) .unwrap_or_default() .lines() .enumerate() .map(|(i, l)| format!(" {:>3} | {l}", i + 1)) .collect::>() .join("\n") } // ------------------------------------------------------------------ driver #[test] fn differential() { let root = Path::new(env!("CARGO_MANIFEST_DIR")); let cases_dir = root.join("tests/cases"); let work = root.join("tests/.work"); let _ = fs::create_dir_all(&work); let nim = match find_nim() { Ok(n) => n, Err(e) => panic!("cannot locate Nim: {e}"), }; let mut cases: Vec = fs::read_dir(&cases_dir) .unwrap_or_else(|e| panic!("no {}: {e}", cases_dir.display())) .filter_map(|e| e.ok().map(|e| e.path())) .filter(|p| p.extension().is_some_and(|x| x == "rs")) .collect(); cases.sort(); // An empty corpus must not read as success. That is the exact failure this // runner exists to catch, and it applies to the runner itself. assert!(!cases.is_empty(), "no cases in {}", cases_dir.display()); let filter = std::env::var("RUSTNIM_CASE").ok(); let mut results: BTreeMap = BTreeMap::new(); for case in &cases { let name = case.file_stem().unwrap().to_string_lossy().into_owned(); if let Some(f) = &filter { if !name.contains(f.as_str()) { continue; } } let outcome = run_case(case, &work, &nim); match &outcome { Outcome::Pass => eprintln!("ok {name}"), Outcome::Skip(why) => eprintln!("skip {name} ({why})"), Outcome::Fail { .. } => eprintln!("FAIL {name}"), } results.insert(name, outcome); } let mut failed = Vec::new(); let (mut pass, mut skip) = (0, 0); for (name, o) in &results { match o { Outcome::Pass => pass += 1, Outcome::Skip(_) => skip += 1, Outcome::Fail { stage, detail } => failed.push(format!( "\n--- {name}: failed at stage `{stage}`\n{}", detail.trim_end() )), } } eprintln!("\n{pass} passed, {} failed, {skip} skipped", failed.len()); if !failed.is_empty() { panic!("{}", failed.join("\n")); } }