| Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 17h ago | 1 | //! Differential test runner: rustc is the oracle. |
| 2 | //! |
| 3 | //! For each `tests/cases/*.rs`: |
| 4 | //! |
| 5 | //! ```text |
| 6 | //! rustc case.rs -o a && ./a -> (stdout_rs, status_rs) |
| 7 | //! rustnim case.rs -o case.nim |
| 8 | //! nim c case.nim -o b && ./b -> (stdout_nim, status_nim) |
| 9 | //! assert stdout_rs == stdout_nim && status_rs == status_nim |
| 10 | //! ``` |
| 11 | //! |
| 12 | //! A case passes only when *both* binaries build and produce identical output. |
| 13 | //! Every intermediate stage is checked explicitly, because the failure mode |
| 14 | //! this project exists to avoid is a transpiler reporting success while |
| 15 | //! emitting nothing: `rustnim` exiting 0 with an empty or Nim-unparseable |
| 16 | //! output file is a hard failure here, not a silent pass. |
| 17 | //! |
| 18 | //! Profile: rustc is invoked *without* `-O`, so debug-profile integer overflow |
| 19 | //! checks are on. Nim's default `nim c` also has overflow checks on. That is |
| 20 | //! the matching pair, and it is the profile this project models. |
| 21 | //! |
| 22 | //! Directives, recognised in `//@ ...` comments at the top of a case: |
| 23 | //! |
| 24 | //! //@ reject: <substring> rustnim must fail, with <substring> in stderr. |
| 25 | //! (The "fail loudly" rule, tested.) |
| 26 | //! //@ skip: <reason> Not run; reported as skipped. |
| 27 | //! //@ args: <argv> Passed to both binaries. |
| 28 | //! //@ stdin: <line> Fed to both binaries on stdin. |
| 29 | |
| 30 | use std::collections::BTreeMap; |
| 31 | use std::fmt::Write as _; |
| 32 | use std::fs; |
| 33 | use std::io::Write as _; |
| 34 | use std::path::{Path, PathBuf}; |
| 35 | use std::process::{Command, Stdio}; |
| 36 | |
| 37 | const RUSTNIM: &str = env!("CARGO_BIN_EXE_rustnim"); |
| 38 | |
| 39 | // ---------------------------------------------------------------- outcomes |
| 40 | |
| 41 | #[derive(Debug)] |
| 42 | enum Outcome { |
| 43 | Pass, |
| 44 | Skip(String), |
| 45 | /// A stage failed. `stage` is where, `detail` is the evidence. |
| 46 | Fail { stage: &'static str, detail: String }, |
| 47 | } |
| 48 | |
| 49 | impl Outcome { |
| 50 | fn fail(stage: &'static str, detail: impl Into<String>) -> Self { |
| 51 | Outcome::Fail { stage, detail: detail.into() } |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | struct Run { |
| 56 | status: Option<i32>, |
| 57 | stdout: Vec<u8>, |
| 58 | stderr: String, |
| 59 | } |
| 60 | |
| 61 | impl Run { |
| 62 | fn ok(&self) -> bool { |
| 63 | self.status == Some(0) |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | // ------------------------------------------------------------------ helpers |
| 68 | |
| 69 | fn run(cmd: &mut Command, stdin: Option<&str>) -> Result<Run, String> { |
| 70 | cmd.stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() }) |
| 71 | .stdout(Stdio::piped()) |
| 72 | .stderr(Stdio::piped()); |
| 73 | let mut child = cmd.spawn().map_err(|e| format!("spawn {:?}: {e}", cmd.get_program()))?; |
| 74 | if let Some(s) = stdin { |
| 75 | child.stdin.as_mut().unwrap().write_all(s.as_bytes()).map_err(|e| e.to_string())?; |
| 76 | } |
| 77 | let out = child.wait_with_output().map_err(|e| e.to_string())?; |
| 78 | Ok(Run { |
| 79 | status: out.status.code(), |
| 80 | stdout: out.stdout, |
| 81 | stderr: String::from_utf8_lossy(&out.stderr).into_owned(), |
| 82 | }) |
| 83 | } |
| 84 | |
| 85 | /// Locate the vendored Nim. It lives at the *repository* root, which is not |
| 86 | /// the manifest dir when running inside a git worktree, so walk upwards. |
| 87 | fn find_nim() -> Result<PathBuf, String> { |
| 88 | if let Ok(p) = std::env::var("RUSTNIM_NIM") { |
| 89 | let p = PathBuf::from(p); |
| 90 | if p.is_file() { |
| 91 | return Ok(p); |
| 92 | } |
| 93 | return Err(format!("RUSTNIM_NIM={} is not a file", p.display())); |
| 94 | } |
| 95 | let mut dir: Option<&Path> = Some(Path::new(env!("CARGO_MANIFEST_DIR"))); |
| 96 | while let Some(d) = dir { |
| 97 | let cand = d.join(".nim-toolchain/bin/nim"); |
| 98 | if cand.is_file() { |
| 99 | return Ok(cand); |
| 100 | } |
| 101 | dir = d.parent(); |
| 102 | } |
| 103 | Err("no .nim-toolchain/bin/nim found in this directory or any parent; \ |
| 104 | set RUSTNIM_NIM to the nim binary" |
| 105 | .into()) |
| 106 | } |
| 107 | |
| 108 | #[derive(Default)] |
| 109 | struct Directives { |
| 110 | reject: Option<String>, |
| 111 | skip: Option<String>, |
| 112 | args: Vec<String>, |
| 113 | stdin: Option<String>, |
| 114 | } |
| 115 | |
| 116 | fn directives(src: &str) -> Directives { |
| 117 | let mut d = Directives::default(); |
| 118 | for line in src.lines() { |
| 119 | let line = line.trim(); |
| 120 | let Some(rest) = line.strip_prefix("//@") else { |
| 121 | // Directives must be in the leading comment block; stop at the |
| 122 | // first line of real code so a `//@` inside a string can't count. |
| 123 | if line.is_empty() || line.starts_with("//") || line.starts_with("#!") { |
| 124 | continue; |
| 125 | } |
| 126 | break; |
| 127 | }; |
| 128 | let rest = rest.trim(); |
| 129 | let (key, val) = match rest.split_once(':') { |
| 130 | Some((k, v)) => (k.trim(), v.trim().to_string()), |
| 131 | None => (rest, String::new()), |
| 132 | }; |
| 133 | match key { |
| 134 | "reject" => d.reject = Some(val), |
| 135 | "skip" => d.skip = Some(val), |
| 136 | "args" => d.args = val.split_whitespace().map(str::to_string).collect(), |
| 137 | "stdin" => d.stdin = Some(format!("{val}\n")), |
| 138 | _ => {} |
| 139 | } |
| 140 | } |
| 141 | d |
| 142 | } |
| 143 | |
| 144 | /// Byte-for-byte diff, rendered readably: show the first differing line with |
| 145 | /// escapes, so a trailing-newline or whitespace difference is visible. |
| 146 | fn diff_report(want: &[u8], got: &[u8]) -> String { |
| 147 | let w = String::from_utf8_lossy(want); |
| 148 | let g = String::from_utf8_lossy(got); |
| 149 | let (wl, gl): (Vec<_>, Vec<_>) = (w.lines().collect(), g.lines().collect()); |
| 150 | let mut out = String::new(); |
| 151 | for i in 0..wl.len().max(gl.len()) { |
| 152 | let (a, b) = (wl.get(i), gl.get(i)); |
| 153 | if a != b { |
| 154 | let _ = writeln!(out, " first difference at line {}:", i + 1); |
| 155 | let _ = writeln!(out, " rustc: {}", a.map(|s| format!("{s:?}")).unwrap_or("<eof>".into())); |
| 156 | let _ = writeln!(out, " nim : {}", b.map(|s| format!("{s:?}")).unwrap_or("<eof>".into())); |
| 157 | break; |
| 158 | } |
| 159 | } |
| 160 | if out.is_empty() { |
| 161 | // Lines all matched, so the difference is in trailing bytes. |
| 162 | let _ = writeln!(out, " lines match; raw bytes differ (trailing newline?)"); |
| 163 | let _ = writeln!(out, " rustc: {:?}", w); |
| 164 | let _ = writeln!(out, " nim : {:?}", g); |
| 165 | } |
| 166 | let _ = writeln!(out, " ({} bytes from rustc, {} from nim)", want.len(), got.len()); |
| 167 | out |
| 168 | } |
| 169 | |
| 170 | fn tail(s: &str, lines: usize) -> String { |
| 171 | let v: Vec<&str> = s.trim_end().lines().collect(); |
| 172 | let start = v.len().saturating_sub(lines); |
| 173 | v[start..] |
| 174 | .iter() |
| 175 | .map(|l| format!(" {l}")) |
| 176 | .collect::<Vec<_>>() |
| 177 | .join("\n") |
| 178 | } |
| 179 | |
| 180 | // --------------------------------------------------------------- one case |
| 181 | |
| 182 | fn run_case(case: &Path, work: &Path, nim: &Path) -> Outcome { |
| 183 | let name = case.file_stem().unwrap().to_string_lossy().into_owned(); |
| 184 | let src = match fs::read_to_string(case) { |
| 185 | Ok(s) => s, |
| 186 | Err(e) => return Outcome::fail("read", e.to_string()), |
| 187 | }; |
| 188 | let d = directives(&src); |
| 189 | if let Some(why) = d.skip { |
| 190 | return Outcome::Skip(why); |
| 191 | } |
| 192 | |
| 193 | let dir = work.join(&name); |
| 194 | let _ = fs::remove_dir_all(&dir); |
| 195 | if let Err(e) = fs::create_dir_all(&dir) { |
| 196 | return Outcome::fail("setup", e.to_string()); |
| 197 | } |
| 198 | |
| 199 | // -- stage: transpile. Checked first, and checked strictly: exit status, |
| 200 | // stderr on failure, and that the file actually has content in it. |
| 201 | // Nim module names must be valid Nim identifiers, which case names |
| 202 | // (`005-base16ct-decode-core`) are not. |
| 203 | let mod_name: String = format!( |
| 204 | "c_{}", |
| 205 | name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect::<String>() |
| 206 | ); |
| 207 | let nim_src = dir.join(format!("{mod_name}.nim")); |
| 208 | let transpile = match run( |
| 209 | Command::new(RUSTNIM).arg(case).arg("-o").arg(&nim_src).env("TMPDIR", &dir), |
| 210 | None, |
| 211 | ) { |
| 212 | Ok(r) => r, |
| 213 | Err(e) => return Outcome::fail("rustnim", e), |
| 214 | }; |
| 215 | |
| 216 | if let Some(want) = &d.reject { |
| 217 | // A `reject` case asserts the "fail loudly" rule. |
| 218 | if transpile.ok() { |
| 219 | return Outcome::fail( |
| 220 | "rustnim", |
| 221 | format!("expected rejection containing {want:?}, but transpile succeeded"), |
| 222 | ); |
| 223 | } |
| 224 | if !transpile.stderr.contains(want.as_str()) { |
| 225 | return Outcome::fail( |
| 226 | "rustnim", |
| 227 | format!("rejected, but message lacks {want:?}:\n{}", tail(&transpile.stderr, 10)), |
| 228 | ); |
| 229 | } |
| 230 | return Outcome::Pass; |
| 231 | } |
| 232 | |
| 233 | if !transpile.ok() { |
| 234 | return Outcome::fail( |
| 235 | "rustnim", |
| 236 | format!("exit {:?}\n{}", transpile.status, tail(&transpile.stderr, 20)), |
| 237 | ); |
| 238 | } |
| 239 | match fs::metadata(&nim_src) { |
| 240 | Err(_) => { |
| 241 | return Outcome::fail("rustnim", "exited 0 but wrote no output file".to_string()); |
| 242 | } |
| 243 | Ok(m) if m.len() == 0 => { |
| 244 | return Outcome::fail("rustnim", "exited 0 but wrote an empty output file".to_string()); |
| 245 | } |
| 246 | Ok(_) => {} |
| 247 | } |
| 248 | |
| 249 | // -- stage: rustc. No -O: debug profile, overflow checks on. |
| 250 | let rs_bin = dir.join("rs.bin"); |
| 251 | let rc = match run( |
| 252 | Command::new("rustc") |
| 253 | .arg("--edition=2021") |
| 254 | .arg("-A").arg("warnings") |
| 255 | .arg(case) |
| 256 | .arg("-o").arg(&rs_bin) |
| 257 | .env("TMPDIR", &dir), |
| 258 | None, |
| 259 | ) { |
| 260 | Ok(r) => r, |
| 261 | Err(e) => return Outcome::fail("rustc", e), |
| 262 | }; |
| 263 | if !rc.ok() { |
| 264 | return Outcome::fail("rustc", format!("case does not compile as Rust:\n{}", tail(&rc.stderr, 20))); |
| 265 | } |
| 266 | |
| 267 | // -- stage: nim c |
| 268 | let nim_bin = dir.join("nim.bin"); |
| 269 | let nc = match run( |
| 270 | Command::new(nim) |
| 271 | .arg("c") |
| 272 | .arg("--hints:off") |
| 273 | .arg("--warnings:off") |
| 274 | .arg("--colors:off") |
| 275 | .arg(format!("--nimcache:{}", dir.join("nimcache").display())) |
| 276 | .arg(format!("-o:{}", nim_bin.display())) |
| 277 | .arg(&nim_src) |
| 278 | .env("TMPDIR", &dir), |
| 279 | None, |
| 280 | ) { |
| 281 | Ok(r) => r, |
| 282 | Err(e) => return Outcome::fail("nim", e), |
| 283 | }; |
| 284 | if !nc.ok() { |
| 285 | // Nim reports compile errors on stdout. |
| 286 | let msg = if nc.stderr.trim().is_empty() { |
| 287 | String::from_utf8_lossy(&nc.stdout).into_owned() |
| 288 | } else { |
| 289 | nc.stderr.clone() |
| 290 | }; |
| 291 | return Outcome::fail( |
| 292 | "nim", |
| 293 | format!("generated Nim does not compile:\n{}\n --- generated ---\n{}", tail(&msg, 20), numbered(&nim_src)), |
| 294 | ); |
| 295 | } |
| 296 | |
| 297 | // -- stage: execute both |
| 298 | let exec = |bin: &Path| { |
| 299 | let mut c = Command::new(bin); |
| 300 | c.args(&d.args).env("TMPDIR", &dir); |
| 301 | run(&mut c, d.stdin.as_deref()) |
| 302 | }; |
| 303 | let (a, b) = match (exec(&rs_bin), exec(&nim_bin)) { |
| 304 | (Ok(a), Ok(b)) => (a, b), |
| 305 | (Err(e), _) | (_, Err(e)) => return Outcome::fail("run", e), |
| 306 | }; |
| 307 | |
| 308 | if a.stdout != b.stdout { |
| 309 | return Outcome::fail("diff", diff_report(&a.stdout, &b.stdout)); |
| 310 | } |
| 311 | if a.status != b.status { |
| 312 | return Outcome::fail( |
| 313 | "diff", |
| 314 | format!( |
| 315 | "stdout matches but exit status differs: rustc {:?}, nim {:?}\n nim stderr:\n{}", |
| 316 | a.status, |
| 317 | b.status, |
| 318 | tail(&b.stderr, 10) |
| 319 | ), |
| 320 | ); |
| 321 | } |
| 322 | Outcome::Pass |
| 323 | } |
| 324 | |
| 325 | fn numbered(p: &Path) -> String { |
| 326 | fs::read_to_string(p) |
| 327 | .unwrap_or_default() |
| 328 | .lines() |
| 329 | .enumerate() |
| 330 | .map(|(i, l)| format!(" {:>3} | {l}", i + 1)) |
| 331 | .collect::<Vec<_>>() |
| 332 | .join("\n") |
| 333 | } |
| 334 | |
| 335 | // ------------------------------------------------------------------ driver |
| 336 | |
| 337 | #[test] |
| 338 | fn differential() { |
| 339 | let root = Path::new(env!("CARGO_MANIFEST_DIR")); |
| 340 | let cases_dir = root.join("tests/cases"); |
| 341 | let work = root.join("tests/.work"); |
| 342 | let _ = fs::create_dir_all(&work); |
| 343 | |
| 344 | let nim = match find_nim() { |
| 345 | Ok(n) => n, |
| 346 | Err(e) => panic!("cannot locate Nim: {e}"), |
| 347 | }; |
| 348 | |
| 349 | let mut cases: Vec<PathBuf> = fs::read_dir(&cases_dir) |
| 350 | .unwrap_or_else(|e| panic!("no {}: {e}", cases_dir.display())) |
| 351 | .filter_map(|e| e.ok().map(|e| e.path())) |
| 352 | .filter(|p| p.extension().is_some_and(|x| x == "rs")) |
| 353 | .collect(); |
| 354 | cases.sort(); |
| 355 | |
| 356 | // An empty corpus must not read as success. That is the exact failure this |
| 357 | // runner exists to catch, and it applies to the runner itself. |
| 358 | assert!(!cases.is_empty(), "no cases in {}", cases_dir.display()); |
| 359 | |
| 360 | let filter = std::env::var("RUSTNIM_CASE").ok(); |
| 361 | let mut results: BTreeMap<String, Outcome> = BTreeMap::new(); |
| 362 | for case in &cases { |
| 363 | let name = case.file_stem().unwrap().to_string_lossy().into_owned(); |
| 364 | if let Some(f) = &filter { |
| 365 | if !name.contains(f.as_str()) { |
| 366 | continue; |
| 367 | } |
| 368 | } |
| 369 | let outcome = run_case(case, &work, &nim); |
| 370 | match &outcome { |
| 371 | Outcome::Pass => eprintln!("ok {name}"), |
| 372 | Outcome::Skip(why) => eprintln!("skip {name} ({why})"), |
| 373 | Outcome::Fail { .. } => eprintln!("FAIL {name}"), |
| 374 | } |
| 375 | results.insert(name, outcome); |
| 376 | } |
| 377 | |
| 378 | let mut failed = Vec::new(); |
| 379 | let (mut pass, mut skip) = (0, 0); |
| 380 | for (name, o) in &results { |
| 381 | match o { |
| 382 | Outcome::Pass => pass += 1, |
| 383 | Outcome::Skip(_) => skip += 1, |
| 384 | Outcome::Fail { stage, detail } => failed.push(format!( |
| 385 | "\n--- {name}: failed at stage `{stage}`\n{}", |
| 386 | detail.trim_end() |
| 387 | )), |
| 388 | } |
| 389 | } |
| 390 | |
| 391 | eprintln!("\n{pass} passed, {} failed, {skip} skipped", failed.len()); |
| 392 | if !failed.is_empty() { |
| 393 | panic!("{}", failed.join("\n")); |
| 394 | } |
| 395 | } |