rickub
nandi/rustnimpublic⑂ Fork 0
⑂ 4e4d09dcdd22d17ba510de5639fc3a952ac73f6e
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/rustnim.git
git clone ssh://git@rickub.com/nandi/rustnim.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

differential.rs · 481 lines · 16.6 KBRust Blame HistoryRaw
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago1//! Differential test runner: rustc is the oracle.
2//!
3//! For each `tests/cases/*.rs`:
4//!
5//! ```text
6//! rustc case.rs -o a && ./a -> (stdout_rs, status_rs)
7//! rustnim case.rs -o case.nim
8//! nim c case.nim -o b && ./b -> (stdout_nim, status_nim)
9//! assert stdout_rs == stdout_nim && status_rs == status_nim
10//! ```
11//!
12//! A case passes only when *both* binaries build and produce identical output.
13//! Every intermediate stage is checked explicitly, because the failure mode
14//! this project exists to avoid is a transpiler reporting success while
15//! emitting nothing: `rustnim` exiting 0 with an empty or Nim-unparseable
16//! output file is a hard failure here, not a silent pass.
17//!
18//! Profile: rustc is invoked *without* `-O`, so debug-profile integer overflow
19//! checks are on. Nim's default `nim c` also has overflow checks on. That is
20//! the matching pair, and it is the profile this project models.
21//!
22//! Directives, recognised in `//@ ...` comments at the top of a case:
23//!
24//! //@ reject: <substring> rustnim must fail, with <substring> in stderr.
25//! (The "fail loudly" rule, tested.)
26//! //@ skip: <reason> Not run; reported as skipped.
27//! //@ args: <argv> Passed to both binaries.
28//! //@ stdin: <line> Fed to both binaries on stdin.
29
30use std::collections::BTreeMap;
31use std::fmt::Write as _;
32use std::fs;
33use std::io::Write as _;
34use std::path::{Path, PathBuf};
35use std::process::{Command, Stdio};
36
37const RUSTNIM: &str = env!("CARGO_BIN_EXE_rustnim");
38
39// ---------------------------------------------------------------- outcomes
40
41#[derive(Debug)]
42enum Outcome {
43 Pass,
44 Skip(String),
45 /// A stage failed. `stage` is where, `detail` is the evidence.
46 Fail { stage: &'static str, detail: String },
47}
48
49impl Outcome {
50 fn fail(stage: &'static str, detail: impl Into<String>) -> Self {
51 Outcome::Fail { stage, detail: detail.into() }
52 }
53}
54
55struct Run {
56 status: Option<i32>,
57 stdout: Vec<u8>,
58 stderr: String,
59}
60
61impl Run {
62 fn ok(&self) -> bool {
63 self.status == Some(0)
64 }
65}
66
67// ------------------------------------------------------------------ helpers
68
69fn run(cmd: &mut Command, stdin: Option<&str>) -> Result<Run, String> {
70 cmd.stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() })
71 .stdout(Stdio::piped())
72 .stderr(Stdio::piped());
73 let mut child = cmd.spawn().map_err(|e| format!("spawn {:?}: {e}", cmd.get_program()))?;
74 if let Some(s) = stdin {
75 child.stdin.as_mut().unwrap().write_all(s.as_bytes()).map_err(|e| e.to_string())?;
76 }
77 let out = child.wait_with_output().map_err(|e| e.to_string())?;
78 Ok(Run {
79 status: out.status.code(),
80 stdout: out.stdout,
81 stderr: String::from_utf8_lossy(&out.stderr).into_owned(),
82 })
83}
84
85/// Locate the vendored Nim. It lives at the *repository* root, which is not
86/// the manifest dir when running inside a git worktree, so walk upwards.
87fn find_nim() -> Result<PathBuf, String> {
88 if let Ok(p) = std::env::var("RUSTNIM_NIM") {
89 let p = PathBuf::from(p);
90 if p.is_file() {
91 return Ok(p);
92 }
93 return Err(format!("RUSTNIM_NIM={} is not a file", p.display()));
94 }
95 let mut dir: Option<&Path> = Some(Path::new(env!("CARGO_MANIFEST_DIR")));
96 while let Some(d) = dir {
97 let cand = d.join(".nim-toolchain/bin/nim");
98 if cand.is_file() {
99 return Ok(cand);
100 }
101 dir = d.parent();
102 }
103 Err("no .nim-toolchain/bin/nim found in this directory or any parent; \
104 set RUSTNIM_NIM to the nim binary"
105 .into())
106}
107
108#[derive(Default)]
109struct Directives {
110 reject: Option<String>,
111 skip: Option<String>,
112 args: Vec<String>,
113 stdin: Option<String>,
Add display.rs and the alloc half: all of base16ct now goes through afb2a6e nandithebull 12h ago114 /// `--cfg` flags for rustnim. rustc gets `--cfg feature="x"` to match.
115 cfg: Vec<String>,
Lower `bitflags!` directly, checked against the real crate af6e50f nandithebull 10h ago116 /// Crates the *oracle* must link. rustnim does not use them — the point
117 /// of such a case is that rustnim reproduces the crate's behaviour
118 /// without it.
119 externs: Vec<String>,
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago120}
121
122fn directives(src: &str) -> Directives {
123 let mut d = Directives::default();
124 for line in src.lines() {
125 let line = line.trim();
126 let Some(rest) = line.strip_prefix("//@") else {
127 // Directives must be in the leading comment block; stop at the
128 // first line of real code so a `//@` inside a string can't count.
129 if line.is_empty() || line.starts_with("//") || line.starts_with("#!") {
130 continue;
131 }
132 break;
133 };
134 let rest = rest.trim();
135 let (key, val) = match rest.split_once(':') {
136 Some((k, v)) => (k.trim(), v.trim().to_string()),
137 None => (rest, String::new()),
138 };
139 match key {
140 "reject" => d.reject = Some(val),
141 "skip" => d.skip = Some(val),
142 "args" => d.args = val.split_whitespace().map(str::to_string).collect(),
Add display.rs and the alloc half: all of base16ct now goes through afb2a6e nandithebull 12h ago143 "cfg" => d.cfg.push(val),
Lower `bitflags!` directly, checked against the real crate af6e50f nandithebull 10h ago144 "extern" => d.externs.push(val),
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago145 "stdin" => d.stdin = Some(format!("{val}\n")),
146 _ => {}
147 }
148 }
149 d
150}
151
Lower `bitflags!` directly, checked against the real crate af6e50f nandithebull 10h ago152/// Locate a dependency's rlib among cargo's build artefacts. It is there
153/// because it is a dev-dependency of this crate, so cargo has already built
154/// it by the time the tests run.
155fn find_rlib(name: &str) -> Result<(PathBuf, PathBuf), String> {
156 // CARGO_BIN_EXE_* points at target/<profile>/<bin>, so deps/ is beside it.
157 let bin = Path::new(RUSTNIM);
158 let deps = bin
159 .parent()
160 .ok_or("no target directory")?
161 .join("deps");
162 let prefix = format!("lib{name}-");
163 let mut best: Option<PathBuf> = None;
164 for e in fs::read_dir(&deps).map_err(|e| format!("{}: {e}", deps.display()))? {
165 let p = e.map_err(|e| e.to_string())?.path();
166 let f = p.file_name().unwrap_or_default().to_string_lossy().into_owned();
167 if f.starts_with(&prefix) && f.ends_with(".rlib") {
168 let newer = match &best {
169 None => true,
170 Some(b) => {
171 fs::metadata(&p).and_then(|m| m.modified()).ok()
172 > fs::metadata(b).and_then(|m| m.modified()).ok()
173 }
174 };
175 if newer {
176 best = Some(p);
177 }
178 }
179 }
180 best.map(|p| (p, deps.clone()))
181 .ok_or_else(|| format!("no {prefix}*.rlib under {}", deps.display()))
182}
183
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago184/// Byte-for-byte diff, rendered readably: show the first differing line with
185/// escapes, so a trailing-newline or whitespace difference is visible.
186fn diff_report(want: &[u8], got: &[u8]) -> String {
187 let w = String::from_utf8_lossy(want);
188 let g = String::from_utf8_lossy(got);
189 let (wl, gl): (Vec<_>, Vec<_>) = (w.lines().collect(), g.lines().collect());
190 let mut out = String::new();
191 for i in 0..wl.len().max(gl.len()) {
192 let (a, b) = (wl.get(i), gl.get(i));
193 if a != b {
194 let _ = writeln!(out, " first difference at line {}:", i + 1);
195 let _ = writeln!(out, " rustc: {}", a.map(|s| format!("{s:?}")).unwrap_or("<eof>".into()));
196 let _ = writeln!(out, " nim : {}", b.map(|s| format!("{s:?}")).unwrap_or("<eof>".into()));
197 break;
198 }
199 }
200 if out.is_empty() {
201 // Lines all matched, so the difference is in trailing bytes.
202 let _ = writeln!(out, " lines match; raw bytes differ (trailing newline?)");
203 let _ = writeln!(out, " rustc: {:?}", w);
204 let _ = writeln!(out, " nim : {:?}", g);
205 }
206 let _ = writeln!(out, " ({} bytes from rustc, {} from nim)", want.len(), got.len());
207 out
208}
209
210fn tail(s: &str, lines: usize) -> String {
211 let v: Vec<&str> = s.trim_end().lines().collect();
212 let start = v.len().saturating_sub(lines);
213 v[start..]
214 .iter()
215 .map(|l| format!(" {l}"))
216 .collect::<Vec<_>>()
217 .join("\n")
218}
219
220// --------------------------------------------------------------- one case
221
222fn run_case(case: &Path, work: &Path, nim: &Path) -> Outcome {
223 let name = case.file_stem().unwrap().to_string_lossy().into_owned();
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 13h ago224 // For a directory case, `main.rs` is the crate root and carries the
225 // directives; every `.rs` beside it is passed to rustnim as well.
226 let (root, extra) = if case.is_dir() {
227 let mut others: Vec<PathBuf> = fs::read_dir(case)
228 .map(|d| {
229 d.filter_map(|e| e.ok().map(|e| e.path()))
230 .filter(|p| {
231 p.extension().is_some_and(|x| x == "rs")
232 && p.file_name().is_some_and(|f| f != "main.rs")
233 })
234 .collect()
235 })
236 .unwrap_or_default();
237 others.sort();
238 (case.join("main.rs"), others)
239 } else {
240 (case.to_path_buf(), Vec::new())
241 };
242 let src = match fs::read_to_string(&root) {
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago243 Ok(s) => s,
244 Err(e) => return Outcome::fail("read", e.to_string()),
245 };
246 let d = directives(&src);
247 if let Some(why) = d.skip {
248 return Outcome::Skip(why);
249 }
250
251 let dir = work.join(&name);
252 let _ = fs::remove_dir_all(&dir);
253 if let Err(e) = fs::create_dir_all(&dir) {
254 return Outcome::fail("setup", e.to_string());
255 }
256
257 // -- stage: transpile. Checked first, and checked strictly: exit status,
258 // stderr on failure, and that the file actually has content in it.
259 // Nim module names must be valid Nim identifiers, which case names
260 // (`005-base16ct-decode-core`) are not.
261 let mod_name: String = format!(
262 "c_{}",
263 name.chars().map(|c| if c.is_alphanumeric() { c } else { '_' }).collect::<String>()
264 );
265 let nim_src = dir.join(format!("{mod_name}.nim"));
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 13h ago266 let mut cmd = Command::new(RUSTNIM);
267 cmd.arg(&root);
268 for e in &extra {
269 cmd.arg(e);
270 }
Add display.rs and the alloc half: all of base16ct now goes through afb2a6e nandithebull 12h ago271 for c in &d.cfg {
272 cmd.arg("--cfg").arg(c);
273 }
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 13h ago274 cmd.arg("-o").arg(&nim_src).env("TMPDIR", &dir);
275 let transpile = match run(&mut cmd, None) {
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago276 Ok(r) => r,
277 Err(e) => return Outcome::fail("rustnim", e),
278 };
279
280 if let Some(want) = &d.reject {
281 // A `reject` case asserts the "fail loudly" rule.
282 if transpile.ok() {
283 return Outcome::fail(
284 "rustnim",
285 format!("expected rejection containing {want:?}, but transpile succeeded"),
286 );
287 }
288 if !transpile.stderr.contains(want.as_str()) {
289 return Outcome::fail(
290 "rustnim",
291 format!("rejected, but message lacks {want:?}:\n{}", tail(&transpile.stderr, 10)),
292 );
293 }
294 return Outcome::Pass;
295 }
296
297 if !transpile.ok() {
298 return Outcome::fail(
299 "rustnim",
300 format!("exit {:?}\n{}", transpile.status, tail(&transpile.stderr, 20)),
301 );
302 }
303 match fs::metadata(&nim_src) {
304 Err(_) => {
305 return Outcome::fail("rustnim", "exited 0 but wrote no output file".to_string());
306 }
307 Ok(m) if m.len() == 0 => {
308 return Outcome::fail("rustnim", "exited 0 but wrote an empty output file".to_string());
309 }
310 Ok(_) => {}
311 }
312
313 // -- stage: rustc. No -O: debug profile, overflow checks on.
314 let rs_bin = dir.join("rs.bin");
Lower `bitflags!` directly, checked against the real crate af6e50f nandithebull 10h ago315 let mut rustc = Command::new("rustc");
316 rustc.arg("--edition=2021").arg("-A").arg("warnings");
317 for c in &d.externs {
318 match find_rlib(c) {
319 Ok((rlib, deps)) => {
320 rustc.arg("--extern").arg(format!("{c}={}", rlib.display()));
321 rustc.arg("-L").arg(format!("dependency={}", deps.display()));
322 }
323 Err(e) => return Outcome::fail("setup", format!("`//@ extern: {c}`: {e}")),
324 }
325 }
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago326 let rc = match run(
Lower `bitflags!` directly, checked against the real crate af6e50f nandithebull 10h ago327 rustc
Add display.rs and the alloc half: all of base16ct now goes through afb2a6e nandithebull 12h ago328 .args(d.cfg.iter().flat_map(|c| {
329 // rustc spells it `feature="x"`; the directive uses the
330 // rustnim form, so it is rewritten here.
331 let c = match c.split_once('=') {
332 Some((k, v)) => format!("{k}=\"{v}\""),
333 None => c.clone(),
334 };
335 ["--cfg".to_string(), c]
336 }))
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 13h ago337 .arg(&root)
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago338 .arg("-o").arg(&rs_bin)
339 .env("TMPDIR", &dir),
340 None,
341 ) {
342 Ok(r) => r,
343 Err(e) => return Outcome::fail("rustc", e),
344 };
345 if !rc.ok() {
346 return Outcome::fail("rustc", format!("case does not compile as Rust:\n{}", tail(&rc.stderr, 20)));
347 }
348
349 // -- stage: nim c
350 let nim_bin = dir.join("nim.bin");
351 let nc = match run(
352 Command::new(nim)
353 .arg("c")
354 .arg("--hints:off")
355 .arg("--warnings:off")
356 .arg("--colors:off")
357 .arg(format!("--nimcache:{}", dir.join("nimcache").display()))
358 .arg(format!("-o:{}", nim_bin.display()))
359 .arg(&nim_src)
360 .env("TMPDIR", &dir),
361 None,
362 ) {
363 Ok(r) => r,
364 Err(e) => return Outcome::fail("nim", e),
365 };
366 if !nc.ok() {
367 // Nim reports compile errors on stdout.
368 let msg = if nc.stderr.trim().is_empty() {
369 String::from_utf8_lossy(&nc.stdout).into_owned()
370 } else {
371 nc.stderr.clone()
372 };
373 return Outcome::fail(
374 "nim",
375 format!("generated Nim does not compile:\n{}\n --- generated ---\n{}", tail(&msg, 20), numbered(&nim_src)),
376 );
377 }
378
379 // -- stage: execute both
380 let exec = |bin: &Path| {
381 let mut c = Command::new(bin);
382 c.args(&d.args).env("TMPDIR", &dir);
383 run(&mut c, d.stdin.as_deref())
384 };
385 let (a, b) = match (exec(&rs_bin), exec(&nim_bin)) {
386 (Ok(a), Ok(b)) => (a, b),
387 (Err(e), _) | (_, Err(e)) => return Outcome::fail("run", e),
388 };
389
390 if a.stdout != b.stdout {
391 return Outcome::fail("diff", diff_report(&a.stdout, &b.stdout));
392 }
393 if a.status != b.status {
394 return Outcome::fail(
395 "diff",
396 format!(
397 "stdout matches but exit status differs: rustc {:?}, nim {:?}\n nim stderr:\n{}",
398 a.status,
399 b.status,
400 tail(&b.stderr, 10)
401 ),
402 );
403 }
404 Outcome::Pass
405}
406
407fn numbered(p: &Path) -> String {
408 fs::read_to_string(p)
409 .unwrap_or_default()
410 .lines()
411 .enumerate()
412 .map(|(i, l)| format!(" {:>3} | {l}", i + 1))
413 .collect::<Vec<_>>()
414 .join("\n")
415}
416
417// ------------------------------------------------------------------ driver
418
419#[test]
420fn differential() {
421 let root = Path::new(env!("CARGO_MANIFEST_DIR"));
422 let cases_dir = root.join("tests/cases");
423 let work = root.join("tests/.work");
424 let _ = fs::create_dir_all(&work);
425
426 let nim = match find_nim() {
427 Ok(n) => n,
428 Err(e) => panic!("cannot locate Nim: {e}"),
429 };
430
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 13h ago431 // A case is either a single `.rs` file or a directory of them whose
432 // crate root is `main.rs` -- which is how a multi-file crate is tested.
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago433 let mut cases: Vec<PathBuf> = fs::read_dir(&cases_dir)
434 .unwrap_or_else(|e| panic!("no {}: {e}", cases_dir.display()))
435 .filter_map(|e| e.ok().map(|e| e.path()))
Add trait impls and slice iterators; base16ct's decoder now goes through ae9f986 nandithebull 13h ago436 .filter(|p| {
437 p.extension().is_some_and(|x| x == "rs") || p.join("main.rs").is_file()
438 })
Add the differential test runner, and a lowering to measure with it 8ac32af nandithebull 14h ago439 .collect();
440 cases.sort();
441
442 // An empty corpus must not read as success. That is the exact failure this
443 // runner exists to catch, and it applies to the runner itself.
444 assert!(!cases.is_empty(), "no cases in {}", cases_dir.display());
445
446 let filter = std::env::var("RUSTNIM_CASE").ok();
447 let mut results: BTreeMap<String, Outcome> = BTreeMap::new();
448 for case in &cases {
449 let name = case.file_stem().unwrap().to_string_lossy().into_owned();
450 if let Some(f) = &filter {
451 if !name.contains(f.as_str()) {
452 continue;
453 }
454 }
455 let outcome = run_case(case, &work, &nim);
456 match &outcome {
457 Outcome::Pass => eprintln!("ok {name}"),
458 Outcome::Skip(why) => eprintln!("skip {name} ({why})"),
459 Outcome::Fail { .. } => eprintln!("FAIL {name}"),
460 }
461 results.insert(name, outcome);
462 }
463
464 let mut failed = Vec::new();
465 let (mut pass, mut skip) = (0, 0);
466 for (name, o) in &results {
467 match o {
468 Outcome::Pass => pass += 1,
469 Outcome::Skip(_) => skip += 1,
470 Outcome::Fail { stage, detail } => failed.push(format!(
471 "\n--- {name}: failed at stage `{stage}`\n{}",
472 detail.trim_end()
473 )),
474 }
475 }
476
477 eprintln!("\n{pass} passed, {} failed, {skip} skipped", failed.len());
478 if !failed.is_empty() {
479 panic!("{}", failed.join("\n"));
480 }
481}