nandi/clonim · 017649f

⑂ 017649f

perf: cache the Nim build so repeat runs skip the backend

`clonim run` pointed --nimcache at a per-PID temp dir that was deleted on
exit, so every run recompiled runtime.nim, core.nim and the Nim stdlib from
scratch. Analysis took 1.8ms; the backend took 5.2s.

The nimcache is now persistent, keyed by the source file's absolute path and
the build flags. On top of that, `run` skips the backend entirely when a stamp
file shows that nothing feeding the binary has changed: the generated Nim, the
sources in src/, the flags, and the Nim compiler itself. Invalidation is
content-based, so touching a file without editing it doesn't force a rebuild.

Repeat runs of examples/intrinsics.clj go from 5.33s to 0.005s.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

nandithebull committed 2026-09-20T18:13:32-07:00 Browse files

017649f parent: 430bb9f

modified src/clonim.nim +36 -6 Viewed

@@ -3,7 +3,7 @@
3	3	## clonim run foo.clj compile and run
4	4	## clonim build foo.clj [-o bin] compile to a native binary
5	5	## clonim emit foo.clj print the generated Nim
6		-import std/[os, osproc, strutils, times]
	6	+import std/[hashes, os, osproc, strutils, times]
7	7	import runtime, compiler
8	8
9	9	proc usage() =
@@ -27,6 +27,17 @@ proc srcDir(): string =
27	27	if fileExists(cand / "runtime.nim"): return cand
28	28	getAppDir()
29	29
	30	+proc buildKey(nimSrc: string, release: bool): string =
	31	+ ## Identifies everything the produced binary depends on. Any change here
	32	+ ## invalidates the cached binary for a source file.
	33	+ var h: Hash = hash(nimSrc) !& hash(release)
	34	+ for f in walkFiles(srcDir() / "*.nim"):
	35	+ h = h !& hash(readFile(f))
	36	+ let nimExe = findExe("nim")
	37	+ if nimExe.len > 0:
	38	+ h = h !& hash($getLastModificationTime(nimExe))
	39	+ $(!$h)
	40	+
30	41	proc main() =
31	42	let argv = commandLineParams()
32	43	if argv.len < 2: usage()
@@ -71,9 +82,15 @@ proc main() =
71	82	for ch in stem:
72	83	modName.add (if ch in {'a'..'z', 'A'..'Z', '0'..'9'}: ch else: '_')
73	84	if modName.len == 0 or modName[0] in {'0'..'9'}: modName = "m" & modName
74		- let work = getTempDir() / ("clonim-" & modName & "-" & $getCurrentProcessId())
	85	+ # The nimcache is persistent and keyed by the source file's absolute path, so
	86	+ # repeated `clonim run` on the same file reuses the compiled runtime/core and
	87	+ # the Nim stdlib instead of rebuilding them from scratch every time. The .nim
	88	+ # file lives in the same directory so its path stays stable across runs too —
	89	+ # Nim keys its cache entries on module paths.
	90	+ let work = getTempDir() / "clonim" /
	91	+ (modName & "-" & toHex(hash(file.absolutePath).uint32, 8) &
	92	+ (if release: "-r" else: ""))
75	93	createDir(work)
76		- defer: removeDir(work)
77	94	let nimFile = work / (modName & ".nim")
78	95	writeFile(nimFile, nimSrc)
79	96
@@ -81,15 +98,27 @@ proc main() =
81	98	outBin = (if cmd == "build": stem else: work / modName)
82	99	outBin = outBin.absolutePath
83	100
	101	+ # Even a warm nimcache costs a second or so of semantic checking and linking.
	102	+ # `run` skips the backend entirely when nothing that feeds the binary has
	103	+ # changed: the generated Nim, the runtime/core sources it imports, the build
	104	+ # flags, and the Nim compiler itself.
	105	+ let stamp = work / "stamp"
	106	+ let cached = cmd == "run" and fileExists(outBin) and fileExists(stamp) and
	107	+ readFile(stamp) == buildKey(nimSrc, release)
	108	+
84	109	var nimCmd = @["nim", "c", "--hints:off", "--warnings:off",
85	110	"--path:" & srcDir(), "--nimcache:" & (work / "cache"),
86	111	"-o:" & outBin]
87	112	if release: nimCmd.add "-d:release"
88	113	nimCmd.add nimFile
89		- if verbose: echo "clonim: " & nimCmd.join(" ")
	114	+ if verbose and not cached: echo "clonim: " & nimCmd.join(" ")
90	115
91	116	let t1 = epochTime()
92		- let (output, code) = execCmdEx(nimCmd.join(" "))
	117	+ var output = ""
	118	+ var code = 0
	119	+ if not cached:
	120	+ (output, code) = execCmdEx(nimCmd.join(" "))
	121	+ if code == 0: writeFile(stamp, buildKey(nimSrc, release))
93	122	let tBuild = epochTime() - t1
94	123	if code != 0:
95	124	stderr.writeLine("clonim: Nim backend failed\n" & output)
@@ -97,7 +126,8 @@ proc main() =
97	126	quit(1)
98	127	if verbose:
99	128	echo "clonim: analyze ", (tCompile * 1000).formatFloat(ffDecimal, 1), "ms ",
100		- "nim ", (tBuild * 1000).formatFloat(ffDecimal, 1), "ms"
	129	+ "nim ", (tBuild * 1000).formatFloat(ffDecimal, 1), "ms",
	130	+ (if cached: " (cached)" else: "")
101	131
102	132	case cmd
103	133	of "build":

  ##   clonim run   foo.clj          compile and run
  ##   clonim build foo.clj [-o bin] compile to a native binary
  ##   clonim emit  foo.clj          print the generated Nim
--import std/[os, osproc, strutils, times]
++import std/[hashes, os, osproc, strutils, times]
  import runtime, compiler
  proc usage() =
      if fileExists(cand / "runtime.nim"): return cand
    getAppDir()
++proc buildKey(nimSrc: string, release: bool): string =
++  ## Identifies everything the produced binary depends on. Any change here
++  ## invalidates the cached binary for a source file.
++  var h: Hash = hash(nimSrc) !& hash(release)
++  for f in walkFiles(srcDir() / "*.nim"):
++    h = h !& hash(readFile(f))
++  let nimExe = findExe("nim")
++  if nimExe.len > 0:
++    h = h !& hash($getLastModificationTime(nimExe))
++  $(!$h)
++
  proc main() =
    let argv = commandLineParams()
    if argv.len < 2: usage()
    for ch in stem:
      modName.add (if ch in {'a'..'z', 'A'..'Z', '0'..'9'}: ch else: '_')
    if modName.len == 0 or modName[0] in {'0'..'9'}: modName = "m" & modName
--  let work = getTempDir() / ("clonim-" & modName & "-" & $getCurrentProcessId())
++  # The nimcache is persistent and keyed by the source file's absolute path, so
++  # repeated `clonim run` on the same file reuses the compiled runtime/core and
++  # the Nim stdlib instead of rebuilding them from scratch every time. The .nim
++  # file lives in the same directory so its path stays stable across runs too —
++  # Nim keys its cache entries on module paths.
++  let work = getTempDir() / "clonim" /
++             (modName & "-" & toHex(hash(file.absolutePath).uint32, 8) &
++              (if release: "-r" else: ""))
    createDir(work)
--  defer: removeDir(work)
    let nimFile = work / (modName & ".nim")
    writeFile(nimFile, nimSrc)
      outBin = (if cmd == "build": stem else: work / modName)
    outBin = outBin.absolutePath
++  # Even a warm nimcache costs a second or so of semantic checking and linking.
++  # `run` skips the backend entirely when nothing that feeds the binary has
++  # changed: the generated Nim, the runtime/core sources it imports, the build
++  # flags, and the Nim compiler itself.
++  let stamp = work / "stamp"
++  let cached = cmd == "run" and fileExists(outBin) and fileExists(stamp) and
++               readFile(stamp) == buildKey(nimSrc, release)
++
    var nimCmd = @["nim", "c", "--hints:off", "--warnings:off",
                   "--path:" & srcDir(), "--nimcache:" & (work / "cache"),
                   "-o:" & outBin]
    if release: nimCmd.add "-d:release"
    nimCmd.add nimFile
--  if verbose: echo "clonim: " & nimCmd.join(" ")
++  if verbose and not cached: echo "clonim: " & nimCmd.join(" ")
    let t1 = epochTime()
--  let (output, code) = execCmdEx(nimCmd.join(" "))
++  var output = ""
++  var code = 0
++  if not cached:
++    (output, code) = execCmdEx(nimCmd.join(" "))
++    if code == 0: writeFile(stamp, buildKey(nimSrc, release))
    let tBuild = epochTime() - t1
    if code != 0:
      stderr.writeLine("clonim: Nim backend failed\n" & output)
      quit(1)
    if verbose:
      echo "clonim: analyze ", (tCompile * 1000).formatFloat(ffDecimal, 1), "ms  ",
--         "nim ", (tBuild * 1000).formatFloat(ffDecimal, 1), "ms"
++         "nim ", (tBuild * 1000).formatFloat(ffDecimal, 1), "ms",
++         (if cached: " (cached)" else: "")
    case cmd
    of "build":