nandi/frq · 013945f

nandi/frqpublic⑂ Fork 0

⑂ 013945f

Stop shipping the build tree, and let a shell be the devShell

Both containers ignore the state a local checkout carries -- 395MB of a
441MB repo, uploaded every start and excluded again on arrival -- and
both name file:///nix-cache in nix.conf, so a nix command typed by hand
reads the cache instead of rebuilding what is already on the volume.
Init went from minutes to 13 seconds.

flutter-dev bakes the devShell with `nix print-dev-env` and sources it
from .bashrc, so a shell attached to that container already is the shell:
no `nix develop`, no re-clone of the flake's git inputs, no 2.7GB fetched
again per Sandbox because the closure is an image layer. `dev` remains
for when the baked env goes stale against a flake edit.

Bare `nix develop` in there resolved to the flake's *default* shell --
libcosmic, jolt-native, a thousand crates of Rust -- which is the wrong
tree and an expensive way to learn it.

The bake is unverified: realising a devShell during an image build is
exactly what `[nix] flake = true` wanted the ptyshim for, and image
builds are still Functions under gVisor. If it cannot, the fallback is to
bake into the volume at run time instead.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

nandi committed 2026-09-13T23:43:45-07:00 Browse files

013945f parent: 85852e8

modified .modal/_loader.py +28 -2 Viewed

@@ -162,6 +162,7 @@ class Container:
162	162	def _build_image(self) -> modal.Image:
163	163	c = self.spec["container"]
164	164	build = self.spec.get("build", {})
	165	+ nix_spec = self.spec.get("nix", {})
165	166
166	167	if c.get("base"):
167	168	image = modal.Image.from_name(c["base"])
@@ -200,14 +201,31 @@ class Container:
200	201	# the container directory -- a container that builds the repo it lives
201	202	# in sets context = "../..", so include = ["."] means the whole repo.
202	203	context = os.path.normpath(os.path.join(self.dir, build.get("context", ".")))
	204	+ # `ignore` is what keeps a build tree out of the image. A checkout that
	205	+ # has been built in locally carries its output -- flutter/build and the
	206	+ # caches beside it were 395MB of a 441MB repo -- and all of it would be
	207	+ # uploaded on every start only to be thrown away, since the container
	208	+ # builds into a volume of its own. Patterns are relative to the copied
	209	+ # directory, as in .dockerignore.
	210	+ ignore = list(build.get("ignore", []))
203	211	for rel in build.get("include", ["."]):
204	212	src = os.path.normpath(os.path.join(context, rel))
205	213	dest = self.workdir if rel == "." else f"{self.workdir}/{rel}"
206	214	if os.path.isdir(src):
207		- image = image.add_local_dir(src, dest, copy=True)
	215	+ image = image.add_local_dir(src, dest, copy=True, ignore=ignore)
208	216	else:
209	217	image = image.add_local_file(src, dest, copy=True)
210	218
	219	+ # Substituters for every nix command in the container, typed by hand or
	220	+ # not. Without this the only things reading a mounted cache are the
	221	+ # scripts that pass --extra-substituters, so an interactive shell
	222	+ # rebuilds from source what the volume beside it already holds.
	223	+ if subs := list(nix_spec.get("substituters", [])):
	224	+ image = image.run_commands(
	225	+ f"echo 'extra-substituters = {' '.join(subs)}'"
	226	+ " >> /etc/nix/nix.conf"
	227	+ )
	228	+
211	229	# A repo copied in brings its `.git` along, and in a worktree that is a
212	230	# file holding `gitdir: <path on the machine that copied it>`. Nix
213	231	# believes it and goes looking for a checkout that is not there --
@@ -345,12 +363,20 @@ class Container:
345	363	return ""
346	364
347	365	def open_sandbox(self) -> "modal.Sandbox":
348		- """Start a Sandbox and leave it running, for `scripts/shell`."""
	366	+ """Start a Sandbox and leave it running, for `scripts/shell`.
	367	+
	368	+ Same workdir and the same [run] env as the real thing: a shell opened
	369	+ to debug a container that does not have the container's environment is
	370	+ a shell that reproduces something else. `command` is the one part left
	371	+ out, because not running it is the point.
	372	+ """
349	373	return modal.Sandbox.create(
350	374	"sleep",
351	375	"infinity",
352	376	app=self.app,
353	377	image=self.image,
	378	+ workdir=self.workdir,
	379	+ env={k: str(v) for k, v in self.env.items()},
354	380	**self.sandbox_kwargs,
355	381	)
356	382

      def _build_image(self) -> modal.Image:
          c = self.spec["container"]
          build = self.spec.get("build", {})
++        nix_spec = self.spec.get("nix", {})
          if c.get("base"):
              image = modal.Image.from_name(c["base"])
          # the container directory -- a container that builds the repo it lives
          # in sets context = "../..", so include = ["."] means the whole repo.
          context = os.path.normpath(os.path.join(self.dir, build.get("context", ".")))
++        # `ignore` is what keeps a build tree out of the image. A checkout that
++        # has been built in locally carries its output -- flutter/build and the
++        # caches beside it were 395MB of a 441MB repo -- and all of it would be
++        # uploaded on every start only to be thrown away, since the container
++        # builds into a volume of its own. Patterns are relative to the copied
++        # directory, as in .dockerignore.
++        ignore = list(build.get("ignore", []))
          for rel in build.get("include", ["."]):
              src = os.path.normpath(os.path.join(context, rel))
              dest = self.workdir if rel == "." else f"{self.workdir}/{rel}"
              if os.path.isdir(src):
--                image = image.add_local_dir(src, dest, copy=True)
++                image = image.add_local_dir(src, dest, copy=True, ignore=ignore)
              else:
                  image = image.add_local_file(src, dest, copy=True)
++        # Substituters for every nix command in the container, typed by hand or
++        # not. Without this the only things reading a mounted cache are the
++        # scripts that pass --extra-substituters, so an interactive shell
++        # rebuilds from source what the volume beside it already holds.
++        if subs := list(nix_spec.get("substituters", [])):
++            image = image.run_commands(
++                f"echo 'extra-substituters = {' '.join(subs)}'"
++                " >> /etc/nix/nix.conf"
++            )
++
          # A repo copied in brings its `.git` along, and in a worktree that is a
          # *file* holding `gitdir: <path on the machine that copied it>`. Nix
          # believes it and goes looking for a checkout that is not there --
          return ""
      def open_sandbox(self) -> "modal.Sandbox":
--        """Start a Sandbox and leave it running, for `scripts/shell`."""
++        """Start a Sandbox and leave it running, for `scripts/shell`.
++
++        Same workdir and the same [run] env as the real thing: a shell opened
++        to debug a container that does not have the container's environment is
++        a shell that reproduces something else. `command` is the one part left
++        out, because not running it is the point.
++        """
          return modal.Sandbox.create(
              "sleep",
              "infinity",
              app=self.app,
              image=self.image,
++            workdir=self.workdir,
++            env={k: str(v) for k, v in self.env.items()},
              **self.sandbox_kwargs,
          )

modified .modal/flutter-dev/container.toml +36 -2 Viewed

@@ -11,6 +11,30 @@ runtime = "sandbox"
11	11	# levels up and `.` is the whole tree.
12	12	context = "../.."
13	13	include = ["."]
	14	+# `dev` is the shell you actually want. Bare `nix develop` here resolves to the
	15	+# flake's default shell -- libcosmic, jolt-native, a thousand crates of Rust --
	16	+# which is the wrong tree and an expensive way to find out.
	17	+# The devShell, baked in rather than entered. `print-dev-env` writes the whole
	18	+# environment out as shell -- PATH, the compiler, every variable mkShell sets
	19	+# -- and realises its inputs on the way, so the closure becomes an image layer
	20	+# instead of a fetch every container pays for. Sourcing it from .bashrc means a
	21	+# shell attached to this container is the devShell: no `nix develop`, no
	22	+# clone of the flake's git inputs, no wait.
	23	+#
	24	+# `dev` stays for the case where the baked env is stale against a flake edit.
	25	+commands = [
	26	+ "nix print-dev-env /app#flutter-desktop --accept-flake-config > /etc/devshell.sh",
	27	+ "echo '. /etc/devshell.sh' >> /root/.bashrc",
	28	+ "printf '#!/bin/sh\\nexec nix develop /app#flutter-desktop \"$@\"\\n' > /usr/local/bin/dev && chmod +x /usr/local/bin/dev",
	29	+]
	30	+# The build state a local checkout carries: 395MB of a 441MB repo, uploaded on
	31	+# every start and wanted by nothing out there. Flutter builds into a volume of
	32	+# its own, and the jolt and clojure caches are this machine's.
	33	+ignore = [
	34	+ "flutter/build", "flutter/.home", "flutter/.dart_tool",
	35	+ "flutter/.clojuredart", "flutter/.cpcache",
	36	+ ".jolt", ".cpcache", "result", "build", ".git",
	37	+]
14	38
15	39	# Two volumes doing two different jobs. `nix-cache` is the binary cache every
16	40	# container here reads from and writes back to. `devshell` is the working
@@ -52,7 +76,7 @@ set -e
52	76	# devShell it keeps the state of. Anything else using this volume picks its
53	77	# own name and the two never meet.
54	78	SHELL_DIR=/devshell/frq-flutter-desktop
55		-mkdir -p "$SHELL_DIR"
	79	+mkdir -p "$SHELL_DIR" "$SHELL_DIR/.cache"
56	80
57	81	# A worktree's `.git` is a file naming a gitdir back on the machine that
58	82	# copied it in, and nix believes it and goes looking for a path that is not
@@ -113,9 +137,19 @@ if [ -f /nix-cache/nix-cache-info ]; then
113	137	nix copy --no-check-sigs --all --to file:///nix-cache
114	138	fi
115	139	"""
116		-env = { }
	140	+# Nix's own cache, on the volume rather than in the container. Without it
	141	+# every Sandbox starts empty and `nix develop` re-clones the flake's git
	142	+# inputs -- jolt-src, jolt-native, glimmer, nixgl and their transitives --
	143	+# because flake.lock pins which revision to fetch, not whether it is already
	144	+# on disk. Set here rather than in the command so an interactive shell into
	145	+# this container gets it too.
	146	+env = { XDG_CACHE_HOME = "/devshell/frq-flutter-desktop/.cache" }
117	147
118	148	[nix]
	149	+# Every nix command in the container reads the mounted cache, including one
	150	+# typed by hand in a shell. Passing --extra-substituters per command only ever
	151	+# covered the scripts.
	152	+substituters = ["file:///nix-cache"]
119	153	# No devShell warming at image build time: this enters `nix develop` at run
120	154	# time, on the VM, where the cache answers for its closure. The ptyshim that
121	155	# warming would need under gVisor is deprecated and does not come back.

  # levels up and `.` is the whole tree.
  context = "../.."
  include = ["."]
++# `dev` is the shell you actually want. Bare `nix develop` here resolves to the
++# flake's default shell -- libcosmic, jolt-native, a thousand crates of Rust --
++# which is the wrong tree and an expensive way to find out.
++# The devShell, baked in rather than entered. `print-dev-env` writes the whole
++# environment out as shell -- PATH, the compiler, every variable mkShell sets
++# -- and realises its inputs on the way, so the closure becomes an image layer
++# instead of a fetch every container pays for. Sourcing it from .bashrc means a
++# shell attached to this container *is* the devShell: no `nix develop`, no
++# clone of the flake's git inputs, no wait.
++#
++# `dev` stays for the case where the baked env is stale against a flake edit.
++commands = [
++  "nix print-dev-env /app#flutter-desktop --accept-flake-config > /etc/devshell.sh",
++  "echo '. /etc/devshell.sh' >> /root/.bashrc",
++  "printf '#!/bin/sh\\nexec nix develop /app#flutter-desktop \"$@\"\\n' > /usr/local/bin/dev && chmod +x /usr/local/bin/dev",
++]
++# The build state a local checkout carries: 395MB of a 441MB repo, uploaded on
++# every start and wanted by nothing out there. Flutter builds into a volume of
++# its own, and the jolt and clojure caches are this machine's.
++ignore  = [
++    "flutter/build", "flutter/.home", "flutter/.dart_tool",
++    "flutter/.clojuredart", "flutter/.cpcache",
++    ".jolt", ".cpcache", "result", "build", ".git",
++]
  # Two volumes doing two different jobs. `nix-cache` is the binary cache every
  # container here reads from and writes back to. `devshell` is the working
  # devShell it keeps the state of. Anything else using this volume picks its
  # own name and the two never meet.
  SHELL_DIR=/devshell/frq-flutter-desktop
--mkdir -p "$SHELL_DIR"
++mkdir -p "$SHELL_DIR" "$SHELL_DIR/.cache"
  # A worktree's `.git` is a *file* naming a gitdir back on the machine that
  # copied it in, and nix believes it and goes looking for a path that is not
      nix copy --no-check-sigs --all --to file:///nix-cache
  fi
  """
--env     = { }
++# Nix's own cache, on the volume rather than in the container. Without it
++# every Sandbox starts empty and `nix develop` re-clones the flake's git
++# inputs -- jolt-src, jolt-native, glimmer, nixgl and their transitives --
++# because flake.lock pins which revision to fetch, not whether it is already
++# on disk. Set here rather than in the command so an interactive shell into
++# this container gets it too.
++env     = { XDG_CACHE_HOME = "/devshell/frq-flutter-desktop/.cache" }
  [nix]
++# Every nix command in the container reads the mounted cache, including one
++# typed by hand in a shell. Passing --extra-substituters per command only ever
++# covered the scripts.
++substituters = ["file:///nix-cache"]
  # No devShell warming at image build time: this enters `nix develop` at run
  # time, on the VM, where the cache answers for its closure. The ptyshim that
  # warming would need under gVisor is deprecated and does not come back.

modified .modal/frq/container.toml +12 -0 Viewed

@@ -12,6 +12,14 @@ runtime = "sandbox"
12	12	# the point of copying rather than fetching.
13	13	context = "../.."
14	14	include = ["."]
	15	+# The build state a local checkout carries: 395MB of a 441MB repo, uploaded on
	16	+# every start and wanted by nothing out there. Flutter builds into a volume of
	17	+# its own, and the jolt and clojure caches are this machine's.
	18	+ignore = [
	19	+ "flutter/build", "flutter/.home", "flutter/.dart_tool",
	20	+ "flutter/.clojuredart", "flutter/.cpcache",
	21	+ ".jolt", ".cpcache", "result", "build", ".git",
	22	+]
15	23
16	24	# The Modal Volume that makes a second build cheap. Mounted at run time, which
17	25	# is when the build happens here -- nothing is written to it while the image is
@@ -95,6 +103,10 @@ fi
95	103	env = { }
96	104
97	105	[nix]
	106	+# Every nix command in the container reads the mounted cache, including one
	107	+# typed by hand in a shell. Passing --extra-substituters per command only ever
	108	+# covered the scripts.
	109	+substituters = ["file:///nix-cache"]
98	110	# No devShell: we want `nix build`, not a shell to run something inside, so
99	111	# there is nothing to warm at build time and no reason for the ptyshim. The
100	112	# build happens at run time, in the Sandbox, on a real VM -- which is the

  # the point of copying rather than fetching.
  context = "../.."
  include = ["."]
++# The build state a local checkout carries: 395MB of a 441MB repo, uploaded on
++# every start and wanted by nothing out there. Flutter builds into a volume of
++# its own, and the jolt and clojure caches are this machine's.
++ignore  = [
++    "flutter/build", "flutter/.home", "flutter/.dart_tool",
++    "flutter/.clojuredart", "flutter/.cpcache",
++    ".jolt", ".cpcache", "result", "build", ".git",
++]
  # The Modal Volume that makes a second build cheap. Mounted at run time, which
  # is when the build happens here -- nothing is written to it while the image is
  env     = { }
  [nix]
++# Every nix command in the container reads the mounted cache, including one
++# typed by hand in a shell. Passing --extra-substituters per command only ever
++# covered the scripts.
++substituters = ["file:///nix-cache"]
  # No devShell: we want `nix build`, not a shell to run something inside, so
  # there is nothing to warm at build time and no reason for the ptyshim. The
  # build happens at run time, in the Sandbox, on a real VM -- which is the