A shell that is already the devShell, and two recipes to reach it
`nix print-dev-env` at image build writes the whole environment out as shell — 124KB of it — and realises its inputs on the way, so the closure is an image layer rather than 2.7GB fetched per Sandbox and the flake's git inputs are cloned once instead of on every start. Sourced from .bashrc, a shell attached to flutter-dev *is* the shell: flutter and cmake resolve to the store without anyone typing `nix develop`. Which matters because bare `nix develop` in there resolves to the flake's *default* shell — libcosmic, jolt-native, a thousand crates of Rust — and finding that out costs a while. The bake reads file:///nix-cache explicitly. It has to: NIX_CONFIG is run env and does not apply while the image builds, and without the cache nix builds cljd-deps rather than fetching it, which gVisor will not do. One local_entrypoint, not two. A second made plain `modal run container.py` ambiguous and Modal refuses to choose, which quietly ate two test runs before the error was read properly. `--shell` is a flag on the one there is. `just modal` and `just modal-shell` are the interface. modal-shell has a recipe of its own rather than a flag because it behaves differently: it blocks, you attach from a second terminal, and it bills until Ctrl-C. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
e853593 parent: 013945f modified
.modal/_loader.py +34 -12 | @@ -44,6 +44,20 @@ class Container: | ||
| 44 | 44 | self.command = run.get("command", "") |
| 45 | 45 | self.env = dict(run.get("env", {})) |
| 46 | 46 | |
| 47 | + # Substituters reach nix through NIX_CONFIG rather than nix.conf, and | |
| 48 | + # at run time rather than build time. Written into the image's nix.conf | |
| 49 | + # instead, nix touches the path while the image is being built and | |
| 50 | + # creates it -- and Modal will not mount a volume over a non-empty | |
| 51 | + # directory, so the container that wanted the cache cannot start. As | |
| 52 | + # env it covers every nix command that runs in the container, the ones | |
| 53 | + # typed by hand in a shell included, and leaves the mount point empty. | |
| 54 | + if subs := list(spec.get("nix", {}).get("substituters", [])): | |
| 55 | + line = "extra-substituters = " + " ".join(subs) | |
| 56 | + self.env["NIX_CONFIG"] = ( | |
| 57 | + f"{self.env['NIX_CONFIG']}\n{line}" if "NIX_CONFIG" in self.env | |
| 58 | + else line | |
| 59 | + ) | |
| 60 | + | |
| 47 | 61 | # "function" (the default) or "sandbox". Sandboxes can run on a real |
| 48 | 62 | # VM, which Functions cannot -- see ../README.md. |
| 49 | 63 | self.runtime = spec.get("container", {}).get("runtime", "function") |
| @@ -162,13 +176,24 @@ class Container: | ||
| 162 | 176 | def _build_image(self) -> modal.Image: |
| 163 | 177 | c = self.spec["container"] |
| 164 | 178 | build = self.spec.get("build", {}) |
| 165 | - nix_spec = self.spec.get("nix", {}) | |
| 166 | 179 | |
| 167 | 180 | if c.get("base"): |
| 168 | 181 | image = modal.Image.from_name(c["base"]) |
| 169 | 182 | else: |
| 170 | 183 | image = modal.Image.from_registry(c["registry"]) |
| 171 | 184 | |
| 185 | + # `nix profile install` puts things in ~/.nix-profile/bin, which is on | |
| 186 | + # nobody's PATH here -- so the install succeeds and the very next line | |
| 187 | + # says `command not found`. Set on the image rather than in a .bashrc, | |
| 188 | + # because a container's command runs under `sh -c` and reads neither. | |
| 189 | + # Spelled out rather than prefixed onto $PATH: an image env is a value, | |
| 190 | + # not a shell expression, so "$PATH" here would be four literal | |
| 191 | + # characters. | |
| 192 | + image = image.env({ | |
| 193 | + "PATH": "/root/.nix-profile/bin:/usr/local/sbin:/usr/local/bin" | |
| 194 | + ":/usr/sbin:/usr/bin:/sbin:/bin", | |
| 195 | + }) | |
| 196 | + | |
| 172 | 197 | if self.use_shim: |
| 173 | 198 | image = image.add_local_file( |
| 174 | 199 | PTYSHIM_C, "/opt/ptyshim.c", copy=True |
| @@ -216,16 +241,6 @@ class Container: | ||
| 216 | 241 | else: |
| 217 | 242 | image = image.add_local_file(src, dest, copy=True) |
| 218 | 243 | |
| 219 | - # Substituters for every nix command in the container, typed by hand or | |
| 220 | - # not. Without this the only things reading a mounted cache are the | |
| 221 | - # scripts that pass --extra-substituters, so an interactive shell | |
| 222 | - # rebuilds from source what the volume beside it already holds. | |
| 223 | - if subs := list(nix_spec.get("substituters", [])): | |
| 224 | - image = image.run_commands( | |
| 225 | - f"echo 'extra-substituters = {' '.join(subs)}'" | |
| 226 | - " >> /etc/nix/nix.conf" | |
| 227 | - ) | |
| 228 | - | |
| 229 | 244 | # A repo copied in brings its `.git` along, and in a worktree that is a |
| 230 | 245 | # *file* holding `gitdir: <path on the machine that copied it>`. Nix |
| 231 | 246 | # believes it and goes looking for a checkout that is not there -- |
| @@ -234,7 +249,14 @@ class Container: | ||
| 234 | 249 | image = image.run_commands(f"rm -rf {self.workdir}/.git") |
| 235 | 250 | |
| 236 | 251 | if commands := build.get("commands", []): |
| 237 | - image = image.run_commands(*commands) | |
| 252 | + # Volumes mounted for the build too, not just the run. A build step | |
| 253 | + # that wants nix to *build* something is the one thing gVisor will | |
| 254 | + # not do -- image builds are Functions underneath, and a derivation | |
| 255 | + # there dies on `unexpected EOF reading a line`. Substituting is | |
| 256 | + # fine, so a step that can reach the cache never has to build, and | |
| 257 | + # the shim stays retired. The mount is not part of the resulting | |
| 258 | + # image; only what the step writes outside it is. | |
| 259 | + image = image.run_commands(*commands, volumes=self.volumes) | |
| 238 | 260 | |
| 239 | 261 | # container.py does `from _loader import Container`, and Modal mounts |
| 240 | 262 | # the entrypoint file alone -- so without this the import that works |
| @@ -44,6 +44,20 @@ class Container: | |||
| 44 | self.command = run.get("command", "") | 44 | self.command = run.get("command", "") |
| 45 | self.env = dict(run.get("env", {})) | 45 | self.env = dict(run.get("env", {})) |
| 46 | 46 | ||
| 47 | + # Substituters reach nix through NIX_CONFIG rather than nix.conf, and | ||
| 48 | + # at run time rather than build time. Written into the image's nix.conf | ||
| 49 | + # instead, nix touches the path while the image is being built and | ||
| 50 | + # creates it -- and Modal will not mount a volume over a non-empty | ||
| 51 | + # directory, so the container that wanted the cache cannot start. As | ||
| 52 | + # env it covers every nix command that runs in the container, the ones | ||
| 53 | + # typed by hand in a shell included, and leaves the mount point empty. | ||
| 54 | + if subs := list(spec.get("nix", {}).get("substituters", [])): | ||
| 55 | + line = "extra-substituters = " + " ".join(subs) | ||
| 56 | + self.env["NIX_CONFIG"] = ( | ||
| 57 | + f"{self.env['NIX_CONFIG']}\n{line}" if "NIX_CONFIG" in self.env | ||
| 58 | + else line | ||
| 59 | + ) | ||
| 60 | + | ||
| 47 | # "function" (the default) or "sandbox". Sandboxes can run on a real | 61 | # "function" (the default) or "sandbox". Sandboxes can run on a real |
| 48 | # VM, which Functions cannot -- see ../README.md. | 62 | # VM, which Functions cannot -- see ../README.md. |
| 49 | self.runtime = spec.get("container", {}).get("runtime", "function") | 63 | self.runtime = spec.get("container", {}).get("runtime", "function") |
| @@ -162,13 +176,24 @@ class Container: | |||
| 162 | def _build_image(self) -> modal.Image: | 176 | def _build_image(self) -> modal.Image: |
| 163 | c = self.spec["container"] | 177 | c = self.spec["container"] |
| 164 | build = self.spec.get("build", {}) | 178 | build = self.spec.get("build", {}) |
| 165 | - nix_spec = self.spec.get("nix", {}) | ||
| 166 | 179 | ||
| 167 | if c.get("base"): | 180 | if c.get("base"): |
| 168 | image = modal.Image.from_name(c["base"]) | 181 | image = modal.Image.from_name(c["base"]) |
| 169 | else: | 182 | else: |
| 170 | image = modal.Image.from_registry(c["registry"]) | 183 | image = modal.Image.from_registry(c["registry"]) |
| 171 | 184 | ||
| 185 | + # `nix profile install` puts things in ~/.nix-profile/bin, which is on | ||
| 186 | + # nobody's PATH here -- so the install succeeds and the very next line | ||
| 187 | + # says `command not found`. Set on the image rather than in a .bashrc, | ||
| 188 | + # because a container's command runs under `sh -c` and reads neither. | ||
| 189 | + # Spelled out rather than prefixed onto $PATH: an image env is a value, | ||
| 190 | + # not a shell expression, so "$PATH" here would be four literal | ||
| 191 | + # characters. | ||
| 192 | + image = image.env({ | ||
| 193 | + "PATH": "/root/.nix-profile/bin:/usr/local/sbin:/usr/local/bin" | ||
| 194 | + ":/usr/sbin:/usr/bin:/sbin:/bin", | ||
| 195 | + }) | ||
| 196 | + | ||
| 172 | if self.use_shim: | 197 | if self.use_shim: |
| 173 | image = image.add_local_file( | 198 | image = image.add_local_file( |
| 174 | PTYSHIM_C, "/opt/ptyshim.c", copy=True | 199 | PTYSHIM_C, "/opt/ptyshim.c", copy=True |
| @@ -216,16 +241,6 @@ class Container: | |||
| 216 | else: | 241 | else: |
| 217 | image = image.add_local_file(src, dest, copy=True) | 242 | image = image.add_local_file(src, dest, copy=True) |
| 218 | 243 | ||
| 219 | - # Substituters for every nix command in the container, typed by hand or | ||
| 220 | - # not. Without this the only things reading a mounted cache are the | ||
| 221 | - # scripts that pass --extra-substituters, so an interactive shell | ||
| 222 | - # rebuilds from source what the volume beside it already holds. | ||
| 223 | - if subs := list(nix_spec.get("substituters", [])): | ||
| 224 | - image = image.run_commands( | ||
| 225 | - f"echo 'extra-substituters = {' '.join(subs)}'" | ||
| 226 | - " >> /etc/nix/nix.conf" | ||
| 227 | - ) | ||
| 228 | - | ||
| 229 | # A repo copied in brings its `.git` along, and in a worktree that is a | 244 | # A repo copied in brings its `.git` along, and in a worktree that is a |
| 230 | # *file* holding `gitdir: <path on the machine that copied it>`. Nix | 245 | # *file* holding `gitdir: <path on the machine that copied it>`. Nix |
| 231 | # believes it and goes looking for a checkout that is not there -- | 246 | # believes it and goes looking for a checkout that is not there -- |
| @@ -234,7 +249,14 @@ class Container: | |||
| 234 | image = image.run_commands(f"rm -rf {self.workdir}/.git") | 249 | image = image.run_commands(f"rm -rf {self.workdir}/.git") |
| 235 | 250 | ||
| 236 | if commands := build.get("commands", []): | 251 | if commands := build.get("commands", []): |
| 237 | - image = image.run_commands(*commands) | 252 | + # Volumes mounted for the build too, not just the run. A build step |
| 253 | + # that wants nix to *build* something is the one thing gVisor will | ||
| 254 | + # not do -- image builds are Functions underneath, and a derivation | ||
| 255 | + # there dies on `unexpected EOF reading a line`. Substituting is | ||
| 256 | + # fine, so a step that can reach the cache never has to build, and | ||
| 257 | + # the shim stays retired. The mount is not part of the resulting | ||
| 258 | + # image; only what the step writes outside it is. | ||
| 259 | + image = image.run_commands(*commands, volumes=self.volumes) | ||
| 238 | 260 | ||
| 239 | # container.py does `from _loader import Container`, and Modal mounts | 261 | # container.py does `from _loader import Container`, and Modal mounts |
| 240 | # the entrypoint file alone -- so without this the import that works | 262 | # the entrypoint file alone -- so without this the import that works |
modified
.modal/flutter-dev/container.py +12 -13 | @@ -17,20 +17,19 @@ image, app = c.image, c.app | ||
| 17 | 17 | # No @app.function here: a Sandbox runs its command as its own process and |
| 18 | 18 | # nothing of this module is imported into it. Registering a Function would be |
| 19 | 19 | # dead weight, and its kwargs are where vm_runtime would be wrongly applied. |
| 20 | +# One entrypoint, not two: a second `@app.local_entrypoint` makes plain | |
| 21 | +# `modal run container.py` ambiguous, and Modal refuses it rather than | |
| 22 | +# picking. `--shell` is a flag on the one there is. | |
| 20 | 23 | @app.local_entrypoint() |
| 21 | -def main(command: str = ""): | |
| 22 | - c.run_sandbox(command) | |
| 23 | - | |
| 24 | - | |
| 25 | -@app.local_entrypoint() | |
| 26 | -def shell(): | |
| 27 | - """Leave a Sandbox running and say how to get into it. | |
| 28 | - | |
| 29 | - `modal shell --image` only takes registry references, so it cannot be | |
| 30 | - pointed at a published Modal image like arch-nix. Attaching to a running | |
| 31 | - Sandbox can, and that Sandbox is this container: same image, same volumes, | |
| 32 | - same resources. It outlives this process, so it also has to be killed. | |
| 33 | - """ | |
| 24 | +def main(command: str = "", shell: bool = False): | |
| 25 | + if not shell: | |
| 26 | + c.run_sandbox(command) | |
| 27 | + return | |
| 28 | + | |
| 29 | + # `modal shell --image` only takes registry references, so it cannot be | |
| 30 | + # pointed at a published Modal image like arch-nix. Attaching to a running | |
| 31 | + # Sandbox can, and that Sandbox is this container: same image, same | |
| 32 | + # volumes, same env. | |
| 34 | 33 | sb = c.open_sandbox() |
| 35 | 34 | print(f"sandbox {sb.object_id} up, with {', '.join(c.volumes) or 'no volumes'}") |
| 36 | 35 | print(f" attach: modal shell {sb.object_id} (from another terminal)") |
| @@ -17,20 +17,19 @@ image, app = c.image, c.app | |||
| 17 | # No @app.function here: a Sandbox runs its command as its own process and | 17 | # No @app.function here: a Sandbox runs its command as its own process and |
| 18 | # nothing of this module is imported into it. Registering a Function would be | 18 | # nothing of this module is imported into it. Registering a Function would be |
| 19 | # dead weight, and its kwargs are where vm_runtime would be wrongly applied. | 19 | # dead weight, and its kwargs are where vm_runtime would be wrongly applied. |
| 20 | +# One entrypoint, not two: a second `@app.local_entrypoint` makes plain | ||
| 21 | +# `modal run container.py` ambiguous, and Modal refuses it rather than | ||
| 22 | +# picking. `--shell` is a flag on the one there is. | ||
| 20 | @app.local_entrypoint() | 23 | @app.local_entrypoint() |
| 21 | -def main(command: str = ""): | 24 | +def main(command: str = "", shell: bool = False): |
| 22 | - c.run_sandbox(command) | 25 | + if not shell: |
| 23 | - | 26 | + c.run_sandbox(command) |
| 24 | - | 27 | + return |
| 25 | -@app.local_entrypoint() | 28 | + |
| 26 | -def shell(): | 29 | + # `modal shell --image` only takes registry references, so it cannot be |
| 27 | - """Leave a Sandbox running and say how to get into it. | 30 | + # pointed at a published Modal image like arch-nix. Attaching to a running |
| 28 | - | 31 | + # Sandbox can, and that Sandbox is this container: same image, same |
| 29 | - `modal shell --image` only takes registry references, so it cannot be | 32 | + # volumes, same env. |
| 30 | - pointed at a published Modal image like arch-nix. Attaching to a running | ||
| 31 | - Sandbox can, and that Sandbox is this container: same image, same volumes, | ||
| 32 | - same resources. It outlives this process, so it also has to be killed. | ||
| 33 | - """ | ||
| 34 | sb = c.open_sandbox() | 33 | sb = c.open_sandbox() |
| 35 | print(f"sandbox {sb.object_id} up, with {', '.join(c.volumes) or 'no volumes'}") | 34 | print(f"sandbox {sb.object_id} up, with {', '.join(c.volumes) or 'no volumes'}") |
| 36 | print(f" attach: modal shell {sb.object_id} (from another terminal)") | 35 | print(f" attach: modal shell {sb.object_id} (from another terminal)") |
modified
.modal/flutter-dev/container.toml +1 -1 | @@ -23,7 +23,7 @@ include = ["."] | ||
| 23 | 23 | # |
| 24 | 24 | # `dev` stays for the case where the baked env is stale against a flake edit. |
| 25 | 25 | commands = [ |
| 26 | - "nix print-dev-env /app#flutter-desktop --accept-flake-config > /etc/devshell.sh", | |
| 26 | + "nix print-dev-env /app#flutter-desktop --accept-flake-config --extra-substituters file:///nix-cache > /etc/devshell.sh", | |
| 27 | 27 | "echo '. /etc/devshell.sh' >> /root/.bashrc", |
| 28 | 28 | "printf '#!/bin/sh\\nexec nix develop /app#flutter-desktop \"$@\"\\n' > /usr/local/bin/dev && chmod +x /usr/local/bin/dev", |
| 29 | 29 | ] |
| @@ -23,7 +23,7 @@ include = ["."] | |||
| 23 | # | 23 | # |
| 24 | # `dev` stays for the case where the baked env is stale against a flake edit. | 24 | # `dev` stays for the case where the baked env is stale against a flake edit. |
| 25 | commands = [ | 25 | commands = [ |
| 26 | - "nix print-dev-env /app#flutter-desktop --accept-flake-config > /etc/devshell.sh", | 26 | + "nix print-dev-env /app#flutter-desktop --accept-flake-config --extra-substituters file:///nix-cache > /etc/devshell.sh", |
| 27 | "echo '. /etc/devshell.sh' >> /root/.bashrc", | 27 | "echo '. /etc/devshell.sh' >> /root/.bashrc", |
| 28 | "printf '#!/bin/sh\\nexec nix develop /app#flutter-desktop \"$@\"\\n' > /usr/local/bin/dev && chmod +x /usr/local/bin/dev", | 28 | "printf '#!/bin/sh\\nexec nix develop /app#flutter-desktop \"$@\"\\n' > /usr/local/bin/dev && chmod +x /usr/local/bin/dev", |
| 29 | ] | 29 | ] |
modified
justfile +34 -0 | @@ -419,3 +419,37 @@ flutter-desktop action="build": | ||
| 419 | 419 | ;; |
| 420 | 420 | *) echo "usage: just flutter-desktop [build|run]" >&2; exit 1 ;; |
| 421 | 421 | esac |
| 422 | + | |
| 423 | +# The containers in `.modal/`, run on Modal rather than here. This machine | |
| 424 | +# evaluates and Modal builds — see CLAUDE.md, which says so rather more | |
| 425 | +# firmly — and these two recipes are the whole interface to that. | |
| 426 | +# | |
| 427 | +# Named for where the work happens, the way `cosmic` and `flutter-desktop` | |
| 428 | +# are named for what paints: there is no re-entry test here because nothing | |
| 429 | +# re-enters, and no `nix` variable because nix runs out there. | |
| 430 | +# | |
| 431 | +# just modal frq build `.#appimage` on Modal | |
| 432 | +# just modal flutter-dev the incremental Flutter loop | |
| 433 | +modal container="frq" *args: | |
| 434 | + #!/usr/bin/env bash | |
| 435 | + set -euo pipefail | |
| 436 | + cd "{{justfile_directory()}}" | |
| 437 | + shift | |
| 438 | + exec modal run ".modal/{{container}}/container.py" "$@" | |
| 439 | + | |
| 440 | +# A sandbox left running with the container's own image, volumes and | |
| 441 | +# environment, and the command to get into it. `modal shell --image` cannot | |
| 442 | +# be pointed at a published Modal image like arch-nix, so attaching to a | |
| 443 | +# running sandbox is the only way to get a shell that is the container. | |
| 444 | +# | |
| 445 | +# It blocks: an ephemeral app stops when its entrypoint returns and takes the | |
| 446 | +# sandbox with it. Attach from a second terminal, and Ctrl-C here when done — | |
| 447 | +# the sandbox bills until you do. | |
| 448 | +# | |
| 449 | +# just modal-shell flutter-dev, the usual one | |
| 450 | +# just modal-shell frq the appimage container | |
| 451 | +modal-shell container="flutter-dev": | |
| 452 | + #!/usr/bin/env bash | |
| 453 | + set -euo pipefail | |
| 454 | + cd "{{justfile_directory()}}" | |
| 455 | + exec modal run ".modal/{{container}}/container.py" --shell | |
| @@ -419,3 +419,37 @@ flutter-desktop action="build": | |||
| 419 | ;; | 419 | ;; |
| 420 | *) echo "usage: just flutter-desktop [build|run]" >&2; exit 1 ;; | 420 | *) echo "usage: just flutter-desktop [build|run]" >&2; exit 1 ;; |
| 421 | esac | 421 | esac |
| 422 | + | ||
| 423 | +# The containers in `.modal/`, run on Modal rather than here. This machine | ||
| 424 | +# evaluates and Modal builds — see CLAUDE.md, which says so rather more | ||
| 425 | +# firmly — and these two recipes are the whole interface to that. | ||
| 426 | +# | ||
| 427 | +# Named for where the work happens, the way `cosmic` and `flutter-desktop` | ||
| 428 | +# are named for what paints: there is no re-entry test here because nothing | ||
| 429 | +# re-enters, and no `nix` variable because nix runs out there. | ||
| 430 | +# | ||
| 431 | +# just modal frq build `.#appimage` on Modal | ||
| 432 | +# just modal flutter-dev the incremental Flutter loop | ||
| 433 | +modal container="frq" *args: | ||
| 434 | + #!/usr/bin/env bash | ||
| 435 | + set -euo pipefail | ||
| 436 | + cd "{{justfile_directory()}}" | ||
| 437 | + shift | ||
| 438 | + exec modal run ".modal/{{container}}/container.py" "$@" | ||
| 439 | + | ||
| 440 | +# A sandbox left running with the container's own image, volumes and | ||
| 441 | +# environment, and the command to get into it. `modal shell --image` cannot | ||
| 442 | +# be pointed at a published Modal image like arch-nix, so attaching to a | ||
| 443 | +# running sandbox is the only way to get a shell that is the container. | ||
| 444 | +# | ||
| 445 | +# It blocks: an ephemeral app stops when its entrypoint returns and takes the | ||
| 446 | +# sandbox with it. Attach from a second terminal, and Ctrl-C here when done — | ||
| 447 | +# the sandbox bills until you do. | ||
| 448 | +# | ||
| 449 | +# just modal-shell flutter-dev, the usual one | ||
| 450 | +# just modal-shell frq the appimage container | ||
| 451 | +modal-shell container="flutter-dev": | ||
| 452 | + #!/usr/bin/env bash | ||
| 453 | + set -euo pipefail | ||
| 454 | + cd "{{justfile_directory()}}" | ||
| 455 | + exec modal run ".modal/{{container}}/container.py" --shell | ||