nandi/frq · toolchain.sh

nandi/frqpublic⑂ Fork 0

⑂ e505dedb6ac7169a867e5e9f7732e997d8a003f9

toolchain.sh · 160 lines · 7.0 KBBash    Blame HistoryRaw
Three tarballs where a devShell was   1#!/usr/bin/env bash
2# The toolchain the web build needs, fetched by hand.
4# Three tarballs — Flutter (which carries Dart), a JDK, and the Clojure CLI —
5# pinned by version and by sha256, unpacked into `.toolchain/`, and put on a
6# PATH. That is the whole of it. No nix, no image, no devShell: a checkout
7# plus this script is a machine that can build the web bundle, and the same
8# script is what the Modal container runs.
10# Why not DotSlash, which the rest of the repo uses for its native libraries:
11# DotSlash hands out an *immutable* cached artifact, and Flutter is not one.
12# `flutter build web` downloads its engine artifacts into `bin/cache/` inside
13# its own SDK directory the first time it runs, so the SDK has to be writable
14# — which is the same thing `just apk` learned when it had to copy the
15# store's Android SDK out to `flutter/.home` before Gradle would touch it.
16# A pinned URL and a checked hash give the reproducibility DotSlash is for;
17# the writability is what it cannot give.
19#   tools/toolchain.sh              fetch whatever is missing
20#   eval "$(tools/toolchain.sh env)"    ...and put it on this shell's PATH
21#   tools/toolchain.sh exec -- flutter --version
23# `FRQ_TOOLCHAIN` says where it all lives; the default is `.toolchain/` at
24# the top of the checkout, and the container points it at a volume so the
25# fetch happens once across runs rather than once across containers.
26set -euo pipefail
28root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
29TC="${FRQ_TOOLCHAIN:-$root/.toolchain}"
31# The pins. A version and its hash, and nothing derived at run time: a
32# toolchain that resolves "latest" is a toolchain that changes under you
33# between two builds of the same commit.
35# Flutter 3.47.0 is the version this tree was building with under nix, and
36# its Dart 3.13.0 is what `flutter/pubspec.yaml` asks for with `sdk: ^3.13.0`.
37# To move it: take `version`, `archive` and `sha256` from
38# https://storage.googleapis.com/flutter_infra_release/releases/releases_linux.json
39FLUTTER_VERSION="3.47.0"
40FLUTTER_URL="https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_${FLUTTER_VERSION}-stable.tar.xz"
41FLUTTER_SHA="26cd99d3d94b1367e6b50535a18aeef0282c10a535bbe3ec493534dcdab75296"
43# Temurin 17, because `clojure` is a JVM program and ClojureDart's compiler
44# runs there. Nothing else in this build wants a JVM.
45JDK_VERSION="17.0.20.1+1"
46JDK_URL="https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.20.1%2B1/OpenJDK17U-jdk_x64_linux_hotspot_17.0.20.1_1.tar.gz"
47JDK_SHA="3808d1d15e3ec6bd5b84057fb5d84c33d8a1536a258146bcea2e603fc726e08e"
49# The Clojure CLI, which is a pair of shell scripts and a jar. Upstream ships
50# an installer; `install_clojure` below is the four lines of it that matter.
51CLOJURE_VERSION="1.12.6.1673"
52CLOJURE_URL="https://github.com/clojure/brew-install/releases/download/${CLOJURE_VERSION}/clojure-tools-${CLOJURE_VERSION}.tar.gz"
53CLOJURE_SHA="fe9194858e75d5af13c2e2aff92d710674d5bc5105f2b42f90a7d94d82ec023c"
55# What the host still has to bring. Small, boring, and on every machine and
56# in every base image that is not deliberately empty — but Flutter shells out
57# to `git` on its own SDK and to `unzip` on its downloads, so a missing one
58# fails somewhere far from here with a much worse message than this.
59require_host_tools() {
60    local missing=()
61    for t in curl tar git unzip; do
62        command -v "$t" >/dev/null 2>&1 || missing+=("$t")
64    if [ ${#missing[@]} -gt 0 ]; then
65        echo "toolchain: this needs ${missing[*]} on PATH and cannot fetch them" >&2
66        exit 1
70# One archive, unpacked once. The stamp holds the hash rather than the
71# version, so re-pointing a pin at the same version with different bytes also
72# refetches, and a half-finished unpack is never mistaken for a finished one:
73# the work happens in `.tmp` and the `mv` at the end is what publishes it.
74install_archive() {
75    local name=$1 url=$2 sha=$3 strip=$4
76    local dest="$TC/$name" stamp="$TC/$name.sha256"
77    if [ -d "$dest" ] && [ "$(cat "$stamp" 2>/dev/null || true)" = "$sha" ]; then
78        return 0
80    echo "toolchain: fetching $name" >&2
81    local dl="$TC/.download.$name"
82    rm -rf "$dest" "$dest.tmp" "$dl"
83    mkdir -p "$dest.tmp"
84    curl -fsSL --retry 3 -o "$dl" "$url"
85    echo "$sha  $dl" | sha256sum -c - >/dev/null
86    tar -xf "$dl" -C "$dest.tmp" --strip-components="$strip"
87    rm -f "$dl"
88    mv "$dest.tmp" "$dest"
89    echo "$sha" > "$stamp"
92# Upstream's install.sh, minus the ruby. The scripts ship with `PREFIX` and
93# `BINDIR` written into them literally and an installer that substitutes the
94# directory it is installing to; this is that, done where the tarball landed.
95install_clojure() {
96    local dest="$TC/clojure"
97    [ -x "$dest/bin/clojure" ] && return 0
98    mkdir -p "$dest/libexec" "$dest/bin"
99    cp "$dest"/*.jar "$dest/libexec/"
100    sed "s|PREFIX|$dest|g" "$dest/clojure" > "$dest/bin/clojure"
101    sed "s|BINDIR|$dest/bin|g" "$dest/clj" > "$dest/bin/clj"
102    chmod +x "$dest/bin/clojure" "$dest/bin/clj"
105install_all() {
106    require_host_tools
107    mkdir -p "$TC"
108    install_archive flutter "$FLUTTER_URL" "$FLUTTER_SHA" 1
109    install_archive jdk "$JDK_URL" "$JDK_SHA" 1
110    install_archive clojure "$CLOJURE_URL" "$CLOJURE_SHA" 1
111    install_clojure
114# The environment, as shell. Everything that would otherwise land in a home
115# directory is named here and kept inside the toolchain instead: the pub
116# cache, the git dependencies tools.deps clones, the local maven repo. One
117# directory to keep on a volume, one directory to delete when it goes wrong.
119# GITLIBS and the maven repo are set for the reason `just apk` sets them —
120# the JVM reads user.home out of /etc/passwd, so neither of them follows HOME.
121print_env() {
122    cat <<ENV
123export FRQ_TOOLCHAIN="$TC"
124# Flutter's SDK tarball is a git checkout, and the tool shells out to git
125# against it for its version -- which fails with "detected dubious ownership"
126# whenever the files' owner is not the user running the build. That is the
127# normal case on a Modal volume, and the failure is not a warning: the dart
128# process ClojureDart's live analyzer talks to dies with it, and the compile
129# ends at 'EOF while reading' with nothing about git in the message.
131# Said through the environment rather than 'git config --global', so it
132# travels with this shell and writes nothing into anyone's ~/.gitconfig.
133export GIT_CONFIG_COUNT=1
134export GIT_CONFIG_KEY_0=safe.directory
135export GIT_CONFIG_VALUE_0="$TC/flutter"
136export JAVA_HOME="$TC/jdk"
137export PUB_CACHE="$TC/pub-cache"
138export GITLIBS="$TC/gitlibs"
139export FRQ_M2="$TC/m2"
140export PATH="$TC/flutter/bin:$TC/jdk/bin:$TC/clojure/bin:\$PATH"
144case "${1:-install}" in
145    install) install_all ;;
146    env)     install_all; print_env ;;
147
148        install_all
149        shift
150        [ "${1:-}" = "--" ] && shift
151        eval "$(print_env)"
152        exec "$@"
154    versions)
155        echo "flutter $FLUTTER_VERSION"
156        echo "jdk     $JDK_VERSION"
157        echo "clojure $CLOJURE_VERSION"
159    *) echo "usage: toolchain.sh [install|env|exec -- cmd...|versions]" >&2; exit 1 ;;

Three tarballs where a devShell was 5ce66d5 nandi 21h ago	1	#!/usr/bin/env bash
	2	# The toolchain the web build needs, fetched by hand.
	3	#
	4	# Three tarballs — Flutter (which carries Dart), a JDK, and the Clojure CLI —
	5	# pinned by version and by sha256, unpacked into `.toolchain/`, and put on a
	6	# PATH. That is the whole of it. No nix, no image, no devShell: a checkout
	7	# plus this script is a machine that can build the web bundle, and the same
	8	# script is what the Modal container runs.
	9	#
	10	# Why not DotSlash, which the rest of the repo uses for its native libraries:
	11	# DotSlash hands out an immutable cached artifact, and Flutter is not one.
	12	# `flutter build web` downloads its engine artifacts into `bin/cache/` inside
	13	# its own SDK directory the first time it runs, so the SDK has to be writable
	14	# — which is the same thing `just apk` learned when it had to copy the
	15	# store's Android SDK out to `flutter/.home` before Gradle would touch it.
	16	# A pinned URL and a checked hash give the reproducibility DotSlash is for;
	17	# the writability is what it cannot give.
	18	#
	19	# tools/toolchain.sh fetch whatever is missing
	20	# eval "$(tools/toolchain.sh env)" ...and put it on this shell's PATH
	21	# tools/toolchain.sh exec -- flutter --version
	22	#
	23	# `FRQ_TOOLCHAIN` says where it all lives; the default is `.toolchain/` at
	24	# the top of the checkout, and the container points it at a volume so the
	25	# fetch happens once across runs rather than once across containers.
	26	set -euo pipefail
	27
	28	root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
	29	TC="${FRQ_TOOLCHAIN:-$root/.toolchain}"
	30
	31	# The pins. A version and its hash, and nothing derived at run time: a
	32	# toolchain that resolves "latest" is a toolchain that changes under you
	33	# between two builds of the same commit.
	34	#
	35	# Flutter 3.47.0 is the version this tree was building with under nix, and
	36	# its Dart 3.13.0 is what `flutter/pubspec.yaml` asks for with `sdk: ^3.13.0`.
	37	# To move it: take `version`, `archive` and `sha256` from
	38	# https://storage.googleapis.com/flutter_infra_release/releases/releases_linux.json
	39	FLUTTER_VERSION="3.47.0"
	40	FLUTTER_URL="https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_${FLUTTER_VERSION}-stable.tar.xz"
	41	FLUTTER_SHA="26cd99d3d94b1367e6b50535a18aeef0282c10a535bbe3ec493534dcdab75296"
	42
	43	# Temurin 17, because `clojure` is a JVM program and ClojureDart's compiler
	44	# runs there. Nothing else in this build wants a JVM.
	45	JDK_VERSION="17.0.20.1+1"
	46	JDK_URL="https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.20.1%2B1/OpenJDK17U-jdk_x64_linux_hotspot_17.0.20.1_1.tar.gz"
	47	JDK_SHA="3808d1d15e3ec6bd5b84057fb5d84c33d8a1536a258146bcea2e603fc726e08e"
	48
	49	# The Clojure CLI, which is a pair of shell scripts and a jar. Upstream ships
	50	# an installer; `install_clojure` below is the four lines of it that matter.
	51	CLOJURE_VERSION="1.12.6.1673"
	52	CLOJURE_URL="https://github.com/clojure/brew-install/releases/download/${CLOJURE_VERSION}/clojure-tools-${CLOJURE_VERSION}.tar.gz"
	53	CLOJURE_SHA="fe9194858e75d5af13c2e2aff92d710674d5bc5105f2b42f90a7d94d82ec023c"
	54
	55	# What the host still has to bring. Small, boring, and on every machine and
	56	# in every base image that is not deliberately empty — but Flutter shells out
	57	# to `git` on its own SDK and to `unzip` on its downloads, so a missing one
	58	# fails somewhere far from here with a much worse message than this.
	59	require_host_tools() {
	60	local missing=()
	61	for t in curl tar git unzip; do
	62	command -v "$t" >/dev/null 2>&1 \|\| missing+=("$t")
	63	done
	64	if [ ${#missing[@]} -gt 0 ]; then
65	echo "toolchain: this needs ${missing[*]} on PATH and cannot fetch them" >&2
66	exit 1
67	fi
68	}
69
70	# One archive, unpacked once. The stamp holds the hash rather than the
71	# version, so re-pointing a pin at the same version with different bytes also
72	# refetches, and a half-finished unpack is never mistaken for a finished one:
73	# the work happens in `.tmp` and the `mv` at the end is what publishes it.
74	install_archive() {
75	local name=$1 url=$2 sha=$3 strip=$4
76	local dest="$TC/$name" stamp="$TC/$name.sha256"
77	if [ -d "$dest" ] && [ "$(cat "$stamp" 2>/dev/null \|\| true)" = "$sha" ]; then
78	return 0
79	fi
80	echo "toolchain: fetching $name" >&2
81	local dl="$TC/.download.$name"
82	rm -rf "$dest" "$dest.tmp" "$dl"
83	mkdir -p "$dest.tmp"
84	curl -fsSL --retry 3 -o "$dl" "$url"
85	echo "$sha $dl" \| sha256sum -c - >/dev/null
86	tar -xf "$dl" -C "$dest.tmp" --strip-components="$strip"
87	rm -f "$dl"
88	mv "$dest.tmp" "$dest"
89	echo "$sha" > "$stamp"
90	}
91
92	# Upstream's install.sh, minus the ruby. The scripts ship with `PREFIX` and
93	# `BINDIR` written into them literally and an installer that substitutes the
94	# directory it is installing to; this is that, done where the tarball landed.
95	install_clojure() {
96	local dest="$TC/clojure"
97	[ -x "$dest/bin/clojure" ] && return 0
98	mkdir -p "$dest/libexec" "$dest/bin"
99	cp "$dest"/*.jar "$dest/libexec/"
100	sed "s\|PREFIX\|$dest\|g" "$dest/clojure" > "$dest/bin/clojure"
101	sed "s\|BINDIR\|$dest/bin\|g" "$dest/clj" > "$dest/bin/clj"
102	chmod +x "$dest/bin/clojure" "$dest/bin/clj"
103	}
104
105	install_all() {
106	require_host_tools
107	mkdir -p "$TC"
108	install_archive flutter "$FLUTTER_URL" "$FLUTTER_SHA" 1
109	install_archive jdk "$JDK_URL" "$JDK_SHA" 1
110	install_archive clojure "$CLOJURE_URL" "$CLOJURE_SHA" 1
111	install_clojure
112	}
113
114	# The environment, as shell. Everything that would otherwise land in a home
115	# directory is named here and kept inside the toolchain instead: the pub
116	# cache, the git dependencies tools.deps clones, the local maven repo. One
117	# directory to keep on a volume, one directory to delete when it goes wrong.
118	#
119	# GITLIBS and the maven repo are set for the reason `just apk` sets them —
120	# the JVM reads user.home out of /etc/passwd, so neither of them follows HOME.
121	print_env() {
122	cat <<ENV
123	export FRQ_TOOLCHAIN="$TC"
124	# Flutter's SDK tarball is a git checkout, and the tool shells out to git
125	# against it for its version -- which fails with "detected dubious ownership"
126	# whenever the files' owner is not the user running the build. That is the
127	# normal case on a Modal volume, and the failure is not a warning: the dart
128	# process ClojureDart's live analyzer talks to dies with it, and the compile
129	# ends at 'EOF while reading' with nothing about git in the message.
130	#
131	# Said through the environment rather than 'git config --global', so it
132	# travels with this shell and writes nothing into anyone's ~/.gitconfig.
133	export GIT_CONFIG_COUNT=1
134	export GIT_CONFIG_KEY_0=safe.directory
135	export GIT_CONFIG_VALUE_0="$TC/flutter"
136	export JAVA_HOME="$TC/jdk"
137	export PUB_CACHE="$TC/pub-cache"
138	export GITLIBS="$TC/gitlibs"
139	export FRQ_M2="$TC/m2"
140	export PATH="$TC/flutter/bin:$TC/jdk/bin:$TC/clojure/bin:\$PATH"
141	ENV
142	}
143
144	case "${1:-install}" in
145	install) install_all ;;
146	env) install_all; print_env ;;
147	exec)
148	install_all
149	shift
150	[ "${1:-}" = "--" ] && shift
151	eval "$(print_env)"
152	exec "$@"
153	;;
154	versions)
155	echo "flutter $FLUTTER_VERSION"
156	echo "jdk $JDK_VERSION"
157	echo "clojure $CLOJURE_VERSION"
158	;;
159	*) echo "usage: toolchain.sh [install\|env\|exec -- cmd...\|versions]" >&2; exit 1 ;;
160	esac