| Build the APK with buck2, from pins rather than from a second checkout a71ef8f nandi 19d ago | 1 | # The APK, as a graph rather than a script. |
| 2 | # |
| 3 | # Every step the old build-apk.sh ran in sequence is a target here, so a change |
| 4 | # to one screen rebuilds the boot image and repackages, and touches neither the |
| 5 | # Rust nor the Java. The two native halves come from jolt-native — libvidya out |
| 6 | # of its buck2 graph, the glue compiled by the NDK its scripts/ pin — and the |
| 7 | # rest is this repo's. |
| 8 | # |
| 9 | # The machine-specific paths are read from .buckconfig.local, which the `buck` |
| 10 | # recipe in the justfile writes. Nothing here is found by looking around the |
| 11 | # machine; if a path is missing the recipe says which. |
| 12 | load(":defs.bzl", "libvidya") |
| 13 | |
| 14 | _JOLT_NATIVE = read_root_config("frq", "jolt_native", "") |
| 15 | _ANDROID_HOME = read_root_config("frq", "android_home", "") |
| 16 | _CHEZ = read_root_config("frq", "chez_android", "") |
| 17 | _OPENSSL = read_root_config("frq", "openssl_android", "") |
| 18 | |
| 19 | _API = "28" |
| 20 | _TOOLS = _ANDROID_HOME + "/build-tools/36.0.0" |
| 21 | _ANDROID_JAR = _ANDROID_HOME + "/platforms/android-36/android.jar" |
| 22 | # `$(...)` in a genrule cmd is a buck macro, not the shell's substitution, so |
| 23 | # everything this file runs at command time uses backticks. |
| 24 | # This repo's own, not the other checkout's: an APK build needs a clang and an |
| 25 | # llvm-objcopy whether or not jolt-native is on the machine. Both resolve the |
| 26 | # NDK through scripts/android-ndk.dotslash, downloading it once. |
| 27 | _NDK_BIN = "ndk=`$(location //scripts:android-ndk-bin)`" |
| 28 | _CC = "$(location //scripts:android-cc)" |
| 29 | |
| 30 | # An absolute path for $OUT, so a command may cd without losing it. buck has |
| 31 | # already made the parent directory. Spelled out step by step because backticks |
| 32 | # do not nest. |
| 33 | # Empty when a checkout answers for it, so the script's own lookup wins. |
| 34 | # The archive's top directory is what strip_prefix took off, so `src` is |
| 35 | # directly inside it. Absolute, because the script is handed this and does its |
| 36 | # own directory changing. |
| 37 | _GLIMMER_VIDYA = "" if read_root_config("frq", "libvidya", "pinned") == "checkout" else \ |
| 38 | "GLIMMER_VIDYA=`realpath $(location toolchains//dist:glimmer-vidya)/src` " |
| 39 | |
| 40 | # jolt_main.c and the ABI's headers. A sibling checkout wins here too, so that |
| 41 | # editing the glue and the library together works; otherwise the release. |
| 42 | _GLUE = _JOLT_NATIVE if read_root_config("frq", "libvidya", "pinned") == "checkout" else None |
| 43 | _GLUE_C = (_GLUE + "/android/jolt_main.c") if _GLUE else "`realpath $(location toolchains//dist:android-glue)/android/jolt_main.c`" |
| 44 | _GLUE_INCLUDE = (_GLUE + "/crates/jolt-vidya/include") if _GLUE else "`realpath $(location toolchains//dist:android-glue)/include`" |
| 45 | |
| 46 | _ABS_OUT = 'd=`dirname "$OUT"` && b=`basename "$OUT"` && d=`cd "$d" && pwd` && out="$d/$b"' |
| 47 | |
| 48 | # Somewhere to assemble in. Removed by the command that made it, on success; |
| 49 | # buck's own scratch space is not this. |
| 50 | _TMP = 'tmp=`mktemp -d`' |
| 51 | |
| 52 | export_file( |
| 53 | name = "build-jolt-boot.sh", |
| 54 | mode = "reference", |
| 55 | ) |
| 56 | |
| 57 | # --- the UI half ------------------------------------------------------------ |
| 58 | # Two ways in, and both make the library's bytes an input rather than a |
| 59 | # command buck would cache forever: |
| 60 | # |
| 61 | # pinned the .so out of jolt-native's release, fetched by digest. What a |
| 62 | # build gets by default, and what makes an APK buildable with no |
| 63 | # jolt-native checkout and no NDK anywhere on the machine. |
| 64 | # checkout the .so a sibling jolt-native just built, staged into this cell |
| 65 | # by the `buck` recipe. Anyone working on both repos at once |
| 66 | # builds what they are editing. |
| 67 | # |
| 68 | # The recipe decides which, by whether that checkout exists, and says so here. |
| 69 | libvidya(name = "libvidya") |
| 70 | |
| 71 | # --- the Jolt half ---------------------------------------------------------- |
| 72 | # The boot image: frq's Scheme, cross-compiled to arm64 by Chez. The sources |
| 73 | # are an input so that editing one rebuilds this; the compile itself reads them |
| 74 | # through the deps.edn the script writes. |
| 75 | # The sources here are frq's own. The other two roots the image is compiled |
| 76 | # from — glimmer, out of the jolt cache, and glimmer-vidya, out of jolt-native |
| 77 | # — are outside this cell, and so are Chez's cross boot files and the pinned |
| 78 | # jolt itself. None of them can be an input, so the `buck` recipe hashes them |
| 79 | # and writes the digest below; naming it in the command is what puts them in |
| 80 | # this action's identity. |
| 81 | # jolt reaches this action as a manifest and a fetcher rather than as a |
| 82 | # program: both are inputs, so nothing here depends on what is installed where |
| 83 | # the compile runs, and a remote worker resolves the same pin — by the same |
| 84 | # digest — from the same place this machine would. |
| 85 | genrule( |
| 86 | name = "jolt-boot", |
| 87 | out = "boot", |
| 88 | srcs = ["//:jolt-sources"], |
| 89 | cmd = "# sources outside this cell: " + read_root_config("frq", "boot_stamp", "unknown") + "\n" + |
| 90 | "DOTSLASH=$(location toolchains//dist:dotslash)/dotslash " + |
| 91 | "JOLT_MANIFEST=$(location //scripts:jolt) " + |
| 92 | # The one source root the script cannot find for itself when there |
| 93 | # is no jolt-native checkout. An input, so a new release recompiles |
| 94 | # the image. |
| 95 | _GLIMMER_VIDYA + |
| 96 | "$(location :build-jolt-boot.sh) \"$OUT\" >&2", |
| 97 | ) |
| 98 | |
| 99 | # The image travels as a blob in an object file's data section; the |
| 100 | # _binary_jolt_boot_{start,end} symbols jolt_main.c reads are named after the |
| 101 | # input *path*, which is why this copies the file somewhere it can be called |
| 102 | # exactly `jolt.boot` before converting it. |
| 103 | genrule( |
| 104 | name = "jolt-boot-obj", |
| 105 | out = "jolt_boot.o", |
| 106 | cmd = _ABS_OUT + " && " + |
| 107 | _TMP + " && " + _NDK_BIN + " && " + |
| 108 | "cp $(location :jolt-boot)/jolt.boot \"$tmp/jolt.boot\" && " + |
| 109 | "( cd \"$tmp\" && \"$ndk\"/llvm-objcopy " + |
| 110 | "--input-target=binary --output-target=elf64-littleaarch64 " + |
| 111 | "--binary-architecture=aarch64 jolt.boot jolt_boot.o ) && " + |
| 112 | "cp \"$tmp/jolt_boot.o\" \"$out\" && rm -rf \"$tmp\"", |
| 113 | ) |
| 114 | |
| 115 | # The glue: jolt-native's android/jolt_main.c over the boot image, linked |
| 116 | # against libvidya by name. --no-undefined is what makes a symbol the Scheme |
| 117 | # side registers but the ABI no longer exports a build failure here rather than |
| 118 | # a crash on the phone. |
| 119 | genrule( |
| 120 | name = "libjoltapp", |
| 121 | out = "libjoltapp.so", |
| 122 | cmd = _ABS_OUT + " && lib=`mktemp -d` && " + |
| 123 | "cp $(location :libvidya) \"$lib/libvidya.so\" && " + |
| 124 | _CC + " -shared -fPIC -O2 -o \"$out\" " + |
| 125 | _GLUE_C + " " + |
| 126 | "$(location :jolt-boot-obj) " + |
| 127 | "-I$(location :jolt-boot) " + |
| 128 | "-I" + _GLUE_INCLUDE + " " + |
| 129 | "-L\"$lib\" " + |
| 130 | _CHEZ + "/tarm64le/boot/tarm64le/libkernel.a " + |
| 131 | _CHEZ + "/lz4/lib/liblz4.a " + |
| 132 | "-lvidya -landroid -llog -lz -ldl -lm -Wl,--no-undefined && " + |
| 133 | "rm -rf \"$lib\"", |
| 134 | ) |
| 135 | |
| 136 | # --- the Java half ---------------------------------------------------------- |
| 137 | # One class: the photo chooser's result has to land somewhere, and native code |
| 138 | # is not somewhere. android.jar on the class path is where every android.* type |
| 139 | # comes from; the JDK's own java.* is what is left. |
| 140 | genrule( |
| 141 | name = "classes-dex", |
| 142 | out = "classes.dex", |
| 143 | srcs = glob(["java/**/*.java"]), |
| 144 | cmd = _ABS_OUT + " && " + _TMP + " && " + |
| 145 | "javac --release 17 --class-path " + _ANDROID_JAR + " -d \"$tmp/classes\" $SRCS && " + |
| 146 | "classes=`find \"$tmp/classes\" -name '*.class'` && " + |
| 147 | _TOOLS + "/d8 --min-api " + _API + " --output \"$tmp\" $classes && " + |
| 148 | "cp \"$tmp/classes.dex\" \"$out\" && rm -rf \"$tmp\"", |
| 149 | ) |
| 150 | |
| 151 | # --- the package ------------------------------------------------------------ |
| 152 | # The libraries are stored rather than deflated: the loader maps them straight |
| 153 | # out of the APK. The dex is read rather than mapped, so it may as well |
| 154 | # compress. OpenSSL travels with the app because the platform's own is not ours |
| 155 | # to load — an app's linker namespace refuses /system/lib64/libssl.so, and |
| 156 | # without one there is no TLS on the phone at all. |
| 157 | genrule( |
| 158 | name = "apk-unsigned", |
| 159 | out = "frq-unsigned.apk", |
| 160 | srcs = ["AndroidManifest.xml"], |
| 161 | cmd = _ABS_OUT + " && stage=`mktemp -d` && " + |
| 162 | "mkdir -p \"$stage/lib/arm64-v8a\" && " + |
| 163 | "cp $(location :libvidya) \"$stage/lib/arm64-v8a/libvidya.so\" && " + |
| 164 | "cp $(location :libjoltapp) \"$stage/lib/arm64-v8a/libjoltapp.so\" && " + |
| 165 | "cp " + _OPENSSL + "/libssl.so " + _OPENSSL + "/libcrypto.so \"$stage/lib/arm64-v8a/\" && " + |
| 166 | "cp $(location :classes-dex) \"$stage/classes.dex\" && " + |
| 167 | _TOOLS + "/aapt2 link -o \"$out\" -I " + _ANDROID_JAR + " " + |
| 168 | "--manifest $SRCS --min-sdk-version " + _API + " --target-sdk-version 36 " + |
| 169 | "--version-code 1 --version-name 0.1.0 >&2 && " + |
| 170 | "( cd \"$stage\" && zip -q -0 \"$out\" lib/arm64-v8a/libvidya.so " + |
| 171 | "lib/arm64-v8a/libjoltapp.so lib/arm64-v8a/libssl.so lib/arm64-v8a/libcrypto.so && " + |
| 172 | "zip -q \"$out\" classes.dex ) && rm -rf \"$stage\"", |
| 173 | ) |
| 174 | |
| 175 | # Aligned and signed with the debug key, which is generated on first use the |
| 176 | # way the SDK's own tools generate it. |
| 177 | genrule( |
| 178 | name = "apk", |
| 179 | out = "frq.apk", |
| 180 | cmd = _ABS_OUT + " && " + _TMP + " && " + |
| 181 | "ks=\"$HOME/.android/debug.keystore\" && " + |
| 182 | "if [ ! -f \"$ks\" ]; then mkdir -p \"$HOME/.android\" && " + |
| 183 | "keytool -genkeypair -v -keystore \"$ks\" -storepass android -keypass android " + |
| 184 | "-alias androiddebugkey -keyalg RSA -keysize 2048 -validity 10000 " + |
| 185 | "-dname 'CN=Android Debug,O=Android,C=US' >&2; fi && " + |
| 186 | _TOOLS + "/zipalign -f -p 4 $(location :apk-unsigned) \"$tmp/aligned.apk\" && " + |
| 187 | _TOOLS + "/apksigner sign --ks \"$ks\" --ks-key-alias androiddebugkey " + |
| 188 | "--ks-pass pass:android --key-pass pass:android --out \"$out\" \"$tmp/aligned.apk\" && " + |
| 189 | _TOOLS + "/apksigner verify \"$out\" >/dev/null && rm -rf \"$tmp\"", |
| 190 | visibility = ["PUBLIC"], |
| 191 | ) |
| 192 | |
| 193 | # Does the pin resolve where the action runs, and does what it resolves to run |
| 194 | # there? Nothing depends on this; it is here to be asked by hand. |
| 195 | # |
| 196 | # buck2 build //android:jolt-fetch-check --remote-only --materializations=all \ |
| 197 | # -c build.execution_platforms=root//platforms:remote -c "parser...->root//platforms:remote" |
| 198 | # |
| 199 | # It is what found the glibc floor: jolt needs 2.38, and the rbe-ubuntu22-04 |
| 200 | # image this platform used to name ships 2.35, so the fetch succeeded and the |
| 201 | # binary would not start. The answer was the newer image, not a different pin. |
| 202 | genrule( |
| 203 | name = "jolt-fetch-check", |
| 204 | out = "report", |
| 205 | cmd = "jolt=`$(location toolchains//dist:dotslash)/dotslash -- fetch $(location //scripts:jolt)`; " + |
| 206 | "{ echo \"fetched: $jolt\"; " + |
| 207 | "cat /etc/os-release 2>/dev/null | head -1 || true; " + |
| 208 | "ldd --version 2>&1 | head -1 || true; " + |
| 209 | "echo '--- run ---'; \"$jolt\" --version; echo \"rc=$?\"; } > \"$OUT\" 2>&1 || true", |
| 210 | ) |