1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

# One job, because the flake is the build.
#
# This used to be a kaniko pipeline whose only purpose was to publish the
# container image buck2's remote executor pulled; nothing about it ran the
# build. `nix flake check` runs fmt, clippy, the test suite and all three
# shared objects, and it pins what it runs with.
#
# The publish step is in the same job for the same reason: a second job would
# get a cold nix store and rebuild everything it was about to upload. The
# branch check is a shell `if` rather than a second job's `rules:` because
# there is nothing to gate but the last two commands.
stages: [check]

check:
  stage: check
  image: nixos/nix:latest
  # A cold run has to vendor ~800 crates and compile the workspace; the default
  # hour is not enough for that even with the builder doing the compiling.
  timeout: 3h
  variables:
    # The flake is fetched from the checkout, so the runner needs the git tree
    # rather than a shallow single commit.
    GIT_DEPTH: "0"
  before_script:
    # Compilation is offloaded to nixbuild.net; the runner evaluates, fetches
    # and uploads. max-jobs must not be zero: at zero the runner refuses to
    # build anything at all, which sends the fixed-output fetches (the rust
    # toolchain tarballs, every crate source) to nixbuild, where the build
    # sandbox has no network and they can only fail. Giving it nproc keeps
    # those fetches and crane's hundreds of trivial cargo-src/cargo-package
    # derivations local and parallel -- each one costs a serial ~1.5s ssh
    # round trip if dispatched -- while the expensive builds still go remote.
    #
    # nixbuild doubles as a substituter, which is what makes a warm pipeline
    # cheap: it already holds every vendor path from previous runs, and fetching
    # one measures ~0.55s against ~1.5s to dispatch it as a build. Its paths are
    # signed, but the key is per-account and only readable from the web UI, so
    # rather than carry it as a second secret we lean on the transport -- the
    # store is reached over SSH authenticated with our own key. cache.nixos.org
    # is verified by its own key either way; require-sigs only relaxes nixbuild.
    # The explicit priority matters: nix picks a substituter by priority rather
    # than by the order listed here, and an ssh-ng store defaults to 0 against
    # cache.nixos.org's 40, so without it nixbuild wins even for stock nixpkgs
    # paths and they crawl through one SSH connection instead of the CDN.
    #
    # Keep the builder's job count modest. nixbuild's sshd caps concurrent
    # sessions and refuses the excess, which nix surfaces as the thoroughly
    # misleading "Nix daemon disconnected unexpectedly (maybe it crashed?)".
    - |
      set -eu
      mkdir -p ~/.ssh
      # base64 because GitLab can only mask a single-line variable.
      printf '%s' "$NIXBUILD_SSH_KEY" | base64 -d > ~/.ssh/nixbuild
      chmod 600 ~/.ssh/nixbuild
      printf 'Host eu.nixbuild.net\n PubkeyAcceptedKeyTypes ssh-ed25519\n IdentityFile ~/.ssh/nixbuild\n IdentitiesOnly yes\n' > ~/.ssh/config
      printf 'eu.nixbuild.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPIQCZc54poJ8vqawd8TraNryQeJnvH1eLpIDgbiqymM\n' > ~/.ssh/known_hosts
      printf '%s\n' \
        'experimental-features = nix-command flakes' \
        'builders = ssh-ng://eu.nixbuild.net x86_64-linux - 16 1 big-parallel,benchmark' \
        'builders-use-substitutes = true' \
        'substituters = https://cache.nixos.org ssh-ng://eu.nixbuild.net?priority=50' \
        'require-sigs = false' \
        "max-jobs = $(nproc)" >> /etc/nix/nix.conf
  script:
    - nix flake check -L
    # The store paths are read-only symlink farms; the runner uploads plain
    # files, so dereference them into a tree it can zip.
    - nix build -L --no-link --print-out-paths .#libs .#android > /tmp/outs
    - mkdir -p artifacts && xargs -a /tmp/outs -I{} cp -rL {}/. artifacts/
    # Staged beside artifacts/ and not into it: libsPortable holds the same
    # four sonames with a different RUNPATH, so merging the two trees would
    # leave whichever was copied last under both names.
    - nix build -L --no-link --print-out-paths .#libsPortable > /tmp/portable
    - mkdir -p portable && xargs -a /tmp/portable -I{} cp -rL {}/. portable/
    # Tarballs rooted at lib/ and include/, so a consumer's `flake = false`
    # input resolves to ${input}/lib/libjoltmoq.so with nothing in between.
    # Three targets, and the split is by what resolves where:
    #
    #   x86_64-linux            the store objects. A RUNPATH into the
    #                           builder's /nix/store, so they are usable from
    #                           nix and from nowhere else.
    #   x86_64-linux-portable   the same objects with their NEEDED closure
    #                           beside them and RUNPATH $ORIGIN. This is the
    #                           one for a consumer without nix -- frq's
    #                           AppImage-less desktop build takes it.
    #   android-arm64-v8a       links nothing but the NDK sysroot, and is what
    #                           an APK packages.
    #
    # libjoltcosmic is named in the desktop tars now. It is the window every
    # desktop consumer paints with and it was the one object never published,
    # because `.#libs` -- what this stages -- did not carry it.
    - |
      set -eu
      if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
        tar czf x86_64-linux.tar.gz -C artifacts include \
          lib/libvidya.so lib/libjolttui.so lib/libjoltmoq.so lib/libjoltcosmic.so
        # `-C portable lib include` and not a file list: the whole point of
        # this one is the libraries that came along, and naming the four we
        # know about would drop them.
        tar czf x86_64-linux-portable.tar.gz -C portable lib include
        tar czf android-arm64-v8a.tar.gz -C artifacts include lib/arm64-v8a
        # Twice: once under the commit, which is what a consumer pins and what
        # never moves, and once under "latest", which is the URL a consumer's
        # `flake = false` input names so that `nix flake update` can find a new
        # build without anyone editing a digest. The lock file still pins the
        # bytes -- "latest" decides what an update *finds*, not what it builds.
        #
        # Uploading the same version and filename again does not replace the
        # old file, it adds a second one and downloads serve the newest, so the
        # "latest" package grows by one set of tarballs per pipeline. A project
        # cleanup policy is what keeps that bounded.
        api="$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/jolt-native"
        nix shell nixpkgs#curl -c sh -eu -c '
          for v in "$2" latest; do
            for f in x86_64-linux.tar.gz x86_64-linux-portable.tar.gz android-arm64-v8a.tar.gz; do
              curl --fail-with-body --header "JOB-TOKEN: $CI_JOB_TOKEN" \
                --upload-file "$f" "$1/$v/$f"
            done
          done
        ' sh "$api" "$CI_COMMIT_SHA"
      fi
  artifacts:
    name: "jolt-native-$CI_COMMIT_SHORT_SHA"
    paths: [artifacts/, portable/]
    expire_in: 1 week
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH