rickub
Sign in Get started
rickub/ci-benchpublic⑂ Fork 0
⑂ main
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/rickub/ci-bench.git
git clone ssh://git@rickub.com/rickub/ci-bench.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

bench.yml · 209 lines · 9.1 KBYAML Blame HistoryRaw
ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions) 0baf736 Olivier Girardot yesterday1name: bench
2
3# THE RICKUB side of the CI race. Runs only on rickub's own CI: once
4# .rickub/workflows/ exists, rickub reads it as the SOLE workflow source and
5# ignores .github/workflows/ entirely (web/ci_dispatch.go walkWorkflows), so
6# this file and the GitHub copy never double-fire on one platform. Keep the
7# STEP ORDER identical to .github/workflows/bench.yml — that is the point of
8# the benchmark.
9#
10# runs-on: large (4 vCPU / 8 GiB / 32 GiB scratch, Docker-in-Docker guest
11# rootfs-docker-glibc.ext4) to roughly match github.com's ubuntu-latest
12# (4 vCPU / 16 GiB). The vCPU count matches; RAM does not — documented in the
13# README as a known asymmetry and a user decision (ubuntu-latest on rickub is
14# 2 vCPU / 4 GiB, cheaper and arguably the fairer "as-consumed" comparison).
15#
16# Deliberately NO `concurrency:` block: queue depth is part of what we
17# measure.
18
19on:
20 push:
21 branches: [main]
22 workflow_dispatch:
23
24permissions:
25 contents: read
26
27defaults:
28 run:
29 shell: bash
30
31env:
32 BENCH_PLATFORM: rickub
33
34jobs:
35 bench:
36 runs-on: large
bench: real-world tier — ripgrep 15.2.0, sqlite 3.45.1, typescript 5.9.3, guava 33.7.1 (pinned, guarded, both twins) a4751cd Olivier Girardot 5h ago37 timeout-minutes: 45
ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions) 0baf736 Olivier Girardot yesterday38 steps:
39 - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
40
41 - name: mark job-start
42 run: |
43 rid="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$(git rev-parse --short HEAD)"
44 echo "BENCH_RUN_ID=$rid" >> "$GITHUB_ENV"
45 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
46 BENCH_RUN_ID="$rid" bench_mark job-start
47
48 # COLD build: every rickub job boots a fresh Firecracker microVM with a
49 # read-only rootfs and a fresh scratch disk, so this cargo run fetches
50 # every crate and compiles the full tree — exactly the GitHub twin's
51 # cold path. `rm -rf target` is belt and braces. No cache action on
52 # purpose.
53 - name: rust cold build
54 run: |
55 set -euo pipefail
56 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
57 bench_step rust-cold-build bash -ec \
58 'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'
59
60 # WARM build: same job, same VM, target/ populated — the incremental
61 # no-op rebuild path.
62 - name: rust warm build
63 run: |
64 set -euo pipefail
65 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
66 bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml
67
68 - name: rust test
69 run: |
70 set -euo pipefail
71 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
72 bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml
73
74 # The large class runs Docker-in-Docker (dockerd in the guest), so this
75 # exercises the guest's dockerd + registry pull path. Still guarded: a
76 # runner without docker records "skipped" instead of failing the job.
77 - name: docker build
78 run: |
79 set -euo pipefail
80 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
81 if command -v docker >/dev/null 2>&1; then
82 bench_step docker-build bash -ec \
83 'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'
84 else
85 bench_skip docker-build "docker not available on runner"
86 fi
87
bench: real-world tier — ripgrep 15.2.0, sqlite 3.45.1, typescript 5.9.3, guava 33.7.1 (pinned, guarded, both twins) a4751cd Olivier Girardot 5h ago88 # --- real-world tier: notorious OSS at pinned tags ----------------------
89 # Each step clones/downloads its project INSIDE the timed window (network
90 # is part of a real build) and builds cold — every run is a fresh VM by
91 # construction. Toolchains are guarded: a runner without the tool records
92 # "skipped", never fails the race. ClickHouse itself is deliberately not
93 # here: a full build needs dozens of cores and ~100 GB, which no shared-
94 # runner tier provides — SQLite's amalgamation stands in for C/C++ (see
95 # README).
96
97 - name: rust real build (ripgrep 15.2.0)
98 run: |
99 set -euo pipefail
100 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
101 if command -v cargo >/dev/null 2>&1; then
102 bench_step rust-ripgrep bash -ec \
103 'rm -rf /tmp/real-ripgrep && git clone -q --depth 1 --branch 15.2.0 https://github.com/BurntSushi/ripgrep /tmp/real-ripgrep && cd /tmp/real-ripgrep && cargo build --release'
104 else
105 bench_skip rust-ripgrep "cargo not available on runner"
106 fi
107
108 - name: c/c++ real build (sqlite 3.45.1 amalgamation)
109 run: |
110 set -euo pipefail
111 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
112 bench_step cpp-sqlite bash -ec \
113 'rm -rf /tmp/real-sqlite && mkdir -p /tmp/real-sqlite && cd /tmp/real-sqlite && curl -fsSL https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz | tar xz --strip-components=1 && ./configure --disable-tcl && make -j"$(nproc)"'
114
115 - name: node real build (typescript v5.9.3)
116 run: |
117 set -euo pipefail
118 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
119 if command -v node >/dev/null 2>&1; then
120 bench_step node-typescript bash -ec \
121 'rm -rf /tmp/real-ts && git clone -q --depth 1 --branch v5.9.3 https://github.com/microsoft/TypeScript /tmp/real-ts && cd /tmp/real-ts && npm ci --no-audit --no-fund && npm run build'
122 else
123 bench_skip node-typescript "node not available on runner"
124 fi
125
126 - name: java real build (guava v33.7.1)
127 run: |
128 set -euo pipefail
129 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
130 if command -v mvn >/dev/null 2>&1; then
131 bench_step java-guava bash -ec \
bench: guava builds the guava module (-pl guava -am) — root compile trips on guava-gwt needing guava-tests:tests from the reactor 7683a17 Olivier Girardot 4h ago132 'rm -rf /tmp/real-guava && git clone -q --depth 1 --branch v33.7.1 https://github.com/google/guava /tmp/real-guava && cd /tmp/real-guava && mvn -B -q -DskipTests -pl guava -am compile'
bench: real-world tier — ripgrep 15.2.0, sqlite 3.45.1, typescript 5.9.3, guava 33.7.1 (pinned, guarded, both twins) a4751cd Olivier Girardot 5h ago133 else
134 bench_skip java-guava "maven not available on runner"
135 fi
136
bench: cache tier — actions/cache@v4.2.4 restore/save timed natively on both platforms, isolated CARGO_HOME warm arm 28dfdd8 Olivier Girardot 1h ago137 # --- cache tier: each platform's NATIVE cache service, as users use it --
138 # actions/cache restore/save split actions @ v4.2.4 — deliberately the
139 # v4/node20 line (the action runs INSIDE the guest; v6 would measure node
140 # absence, not cache). Identical key and paths on both platforms; v4 is
141 # also the exact protocol generation rickub's cache service implements
142 # (verified against the actions/cache@v4 client, web/ci_cache_api.go).
143 # The cached build uses an ISOLATED CARGO_HOME + target dir so it measures
144 # the warm path from cache alone — the earlier steps' registry downloads
145 # cannot leak into it. First run on a lockfile change is a MISS (primes
146 # the cache); the interleaved rounds then measure warm hits. `uses:`
147 # steps cannot be wrapped by bench_step, so restore/save are timed via an
148 # env-carried start stamp + a direct _bench_emit, with the hit flag as a
149 # JSON boolean in the value field.
150
151 - name: stamp cache-restore start
152 run: |
153 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
154 echo "CACHE_RESTORE_T0=$(_bench_now_iso)" >> "$GITHUB_ENV"
155
156 - name: cache restore (cargo)
157 id: cache-restore
158 uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
159 with:
160 path: |
161 workloads/rust-build/.cargo-cached/registry
162 workloads/rust-build/target-cached
163 key: ci-bench-cached-rust-${{ hashFiles('workloads/rust-build/Cargo.lock') }}
164
165 - name: time cache restore
166 run: |
167 set -euo pipefail
168 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
169 if [ "${{ steps.cache-restore.outputs.cache-hit }}" = "true" ]; then hit=true; else hit=false; fi
170 _bench_emit cache-restore-rust ok "$CACHE_RESTORE_T0" "$(_bench_now_iso)" "$hit" "cache_hit"
171
172 - name: rust cached build
173 run: |
174 set -euo pipefail
175 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
176 cd workloads/rust-build
177 bench_step rust-cached-build env \
178 CARGO_HOME="$PWD/.cargo-cached" CARGO_TARGET_DIR="$PWD/target-cached" \
179 cargo build --manifest-path Cargo.toml
180
181 - name: stamp cache-save start
182 run: |
183 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
184 echo "CACHE_SAVE_T0=$(_bench_now_iso)" >> "$GITHUB_ENV"
185
186 - name: cache save (cargo)
187 uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
188 with:
189 path: |
190 workloads/rust-build/.cargo-cached/registry
191 workloads/rust-build/target-cached
192 key: ci-bench-cached-rust-${{ hashFiles('workloads/rust-build/Cargo.lock') }}
193
194 - name: time cache save
195 run: |
196 set -euo pipefail
197 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
198 _bench_emit cache-save-rust ok "$CACHE_SAVE_T0" "$(_bench_now_iso)"
199
ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions) 0baf736 Olivier Girardot yesterday200 - name: probe runner
201 run: bash workloads/probe/probe.sh
202
203 - name: show results
204 run: cat "${GITHUB_WORKSPACE:-$PWD}/results.jsonl"
205
206 - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
207 with:
208 name: bench-results
209 path: results.jsonl