1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
name: bench
# THE GITHUB side of the CI race. Runs on github.com only: rickub reads
# .rickub/workflows/ as its sole workflow source when that directory exists
# and ignores .github/workflows/ entirely (rickub web/ci_dispatch.go
# walkWorkflows), so the two files never double-fire. Keep the STEP ORDER
# identical to .rickub/workflows/bench.yml — that is the point of the
# benchmark.
#
# Deliberately NO `concurrency:` block: queue depth is part of what we
# measure, and cancel-in-progress would destroy rounds.
on:
push:
branches: [main]
workflow_dispatch:
permissions:
contents: read
defaults:
run:
shell: bash
env:
BENCH_PLATFORM: github
jobs:
bench:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
# Pin comment policy mirrors rickub's own ci.yml: SHA-pinned actions.
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- name: mark job-start
run: |
rid="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$(git rev-parse --short HEAD)"
echo "BENCH_RUN_ID=$rid" >> "$GITHUB_ENV"
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
BENCH_RUN_ID="$rid" bench_mark job-start
# COLD build: hosted runners start empty, so this run of cargo fetches
# every crate and compiles the full tree. `rm -rf target` is belt and
# braces (it should never exist here). No actions/cache on purpose.
- name: rust cold build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step rust-cold-build bash -ec \
'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'
# WARM build: same job, target/ populated — measures the incremental
# no-op rebuild path (link + freshness checks only).
- name: rust warm build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml
- name: rust test
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml
# docker-build: skipped (recorded, not failed) when docker is absent.
- name: docker build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v docker >/dev/null 2>&1; then
bench_step docker-build bash -ec \
'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'
else
bench_skip docker-build "docker not available on runner"
fi
# --- real-world tier: notorious OSS at pinned tags ----------------------
# Each step clones/downloads its project INSIDE the timed window (network
# is part of a real build) and builds cold — every run is a fresh VM by
# construction. Toolchains are guarded: a runner without the tool records
# "skipped", never fails the race. ClickHouse itself is deliberately not
# here: a full build needs dozens of cores and ~100 GB, which no shared-
# runner tier provides — SQLite's amalgamation stands in for C/C++ (see
# README).
- name: rust real build (ripgrep 15.2.0)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v cargo >/dev/null 2>&1; then
bench_step rust-ripgrep bash -ec \
'rm -rf /tmp/real-ripgrep && git clone -q --depth 1 --branch 15.2.0 https://github.com/BurntSushi/ripgrep /tmp/real-ripgrep && cd /tmp/real-ripgrep && cargo build --release'
else
bench_skip rust-ripgrep "cargo not available on runner"
fi
- name: c/c++ real build (sqlite 3.45.1 amalgamation)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step cpp-sqlite bash -ec \
'rm -rf /tmp/real-sqlite && mkdir -p /tmp/real-sqlite && cd /tmp/real-sqlite && curl -fsSL https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz | tar xz --strip-components=1 && ./configure --disable-tcl && make -j"$(nproc)"'
- name: node real build (typescript v5.9.3)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v node >/dev/null 2>&1; then
bench_step node-typescript bash -ec \
'rm -rf /tmp/real-ts && git clone -q --depth 1 --branch v5.9.3 https://github.com/microsoft/TypeScript /tmp/real-ts && cd /tmp/real-ts && npm ci --no-audit --no-fund && npm run build'
else
bench_skip node-typescript "node not available on runner"
fi
- name: java real build (guava v33.7.1)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v mvn >/dev/null 2>&1; then
bench_step java-guava bash -ec \
'rm -rf /tmp/real-guava && git clone -q --depth 1 --branch v33.7.1 https://github.com/google/guava /tmp/real-guava && cd /tmp/real-guava && mvn -B -q -DskipTests -pl guava -am compile'
else
bench_skip java-guava "maven not available on runner"
fi
# --- cache tier: each platform's NATIVE cache service, as users use it --
# actions/cache restore/save split actions @ v4.2.4 — deliberately the
# v4/node20 line (the action runs INSIDE the guest; v6 would measure node
# absence, not cache). Identical key and paths on both platforms; v4 is
# also the exact protocol generation rickub's cache service implements
# (verified against the actions/cache@v4 client, web/ci_cache_api.go).
# The cached build uses an ISOLATED CARGO_HOME + target dir so it measures
# the warm path from cache alone — the earlier steps' registry downloads
# cannot leak into it. First run on a lockfile change is a MISS (primes
# the cache); the interleaved rounds then measure warm hits. `uses:`
# steps cannot be wrapped by bench_step, so restore/save are timed via an
# env-carried start stamp + a direct _bench_emit, with the hit flag as a
# JSON boolean in the value field.
- name: stamp cache-restore start
run: |
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
echo "CACHE_RESTORE_T0=$(_bench_now_iso)" >> "$GITHUB_ENV"
- name: cache restore (cargo)
id: cache-restore
uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
with:
path: |
workloads/rust-build/.cargo-cached/registry
workloads/rust-build/target-cached
key: ci-bench-cached-rust-${{ hashFiles('workloads/rust-build/Cargo.lock') }}
- name: time cache restore
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if [ "${{ steps.cache-restore.outputs.cache-hit }}" = "true" ]; then hit=true; else hit=false; fi
_bench_emit cache-restore-rust ok "$CACHE_RESTORE_T0" "$(_bench_now_iso)" "$hit" "cache_hit"
- name: rust cached build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
cd workloads/rust-build
bench_step rust-cached-build env \
CARGO_HOME="$PWD/.cargo-cached" CARGO_TARGET_DIR="$PWD/target-cached" \
cargo build --manifest-path Cargo.toml
- name: stamp cache-save start
run: |
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
echo "CACHE_SAVE_T0=$(_bench_now_iso)" >> "$GITHUB_ENV"
- name: cache save (cargo)
uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
with:
path: |
workloads/rust-build/.cargo-cached/registry
workloads/rust-build/target-cached
key: ci-bench-cached-rust-${{ hashFiles('workloads/rust-build/Cargo.lock') }}
- name: time cache save
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
_bench_emit cache-save-rust ok "$CACHE_SAVE_T0" "$(_bench_now_iso)"
- name: probe runner
run: bash workloads/probe/probe.sh
- name: show results
run: cat "${GITHUB_WORKSPACE:-$PWD}/results.jsonl"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: bench-results
path: results.jsonl
|