1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
name: bench
# THE RICKUB side of the CI race. Runs only on rickub's own CI: once
# .rickub/workflows/ exists, rickub reads it as the SOLE workflow source and
# ignores .github/workflows/ entirely (web/ci_dispatch.go walkWorkflows), so
# this file and the GitHub copy never double-fire on one platform. Keep the
# STEP ORDER identical to .github/workflows/bench.yml — that is the point of
# the benchmark.
#
# runs-on: large (4 vCPU / 8 GiB / 32 GiB scratch, Docker-in-Docker guest
# rootfs-docker-glibc.ext4) to roughly match github.com's ubuntu-latest
# (4 vCPU / 16 GiB). The vCPU count matches; RAM does not — documented in the
# README as a known asymmetry and a user decision (ubuntu-latest on rickub is
# 2 vCPU / 4 GiB, cheaper and arguably the fairer "as-consumed" comparison).
#
# Deliberately NO `concurrency:` block: queue depth is part of what we
# measure.
on:
push:
branches: [main]
workflow_dispatch:
permissions:
contents: read
defaults:
run:
shell: bash
env:
BENCH_PLATFORM: rickub
jobs:
bench:
runs-on: large
timeout-minutes: 45
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- name: mark job-start
run: |
rid="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$(git rev-parse --short HEAD)"
echo "BENCH_RUN_ID=$rid" >> "$GITHUB_ENV"
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
BENCH_RUN_ID="$rid" bench_mark job-start
# COLD build: every rickub job boots a fresh Firecracker microVM with a
# read-only rootfs and a fresh scratch disk, so this cargo run fetches
# every crate and compiles the full tree — exactly the GitHub twin's
# cold path. `rm -rf target` is belt and braces. No cache action on
# purpose.
- name: rust cold build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step rust-cold-build bash -ec \
'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'
# WARM build: same job, same VM, target/ populated — the incremental
# no-op rebuild path.
- name: rust warm build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml
- name: rust test
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml
# The large class runs Docker-in-Docker (dockerd in the guest), so this
# exercises the guest's dockerd + registry pull path. Still guarded: a
# runner without docker records "skipped" instead of failing the job.
- name: docker build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v docker >/dev/null 2>&1; then
bench_step docker-build bash -ec \
'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'
else
bench_skip docker-build "docker not available on runner"
fi
# --- real-world tier: notorious OSS at pinned tags ----------------------
# Each step clones/downloads its project INSIDE the timed window (network
# is part of a real build) and builds cold — every run is a fresh VM by
# construction. Toolchains are guarded: a runner without the tool records
# "skipped", never fails the race. ClickHouse itself is deliberately not
# here: a full build needs dozens of cores and ~100 GB, which no shared-
# runner tier provides — SQLite's amalgamation stands in for C/C++ (see
# README).
- name: rust real build (ripgrep 15.2.0)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v cargo >/dev/null 2>&1; then
bench_step rust-ripgrep bash -ec \
'rm -rf /tmp/real-ripgrep && git clone -q --depth 1 --branch 15.2.0 https://github.com/BurntSushi/ripgrep /tmp/real-ripgrep && cd /tmp/real-ripgrep && cargo build --release'
else
bench_skip rust-ripgrep "cargo not available on runner"
fi
- name: c/c++ real build (sqlite 3.45.1 amalgamation)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step cpp-sqlite bash -ec \
'rm -rf /tmp/real-sqlite && mkdir -p /tmp/real-sqlite && cd /tmp/real-sqlite && curl -fsSL https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz | tar xz --strip-components=1 && ./configure --disable-tcl && make -j"$(nproc)"'
- name: node real build (typescript v5.9.3)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v node >/dev/null 2>&1; then
bench_step node-typescript bash -ec \
'rm -rf /tmp/real-ts && git clone -q --depth 1 --branch v5.9.3 https://github.com/microsoft/TypeScript /tmp/real-ts && cd /tmp/real-ts && npm ci --no-audit --no-fund && npm run build'
else
bench_skip node-typescript "node not available on runner"
fi
- name: java real build (guava v33.7.1)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v mvn >/dev/null 2>&1; then
bench_step java-guava bash -ec \
'rm -rf /tmp/real-guava && git clone -q --depth 1 --branch v33.7.1 https://github.com/google/guava /tmp/real-guava && cd /tmp/real-guava && mvn -B -q -DskipTests -pl guava -am compile'
else
bench_skip java-guava "maven not available on runner"
fi
- name: probe runner
run: bash workloads/probe/probe.sh
- name: show results
run: cat "${GITHUB_WORKSPACE:-$PWD}/results.jsonl"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: bench-results
path: results.jsonl
|