1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
name: bench
# THE GITHUB side of the CI race. Runs on github.com only: rickub reads
# .rickub/workflows/ as its sole workflow source when that directory exists
# and ignores .github/workflows/ entirely (rickub web/ci_dispatch.go
# walkWorkflows), so the two files never double-fire. Keep the STEP ORDER
# identical to .rickub/workflows/bench.yml — that is the point of the
# benchmark.
#
# Deliberately NO `concurrency:` block: queue depth is part of what we
# measure, and cancel-in-progress would destroy rounds.
on:
push:
branches: [main]
workflow_dispatch:
permissions:
contents: read
defaults:
run:
shell: bash
env:
BENCH_PLATFORM: github
jobs:
bench:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
# Pin comment policy mirrors rickub's own ci.yml: SHA-pinned actions.
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- name: mark job-start
run: |
rid="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$(git rev-parse --short HEAD)"
echo "BENCH_RUN_ID=$rid" >> "$GITHUB_ENV"
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
BENCH_RUN_ID="$rid" bench_mark job-start
# COLD build: hosted runners start empty, so this run of cargo fetches
# every crate and compiles the full tree. `rm -rf target` is belt and
# braces (it should never exist here). No actions/cache on purpose.
- name: rust cold build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step rust-cold-build bash -ec \
'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'
# WARM build: same job, target/ populated — measures the incremental
# no-op rebuild path (link + freshness checks only).
- name: rust warm build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml
- name: rust test
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml
# docker-build: skipped (recorded, not failed) when docker is absent.
- name: docker build
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v docker >/dev/null 2>&1; then
bench_step docker-build bash -ec \
'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'
else
bench_skip docker-build "docker not available on runner"
fi
# --- real-world tier: notorious OSS at pinned tags ----------------------
# Each step clones/downloads its project INSIDE the timed window (network
# is part of a real build) and builds cold — every run is a fresh VM by
# construction. Toolchains are guarded: a runner without the tool records
# "skipped", never fails the race. ClickHouse itself is deliberately not
# here: a full build needs dozens of cores and ~100 GB, which no shared-
# runner tier provides — SQLite's amalgamation stands in for C/C++ (see
# README).
- name: rust real build (ripgrep 15.2.0)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v cargo >/dev/null 2>&1; then
bench_step rust-ripgrep bash -ec \
'rm -rf /tmp/real-ripgrep && git clone -q --depth 1 --branch 15.2.0 https://github.com/BurntSushi/ripgrep /tmp/real-ripgrep && cd /tmp/real-ripgrep && cargo build --release'
else
bench_skip rust-ripgrep "cargo not available on runner"
fi
- name: c/c++ real build (sqlite 3.45.1 amalgamation)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
bench_step cpp-sqlite bash -ec \
'rm -rf /tmp/real-sqlite && mkdir -p /tmp/real-sqlite && cd /tmp/real-sqlite && curl -fsSL https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz | tar xz --strip-components=1 && ./configure --disable-tcl && make -j"$(nproc)"'
- name: node real build (typescript v5.9.3)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v node >/dev/null 2>&1; then
bench_step node-typescript bash -ec \
'rm -rf /tmp/real-ts && git clone -q --depth 1 --branch v5.9.3 https://github.com/microsoft/TypeScript /tmp/real-ts && cd /tmp/real-ts && npm ci --no-audit --no-fund && npm run build'
else
bench_skip node-typescript "node not available on runner"
fi
- name: java real build (guava v33.7.1)
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
if command -v mvn >/dev/null 2>&1; then
bench_step java-guava bash -ec \
'rm -rf /tmp/real-guava && git clone -q --depth 1 --branch v33.7.1 https://github.com/google/guava /tmp/real-guava && cd /tmp/real-guava && mvn -B -q -DskipTests compile'
else
bench_skip java-guava "maven not available on runner"
fi
- name: probe runner
run: bash workloads/probe/probe.sh
- name: show results
run: cat "${GITHUB_WORKSPACE:-$PWD}/results.jsonl"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: bench-results
path: results.jsonl
|