rickub
Sign in Get started
rickub/ci-benchpublic⑂ Fork 0
⑂ acae4f95ec28e6b109e55165077373ea0587870e
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/rickub/ci-bench.git
git clone ssh://git@rickub.com/rickub/ci-bench.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

bench.yml · 146 lines · 6.2 KBYAML Blame HistoryRaw
ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions) 0baf736 Olivier Girardot yesterday1name: bench
2
3# THE RICKUB side of the CI race. Runs only on rickub's own CI: once
4# .rickub/workflows/ exists, rickub reads it as the SOLE workflow source and
5# ignores .github/workflows/ entirely (web/ci_dispatch.go walkWorkflows), so
6# this file and the GitHub copy never double-fire on one platform. Keep the
7# STEP ORDER identical to .github/workflows/bench.yml — that is the point of
8# the benchmark.
9#
10# runs-on: large (4 vCPU / 8 GiB / 32 GiB scratch, Docker-in-Docker guest
11# rootfs-docker-glibc.ext4) to roughly match github.com's ubuntu-latest
12# (4 vCPU / 16 GiB). The vCPU count matches; RAM does not — documented in the
13# README as a known asymmetry and a user decision (ubuntu-latest on rickub is
14# 2 vCPU / 4 GiB, cheaper and arguably the fairer "as-consumed" comparison).
15#
16# Deliberately NO `concurrency:` block: queue depth is part of what we
17# measure.
18
19on:
20 push:
21 branches: [main]
22 workflow_dispatch:
23
24permissions:
25 contents: read
26
27defaults:
28 run:
29 shell: bash
30
31env:
32 BENCH_PLATFORM: rickub
33
34jobs:
35 bench:
36 runs-on: large
bench: real-world tier — ripgrep 15.2.0, sqlite 3.45.1, typescript 5.9.3, guava 33.7.1 (pinned, guarded, both twins) a4751cd Olivier Girardot 9h ago37 timeout-minutes: 45
ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions) 0baf736 Olivier Girardot yesterday38 steps:
39 - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
40
41 - name: mark job-start
42 run: |
43 rid="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$(git rev-parse --short HEAD)"
44 echo "BENCH_RUN_ID=$rid" >> "$GITHUB_ENV"
45 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
46 BENCH_RUN_ID="$rid" bench_mark job-start
47
48 # COLD build: every rickub job boots a fresh Firecracker microVM with a
49 # read-only rootfs and a fresh scratch disk, so this cargo run fetches
50 # every crate and compiles the full tree — exactly the GitHub twin's
51 # cold path. `rm -rf target` is belt and braces. No cache action on
52 # purpose.
53 - name: rust cold build
54 run: |
55 set -euo pipefail
56 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
57 bench_step rust-cold-build bash -ec \
58 'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'
59
60 # WARM build: same job, same VM, target/ populated — the incremental
61 # no-op rebuild path.
62 - name: rust warm build
63 run: |
64 set -euo pipefail
65 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
66 bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml
67
68 - name: rust test
69 run: |
70 set -euo pipefail
71 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
72 bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml
73
74 # The large class runs Docker-in-Docker (dockerd in the guest), so this
75 # exercises the guest's dockerd + registry pull path. Still guarded: a
76 # runner without docker records "skipped" instead of failing the job.
77 - name: docker build
78 run: |
79 set -euo pipefail
80 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
81 if command -v docker >/dev/null 2>&1; then
82 bench_step docker-build bash -ec \
83 'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'
84 else
85 bench_skip docker-build "docker not available on runner"
86 fi
87
bench: real-world tier — ripgrep 15.2.0, sqlite 3.45.1, typescript 5.9.3, guava 33.7.1 (pinned, guarded, both twins) a4751cd Olivier Girardot 9h ago88 # --- real-world tier: notorious OSS at pinned tags ----------------------
89 # Each step clones/downloads its project INSIDE the timed window (network
90 # is part of a real build) and builds cold — every run is a fresh VM by
91 # construction. Toolchains are guarded: a runner without the tool records
92 # "skipped", never fails the race. ClickHouse itself is deliberately not
93 # here: a full build needs dozens of cores and ~100 GB, which no shared-
94 # runner tier provides — SQLite's amalgamation stands in for C/C++ (see
95 # README).
96
97 - name: rust real build (ripgrep 15.2.0)
98 run: |
99 set -euo pipefail
100 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
101 if command -v cargo >/dev/null 2>&1; then
102 bench_step rust-ripgrep bash -ec \
103 'rm -rf /tmp/real-ripgrep && git clone -q --depth 1 --branch 15.2.0 https://github.com/BurntSushi/ripgrep /tmp/real-ripgrep && cd /tmp/real-ripgrep && cargo build --release'
104 else
105 bench_skip rust-ripgrep "cargo not available on runner"
106 fi
107
108 - name: c/c++ real build (sqlite 3.45.1 amalgamation)
109 run: |
110 set -euo pipefail
111 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
112 bench_step cpp-sqlite bash -ec \
113 'rm -rf /tmp/real-sqlite && mkdir -p /tmp/real-sqlite && cd /tmp/real-sqlite && curl -fsSL https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz | tar xz --strip-components=1 && ./configure --disable-tcl && make -j"$(nproc)"'
114
115 - name: node real build (typescript v5.9.3)
116 run: |
117 set -euo pipefail
118 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
119 if command -v node >/dev/null 2>&1; then
120 bench_step node-typescript bash -ec \
121 'rm -rf /tmp/real-ts && git clone -q --depth 1 --branch v5.9.3 https://github.com/microsoft/TypeScript /tmp/real-ts && cd /tmp/real-ts && npm ci --no-audit --no-fund && npm run build'
122 else
123 bench_skip node-typescript "node not available on runner"
124 fi
125
126 - name: java real build (guava v33.7.1)
127 run: |
128 set -euo pipefail
129 source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
130 if command -v mvn >/dev/null 2>&1; then
131 bench_step java-guava bash -ec \
bench: guava builds the guava module (-pl guava -am) — root compile trips on guava-gwt needing guava-tests:tests from the reactor 7683a17 Olivier Girardot 8h ago132 'rm -rf /tmp/real-guava && git clone -q --depth 1 --branch v33.7.1 https://github.com/google/guava /tmp/real-guava && cd /tmp/real-guava && mvn -B -q -DskipTests -pl guava -am compile'
bench: real-world tier — ripgrep 15.2.0, sqlite 3.45.1, typescript 5.9.3, guava 33.7.1 (pinned, guarded, both twins) a4751cd Olivier Girardot 9h ago133 else
134 bench_skip java-guava "maven not available on runner"
135 fi
136
ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions) 0baf736 Olivier Girardot yesterday137 - name: probe runner
138 run: bash workloads/probe/probe.sh
139
140 - name: show results
141 run: cat "${GITHUB_WORKSPACE:-$PWD}/results.jsonl"
142
143 - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
144 with:
145 name: bench-results
146 path: results.jsonl