rickub
Sign in Get started
rickub/ci-benchpublic⑂ Fork 0
⑂ 0baf736
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/rickub/ci-bench.git
git clone ssh://git@rickub.com/rickub/ci-bench.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions)Unverified

Identical pinned workload on both platforms, one workflow file per platform:
.github/workflows/bench.yml (GitHub) and .rickub/workflows/bench.yml
(rickub reads .rickub/workflows/ as its sole source when present), same steps
in the same order: rust cold build, rust warm build, rust test, docker build,
runner probe. Steps emit one JSON record each via scripts/emit_timing.sh;
collect.sh + compare.py produce median/p95 tables; Makefile wires
bootstrap/run-local/collect/compare. Everything degrades to recorded
'skipped'/'null' when docker or a probe tool is missing.

Rust workload pins toolchain 1.85.0 and an exact-pinned dep set
(serde/tokio/clap/regex/anyhow + serde_json) with a generated Cargo.lock.
Docker workload pins alpine:3.20 by digest. Methodology, clock boundaries,
interleaving and variance pitfalls documented in README.md.
Olivier Girardot committed 2026-09-11T22:54:32+02:00 Browse files
0baf736
added .github/workflows/bench.yml +88 -0
new file mode 100644
@@ -0,0 +1,88 @@
1+name: bench
2+
3+# THE GITHUB side of the CI race. Runs on github.com only: rickub reads
4+# .rickub/workflows/ as its sole workflow source when that directory exists
5+# and ignores .github/workflows/ entirely (rickub web/ci_dispatch.go
6+# walkWorkflows), so the two files never double-fire. Keep the STEP ORDER
7+# identical to .rickub/workflows/bench.yml — that is the point of the
8+# benchmark.
9+#
10+# Deliberately NO `concurrency:` block: queue depth is part of what we
11+# measure, and cancel-in-progress would destroy rounds.
12+
13+on:
14+ push:
15+ branches: [main]
16+ workflow_dispatch:
17+
18+permissions:
19+ contents: read
20+
21+defaults:
22+ run:
23+ shell: bash
24+
25+env:
26+ BENCH_PLATFORM: github
27+
28+jobs:
29+ bench:
30+ runs-on: ubuntu-latest
31+ timeout-minutes: 30
32+ steps:
33+ # Pin comment policy mirrors rickub's own ci.yml: SHA-pinned actions.
34+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
35+
36+ - name: mark job-start
37+ run: |
38+ rid="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$(git rev-parse --short HEAD)"
39+ echo "BENCH_RUN_ID=$rid" >> "$GITHUB_ENV"
40+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
41+ BENCH_RUN_ID="$rid" bench_mark job-start
42+
43+ # COLD build: hosted runners start empty, so this run of cargo fetches
44+ # every crate and compiles the full tree. `rm -rf target` is belt and
45+ # braces (it should never exist here). No actions/cache on purpose.
46+ - name: rust cold build
47+ run: |
48+ set -euo pipefail
49+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
50+ bench_step rust-cold-build bash -ec \
51+ 'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'
52+
53+ # WARM build: same job, target/ populated — measures the incremental
54+ # no-op rebuild path (link + freshness checks only).
55+ - name: rust warm build
56+ run: |
57+ set -euo pipefail
58+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
59+ bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml
60+
61+ - name: rust test
62+ run: |
63+ set -euo pipefail
64+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
65+ bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml
66+
67+ # docker-build: skipped (recorded, not failed) when docker is absent.
68+ - name: docker build
69+ run: |
70+ set -euo pipefail
71+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
72+ if command -v docker >/dev/null 2>&1; then
73+ bench_step docker-build bash -ec \
74+ 'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'
75+ else
76+ bench_skip docker-build "docker not available on runner"
77+ fi
78+
79+ - name: probe runner
80+ run: bash workloads/probe/probe.sh
81+
82+ - name: show results
83+ run: cat "${GITHUB_WORKSPACE:-$PWD}/results.jsonl"
84+
85+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
86+ with:
87+ name: bench-results
88+ path: results.jsonl
new file mode 100644
@@ -0,0 +1,88 @@
1+name: bench
2+
3+# THE GITHUB side of the CI race. Runs on github.com only: rickub reads
4+# .rickub/workflows/ as its sole workflow source when that directory exists
5+# and ignores .github/workflows/ entirely (rickub web/ci_dispatch.go
6+# walkWorkflows), so the two files never double-fire. Keep the STEP ORDER
7+# identical to .rickub/workflows/bench.yml — that is the point of the
8+# benchmark.
9+#
10+# Deliberately NO `concurrency:` block: queue depth is part of what we
11+# measure, and cancel-in-progress would destroy rounds.
12+
13+on:
14+ push:
15+ branches: [main]
16+ workflow_dispatch:
17+
18+permissions:
19+ contents: read
20+
21+defaults:
22+ run:
23+ shell: bash
24+
25+env:
26+ BENCH_PLATFORM: github
27+
28+jobs:
29+ bench:
30+ runs-on: ubuntu-latest
31+ timeout-minutes: 30
32+ steps:
33+ # Pin comment policy mirrors rickub's own ci.yml: SHA-pinned actions.
34+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
35+
36+ - name: mark job-start
37+ run: |
38+ rid="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$(git rev-parse --short HEAD)"
39+ echo "BENCH_RUN_ID=$rid" >> "$GITHUB_ENV"
40+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
41+ BENCH_RUN_ID="$rid" bench_mark job-start
42+
43+ # COLD build: hosted runners start empty, so this run of cargo fetches
44+ # every crate and compiles the full tree. `rm -rf target` is belt and
45+ # braces (it should never exist here). No actions/cache on purpose.
46+ - name: rust cold build
47+ run: |
48+ set -euo pipefail
49+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
50+ bench_step rust-cold-build bash -ec \
51+ 'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'
52+
53+ # WARM build: same job, target/ populated — measures the incremental
54+ # no-op rebuild path (link + freshness checks only).
55+ - name: rust warm build
56+ run: |
57+ set -euo pipefail
58+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
59+ bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml
60+
61+ - name: rust test
62+ run: |
63+ set -euo pipefail
64+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
65+ bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml
66+
67+ # docker-build: skipped (recorded, not failed) when docker is absent.
68+ - name: docker build
69+ run: |
70+ set -euo pipefail
71+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
72+ if command -v docker >/dev/null 2>&1; then
73+ bench_step docker-build bash -ec \
74+ 'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'
75+ else
76+ bench_skip docker-build "docker not available on runner"
77+ fi
78+
79+ - name: probe runner
80+ run: bash workloads/probe/probe.sh
81+
82+ - name: show results
83+ run: cat "${GITHUB_WORKSPACE:-$PWD}/results.jsonl"
84+
85+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
86+ with:
87+ name: bench-results
88+ path: results.jsonl
added .gitignore +14 -0
new file mode 100644
@@ -0,0 +1,14 @@
1+# rust build artifacts
2+workloads/rust-build/target/
3+
4+# local benchmark run output (collected results/ is COMMITTED-free but kept
5+# out of git too — it is machine-local analysis state)
6+/results.jsonl
7+results/
8+
9+# probe scratch output (written to TMPDIR, guarded just in case)
10+probe.bin
11+*.probe.bin
12+
13+# macOS noise
14+.DS_Store
new file mode 100644
@@ -0,0 +1,14 @@
1+# rust build artifacts
2+workloads/rust-build/target/
3+
4+# local benchmark run output (collected results/ is COMMITTED-free but kept
5+# out of git too — it is machine-local analysis state)
6+/results.jsonl
7+results/
8+
9+# probe scratch output (written to TMPDIR, guarded just in case)
10+probe.bin
11+*.probe.bin
12+
13+# macOS noise
14+.DS_Store
added .rickub/workflows/bench.yml +97 -0
new file mode 100644
@@ -0,0 +1,97 @@
1+name: bench
2+
3+# THE RICKUB side of the CI race. Runs only on rickub's own CI: once
4+# .rickub/workflows/ exists, rickub reads it as the SOLE workflow source and
5+# ignores .github/workflows/ entirely (web/ci_dispatch.go walkWorkflows), so
6+# this file and the GitHub copy never double-fire on one platform. Keep the
7+# STEP ORDER identical to .github/workflows/bench.yml — that is the point of
8+# the benchmark.
9+#
10+# runs-on: large (4 vCPU / 8 GiB / 32 GiB scratch, Docker-in-Docker guest
11+# rootfs-docker-glibc.ext4) to roughly match github.com's ubuntu-latest
12+# (4 vCPU / 16 GiB). The vCPU count matches; RAM does not — documented in the
13+# README as a known asymmetry and a user decision (ubuntu-latest on rickub is
14+# 2 vCPU / 4 GiB, cheaper and arguably the fairer "as-consumed" comparison).
15+#
16+# Deliberately NO `concurrency:` block: queue depth is part of what we
17+# measure.
18+
19+on:
20+ push:
21+ branches: [main]
22+ workflow_dispatch:
23+
24+permissions:
25+ contents: read
26+
27+defaults:
28+ run:
29+ shell: bash
30+
31+env:
32+ BENCH_PLATFORM: rickub
33+
34+jobs:
35+ bench:
36+ runs-on: large
37+ timeout-minutes: 30
38+ steps:
39+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
40+
41+ - name: mark job-start
42+ run: |
43+ rid="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$(git rev-parse --short HEAD)"
44+ echo "BENCH_RUN_ID=$rid" >> "$GITHUB_ENV"
45+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
46+ BENCH_RUN_ID="$rid" bench_mark job-start
47+
48+ # COLD build: every rickub job boots a fresh Firecracker microVM with a
49+ # read-only rootfs and a fresh scratch disk, so this cargo run fetches
50+ # every crate and compiles the full tree — exactly the GitHub twin's
51+ # cold path. `rm -rf target` is belt and braces. No cache action on
52+ # purpose.
53+ - name: rust cold build
54+ run: |
55+ set -euo pipefail
56+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
57+ bench_step rust-cold-build bash -ec \
58+ 'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'
59+
60+ # WARM build: same job, same VM, target/ populated — the incremental
61+ # no-op rebuild path.
62+ - name: rust warm build
63+ run: |
64+ set -euo pipefail
65+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
66+ bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml
67+
68+ - name: rust test
69+ run: |
70+ set -euo pipefail
71+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
72+ bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml
73+
74+ # The large class runs Docker-in-Docker (dockerd in the guest), so this
75+ # exercises the guest's dockerd + registry pull path. Still guarded: a
76+ # runner without docker records "skipped" instead of failing the job.
77+ - name: docker build
78+ run: |
79+ set -euo pipefail
80+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
81+ if command -v docker >/dev/null 2>&1; then
82+ bench_step docker-build bash -ec \
83+ 'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'
84+ else
85+ bench_skip docker-build "docker not available on runner"
86+ fi
87+
88+ - name: probe runner
89+ run: bash workloads/probe/probe.sh
90+
91+ - name: show results
92+ run: cat "${GITHUB_WORKSPACE:-$PWD}/results.jsonl"
93+
94+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
95+ with:
96+ name: bench-results
97+ path: results.jsonl
new file mode 100644
@@ -0,0 +1,97 @@
1+name: bench
2+
3+# THE RICKUB side of the CI race. Runs only on rickub's own CI: once
4+# .rickub/workflows/ exists, rickub reads it as the SOLE workflow source and
5+# ignores .github/workflows/ entirely (web/ci_dispatch.go walkWorkflows), so
6+# this file and the GitHub copy never double-fire on one platform. Keep the
7+# STEP ORDER identical to .github/workflows/bench.yml — that is the point of
8+# the benchmark.
9+#
10+# runs-on: large (4 vCPU / 8 GiB / 32 GiB scratch, Docker-in-Docker guest
11+# rootfs-docker-glibc.ext4) to roughly match github.com's ubuntu-latest
12+# (4 vCPU / 16 GiB). The vCPU count matches; RAM does not — documented in the
13+# README as a known asymmetry and a user decision (ubuntu-latest on rickub is
14+# 2 vCPU / 4 GiB, cheaper and arguably the fairer "as-consumed" comparison).
15+#
16+# Deliberately NO `concurrency:` block: queue depth is part of what we
17+# measure.
18+
19+on:
20+ push:
21+ branches: [main]
22+ workflow_dispatch:
23+
24+permissions:
25+ contents: read
26+
27+defaults:
28+ run:
29+ shell: bash
30+
31+env:
32+ BENCH_PLATFORM: rickub
33+
34+jobs:
35+ bench:
36+ runs-on: large
37+ timeout-minutes: 30
38+ steps:
39+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
40+
41+ - name: mark job-start
42+ run: |
43+ rid="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$(git rev-parse --short HEAD)"
44+ echo "BENCH_RUN_ID=$rid" >> "$GITHUB_ENV"
45+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
46+ BENCH_RUN_ID="$rid" bench_mark job-start
47+
48+ # COLD build: every rickub job boots a fresh Firecracker microVM with a
49+ # read-only rootfs and a fresh scratch disk, so this cargo run fetches
50+ # every crate and compiles the full tree — exactly the GitHub twin's
51+ # cold path. `rm -rf target` is belt and braces. No cache action on
52+ # purpose.
53+ - name: rust cold build
54+ run: |
55+ set -euo pipefail
56+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
57+ bench_step rust-cold-build bash -ec \
58+ 'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'
59+
60+ # WARM build: same job, same VM, target/ populated — the incremental
61+ # no-op rebuild path.
62+ - name: rust warm build
63+ run: |
64+ set -euo pipefail
65+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
66+ bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml
67+
68+ - name: rust test
69+ run: |
70+ set -euo pipefail
71+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
72+ bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml
73+
74+ # The large class runs Docker-in-Docker (dockerd in the guest), so this
75+ # exercises the guest's dockerd + registry pull path. Still guarded: a
76+ # runner without docker records "skipped" instead of failing the job.
77+ - name: docker build
78+ run: |
79+ set -euo pipefail
80+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
81+ if command -v docker >/dev/null 2>&1; then
82+ bench_step docker-build bash -ec \
83+ 'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'
84+ else
85+ bench_skip docker-build "docker not available on runner"
86+ fi
87+
88+ - name: probe runner
89+ run: bash workloads/probe/probe.sh
90+
91+ - name: show results
92+ run: cat "${GITHUB_WORKSPACE:-$PWD}/results.jsonl"
93+
94+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
95+ with:
96+ name: bench-results
97+ path: results.jsonl
added Makefile +56 -0
new file mode 100644
@@ -0,0 +1,56 @@
1+# ci-bench — cross-platform CI race benchmark (rickub CI vs GitHub Actions).
2+# bash + python3(stdlib) only; everything degrades gracefully.
3+
4+SHELL := /usr/bin/env bash
5+
6+PLATFORM ?= local
7+RUN_ID ?= local-$(shell date -u +%Y%m%dT%H%M%SZ)
8+
9+.PHONY: bootstrap run-local collect compare clean
10+
11+## bootstrap — make scripts executable and create results/ (safe to re-run)
12+bootstrap:
13+ chmod +x scripts/*.sh workloads/probe/probe.sh
14+ mkdir -p results
15+
16+## run-local — run the benchmark step sequence on THIS machine (smoke-test /
17+## local baseline). Emits ./results.jsonl, mirroring the CI step order.
18+run-local: bootstrap
19+ rm -f results.jsonl
20+ set -euo pipefail; \
21+ export BENCH_RUN_ID="$(RUN_ID)"; \
22+ source scripts/emit_timing.sh; \
23+ bench_mark job-start; \
24+ if command -v cargo >/dev/null 2>&1; then \
25+ bench_step rust-cold-build bash -ec \
26+ 'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'; \
27+ bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml; \
28+ bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml; \
29+ else \
30+ bench_skip rust-cold-build "cargo not found"; \
31+ bench_skip rust-warm-build "cargo not found"; \
32+ bench_skip rust-test "cargo not found"; \
33+ fi; \
34+ if command -v docker >/dev/null 2>&1; then \
35+ bench_step docker-build bash -ec \
36+ 'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'; \
37+ else \
38+ bench_skip docker-build "docker not found"; \
39+ fi; \
40+ bash workloads/probe/probe.sh; \
41+ cat results.jsonl
42+
43+## collect — fold a results.jsonl into the results tree:
44+## make collect PLATFORM=github RUN_ID=<id> SRC=/path/to/results.jsonl
45+collect:
46+ @test -n "$(SRC)" || { echo "usage: make collect PLATFORM=<p> RUN_ID=<id> SRC=<results.jsonl>"; exit 2; }
47+ scripts/collect.sh "$(PLATFORM)" "$(RUN_ID)" "$(SRC)"
48+
49+## compare — median/p95 table from everything collected so far
50+compare:
51+ python3 scripts/compare.py
52+
53+## clean — drop local run outputs (not the collected results/ tree)
54+clean:
55+ rm -f results.jsonl
56+ rm -rf workloads/rust-build/target
new file mode 100644
@@ -0,0 +1,56 @@
1+# ci-bench — cross-platform CI race benchmark (rickub CI vs GitHub Actions).
2+# bash + python3(stdlib) only; everything degrades gracefully.
3+
4+SHELL := /usr/bin/env bash
5+
6+PLATFORM ?= local
7+RUN_ID ?= local-$(shell date -u +%Y%m%dT%H%M%SZ)
8+
9+.PHONY: bootstrap run-local collect compare clean
10+
11+## bootstrap — make scripts executable and create results/ (safe to re-run)
12+bootstrap:
13+ chmod +x scripts/*.sh workloads/probe/probe.sh
14+ mkdir -p results
15+
16+## run-local — run the benchmark step sequence on THIS machine (smoke-test /
17+## local baseline). Emits ./results.jsonl, mirroring the CI step order.
18+run-local: bootstrap
19+ rm -f results.jsonl
20+ set -euo pipefail; \
21+ export BENCH_RUN_ID="$(RUN_ID)"; \
22+ source scripts/emit_timing.sh; \
23+ bench_mark job-start; \
24+ if command -v cargo >/dev/null 2>&1; then \
25+ bench_step rust-cold-build bash -ec \
26+ 'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'; \
27+ bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml; \
28+ bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml; \
29+ else \
30+ bench_skip rust-cold-build "cargo not found"; \
31+ bench_skip rust-warm-build "cargo not found"; \
32+ bench_skip rust-test "cargo not found"; \
33+ fi; \
34+ if command -v docker >/dev/null 2>&1; then \
35+ bench_step docker-build bash -ec \
36+ 'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'; \
37+ else \
38+ bench_skip docker-build "docker not found"; \
39+ fi; \
40+ bash workloads/probe/probe.sh; \
41+ cat results.jsonl
42+
43+## collect — fold a results.jsonl into the results tree:
44+## make collect PLATFORM=github RUN_ID=<id> SRC=/path/to/results.jsonl
45+collect:
46+ @test -n "$(SRC)" || { echo "usage: make collect PLATFORM=<p> RUN_ID=<id> SRC=<results.jsonl>"; exit 2; }
47+ scripts/collect.sh "$(PLATFORM)" "$(RUN_ID)" "$(SRC)"
48+
49+## compare — median/p95 table from everything collected so far
50+compare:
51+ python3 scripts/compare.py
52+
53+## clean — drop local run outputs (not the collected results/ tree)
54+clean:
55+ rm -f results.jsonl
56+ rm -rf workloads/rust-build/target
added README.md +199 -0
new file mode 100644
@@ -0,0 +1,199 @@
1+# ci-bench — the CI race
2+
3+An identical, pinned workload pushed to **GitHub Actions** and **rickub CI**
4+(Firecracker microVM fleet), raced to answer: who starts jobs faster, whose
5+steps run faster, and what the runner hardware itself is worth?
6+
7+Everything here is bash + python3 stdlib. No package installs, no external
8+services beyond the two CI platforms themselves.
9+
10+## Layout
11+
12+ .github/workflows/bench.yml GitHub side of the race
13+ .rickub/workflows/bench.yml rickub side (same steps, same order)
14+ workloads/rust-build/ pinned Cargo project (cold/warm/test)
15+ workloads/docker-build/ digest-pinned multi-stage Dockerfile
16+ workloads/probe/probe.sh runner capability probe (cpu/disk/fs)
17+ scripts/emit_timing.sh sourced by steps; one JSON line per step
18+ scripts/collect.sh results.jsonl -> results/<platform>/<run>.json
19+ scripts/compare.py median/p95 table per step per platform
20+ Makefile bootstrap / run-local / collect / compare
21+
22+Why two workflow files and not one: rickub reads `.rickub/workflows/` as its
23+SOLE workflow source when that directory exists and ignores
24+`.github/workflows/` entirely, while GitHub only ever reads `.github/`. One
25+repo therefore carries one file per platform and neither platform sees the
26+other's copy. The two files run the same steps in the same order; the only
27+differences are `BENCH_PLATFORM`, `runs-on:` (see *Known asymmetries*), and
28+comments.
29+
30+## The workload (identical on both platforms)
31+
32+| step | what it does |
33+|-----------------|--------------------------------------------------------------------------|
34+| `job-start` | zero-duration marker emitted by the first step after checkout |
35+| `rust-cold-build` | `rm -rf target && cargo build` — full crate download + compile |
36+| `rust-warm-build` | immediate second `cargo build` — incremental no-op rebuild |
37+| `rust-test` | `cargo test` (5 real tests) over the warm build |
38+| `docker-build` | multi-stage `docker build` (alpine:3.20 pinned by digest) + `docker run` |
39+| `probe-*` | cpu loop, 2 GiB sequential write/read to `$RUNNER_TEMP`, fs type + free |
40+
41+Pins: Rust toolchain 1.85.0 (`workloads/rust-build/rust-toolchain.toml`), the
42+full dependency tree (`workloads/rust-build/Cargo.lock`, generated offline
43+against a real cargo registry cache, versions exact-pinned in `Cargo.toml`
44+with `=`), Docker base `alpine:3.20@sha256:d9e853e87e...` (the digest rickub
45+itself pins for its CI images). The workload is not bit-frozen against
46+`apk`/crates.io mirrors moving — see *Variance pitfalls*.
47+
48+### Cold vs warm — definitions
49+
50+Both platforms give every job a **fresh machine** (GitHub hosted runners;
51+rickub boots a new Firecracker microVM with a read-only rootfs and a fresh
52+scratch disk). Therefore:
53+
54+- **cold** = the FIRST `cargo build` in a job: crates.io fetch + full compile
55+ of the transitive tree. Never accelerated by any cache (there is no
56+ `actions/cache` and no cargo cache persistence — on purpose).
57+- **warm** = the SECOND `cargo build` in the SAME job: `target/` exists, the
58+ no-op incremental path (metadata checks + link of nothing).
59+- Docker build is always cold (no image cache survives the machine).
60+- A "cold docker" vs "warm docker" axis would need an in-job second build of
61+ the same Dockerfile; not in scope for v1.
62+
63+## Metrics — exact clock boundaries
64+
65+All records are JSON lines in `results.jsonl`:
66+
67+ {"step": "...", "platform": "...", "run_id": "...",
68+ "start": "2026-09-11T12:34:56.789Z", "end": "2026-09-11T12:35:40.123Z",
69+ "status": "ok|fail|skipped", "duration_ms": 43334,
70+ "value": <optional measurement>, "unit": "...", "reason": "..."}
71+
72+`start`/`end` are guest wall clock, UTC, millisecond precision (GNU `date
73+%3N`; python3 fallback on BSD; whole-second last resort — the fallback in use
74+is visible in the timestamps themselves: `.000Z` means coarse).
75+
76+1. **push→job-start** = `start` of the `job-start` record − **push timestamp
77+ from the platform API** (not from the guest).
78+ - GitHub: `gh api repos/:o/:r/actions/runs/<run_id>` → `created_at`
79+ (when the push event was accepted server-side). Alternative, fully
80+ server-side cross-check: `run_started_at − created_at` (queue +
81+ provisioning, no guest clock involved).
82+ - rickub: the run's created/queued timestamp from its CI API for the same
83+ commit.
84+ - CAVEAT: this metric subtracts a **server-side** timestamp from a
85+ **guest-side** timestamp. Guest clock skew shifts both platforms' numbers
86+ unpredictably (NTP inside a just-booted microVM can be off by seconds).
87+ Treat it as indicative; prefer the server-side-only variant above when
88+ the two disagree, and record both when possible.
89+2. **per-step duration** = `end − start` of the step's record, from the same
90+ guest clock, so skew cancels. Millisecond precision.
91+3. **end-to-end pipeline time** (two views):
92+ - job execution = last record's `end` − `job-start` record's `start`;
93+ - run wall clock = last record's `end` − push timestamp (carries the same
94+ cross-clock caveat as metric 1).
95+4. **cold vs warm cache** = `rust-cold-build` vs `rust-warm-build` durations
96+ as defined above.
97+5. **disk-write throughput** = `probe-write` value (MB/s, 2 GiB `dd
98+ bs=1M count=2048` to the job scratch dir, then removed); `probe-read` is
99+ the same file read back (warm page cache — still comparable across
100+ platforms); `probe-cpu` is shell-loop kops/s; `probe-fs-type`/`probe-fs-free`
101+ describe the filesystem the runner gave the job.
102+
103+Anything unavailable degrades to `"value": null` or a `skipped` record; the
104+job itself never fails from the probe, and absent docker skips
105+`docker-build` with a recorded reason.
106+
107+## Running the race
108+
109+### One round
110+
111+1. Push the same commit to both remotes (e.g. an empty commit:
112+ `git commit --allow-empty -m "bench round N"`; or use
113+ `workflow_dispatch` on both — but record the dispatch time as the "push"
114+ timestamp then).
115+2. When both runs finish, download `results.jsonl` from each:
116+ - GitHub: `gh run download <run-id> -n bench-results` (or the run page);
117+ - rickub: the run's artifacts panel.
118+3. Collect:
119+ ```
120+ scripts/collect.sh github <run-id> /path/to/github-results/results.jsonl
121+ scripts/collect.sh rickub <run-id> /path/to/rickub-results/results.jsonl
122+ ```
123+4. Record the push timestamps (server-side, per platform API) into a
124+ `push-times.json` map `"github/<run-id>": "<iso>"` if you want the
125+ push→job-start row.
126+
127+### Interleaving (mandatory)
128+
129+Run rounds **alternating platforms** — G, R, G, R, … — at least **10 rounds
130+per platform**, serialized (never two benchmark runs in flight at once: on
131+rickub your own queued job would inflate the other platform's queue metric).
132+Cloud CI variance across hours and neighbours is well documented; a single
133+run is noise. Alternation decorrelates the two platforms from time-of-day
134+effects (fleet load, mirror warmth, co-tenants). Optionally discard round 1
135+per platform as image/toolchain warmup for the *operators*, not the runners.
136+
137+### Reporting
138+
139+```
140+python3 scripts/compare.py [--push-times push-times.json]
141+```
142+
143+Median (p50) and p95 per step per platform, linear-interpolated percentiles
144+(the numpy method) over `ok` records only; `skipped`/`fail` counts are shown
145+next to each cell so a platform quietly skipping docker is visible. The
146+table prints n per cell — do not compare anything with n < 10.
147+
148+### Local smoke run
149+
150+`make run-local` executes the same step sequence on your machine (macOS
151+works: the emitter falls back to python3 for millisecond timestamps and the
152+probe degrades gracefully). Collect with
153+`make collect PLATFORM=local RUN_ID=<id> SRC=results.jsonl`.
154+
155+## Known variance pitfalls
156+
157+- **Cross-clock subtraction** (push→job-start): server vs guest clock skew;
158+ prefer server-side-only `run_started_at − created_at` as cross-check.
159+- **Queue contamination**: rickub queues runs; benchmark rounds must be
160+ serialized or you measure your own backlog. Same for GitHub concurrency
161+ groups (none set here, on purpose).
162+- **Runner placement**: GitHub assigns runners across regions/hosts;
163+ crates.io and the Alpine mirror are at different RTTs from each placement.
164+ Only medians over many rounds are meaningful.
165+- **Floating package indexes**: `apk add build-base` in the docker workload
166+ floats with the mirror; crate downloads float with crates.io. A changed
167+ upstream version shifts a cold build by minutes. If a round's cold build
168+ moves >2x the running median, check upstream before believing it.
169+- **Toolchain download**: 1.85.0 is rustup-installed at job start on both
170+ platforms (inside the timed cold-build step? No — rustup resolves it when
171+ `cargo` first runs, so it IS inside the cold-build timing; identical on
172+ both platforms by construction).
173+- **Warm build is nearly zero**: it measures scheduler/link noise; treat p95,
174+ not median, as the signal.
175+- **probe-read is page-cache warm** by design (comparable, not absolute).
176+- **vCPU asymmetry**: rickub `large` = 4 vCPU / 8 GiB vs GitHub ubuntu-latest
177+ 4 vCPU / 16 GiB. See below.
178+
179+## Known asymmetries (decisions to revisit)
180+
181+1. **Runner class**: the rickub workflow uses `runs-on: large` for vCPU parity
182+ with GitHub's ubuntu-latest. `ubuntu-latest` on rickub (2 vCPU / 4 GiB) is
183+ the "as-consumed, 1x minutes" alternative. This is a user decision.
184+2. **Where the repo lives**: needs one GitHub repo and one rickub repo (or the
185+ same repo mirrored to both forges). Not created by this scaffold — no
186+ network was touched.
187+3. **push-times.json**: the push timestamp per run is currently recorded by
188+ the operator from each platform's API; automating it needs API tokens on
189+ both sides.
190+
191+## Degrade behaviour
192+
193+| missing thing | behaviour |
194+|---------------|------------------------------------------------------|
195+| docker | `docker-build` step records `skipped` with reason |
196+| cargo | (local runs) rust steps record `skipped` |
197+| GNU date | python3 millis fallback, then whole-second `.000Z` |
198+| dd / stat / df| probe records `null` values, job stays green |
199+| python3 | emitter falls back to whole-second timestamps |
new file mode 100644
@@ -0,0 +1,199 @@
1+# ci-bench — the CI race
2+
3+An identical, pinned workload pushed to **GitHub Actions** and **rickub CI**
4+(Firecracker microVM fleet), raced to answer: who starts jobs faster, whose
5+steps run faster, and what the runner hardware itself is worth?
6+
7+Everything here is bash + python3 stdlib. No package installs, no external
8+services beyond the two CI platforms themselves.
9+
10+## Layout
11+
12+ .github/workflows/bench.yml GitHub side of the race
13+ .rickub/workflows/bench.yml rickub side (same steps, same order)
14+ workloads/rust-build/ pinned Cargo project (cold/warm/test)
15+ workloads/docker-build/ digest-pinned multi-stage Dockerfile
16+ workloads/probe/probe.sh runner capability probe (cpu/disk/fs)
17+ scripts/emit_timing.sh sourced by steps; one JSON line per step
18+ scripts/collect.sh results.jsonl -> results/<platform>/<run>.json
19+ scripts/compare.py median/p95 table per step per platform
20+ Makefile bootstrap / run-local / collect / compare
21+
22+Why two workflow files and not one: rickub reads `.rickub/workflows/` as its
23+SOLE workflow source when that directory exists and ignores
24+`.github/workflows/` entirely, while GitHub only ever reads `.github/`. One
25+repo therefore carries one file per platform and neither platform sees the
26+other's copy. The two files run the same steps in the same order; the only
27+differences are `BENCH_PLATFORM`, `runs-on:` (see *Known asymmetries*), and
28+comments.
29+
30+## The workload (identical on both platforms)
31+
32+| step | what it does |
33+|-----------------|--------------------------------------------------------------------------|
34+| `job-start` | zero-duration marker emitted by the first step after checkout |
35+| `rust-cold-build` | `rm -rf target && cargo build` — full crate download + compile |
36+| `rust-warm-build` | immediate second `cargo build` — incremental no-op rebuild |
37+| `rust-test` | `cargo test` (5 real tests) over the warm build |
38+| `docker-build` | multi-stage `docker build` (alpine:3.20 pinned by digest) + `docker run` |
39+| `probe-*` | cpu loop, 2 GiB sequential write/read to `$RUNNER_TEMP`, fs type + free |
40+
41+Pins: Rust toolchain 1.85.0 (`workloads/rust-build/rust-toolchain.toml`), the
42+full dependency tree (`workloads/rust-build/Cargo.lock`, generated offline
43+against a real cargo registry cache, versions exact-pinned in `Cargo.toml`
44+with `=`), Docker base `alpine:3.20@sha256:d9e853e87e...` (the digest rickub
45+itself pins for its CI images). The workload is not bit-frozen against
46+`apk`/crates.io mirrors moving — see *Variance pitfalls*.
47+
48+### Cold vs warm — definitions
49+
50+Both platforms give every job a **fresh machine** (GitHub hosted runners;
51+rickub boots a new Firecracker microVM with a read-only rootfs and a fresh
52+scratch disk). Therefore:
53+
54+- **cold** = the FIRST `cargo build` in a job: crates.io fetch + full compile
55+ of the transitive tree. Never accelerated by any cache (there is no
56+ `actions/cache` and no cargo cache persistence — on purpose).
57+- **warm** = the SECOND `cargo build` in the SAME job: `target/` exists, the
58+ no-op incremental path (metadata checks + link of nothing).
59+- Docker build is always cold (no image cache survives the machine).
60+- A "cold docker" vs "warm docker" axis would need an in-job second build of
61+ the same Dockerfile; not in scope for v1.
62+
63+## Metrics — exact clock boundaries
64+
65+All records are JSON lines in `results.jsonl`:
66+
67+ {"step": "...", "platform": "...", "run_id": "...",
68+ "start": "2026-09-11T12:34:56.789Z", "end": "2026-09-11T12:35:40.123Z",
69+ "status": "ok|fail|skipped", "duration_ms": 43334,
70+ "value": <optional measurement>, "unit": "...", "reason": "..."}
71+
72+`start`/`end` are guest wall clock, UTC, millisecond precision (GNU `date
73+%3N`; python3 fallback on BSD; whole-second last resort — the fallback in use
74+is visible in the timestamps themselves: `.000Z` means coarse).
75+
76+1. **push→job-start** = `start` of the `job-start` record − **push timestamp
77+ from the platform API** (not from the guest).
78+ - GitHub: `gh api repos/:o/:r/actions/runs/<run_id>` → `created_at`
79+ (when the push event was accepted server-side). Alternative, fully
80+ server-side cross-check: `run_started_at − created_at` (queue +
81+ provisioning, no guest clock involved).
82+ - rickub: the run's created/queued timestamp from its CI API for the same
83+ commit.
84+ - CAVEAT: this metric subtracts a **server-side** timestamp from a
85+ **guest-side** timestamp. Guest clock skew shifts both platforms' numbers
86+ unpredictably (NTP inside a just-booted microVM can be off by seconds).
87+ Treat it as indicative; prefer the server-side-only variant above when
88+ the two disagree, and record both when possible.
89+2. **per-step duration** = `end − start` of the step's record, from the same
90+ guest clock, so skew cancels. Millisecond precision.
91+3. **end-to-end pipeline time** (two views):
92+ - job execution = last record's `end` − `job-start` record's `start`;
93+ - run wall clock = last record's `end` − push timestamp (carries the same
94+ cross-clock caveat as metric 1).
95+4. **cold vs warm cache** = `rust-cold-build` vs `rust-warm-build` durations
96+ as defined above.
97+5. **disk-write throughput** = `probe-write` value (MB/s, 2 GiB `dd
98+ bs=1M count=2048` to the job scratch dir, then removed); `probe-read` is
99+ the same file read back (warm page cache — still comparable across
100+ platforms); `probe-cpu` is shell-loop kops/s; `probe-fs-type`/`probe-fs-free`
101+ describe the filesystem the runner gave the job.
102+
103+Anything unavailable degrades to `"value": null` or a `skipped` record; the
104+job itself never fails from the probe, and absent docker skips
105+`docker-build` with a recorded reason.
106+
107+## Running the race
108+
109+### One round
110+
111+1. Push the same commit to both remotes (e.g. an empty commit:
112+ `git commit --allow-empty -m "bench round N"`; or use
113+ `workflow_dispatch` on both — but record the dispatch time as the "push"
114+ timestamp then).
115+2. When both runs finish, download `results.jsonl` from each:
116+ - GitHub: `gh run download <run-id> -n bench-results` (or the run page);
117+ - rickub: the run's artifacts panel.
118+3. Collect:
119+ ```
120+ scripts/collect.sh github <run-id> /path/to/github-results/results.jsonl
121+ scripts/collect.sh rickub <run-id> /path/to/rickub-results/results.jsonl
122+ ```
123+4. Record the push timestamps (server-side, per platform API) into a
124+ `push-times.json` map `"github/<run-id>": "<iso>"` if you want the
125+ push→job-start row.
126+
127+### Interleaving (mandatory)
128+
129+Run rounds **alternating platforms** — G, R, G, R, … — at least **10 rounds
130+per platform**, serialized (never two benchmark runs in flight at once: on
131+rickub your own queued job would inflate the other platform's queue metric).
132+Cloud CI variance across hours and neighbours is well documented; a single
133+run is noise. Alternation decorrelates the two platforms from time-of-day
134+effects (fleet load, mirror warmth, co-tenants). Optionally discard round 1
135+per platform as image/toolchain warmup for the *operators*, not the runners.
136+
137+### Reporting
138+
139+```
140+python3 scripts/compare.py [--push-times push-times.json]
141+```
142+
143+Median (p50) and p95 per step per platform, linear-interpolated percentiles
144+(the numpy method) over `ok` records only; `skipped`/`fail` counts are shown
145+next to each cell so a platform quietly skipping docker is visible. The
146+table prints n per cell — do not compare anything with n < 10.
147+
148+### Local smoke run
149+
150+`make run-local` executes the same step sequence on your machine (macOS
151+works: the emitter falls back to python3 for millisecond timestamps and the
152+probe degrades gracefully). Collect with
153+`make collect PLATFORM=local RUN_ID=<id> SRC=results.jsonl`.
154+
155+## Known variance pitfalls
156+
157+- **Cross-clock subtraction** (push→job-start): server vs guest clock skew;
158+ prefer server-side-only `run_started_at − created_at` as cross-check.
159+- **Queue contamination**: rickub queues runs; benchmark rounds must be
160+ serialized or you measure your own backlog. Same for GitHub concurrency
161+ groups (none set here, on purpose).
162+- **Runner placement**: GitHub assigns runners across regions/hosts;
163+ crates.io and the Alpine mirror are at different RTTs from each placement.
164+ Only medians over many rounds are meaningful.
165+- **Floating package indexes**: `apk add build-base` in the docker workload
166+ floats with the mirror; crate downloads float with crates.io. A changed
167+ upstream version shifts a cold build by minutes. If a round's cold build
168+ moves >2x the running median, check upstream before believing it.
169+- **Toolchain download**: 1.85.0 is rustup-installed at job start on both
170+ platforms (inside the timed cold-build step? No — rustup resolves it when
171+ `cargo` first runs, so it IS inside the cold-build timing; identical on
172+ both platforms by construction).
173+- **Warm build is nearly zero**: it measures scheduler/link noise; treat p95,
174+ not median, as the signal.
175+- **probe-read is page-cache warm** by design (comparable, not absolute).
176+- **vCPU asymmetry**: rickub `large` = 4 vCPU / 8 GiB vs GitHub ubuntu-latest
177+ 4 vCPU / 16 GiB. See below.
178+
179+## Known asymmetries (decisions to revisit)
180+
181+1. **Runner class**: the rickub workflow uses `runs-on: large` for vCPU parity
182+ with GitHub's ubuntu-latest. `ubuntu-latest` on rickub (2 vCPU / 4 GiB) is
183+ the "as-consumed, 1x minutes" alternative. This is a user decision.
184+2. **Where the repo lives**: needs one GitHub repo and one rickub repo (or the
185+ same repo mirrored to both forges). Not created by this scaffold — no
186+ network was touched.
187+3. **push-times.json**: the push timestamp per run is currently recorded by
188+ the operator from each platform's API; automating it needs API tokens on
189+ both sides.
190+
191+## Degrade behaviour
192+
193+| missing thing | behaviour |
194+|---------------|------------------------------------------------------|
195+| docker | `docker-build` step records `skipped` with reason |
196+| cargo | (local runs) rust steps record `skipped` |
197+| GNU date | python3 millis fallback, then whole-second `.000Z` |
198+| dd / stat / df| probe records `null` values, job stays green |
199+| python3 | emitter falls back to whole-second timestamps |
added scripts/collect.sh +60 -0
new file mode 100755
@@ -0,0 +1,60 @@
1+#!/usr/bin/env bash
2+# collect.sh — gather one run's results.jsonl into the results tree.
3+#
4+# scripts/collect.sh PLATFORM RUN_ID SRC [SRC...]
5+#
6+# PLATFORM github | rickub | local
7+# RUN_ID the run's identifier (bench run_id / artifact run number)
8+# SRC one or more results.jsonl files (e.g. a downloaded artifact,
9+# extracted anywhere; multiple are concatenated in order)
10+#
11+# Output: results/<platform>/<run-id>.json — a JSON array of the run's records,
12+# normalized (blank lines dropped). Refuses to overwrite an existing file: two
13+# collected runs with the same id are almost certainly a copy/paste mistake.
14+
15+set -euo pipefail
16+
17+usage() {
18+ sed -n '2,12p' "$0" >&2
19+ exit 2
20+}
21+
22+[ $# -ge 3 ] || usage
23+
24+platform=$1
25+run_id=$2
26+shift 2
27+
28+out="results/$platform/$run_id.json"
29+if [ -e "$out" ]; then
30+ echo "collect: refusing to overwrite $out (delete it first if intentional)" >&2
31+ exit 1
32+fi
33+
34+mkdir -p "results/$platform"
35+
36+# Concatenate the sources, drop blanks, and re-emit as a JSON array via
37+# python3 (stdlib) so the output is valid JSON even from partial files.
38+python3 - "$out" "$@" <<'PY'
39+import json, sys
40+
41+out_path, sources = sys.argv[1], sys.argv[2:]
42+records = []
43+for src in sources:
44+ with open(src, "r", encoding="utf-8", errors="replace") as fh:
45+ for line in fh:
46+ line = line.strip()
47+ if not line:
48+ continue
49+ try:
50+ records.append(json.loads(line))
51+ except json.JSONDecodeError:
52+ # A torn last line (job killed mid-write): keep it out but say so.
53+ print(f"collect: skipping unparseable line in {src}: {line[:80]!r}", file=sys.stderr)
54+if not records:
55+ sys.exit(f"collect: no records found in {sources}")
56+with open(out_path, "w", encoding="utf-8") as fh:
57+ json.dump(records, fh, indent=1)
58+ fh.write("\n")
59+print(f"collect: {len(records)} records -> {out_path}")
60+PY
new file mode 100755
@@ -0,0 +1,60 @@
1+#!/usr/bin/env bash
2+# collect.sh — gather one run's results.jsonl into the results tree.
3+#
4+# scripts/collect.sh PLATFORM RUN_ID SRC [SRC...]
5+#
6+# PLATFORM github | rickub | local
7+# RUN_ID the run's identifier (bench run_id / artifact run number)
8+# SRC one or more results.jsonl files (e.g. a downloaded artifact,
9+# extracted anywhere; multiple are concatenated in order)
10+#
11+# Output: results/<platform>/<run-id>.json — a JSON array of the run's records,
12+# normalized (blank lines dropped). Refuses to overwrite an existing file: two
13+# collected runs with the same id are almost certainly a copy/paste mistake.
14+
15+set -euo pipefail
16+
17+usage() {
18+ sed -n '2,12p' "$0" >&2
19+ exit 2
20+}
21+
22+[ $# -ge 3 ] || usage
23+
24+platform=$1
25+run_id=$2
26+shift 2
27+
28+out="results/$platform/$run_id.json"
29+if [ -e "$out" ]; then
30+ echo "collect: refusing to overwrite $out (delete it first if intentional)" >&2
31+ exit 1
32+fi
33+
34+mkdir -p "results/$platform"
35+
36+# Concatenate the sources, drop blanks, and re-emit as a JSON array via
37+# python3 (stdlib) so the output is valid JSON even from partial files.
38+python3 - "$out" "$@" <<'PY'
39+import json, sys
40+
41+out_path, sources = sys.argv[1], sys.argv[2:]
42+records = []
43+for src in sources:
44+ with open(src, "r", encoding="utf-8", errors="replace") as fh:
45+ for line in fh:
46+ line = line.strip()
47+ if not line:
48+ continue
49+ try:
50+ records.append(json.loads(line))
51+ except json.JSONDecodeError:
52+ # A torn last line (job killed mid-write): keep it out but say so.
53+ print(f"collect: skipping unparseable line in {src}: {line[:80]!r}", file=sys.stderr)
54+if not records:
55+ sys.exit(f"collect: no records found in {sources}")
56+with open(out_path, "w", encoding="utf-8") as fh:
57+ json.dump(records, fh, indent=1)
58+ fh.write("\n")
59+print(f"collect: {len(records)} records -> {out_path}")
60+PY
added scripts/compare.py +197 -0
new file mode 100644
@@ -0,0 +1,197 @@
1+#!/usr/bin/env python3
2+"""compare.py — median/p95 table per step per platform, from collected runs.
3+
4+Reads results/<platform>/<run-id>.json files (produced by scripts/collect.sh).
5+python3 stdlib only.
6+
7+Usage:
8+ scripts/compare.py [--push-times FILE]
9+
10+Output:
11+ - per step x platform: n, median, p95 of duration_ms (status ok only;
12+ "fail"/"skipped" counted separately and shown)
13+ - for metric-bearing steps (value field: probe-*), median of the value
14+ - with --push-times: push -> job-start latency per platform, where push
15+ timestamps come from FILE: {"<platform>/<run-id>": "<iso8601>", ...}
16+ (see README "push->job-start" for where each timestamp comes from)
17+
18+Percentiles: linear interpolation between order statistics (the method numpy
19+percentile uses), reported at p50 (median) and p95.
20+"""
21+
22+import argparse
23+import json
24+import sys
25+from pathlib import Path
26+
27+RESULTS_DIR = Path(__file__).resolve().parent.parent / "results"
28+JOB_START_STEP = "job-start"
29+DURATION_UNITS = {"duration_ms": "ms"}
30+
31+
32+def percentile(sorted_vals, q):
33+ """Linear-interpolation percentile. sorted_vals non-empty, q in [0,100]."""
34+ if len(sorted_vals) == 1:
35+ return sorted_vals[0]
36+ pos = (len(sorted_vals) - 1) * q / 100.0
37+ lo = int(pos)
38+ hi = min(lo + 1, len(sorted_vals) - 1)
39+ frac = pos - lo
40+ return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac
41+
42+
43+def fmt_ms(v):
44+ return f"{v / 1000.0:8.2f}s" if v >= 1000 else f"{v:8.0f}ms"
45+
46+
47+def fmt_v(v):
48+ if v is None:
49+ return " null"
50+ return f"{v:8.1f}"
51+
52+
53+def load_platforms():
54+ platforms = {}
55+ if not RESULTS_DIR.is_dir():
56+ return platforms
57+ for pdir in sorted(RESULTS_DIR.iterdir()):
58+ if not pdir.is_dir():
59+ continue
60+ runs = []
61+ for rfile in sorted(pdir.glob("*.json")):
62+ try:
63+ with open(rfile, encoding="utf-8") as fh:
64+ runs.append(json.load(fh))
65+ except (json.JSONDecodeError, OSError) as exc:
66+ print(f"compare: skipping unreadable {rfile}: {exc}", file=sys.stderr)
67+ if runs:
68+ platforms[pdir.name] = runs
69+ return platforms
70+
71+
72+def parse_iso(ts):
73+ from datetime import datetime, timezone
74+
75+ ts = ts.strip()
76+ if ts.endswith("Z"):
77+ ts = ts[:-1] + "+00:00"
78+ dt = datetime.fromisoformat(ts)
79+ if dt.tzinfo is None:
80+ dt = dt.replace(tzinfo=timezone.utc)
81+ return dt
82+
83+
84+def main():
85+ ap = argparse.ArgumentParser()
86+ ap.add_argument("--push-times", metavar="FILE",
87+ help="JSON map of '<platform>/<run-id>' -> push ISO timestamp")
88+ args = ap.parse_args()
89+
90+ platforms = load_platforms()
91+ if not platforms:
92+ print("compare: no collected results under results/ — run scripts/collect.sh first")
93+ return 1
94+
95+ # step -> platform -> list of (record); plus counters
96+ steps = {}
97+ for platform, runs in platforms.items():
98+ for run in runs:
99+ for rec in run:
100+ step = rec.get("step", "?")
101+ steps.setdefault(step, {}).setdefault(platform, []).append(rec)
102+
103+ step_names = sorted(steps)
104+ plat_names = sorted(platforms)
105+
106+ header = f"{'step':<22}" + "".join(f"{p:>26}" for p in plat_names)
107+ sub = f"{'':<22}" + "".join(f"{'n / median / p95':>26}" for p in plat_names)
108+ print(header)
109+ print(sub)
110+ print("-" * len(header))
111+
112+ for step in step_names:
113+ row = f"{step:<22}"
114+ for plat in plat_names:
115+ recs = steps[step].get(plat, [])
116+ durs = sorted(r["duration_ms"] for r in recs
117+ if r.get("status") == "ok" and isinstance(r.get("duration_ms"), (int, float)))
118+ n_skipped = sum(1 for r in recs if r.get("status") == "skipped")
119+ n_fail = sum(1 for r in recs if r.get("status") == "fail")
120+ cell = f"{len(durs):>3} /"
121+ if durs:
122+ cell += f" {fmt_ms(percentile(durs, 50))} / {fmt_ms(percentile(durs, 95))}"
123+ else:
124+ cell += " - / -"
125+ if n_skipped:
126+ cell += f" (+{n_skipped}skip)"
127+ if n_fail:
128+ cell += f" (+{n_fail}FAIL)"
129+ row += f"{cell:>26}"
130+ print(row)
131+
132+ # Metric-bearing steps: median of value.
133+ metric_steps = [s for s in step_names
134+ if any("value" in r for recs in steps[s].values() for r in recs)]
135+ if metric_steps:
136+ print()
137+ print(f"{'metric (median value)':<22}" + "".join(f"{p:>26}" for p in plat_names))
138+ print("-" * len(header))
139+ for step in metric_steps:
140+ row = f"{step:<22}"
141+ unit = ""
142+ for plat in plat_names:
143+ recs = [r for r in steps[step].get(plat, []) if "value" in r]
144+ nums = sorted(r["value"] for r in recs
145+ if isinstance(r.get("value"), (int, float)))
146+ strs = [r["value"] for r in recs if isinstance(r.get("value"), str)]
147+ if nums:
148+ unit = next((r.get("unit", "") for r in recs if r.get("unit")), "")
149+ cell = f"{percentile(nums, 50):.1f} {unit}"
150+ elif strs:
151+ # categorical metric (e.g. fs type): show the most common value
152+ cell = max(set(strs), key=strs.count)[:16]
153+ else:
154+ cell = "null"
155+ row += f"{cell:>26}"
156+ print(row)
157+
158+ # push -> job-start latency.
159+ if args.push_times:
160+ print()
161+ try:
162+ with open(args.push_times, encoding="utf-8") as fh:
163+ push_times = json.load(fh)
164+ except (json.JSONDecodeError, OSError) as exc:
165+ print(f"compare: cannot read --push-times: {exc}", file=sys.stderr)
166+ return 1
167+ row = f"{'push->job-start':<22}"
168+ for plat in plat_names:
169+ lat = []
170+ for run in platforms[plat]:
171+ run_id = next((r.get("run_id") for r in run if r.get("run_id")), None)
172+ key = f"{plat}/{run_id}"
173+ if key not in push_times:
174+ continue
175+ mark = next((r for r in run if r.get("step") == JOB_START_STEP), None)
176+ if not mark:
177+ continue
178+ try:
179+ delta = (parse_iso(mark["start"]) - parse_iso(push_times[key])).total_seconds() * 1000
180+ except (ValueError, KeyError):
181+ continue
182+ if delta >= 0:
183+ lat.append(delta)
184+ cell = f"{len(lat):>3} /"
185+ cell += (f" {fmt_ms(percentile(sorted(lat), 50))} / {fmt_ms(percentile(sorted(lat), 95))}"
186+ if lat else " - / -")
187+ row += f"{cell:>26}"
188+ print(row + " (cross-clock; see README caveats)")
189+
190+ print()
191+ print("durations = emitted end-start per step; percentiles linear-interpolated;")
192+ print("report n>=10 per platform before trusting any comparison (README: variance).")
193+ return 0
194+
195+
196+if __name__ == "__main__":
197+ sys.exit(main())
new file mode 100644
@@ -0,0 +1,197 @@
1+#!/usr/bin/env python3
2+"""compare.py — median/p95 table per step per platform, from collected runs.
3+
4+Reads results/<platform>/<run-id>.json files (produced by scripts/collect.sh).
5+python3 stdlib only.
6+
7+Usage:
8+ scripts/compare.py [--push-times FILE]
9+
10+Output:
11+ - per step x platform: n, median, p95 of duration_ms (status ok only;
12+ "fail"/"skipped" counted separately and shown)
13+ - for metric-bearing steps (value field: probe-*), median of the value
14+ - with --push-times: push -> job-start latency per platform, where push
15+ timestamps come from FILE: {"<platform>/<run-id>": "<iso8601>", ...}
16+ (see README "push->job-start" for where each timestamp comes from)
17+
18+Percentiles: linear interpolation between order statistics (the method numpy
19+percentile uses), reported at p50 (median) and p95.
20+"""
21+
22+import argparse
23+import json
24+import sys
25+from pathlib import Path
26+
27+RESULTS_DIR = Path(__file__).resolve().parent.parent / "results"
28+JOB_START_STEP = "job-start"
29+DURATION_UNITS = {"duration_ms": "ms"}
30+
31+
32+def percentile(sorted_vals, q):
33+ """Linear-interpolation percentile. sorted_vals non-empty, q in [0,100]."""
34+ if len(sorted_vals) == 1:
35+ return sorted_vals[0]
36+ pos = (len(sorted_vals) - 1) * q / 100.0
37+ lo = int(pos)
38+ hi = min(lo + 1, len(sorted_vals) - 1)
39+ frac = pos - lo
40+ return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac
41+
42+
43+def fmt_ms(v):
44+ return f"{v / 1000.0:8.2f}s" if v >= 1000 else f"{v:8.0f}ms"
45+
46+
47+def fmt_v(v):
48+ if v is None:
49+ return " null"
50+ return f"{v:8.1f}"
51+
52+
53+def load_platforms():
54+ platforms = {}
55+ if not RESULTS_DIR.is_dir():
56+ return platforms
57+ for pdir in sorted(RESULTS_DIR.iterdir()):
58+ if not pdir.is_dir():
59+ continue
60+ runs = []
61+ for rfile in sorted(pdir.glob("*.json")):
62+ try:
63+ with open(rfile, encoding="utf-8") as fh:
64+ runs.append(json.load(fh))
65+ except (json.JSONDecodeError, OSError) as exc:
66+ print(f"compare: skipping unreadable {rfile}: {exc}", file=sys.stderr)
67+ if runs:
68+ platforms[pdir.name] = runs
69+ return platforms
70+
71+
72+def parse_iso(ts):
73+ from datetime import datetime, timezone
74+
75+ ts = ts.strip()
76+ if ts.endswith("Z"):
77+ ts = ts[:-1] + "+00:00"
78+ dt = datetime.fromisoformat(ts)
79+ if dt.tzinfo is None:
80+ dt = dt.replace(tzinfo=timezone.utc)
81+ return dt
82+
83+
84+def main():
85+ ap = argparse.ArgumentParser()
86+ ap.add_argument("--push-times", metavar="FILE",
87+ help="JSON map of '<platform>/<run-id>' -> push ISO timestamp")
88+ args = ap.parse_args()
89+
90+ platforms = load_platforms()
91+ if not platforms:
92+ print("compare: no collected results under results/ — run scripts/collect.sh first")
93+ return 1
94+
95+ # step -> platform -> list of (record); plus counters
96+ steps = {}
97+ for platform, runs in platforms.items():
98+ for run in runs:
99+ for rec in run:
100+ step = rec.get("step", "?")
101+ steps.setdefault(step, {}).setdefault(platform, []).append(rec)
102+
103+ step_names = sorted(steps)
104+ plat_names = sorted(platforms)
105+
106+ header = f"{'step':<22}" + "".join(f"{p:>26}" for p in plat_names)
107+ sub = f"{'':<22}" + "".join(f"{'n / median / p95':>26}" for p in plat_names)
108+ print(header)
109+ print(sub)
110+ print("-" * len(header))
111+
112+ for step in step_names:
113+ row = f"{step:<22}"
114+ for plat in plat_names:
115+ recs = steps[step].get(plat, [])
116+ durs = sorted(r["duration_ms"] for r in recs
117+ if r.get("status") == "ok" and isinstance(r.get("duration_ms"), (int, float)))
118+ n_skipped = sum(1 for r in recs if r.get("status") == "skipped")
119+ n_fail = sum(1 for r in recs if r.get("status") == "fail")
120+ cell = f"{len(durs):>3} /"
121+ if durs:
122+ cell += f" {fmt_ms(percentile(durs, 50))} / {fmt_ms(percentile(durs, 95))}"
123+ else:
124+ cell += " - / -"
125+ if n_skipped:
126+ cell += f" (+{n_skipped}skip)"
127+ if n_fail:
128+ cell += f" (+{n_fail}FAIL)"
129+ row += f"{cell:>26}"
130+ print(row)
131+
132+ # Metric-bearing steps: median of value.
133+ metric_steps = [s for s in step_names
134+ if any("value" in r for recs in steps[s].values() for r in recs)]
135+ if metric_steps:
136+ print()
137+ print(f"{'metric (median value)':<22}" + "".join(f"{p:>26}" for p in plat_names))
138+ print("-" * len(header))
139+ for step in metric_steps:
140+ row = f"{step:<22}"
141+ unit = ""
142+ for plat in plat_names:
143+ recs = [r for r in steps[step].get(plat, []) if "value" in r]
144+ nums = sorted(r["value"] for r in recs
145+ if isinstance(r.get("value"), (int, float)))
146+ strs = [r["value"] for r in recs if isinstance(r.get("value"), str)]
147+ if nums:
148+ unit = next((r.get("unit", "") for r in recs if r.get("unit")), "")
149+ cell = f"{percentile(nums, 50):.1f} {unit}"
150+ elif strs:
151+ # categorical metric (e.g. fs type): show the most common value
152+ cell = max(set(strs), key=strs.count)[:16]
153+ else:
154+ cell = "null"
155+ row += f"{cell:>26}"
156+ print(row)
157+
158+ # push -> job-start latency.
159+ if args.push_times:
160+ print()
161+ try:
162+ with open(args.push_times, encoding="utf-8") as fh:
163+ push_times = json.load(fh)
164+ except (json.JSONDecodeError, OSError) as exc:
165+ print(f"compare: cannot read --push-times: {exc}", file=sys.stderr)
166+ return 1
167+ row = f"{'push->job-start':<22}"
168+ for plat in plat_names:
169+ lat = []
170+ for run in platforms[plat]:
171+ run_id = next((r.get("run_id") for r in run if r.get("run_id")), None)
172+ key = f"{plat}/{run_id}"
173+ if key not in push_times:
174+ continue
175+ mark = next((r for r in run if r.get("step") == JOB_START_STEP), None)
176+ if not mark:
177+ continue
178+ try:
179+ delta = (parse_iso(mark["start"]) - parse_iso(push_times[key])).total_seconds() * 1000
180+ except (ValueError, KeyError):
181+ continue
182+ if delta >= 0:
183+ lat.append(delta)
184+ cell = f"{len(lat):>3} /"
185+ cell += (f" {fmt_ms(percentile(sorted(lat), 50))} / {fmt_ms(percentile(sorted(lat), 95))}"
186+ if lat else " - / -")
187+ row += f"{cell:>26}"
188+ print(row + " (cross-clock; see README caveats)")
189+
190+ print()
191+ print("durations = emitted end-start per step; percentiles linear-interpolated;")
192+ print("report n>=10 per platform before trusting any comparison (README: variance).")
193+ return 0
194+
195+
196+if __name__ == "__main__":
197+ sys.exit(main())
added scripts/emit_timing.sh +143 -0
new file mode 100755
@@ -0,0 +1,143 @@
1+#!/usr/bin/env bash
2+# emit_timing.sh — sourced by every benchmark step (CI and local).
3+#
4+# Emits ONE JSON line per step to $GITHUB_WORKSPACE/results.jsonl (falling back
5+# to ./results.jsonl). Schema:
6+# {"step": <name>, "platform": <github|rickub|local>, "run_id": <id>,
7+# "start": <iso8601>, "end": <iso8601>, "status": <ok|fail|skipped>,
8+# "duration_ms": <int>, # convenience, = end - start
9+# "value": <json>, "unit": <string>, "reason": <string>} # optional extras
10+#
11+# Public API:
12+# bench_step STEP CMD [ARGS...] run CMD, record ok/fail (compound bodies:
13+# bench_step x bash -c '...; ...')
14+# bench_skip STEP REASON record a skipped step (e.g. no docker)
15+# bench_metric STEP VALUE UNIT record a measurement now (value is raw JSON:
16+# number, "string", or null)
17+# bench_mark STEP zero-duration marker (e.g. job-start)
18+#
19+# Determined at source time (env overrides win):
20+# BENCH_PLATFORM github | rickub | local (default: inferred)
21+# BENCH_RUN_ID stable identifier for the whole run (default: GITHUB_RUN_ID,
22+# else local-<utcstamp>-<pid>)
23+#
24+# This file sets NO shell options on purpose: it is sourced into the caller's
25+# shell and must not change its behaviour.
26+
27+# --- clock helpers -----------------------------------------------------------
28+# Prefer GNU date's %N (all Linux CI guests); BSD date (macOS) prints a literal
29+# "N" which the regex rejects; then python3; then whole-second granularity.
30+_bench_now_iso() {
31+ local t
32+ t=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ 2>/dev/null || true)
33+ # A valid timestamp contains no "N"; BSD date prints a literal one for %3N.
34+ case $t in
35+ *N*) ;; # %3N unsupported (BSD date) — fall through
36+ *) printf '%s\n' "$t"; return 0 ;;
37+ esac
38+ t=$(python3 -c 'import datetime
39+print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]+"Z")' 2>/dev/null || true)
40+ case $t in
41+ 2[0-9][0-9][0-9]-*) printf '%s\n' "$t"; return 0 ;;
42+ esac
43+ date -u +%Y-%m-%dT%H:%M:%S.000Z
44+}
45+
46+_bench_iso_to_ms() { # ISO (above format) -> epoch ms; python3 when available
47+ python3 - "$1" <<'PY' 2>/dev/null || printf '%s\n' "$(_bench_iso_to_ms_shell "$1")"
48+import sys, datetime
49+s = sys.argv[1]
50+print(int(datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=datetime.timezone.utc).timestamp() * 1000))
51+PY
52+}
53+
54+_bench_iso_to_ms_shell() { # fallback: second granularity (documented coarser)
55+ local s=${1%%.*}
56+ date -u -j -f '%Y-%m-%dT%H:%M:%S' "$s" +%s 2>/dev/null \
57+ || date -u -d "$s" +%s 2>/dev/null \
58+ || printf '0\n'
59+}
60+
61+# --- identity ----------------------------------------------------------------
62+_bench_platform() {
63+ if [ -n "${BENCH_PLATFORM:-}" ]; then
64+ printf '%s\n' "$BENCH_PLATFORM"
65+ elif [ -n "${GITHUB_SERVER_URL:-}" ]; then
66+ case $GITHUB_SERVER_URL in
67+ *github.com*) printf 'github\n' ;;
68+ *) printf 'rickub\n' ;;
69+ esac
70+ else
71+ printf 'local\n'
72+ fi
73+}
74+
75+_bench_run_id() {
76+ if [ -n "${BENCH_RUN_ID:-}" ]; then printf '%s\n' "$BENCH_RUN_ID"
77+ elif [ -n "${GITHUB_RUN_ID:-}" ]; then
78+ printf '%s-attempt%s\n' "${GITHUB_RUN_ID}" "${GITHUB_RUN_ATTEMPT:-1}"
79+ else
80+ printf 'local-%s-%s\n' "$(date -u +%Y%m%dT%H%M%SZ)" "$$"
81+ fi
82+}
83+
84+# --- record emission ---------------------------------------------------------
85+_bench_results_file() {
86+ local dir=${GITHUB_WORKSPACE:-$PWD}
87+ printf '%s/results.jsonl\n' "$dir"
88+}
89+
90+_bench_esc() { # minimal JSON string escaping (control chars flattened)
91+ printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' | tr '\n\r\t' ' '
92+}
93+
94+# _bench_emit STEP STATUS START END [VALUE UNIT REASON]
95+_bench_emit() {
96+ local step=$1 status=$2 start=$3 end=$4 value=${5-} unit=${6-} reason=${7-}
97+ local line f dur_ms
98+ f=$(_bench_results_file)
99+ dur_ms=$(( $(_bench_iso_to_ms "$end") - $(_bench_iso_to_ms "$start") ))
100+ [ "$dur_ms" -lt 0 ] && dur_ms=0
101+ line='{"step": "'$(_bench_esc "$step")'"'
102+ line+=', "platform": "'$(_bench_esc "$(_bench_platform)")'"'
103+ line+=', "run_id": "'$(_bench_esc "$(_bench_run_id)")'"'
104+ line+=', "start": "'$(_bench_esc "$start")'"'
105+ line+=', "end": "'$(_bench_esc "$end")'"'
106+ line+=', "status": "'$(_bench_esc "$status")'"'
107+ line+=', "duration_ms": '"$dur_ms"
108+ [ -n "$value" ] && line+=', "value": '"$value"
109+ [ -n "$unit" ] && line+=', "unit": "'$(_bench_esc "$unit")'"'
110+ [ -n "$reason" ] && line+=', "reason": "'$(_bench_esc "$reason")'"'
111+ line+='}'
112+ printf '%s\n' "$line" >>"$f"
113+}
114+
115+# --- public API --------------------------------------------------------------
116+bench_step() { # STEP CMD [ARGS...]
117+ # Records ok/fail AND propagates the command's exit code, so a failed
118+ # workload fails the CI step (not silently recorded as a red data point).
119+ local step=$1; shift
120+ local start end status rc
121+ start=$(_bench_now_iso)
122+ rc=0
123+ "$@" || rc=$?
124+ [ "$rc" -eq 0 ] && status=ok || status=fail
125+ end=$(_bench_now_iso)
126+ _bench_emit "$step" "$status" "$start" "$end"
127+ return "$rc"
128+}
129+
130+bench_skip() { # STEP REASON
131+ local now=$(_bench_now_iso)
132+ _bench_emit "$1" skipped "$now" "$now" '' '' "$2"
133+}
134+
135+bench_metric() { # STEP VALUE UNIT (VALUE is raw JSON: 123, "ext4", null)
136+ local now=$(_bench_now_iso)
137+ _bench_emit "$1" ok "$now" "$now" "$2" "$3"
138+}
139+
140+bench_mark() { # STEP
141+ local now=$(_bench_now_iso)
142+ _bench_emit "$1" ok "$now" "$now"
143+}
new file mode 100755
@@ -0,0 +1,143 @@
1+#!/usr/bin/env bash
2+# emit_timing.sh — sourced by every benchmark step (CI and local).
3+#
4+# Emits ONE JSON line per step to $GITHUB_WORKSPACE/results.jsonl (falling back
5+# to ./results.jsonl). Schema:
6+# {"step": <name>, "platform": <github|rickub|local>, "run_id": <id>,
7+# "start": <iso8601>, "end": <iso8601>, "status": <ok|fail|skipped>,
8+# "duration_ms": <int>, # convenience, = end - start
9+# "value": <json>, "unit": <string>, "reason": <string>} # optional extras
10+#
11+# Public API:
12+# bench_step STEP CMD [ARGS...] run CMD, record ok/fail (compound bodies:
13+# bench_step x bash -c '...; ...')
14+# bench_skip STEP REASON record a skipped step (e.g. no docker)
15+# bench_metric STEP VALUE UNIT record a measurement now (value is raw JSON:
16+# number, "string", or null)
17+# bench_mark STEP zero-duration marker (e.g. job-start)
18+#
19+# Determined at source time (env overrides win):
20+# BENCH_PLATFORM github | rickub | local (default: inferred)
21+# BENCH_RUN_ID stable identifier for the whole run (default: GITHUB_RUN_ID,
22+# else local-<utcstamp>-<pid>)
23+#
24+# This file sets NO shell options on purpose: it is sourced into the caller's
25+# shell and must not change its behaviour.
26+
27+# --- clock helpers -----------------------------------------------------------
28+# Prefer GNU date's %N (all Linux CI guests); BSD date (macOS) prints a literal
29+# "N" which the regex rejects; then python3; then whole-second granularity.
30+_bench_now_iso() {
31+ local t
32+ t=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ 2>/dev/null || true)
33+ # A valid timestamp contains no "N"; BSD date prints a literal one for %3N.
34+ case $t in
35+ *N*) ;; # %3N unsupported (BSD date) — fall through
36+ *) printf '%s\n' "$t"; return 0 ;;
37+ esac
38+ t=$(python3 -c 'import datetime
39+print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]+"Z")' 2>/dev/null || true)
40+ case $t in
41+ 2[0-9][0-9][0-9]-*) printf '%s\n' "$t"; return 0 ;;
42+ esac
43+ date -u +%Y-%m-%dT%H:%M:%S.000Z
44+}
45+
46+_bench_iso_to_ms() { # ISO (above format) -> epoch ms; python3 when available
47+ python3 - "$1" <<'PY' 2>/dev/null || printf '%s\n' "$(_bench_iso_to_ms_shell "$1")"
48+import sys, datetime
49+s = sys.argv[1]
50+print(int(datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=datetime.timezone.utc).timestamp() * 1000))
51+PY
52+}
53+
54+_bench_iso_to_ms_shell() { # fallback: second granularity (documented coarser)
55+ local s=${1%%.*}
56+ date -u -j -f '%Y-%m-%dT%H:%M:%S' "$s" +%s 2>/dev/null \
57+ || date -u -d "$s" +%s 2>/dev/null \
58+ || printf '0\n'
59+}
60+
61+# --- identity ----------------------------------------------------------------
62+_bench_platform() {
63+ if [ -n "${BENCH_PLATFORM:-}" ]; then
64+ printf '%s\n' "$BENCH_PLATFORM"
65+ elif [ -n "${GITHUB_SERVER_URL:-}" ]; then
66+ case $GITHUB_SERVER_URL in
67+ *github.com*) printf 'github\n' ;;
68+ *) printf 'rickub\n' ;;
69+ esac
70+ else
71+ printf 'local\n'
72+ fi
73+}
74+
75+_bench_run_id() {
76+ if [ -n "${BENCH_RUN_ID:-}" ]; then printf '%s\n' "$BENCH_RUN_ID"
77+ elif [ -n "${GITHUB_RUN_ID:-}" ]; then
78+ printf '%s-attempt%s\n' "${GITHUB_RUN_ID}" "${GITHUB_RUN_ATTEMPT:-1}"
79+ else
80+ printf 'local-%s-%s\n' "$(date -u +%Y%m%dT%H%M%SZ)" "$$"
81+ fi
82+}
83+
84+# --- record emission ---------------------------------------------------------
85+_bench_results_file() {
86+ local dir=${GITHUB_WORKSPACE:-$PWD}
87+ printf '%s/results.jsonl\n' "$dir"
88+}
89+
90+_bench_esc() { # minimal JSON string escaping (control chars flattened)
91+ printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' | tr '\n\r\t' ' '
92+}
93+
94+# _bench_emit STEP STATUS START END [VALUE UNIT REASON]
95+_bench_emit() {
96+ local step=$1 status=$2 start=$3 end=$4 value=${5-} unit=${6-} reason=${7-}
97+ local line f dur_ms
98+ f=$(_bench_results_file)
99+ dur_ms=$(( $(_bench_iso_to_ms "$end") - $(_bench_iso_to_ms "$start") ))
100+ [ "$dur_ms" -lt 0 ] && dur_ms=0
101+ line='{"step": "'$(_bench_esc "$step")'"'
102+ line+=', "platform": "'$(_bench_esc "$(_bench_platform)")'"'
103+ line+=', "run_id": "'$(_bench_esc "$(_bench_run_id)")'"'
104+ line+=', "start": "'$(_bench_esc "$start")'"'
105+ line+=', "end": "'$(_bench_esc "$end")'"'
106+ line+=', "status": "'$(_bench_esc "$status")'"'
107+ line+=', "duration_ms": '"$dur_ms"
108+ [ -n "$value" ] && line+=', "value": '"$value"
109+ [ -n "$unit" ] && line+=', "unit": "'$(_bench_esc "$unit")'"'
110+ [ -n "$reason" ] && line+=', "reason": "'$(_bench_esc "$reason")'"'
111+ line+='}'
112+ printf '%s\n' "$line" >>"$f"
113+}
114+
115+# --- public API --------------------------------------------------------------
116+bench_step() { # STEP CMD [ARGS...]
117+ # Records ok/fail AND propagates the command's exit code, so a failed
118+ # workload fails the CI step (not silently recorded as a red data point).
119+ local step=$1; shift
120+ local start end status rc
121+ start=$(_bench_now_iso)
122+ rc=0
123+ "$@" || rc=$?
124+ [ "$rc" -eq 0 ] && status=ok || status=fail
125+ end=$(_bench_now_iso)
126+ _bench_emit "$step" "$status" "$start" "$end"
127+ return "$rc"
128+}
129+
130+bench_skip() { # STEP REASON
131+ local now=$(_bench_now_iso)
132+ _bench_emit "$1" skipped "$now" "$now" '' '' "$2"
133+}
134+
135+bench_metric() { # STEP VALUE UNIT (VALUE is raw JSON: 123, "ext4", null)
136+ local now=$(_bench_now_iso)
137+ _bench_emit "$1" ok "$now" "$now" "$2" "$3"
138+}
139+
140+bench_mark() { # STEP
141+ local now=$(_bench_now_iso)
142+ _bench_emit "$1" ok "$now" "$now"
143+}
added workloads/docker-build/Dockerfile +25 -0
new file mode 100644
@@ -0,0 +1,25 @@
1+# docker-build workload — multi-stage build against a digest-pinned base.
2+#
3+# The base is pinned to the exact alpine:3.20 digest rickub itself pins for its
4+# own CI images (images/firecracker/rootfs/Dockerfile in the rickub monorepo),
5+# i.e. a digest known to be stable and public. CAVEAT (documented in README):
6+# `apk add` inside the builder still floats with the Alpine mirror at run time;
7+# a truly bit-frozen build would vendored-package the toolchain, which would
8+# remove the network component this workload deliberately measures.
9+
10+# syntax=docker/dockerfile:1
11+ARG ALPINE_IMAGE=alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc
12+
13+# --- builder: fetch toolchain, compile --------------------------------------
14+FROM ${ALPINE_IMAGE} AS builder
15+RUN apk add --no-cache build-base
16+WORKDIR /src
17+COPY main.c .
18+RUN cc -O2 -static -o ci-bench-docker main.c
19+
20+# --- final: tiny image, just the static binary ------------------------------
21+FROM ${ALPINE_IMAGE}
22+WORKDIR /
23+COPY --from=builder /src/ci-bench-docker /usr/local/bin/ci-bench-docker
24+# Expected output: ci-bench-docker-a56ee6092483
25+CMD ["/usr/local/bin/ci-bench-docker"]
new file mode 100644
@@ -0,0 +1,25 @@
1+# docker-build workload — multi-stage build against a digest-pinned base.
2+#
3+# The base is pinned to the exact alpine:3.20 digest rickub itself pins for its
4+# own CI images (images/firecracker/rootfs/Dockerfile in the rickub monorepo),
5+# i.e. a digest known to be stable and public. CAVEAT (documented in README):
6+# `apk add` inside the builder still floats with the Alpine mirror at run time;
7+# a truly bit-frozen build would vendored-package the toolchain, which would
8+# remove the network component this workload deliberately measures.
9+
10+# syntax=docker/dockerfile:1
11+ARG ALPINE_IMAGE=alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc
12+
13+# --- builder: fetch toolchain, compile --------------------------------------
14+FROM ${ALPINE_IMAGE} AS builder
15+RUN apk add --no-cache build-base
16+WORKDIR /src
17+COPY main.c .
18+RUN cc -O2 -static -o ci-bench-docker main.c
19+
20+# --- final: tiny image, just the static binary ------------------------------
21+FROM ${ALPINE_IMAGE}
22+WORKDIR /
23+COPY --from=builder /src/ci-bench-docker /usr/local/bin/ci-bench-docker
24+# Expected output: ci-bench-docker-a56ee6092483
25+CMD ["/usr/local/bin/ci-bench-docker"]
added workloads/docker-build/main.c +41 -0
new file mode 100644
@@ -0,0 +1,41 @@
1+/*
2+ * main.c — the tiny C program the docker-build workload compiles.
3+ *
4+ * The BUILD is the benchmark (multi-stage, network fetch of build-base, gcc
5+ * invocation, final image assembly). The program itself just does a
6+ * deterministic FNV-1a pass so the binary is non-trivial and its output is
7+ * stable across platforms: ci-bench-docker-a56ee6092483
8+ */
9+
10+#include <stdio.h>
11+#include <stdint.h>
12+#include <string.h>
13+
14+#define ROUNDS 2000000u
15+#define SEED_LEN 64
16+
17+static uint64_t fnv1a(uint64_t h, const unsigned char *data, size_t len) {
18+ for (size_t i = 0; i < len; i++) {
19+ h ^= data[i];
20+ h *= 1099511628211ULL; /* FNV prime */
21+ }
22+ return h;
23+}
24+
25+int main(void) {
26+ unsigned char seed[SEED_LEN];
27+ for (size_t i = 0; i < SEED_LEN; i++)
28+ seed[i] = (unsigned char)(i * 31 + 7);
29+
30+ uint64_t h = 1469598103934665603ULL; /* FNV offset basis */
31+ for (uint32_t r = 0; r < ROUNDS; r++) {
32+ seed[0] = (unsigned char)(r & 0xff);
33+ seed[1] = (unsigned char)((r >> 8) & 0xff);
34+ h = fnv1a(h, seed, SEED_LEN);
35+ }
36+
37+ /* Expected: stable for the fixed seed/rounds above; printed so a runner
38+ * log shows the binary actually executed. */
39+ printf("ci-bench-docker-%012llx\n", (unsigned long long)(h & 0xffffffffffffULL));
40+ return 0;
41+}
new file mode 100644
@@ -0,0 +1,41 @@
1+/*
2+ * main.c — the tiny C program the docker-build workload compiles.
3+ *
4+ * The BUILD is the benchmark (multi-stage, network fetch of build-base, gcc
5+ * invocation, final image assembly). The program itself just does a
6+ * deterministic FNV-1a pass so the binary is non-trivial and its output is
7+ * stable across platforms: ci-bench-docker-a56ee6092483
8+ */
9+
10+#include <stdio.h>
11+#include <stdint.h>
12+#include <string.h>
13+
14+#define ROUNDS 2000000u
15+#define SEED_LEN 64
16+
17+static uint64_t fnv1a(uint64_t h, const unsigned char *data, size_t len) {
18+ for (size_t i = 0; i < len; i++) {
19+ h ^= data[i];
20+ h *= 1099511628211ULL; /* FNV prime */
21+ }
22+ return h;
23+}
24+
25+int main(void) {
26+ unsigned char seed[SEED_LEN];
27+ for (size_t i = 0; i < SEED_LEN; i++)
28+ seed[i] = (unsigned char)(i * 31 + 7);
29+
30+ uint64_t h = 1469598103934665603ULL; /* FNV offset basis */
31+ for (uint32_t r = 0; r < ROUNDS; r++) {
32+ seed[0] = (unsigned char)(r & 0xff);
33+ seed[1] = (unsigned char)((r >> 8) & 0xff);
34+ h = fnv1a(h, seed, SEED_LEN);
35+ }
36+
37+ /* Expected: stable for the fixed seed/rounds above; printed so a runner
38+ * log shows the binary actually executed. */
39+ printf("ci-bench-docker-%012llx\n", (unsigned long long)(h & 0xffffffffffffULL));
40+ return 0;
41+}
added workloads/probe/probe.sh +132 -0
new file mode 100755
@@ -0,0 +1,132 @@
1+#!/usr/bin/env bash
2+# probe.sh — deterministic runner-capability probe. Every measurement degrades
3+# to a "null" value (still a recorded line) when a tool is missing; the job
4+# NEVER fails from the probe itself.
5+#
6+# Measurements:
7+# probe-cpu single-core shell-arithmetic loop throughput (kops/s over a
8+# fixed 1,000,000-iteration loop)
9+# probe-write sequential write throughput (dd 2 GiB to the job scratch
10+# dir, then removed)
11+# probe-read sequential read throughput of that same file
12+# probe-fs-type filesystem type of the scratch dir
13+# probe-fs-free free space of the scratch dir (MB)
14+#
15+# Usage: run from the repo root (the scripts/ dir next to workloads/):
16+# bash workloads/probe/probe.sh
17+
18+set -u
19+
20+REPO_ROOT=$(cd "$(dirname "$0")/../.." && pwd)
21+# shellcheck disable=SC1091
22+. "$REPO_ROOT/scripts/emit_timing.sh"
23+
24+# Scratch dir: the runner's temp when we have one, else mktemp -d.
25+PROBE_DIR=${RUNNER_TEMP:-${TMPDIR:-/tmp}}
26+mkdir -p "$PROBE_DIR" 2>/dev/null || PROBE_DIR=$(mktemp -d)
27+PROBE_BIN=$PROBE_DIR/probe.bin
28+
29+now_s() { # fractional seconds, best available clock
30+ local t
31+ t=$(date -u +%s.%N 2>/dev/null || true)
32+ case $t in
33+ *N) t=$(python3 -c 'import time; print(f"{time.time():.6f}")' 2>/dev/null || echo '') ;;
34+ esac
35+ [ -n "$t" ] || t=$(date +%s).000
36+ printf '%s\n' "$t"
37+}
38+
39+probe_cpu() {
40+ if ! command -v python3 >/dev/null 2>&1 && ! command -v bc >/dev/null 2>&1; then
41+ bench_metric probe-cpu null kops_per_s # no fractional clock available
42+ return 0
43+ fi
44+ local iters=1000000 t0 t1 elapsed
45+ t0=$(now_s)
46+ local i=0
47+ while [ "$i" -lt "$iters" ]; do
48+ i=$((i + 1))
49+ done
50+ t1=$(now_s)
51+ if command -v python3 >/dev/null 2>&1; then
52+ elapsed=$(python3 -c "print(f'{$t1 - $t0:.6f}')")
53+ else
54+ elapsed=$(echo "$t1 - $t0" | bc -l)
55+ fi
56+ bench_metric probe-cpu "$(python3 -c "print(f'{$iters / $elapsed / 1000:.2f}')" 2>/dev/null \
57+ || echo "$iters $elapsed" | awk '{printf "%.2f", $1 / $2 / 1000}')" kops_per_s
58+}
59+
60+# Parse the "bytes ... copied/transferred in N s" line dd writes to stderr.
61+# GNU: "X bytes (...) copied, 3.123 s, ...". BSD: "X bytes transferred in
62+# 3.123456 secs". Prints "BYTES SECONDS" on stdout, nothing when unmatched.
63+# ERE (-E) because BSD sed has no BRE alternation.
64+parse_dd() {
65+ sed -nE 's/^([0-9]+) bytes.*[ ,] ?([0-9.]+) s.*/\1 \2/p'
66+}
67+
68+probe_write() {
69+ command -v dd >/dev/null 2>&1 || { bench_metric probe-write null MB_per_s; return 0; }
70+ local out
71+ out=$(dd if=/dev/zero of="$PROBE_BIN" bs=1M count=2048 2>&1) || true
72+ local parsed
73+ parsed=$(printf '%s\n' "$out" | parse_dd)
74+ if [ -z "$parsed" ]; then
75+ bench_metric probe-write null MB_per_s
76+ return 0
77+ fi
78+ set -- $parsed
79+ bench_metric probe-write "$(awk -v b="$1" -v s="$2" 'BEGIN { printf "%.1f", b / s / 1000000 }')" MB_per_s
80+}
81+
82+probe_read() {
83+ if [ ! -f "$PROBE_BIN" ]; then
84+ bench_metric probe-read null MB_per_s # write probe never produced it
85+ return 0
86+ fi
87+ command -v dd >/dev/null 2>&1 || { bench_metric probe-read null MB_per_s; return 0; }
88+ # Drop the page cache when we can (best effort, usually not permitted);
89+ # otherwise this measures warm cache — still comparable across platforms.
90+ sync 2>/dev/null || true
91+ local out
92+ out=$(dd if="$PROBE_BIN" of=/dev/null bs=1M 2>&1) || true
93+ local parsed
94+ parsed=$(printf '%s\n' "$out" | parse_dd)
95+ if [ -z "$parsed" ]; then
96+ bench_metric probe-read null MB_per_s
97+ return 0
98+ fi
99+ set -- $parsed
100+ bench_metric probe-read "$(awk -v b="$1" -v s="$2" 'BEGIN { printf "%.1f", b / s / 1000000 }')" MB_per_s
101+}
102+
103+probe_fs() {
104+ # Filesystem type: GNU stat -f -c %T, BSD stat -f %T, else null.
105+ local fstype=null
106+ if stat -f -c %T "$PROBE_DIR" >/dev/null 2>&1; then
107+ fstype='"'$(stat -f -c %T "$PROBE_DIR" 2>/dev/null)'"'
108+ elif stat -f %T "$PROBE_DIR" >/dev/null 2>&1; then
109+ fstype='"'$(stat -f %T "$PROBE_DIR" 2>/dev/null | tr -d '"')'"'
110+ fi
111+ bench_metric probe-fs-type "$fstype" fstype
112+
113+ # Free space in MB via df -k (POSIX), else null.
114+ if command -v df >/dev/null 2>&1; then
115+ local free_kb
116+ free_kb=$(df -Pk "$PROBE_DIR" 2>/dev/null | awk 'NR==2 {print $4}') || free_kb=''
117+ case $free_kb in
118+ ''|*[!0-9]*) bench_metric probe-fs-free null MB ;;
119+ *) bench_metric probe-fs-free "$((free_kb / 1024))" MB ;;
120+ esac
121+ else
122+ bench_metric probe-fs-free null MB
123+ fi
124+}
125+
126+probe_cpu
127+probe_write
128+probe_read
129+rm -f "$PROBE_BIN" 2>/dev/null || true
130+probe_fs
131+
132+exit 0
new file mode 100755
@@ -0,0 +1,132 @@
1+#!/usr/bin/env bash
2+# probe.sh — deterministic runner-capability probe. Every measurement degrades
3+# to a "null" value (still a recorded line) when a tool is missing; the job
4+# NEVER fails from the probe itself.
5+#
6+# Measurements:
7+# probe-cpu single-core shell-arithmetic loop throughput (kops/s over a
8+# fixed 1,000,000-iteration loop)
9+# probe-write sequential write throughput (dd 2 GiB to the job scratch
10+# dir, then removed)
11+# probe-read sequential read throughput of that same file
12+# probe-fs-type filesystem type of the scratch dir
13+# probe-fs-free free space of the scratch dir (MB)
14+#
15+# Usage: run from the repo root (the scripts/ dir next to workloads/):
16+# bash workloads/probe/probe.sh
17+
18+set -u
19+
20+REPO_ROOT=$(cd "$(dirname "$0")/../.." && pwd)
21+# shellcheck disable=SC1091
22+. "$REPO_ROOT/scripts/emit_timing.sh"
23+
24+# Scratch dir: the runner's temp when we have one, else mktemp -d.
25+PROBE_DIR=${RUNNER_TEMP:-${TMPDIR:-/tmp}}
26+mkdir -p "$PROBE_DIR" 2>/dev/null || PROBE_DIR=$(mktemp -d)
27+PROBE_BIN=$PROBE_DIR/probe.bin
28+
29+now_s() { # fractional seconds, best available clock
30+ local t
31+ t=$(date -u +%s.%N 2>/dev/null || true)
32+ case $t in
33+ *N) t=$(python3 -c 'import time; print(f"{time.time():.6f}")' 2>/dev/null || echo '') ;;
34+ esac
35+ [ -n "$t" ] || t=$(date +%s).000
36+ printf '%s\n' "$t"
37+}
38+
39+probe_cpu() {
40+ if ! command -v python3 >/dev/null 2>&1 && ! command -v bc >/dev/null 2>&1; then
41+ bench_metric probe-cpu null kops_per_s # no fractional clock available
42+ return 0
43+ fi
44+ local iters=1000000 t0 t1 elapsed
45+ t0=$(now_s)
46+ local i=0
47+ while [ "$i" -lt "$iters" ]; do
48+ i=$((i + 1))
49+ done
50+ t1=$(now_s)
51+ if command -v python3 >/dev/null 2>&1; then
52+ elapsed=$(python3 -c "print(f'{$t1 - $t0:.6f}')")
53+ else
54+ elapsed=$(echo "$t1 - $t0" | bc -l)
55+ fi
56+ bench_metric probe-cpu "$(python3 -c "print(f'{$iters / $elapsed / 1000:.2f}')" 2>/dev/null \
57+ || echo "$iters $elapsed" | awk '{printf "%.2f", $1 / $2 / 1000}')" kops_per_s
58+}
59+
60+# Parse the "bytes ... copied/transferred in N s" line dd writes to stderr.
61+# GNU: "X bytes (...) copied, 3.123 s, ...". BSD: "X bytes transferred in
62+# 3.123456 secs". Prints "BYTES SECONDS" on stdout, nothing when unmatched.
63+# ERE (-E) because BSD sed has no BRE alternation.
64+parse_dd() {
65+ sed -nE 's/^([0-9]+) bytes.*[ ,] ?([0-9.]+) s.*/\1 \2/p'
66+}
67+
68+probe_write() {
69+ command -v dd >/dev/null 2>&1 || { bench_metric probe-write null MB_per_s; return 0; }
70+ local out
71+ out=$(dd if=/dev/zero of="$PROBE_BIN" bs=1M count=2048 2>&1) || true
72+ local parsed
73+ parsed=$(printf '%s\n' "$out" | parse_dd)
74+ if [ -z "$parsed" ]; then
75+ bench_metric probe-write null MB_per_s
76+ return 0
77+ fi
78+ set -- $parsed
79+ bench_metric probe-write "$(awk -v b="$1" -v s="$2" 'BEGIN { printf "%.1f", b / s / 1000000 }')" MB_per_s
80+}
81+
82+probe_read() {
83+ if [ ! -f "$PROBE_BIN" ]; then
84+ bench_metric probe-read null MB_per_s # write probe never produced it
85+ return 0
86+ fi
87+ command -v dd >/dev/null 2>&1 || { bench_metric probe-read null MB_per_s; return 0; }
88+ # Drop the page cache when we can (best effort, usually not permitted);
89+ # otherwise this measures warm cache — still comparable across platforms.
90+ sync 2>/dev/null || true
91+ local out
92+ out=$(dd if="$PROBE_BIN" of=/dev/null bs=1M 2>&1) || true
93+ local parsed
94+ parsed=$(printf '%s\n' "$out" | parse_dd)
95+ if [ -z "$parsed" ]; then
96+ bench_metric probe-read null MB_per_s
97+ return 0
98+ fi
99+ set -- $parsed
100+ bench_metric probe-read "$(awk -v b="$1" -v s="$2" 'BEGIN { printf "%.1f", b / s / 1000000 }')" MB_per_s
101+}
102+
103+probe_fs() {
104+ # Filesystem type: GNU stat -f -c %T, BSD stat -f %T, else null.
105+ local fstype=null
106+ if stat -f -c %T "$PROBE_DIR" >/dev/null 2>&1; then
107+ fstype='"'$(stat -f -c %T "$PROBE_DIR" 2>/dev/null)'"'
108+ elif stat -f %T "$PROBE_DIR" >/dev/null 2>&1; then
109+ fstype='"'$(stat -f %T "$PROBE_DIR" 2>/dev/null | tr -d '"')'"'
110+ fi
111+ bench_metric probe-fs-type "$fstype" fstype
112+
113+ # Free space in MB via df -k (POSIX), else null.
114+ if command -v df >/dev/null 2>&1; then
115+ local free_kb
116+ free_kb=$(df -Pk "$PROBE_DIR" 2>/dev/null | awk 'NR==2 {print $4}') || free_kb=''
117+ case $free_kb in
118+ ''|*[!0-9]*) bench_metric probe-fs-free null MB ;;
119+ *) bench_metric probe-fs-free "$((free_kb / 1024))" MB ;;
120+ esac
121+ else
122+ bench_metric probe-fs-free null MB
123+ fi
124+}
125+
126+probe_cpu
127+probe_write
128+probe_read
129+rm -f "$PROBE_BIN" 2>/dev/null || true
130+probe_fs
131+
132+exit 0
added workloads/rust-build/Cargo.lock +341 -0
new file mode 100644
@@ -0,0 +1,341 @@
1+# This file is automatically @generated by Cargo.
2+# It is not intended for manual editing.
3+version = 4
4+
5+[[package]]
6+name = "aho-corasick"
7+version = "1.1.5"
8+source = "registry+https://github.com/rust-lang/crates.io-index"
9+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
10+dependencies = [
11+ "memchr",
12+]
13+
14+[[package]]
15+name = "anstream"
16+version = "1.0.0"
17+source = "registry+https://github.com/rust-lang/crates.io-index"
18+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
19+dependencies = [
20+ "anstyle",
21+ "anstyle-parse",
22+ "anstyle-query",
23+ "anstyle-wincon",
24+ "colorchoice",
25+ "is_terminal_polyfill",
26+ "utf8parse",
27+]
28+
29+[[package]]
30+name = "anstyle"
31+version = "1.0.14"
32+source = "registry+https://github.com/rust-lang/crates.io-index"
33+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
34+
35+[[package]]
36+name = "anstyle-parse"
37+version = "1.0.0"
38+source = "registry+https://github.com/rust-lang/crates.io-index"
39+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
40+dependencies = [
41+ "utf8parse",
42+]
43+
44+[[package]]
45+name = "anstyle-query"
46+version = "1.1.5"
47+source = "registry+https://github.com/rust-lang/crates.io-index"
48+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
49+dependencies = [
50+ "windows-sys",
51+]
52+
53+[[package]]
54+name = "anstyle-wincon"
55+version = "3.0.11"
56+source = "registry+https://github.com/rust-lang/crates.io-index"
57+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
58+dependencies = [
59+ "anstyle",
60+ "once_cell_polyfill",
61+ "windows-sys",
62+]
63+
64+[[package]]
65+name = "anyhow"
66+version = "1.0.103"
67+source = "registry+https://github.com/rust-lang/crates.io-index"
68+checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
69+
70+[[package]]
71+name = "bench-rust-build"
72+version = "0.1.0"
73+dependencies = [
74+ "anyhow",
75+ "clap",
76+ "regex",
77+ "serde",
78+ "serde_json",
79+ "tokio",
80+]
81+
82+[[package]]
83+name = "bytes"
84+version = "1.12.1"
85+source = "registry+https://github.com/rust-lang/crates.io-index"
86+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
87+
88+[[package]]
89+name = "clap"
90+version = "4.6.1"
91+source = "registry+https://github.com/rust-lang/crates.io-index"
92+checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
93+dependencies = [
94+ "clap_builder",
95+ "clap_derive",
96+]
97+
98+[[package]]
99+name = "clap_builder"
100+version = "4.6.0"
101+source = "registry+https://github.com/rust-lang/crates.io-index"
102+checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
103+dependencies = [
104+ "anstream",
105+ "anstyle",
106+ "clap_lex",
107+ "strsim",
108+]
109+
110+[[package]]
111+name = "clap_derive"
112+version = "4.6.1"
113+source = "registry+https://github.com/rust-lang/crates.io-index"
114+checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
115+dependencies = [
116+ "heck",
117+ "proc-macro2",
118+ "quote",
119+ "syn 2.0.119",
120+]
121+
122+[[package]]
123+name = "clap_lex"
124+version = "1.1.0"
125+source = "registry+https://github.com/rust-lang/crates.io-index"
126+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
127+
128+[[package]]
129+name = "colorchoice"
130+version = "1.0.5"
131+source = "registry+https://github.com/rust-lang/crates.io-index"
132+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
133+
134+[[package]]
135+name = "heck"
136+version = "0.5.0"
137+source = "registry+https://github.com/rust-lang/crates.io-index"
138+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
139+
140+[[package]]
141+name = "is_terminal_polyfill"
142+version = "1.70.2"
143+source = "registry+https://github.com/rust-lang/crates.io-index"
144+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
145+
146+[[package]]
147+name = "itoa"
148+version = "1.0.18"
149+source = "registry+https://github.com/rust-lang/crates.io-index"
150+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
151+
152+[[package]]
153+name = "memchr"
154+version = "2.8.3"
155+source = "registry+https://github.com/rust-lang/crates.io-index"
156+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
157+
158+[[package]]
159+name = "once_cell_polyfill"
160+version = "1.70.2"
161+source = "registry+https://github.com/rust-lang/crates.io-index"
162+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
163+
164+[[package]]
165+name = "pin-project-lite"
166+version = "0.2.17"
167+source = "registry+https://github.com/rust-lang/crates.io-index"
168+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
169+
170+[[package]]
171+name = "proc-macro2"
172+version = "1.0.107"
173+source = "registry+https://github.com/rust-lang/crates.io-index"
174+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
175+dependencies = [
176+ "unicode-ident",
177+]
178+
179+[[package]]
180+name = "quote"
181+version = "1.0.47"
182+source = "registry+https://github.com/rust-lang/crates.io-index"
183+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
184+dependencies = [
185+ "proc-macro2",
186+]
187+
188+[[package]]
189+name = "regex"
190+version = "1.13.1"
191+source = "registry+https://github.com/rust-lang/crates.io-index"
192+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
193+dependencies = [
194+ "aho-corasick",
195+ "memchr",
196+ "regex-automata",
197+ "regex-syntax",
198+]
199+
200+[[package]]
201+name = "regex-automata"
202+version = "0.4.18"
203+source = "registry+https://github.com/rust-lang/crates.io-index"
204+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
205+dependencies = [
206+ "aho-corasick",
207+ "memchr",
208+ "regex-syntax",
209+]
210+
211+[[package]]
212+name = "regex-syntax"
213+version = "0.8.11"
214+source = "registry+https://github.com/rust-lang/crates.io-index"
215+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
216+
217+[[package]]
218+name = "serde"
219+version = "1.0.228"
220+source = "registry+https://github.com/rust-lang/crates.io-index"
221+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
222+dependencies = [
223+ "serde_core",
224+ "serde_derive",
225+]
226+
227+[[package]]
228+name = "serde_core"
229+version = "1.0.228"
230+source = "registry+https://github.com/rust-lang/crates.io-index"
231+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
232+dependencies = [
233+ "serde_derive",
234+]
235+
236+[[package]]
237+name = "serde_derive"
238+version = "1.0.228"
239+source = "registry+https://github.com/rust-lang/crates.io-index"
240+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
241+dependencies = [
242+ "proc-macro2",
243+ "quote",
244+ "syn 2.0.119",
245+]
246+
247+[[package]]
248+name = "serde_json"
249+version = "1.0.150"
250+source = "registry+https://github.com/rust-lang/crates.io-index"
251+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
252+dependencies = [
253+ "itoa",
254+ "memchr",
255+ "serde",
256+ "serde_core",
257+ "zmij",
258+]
259+
260+[[package]]
261+name = "strsim"
262+version = "0.11.1"
263+source = "registry+https://github.com/rust-lang/crates.io-index"
264+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
265+
266+[[package]]
267+name = "syn"
268+version = "2.0.119"
269+source = "registry+https://github.com/rust-lang/crates.io-index"
270+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
271+dependencies = [
272+ "proc-macro2",
273+ "quote",
274+ "unicode-ident",
275+]
276+
277+[[package]]
278+name = "syn"
279+version = "3.0.4"
280+source = "registry+https://github.com/rust-lang/crates.io-index"
281+checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f"
282+dependencies = [
283+ "proc-macro2",
284+ "quote",
285+ "unicode-ident",
286+]
287+
288+[[package]]
289+name = "tokio"
290+version = "1.53.0"
291+source = "registry+https://github.com/rust-lang/crates.io-index"
292+checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
293+dependencies = [
294+ "bytes",
295+ "pin-project-lite",
296+ "tokio-macros",
297+]
298+
299+[[package]]
300+name = "tokio-macros"
301+version = "2.7.2"
302+source = "registry+https://github.com/rust-lang/crates.io-index"
303+checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
304+dependencies = [
305+ "proc-macro2",
306+ "quote",
307+ "syn 3.0.4",
308+]
309+
310+[[package]]
311+name = "unicode-ident"
312+version = "1.0.24"
313+source = "registry+https://github.com/rust-lang/crates.io-index"
314+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
315+
316+[[package]]
317+name = "utf8parse"
318+version = "0.2.2"
319+source = "registry+https://github.com/rust-lang/crates.io-index"
320+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
321+
322+[[package]]
323+name = "windows-link"
324+version = "0.2.1"
325+source = "registry+https://github.com/rust-lang/crates.io-index"
326+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
327+
328+[[package]]
329+name = "windows-sys"
330+version = "0.61.2"
331+source = "registry+https://github.com/rust-lang/crates.io-index"
332+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
333+dependencies = [
334+ "windows-link",
335+]
336+
337+[[package]]
338+name = "zmij"
339+version = "1.0.23"
340+source = "registry+https://github.com/rust-lang/crates.io-index"
341+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
new file mode 100644
@@ -0,0 +1,341 @@
1+# This file is automatically @generated by Cargo.
2+# It is not intended for manual editing.
3+version = 4
4+
5+[[package]]
6+name = "aho-corasick"
7+version = "1.1.5"
8+source = "registry+https://github.com/rust-lang/crates.io-index"
9+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
10+dependencies = [
11+ "memchr",
12+]
13+
14+[[package]]
15+name = "anstream"
16+version = "1.0.0"
17+source = "registry+https://github.com/rust-lang/crates.io-index"
18+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
19+dependencies = [
20+ "anstyle",
21+ "anstyle-parse",
22+ "anstyle-query",
23+ "anstyle-wincon",
24+ "colorchoice",
25+ "is_terminal_polyfill",
26+ "utf8parse",
27+]
28+
29+[[package]]
30+name = "anstyle"
31+version = "1.0.14"
32+source = "registry+https://github.com/rust-lang/crates.io-index"
33+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
34+
35+[[package]]
36+name = "anstyle-parse"
37+version = "1.0.0"
38+source = "registry+https://github.com/rust-lang/crates.io-index"
39+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
40+dependencies = [
41+ "utf8parse",
42+]
43+
44+[[package]]
45+name = "anstyle-query"
46+version = "1.1.5"
47+source = "registry+https://github.com/rust-lang/crates.io-index"
48+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
49+dependencies = [
50+ "windows-sys",
51+]
52+
53+[[package]]
54+name = "anstyle-wincon"
55+version = "3.0.11"
56+source = "registry+https://github.com/rust-lang/crates.io-index"
57+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
58+dependencies = [
59+ "anstyle",
60+ "once_cell_polyfill",
61+ "windows-sys",
62+]
63+
64+[[package]]
65+name = "anyhow"
66+version = "1.0.103"
67+source = "registry+https://github.com/rust-lang/crates.io-index"
68+checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
69+
70+[[package]]
71+name = "bench-rust-build"
72+version = "0.1.0"
73+dependencies = [
74+ "anyhow",
75+ "clap",
76+ "regex",
77+ "serde",
78+ "serde_json",
79+ "tokio",
80+]
81+
82+[[package]]
83+name = "bytes"
84+version = "1.12.1"
85+source = "registry+https://github.com/rust-lang/crates.io-index"
86+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
87+
88+[[package]]
89+name = "clap"
90+version = "4.6.1"
91+source = "registry+https://github.com/rust-lang/crates.io-index"
92+checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
93+dependencies = [
94+ "clap_builder",
95+ "clap_derive",
96+]
97+
98+[[package]]
99+name = "clap_builder"
100+version = "4.6.0"
101+source = "registry+https://github.com/rust-lang/crates.io-index"
102+checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
103+dependencies = [
104+ "anstream",
105+ "anstyle",
106+ "clap_lex",
107+ "strsim",
108+]
109+
110+[[package]]
111+name = "clap_derive"
112+version = "4.6.1"
113+source = "registry+https://github.com/rust-lang/crates.io-index"
114+checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
115+dependencies = [
116+ "heck",
117+ "proc-macro2",
118+ "quote",
119+ "syn 2.0.119",
120+]
121+
122+[[package]]
123+name = "clap_lex"
124+version = "1.1.0"
125+source = "registry+https://github.com/rust-lang/crates.io-index"
126+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
127+
128+[[package]]
129+name = "colorchoice"
130+version = "1.0.5"
131+source = "registry+https://github.com/rust-lang/crates.io-index"
132+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
133+
134+[[package]]
135+name = "heck"
136+version = "0.5.0"
137+source = "registry+https://github.com/rust-lang/crates.io-index"
138+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
139+
140+[[package]]
141+name = "is_terminal_polyfill"
142+version = "1.70.2"
143+source = "registry+https://github.com/rust-lang/crates.io-index"
144+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
145+
146+[[package]]
147+name = "itoa"
148+version = "1.0.18"
149+source = "registry+https://github.com/rust-lang/crates.io-index"
150+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
151+
152+[[package]]
153+name = "memchr"
154+version = "2.8.3"
155+source = "registry+https://github.com/rust-lang/crates.io-index"
156+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
157+
158+[[package]]
159+name = "once_cell_polyfill"
160+version = "1.70.2"
161+source = "registry+https://github.com/rust-lang/crates.io-index"
162+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
163+
164+[[package]]
165+name = "pin-project-lite"
166+version = "0.2.17"
167+source = "registry+https://github.com/rust-lang/crates.io-index"
168+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
169+
170+[[package]]
171+name = "proc-macro2"
172+version = "1.0.107"
173+source = "registry+https://github.com/rust-lang/crates.io-index"
174+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
175+dependencies = [
176+ "unicode-ident",
177+]
178+
179+[[package]]
180+name = "quote"
181+version = "1.0.47"
182+source = "registry+https://github.com/rust-lang/crates.io-index"
183+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
184+dependencies = [
185+ "proc-macro2",
186+]
187+
188+[[package]]
189+name = "regex"
190+version = "1.13.1"
191+source = "registry+https://github.com/rust-lang/crates.io-index"
192+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
193+dependencies = [
194+ "aho-corasick",
195+ "memchr",
196+ "regex-automata",
197+ "regex-syntax",
198+]
199+
200+[[package]]
201+name = "regex-automata"
202+version = "0.4.18"
203+source = "registry+https://github.com/rust-lang/crates.io-index"
204+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
205+dependencies = [
206+ "aho-corasick",
207+ "memchr",
208+ "regex-syntax",
209+]
210+
211+[[package]]
212+name = "regex-syntax"
213+version = "0.8.11"
214+source = "registry+https://github.com/rust-lang/crates.io-index"
215+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
216+
217+[[package]]
218+name = "serde"
219+version = "1.0.228"
220+source = "registry+https://github.com/rust-lang/crates.io-index"
221+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
222+dependencies = [
223+ "serde_core",
224+ "serde_derive",
225+]
226+
227+[[package]]
228+name = "serde_core"
229+version = "1.0.228"
230+source = "registry+https://github.com/rust-lang/crates.io-index"
231+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
232+dependencies = [
233+ "serde_derive",
234+]
235+
236+[[package]]
237+name = "serde_derive"
238+version = "1.0.228"
239+source = "registry+https://github.com/rust-lang/crates.io-index"
240+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
241+dependencies = [
242+ "proc-macro2",
243+ "quote",
244+ "syn 2.0.119",
245+]
246+
247+[[package]]
248+name = "serde_json"
249+version = "1.0.150"
250+source = "registry+https://github.com/rust-lang/crates.io-index"
251+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
252+dependencies = [
253+ "itoa",
254+ "memchr",
255+ "serde",
256+ "serde_core",
257+ "zmij",
258+]
259+
260+[[package]]
261+name = "strsim"
262+version = "0.11.1"
263+source = "registry+https://github.com/rust-lang/crates.io-index"
264+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
265+
266+[[package]]
267+name = "syn"
268+version = "2.0.119"
269+source = "registry+https://github.com/rust-lang/crates.io-index"
270+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
271+dependencies = [
272+ "proc-macro2",
273+ "quote",
274+ "unicode-ident",
275+]
276+
277+[[package]]
278+name = "syn"
279+version = "3.0.4"
280+source = "registry+https://github.com/rust-lang/crates.io-index"
281+checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f"
282+dependencies = [
283+ "proc-macro2",
284+ "quote",
285+ "unicode-ident",
286+]
287+
288+[[package]]
289+name = "tokio"
290+version = "1.53.0"
291+source = "registry+https://github.com/rust-lang/crates.io-index"
292+checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
293+dependencies = [
294+ "bytes",
295+ "pin-project-lite",
296+ "tokio-macros",
297+]
298+
299+[[package]]
300+name = "tokio-macros"
301+version = "2.7.2"
302+source = "registry+https://github.com/rust-lang/crates.io-index"
303+checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
304+dependencies = [
305+ "proc-macro2",
306+ "quote",
307+ "syn 3.0.4",
308+]
309+
310+[[package]]
311+name = "unicode-ident"
312+version = "1.0.24"
313+source = "registry+https://github.com/rust-lang/crates.io-index"
314+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
315+
316+[[package]]
317+name = "utf8parse"
318+version = "0.2.2"
319+source = "registry+https://github.com/rust-lang/crates.io-index"
320+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
321+
322+[[package]]
323+name = "windows-link"
324+version = "0.2.1"
325+source = "registry+https://github.com/rust-lang/crates.io-index"
326+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
327+
328+[[package]]
329+name = "windows-sys"
330+version = "0.61.2"
331+source = "registry+https://github.com/rust-lang/crates.io-index"
332+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
333+dependencies = [
334+ "windows-link",
335+]
336+
337+[[package]]
338+name = "zmij"
339+version = "1.0.23"
340+source = "registry+https://github.com/rust-lang/crates.io-index"
341+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
added workloads/rust-build/Cargo.toml +18 -0
new file mode 100644
@@ -0,0 +1,18 @@
1+[package]
2+name = "bench-rust-build"
3+version = "0.1.0"
4+edition = "2021"
5+description = "Cold/warm build + test workload for the ci-bench CI race benchmark."
6+
7+# Versions are EXACT-pinned (=) so both platforms resolve the identical set; the
8+# generated Cargo.lock is committed alongside and is the real pin.
9+[dependencies]
10+anyhow = "=1.0.103"
11+clap = { version = "=4.6.1", features = ["derive"] }
12+regex = "=1.13.1"
13+serde = { version = "=1.0.228", features = ["derive"] }
14+serde_json = "=1.0.150"
15+tokio = { version = "=1.53.0", features = ["rt-multi-thread", "macros", "time", "sync", "io-util"] }
16+
17+[profile.release]
18+debug = 1
new file mode 100644
@@ -0,0 +1,18 @@
1+[package]
2+name = "bench-rust-build"
3+version = "0.1.0"
4+edition = "2021"
5+description = "Cold/warm build + test workload for the ci-bench CI race benchmark."
6+
7+# Versions are EXACT-pinned (=) so both platforms resolve the identical set; the
8+# generated Cargo.lock is committed alongside and is the real pin.
9+[dependencies]
10+anyhow = "=1.0.103"
11+clap = { version = "=4.6.1", features = ["derive"] }
12+regex = "=1.13.1"
13+serde = { version = "=1.0.228", features = ["derive"] }
14+serde_json = "=1.0.150"
15+tokio = { version = "=1.53.0", features = ["rt-multi-thread", "macros", "time", "sync", "io-util"] }
16+
17+[profile.release]
18+debug = 1
added workloads/rust-build/rust-toolchain.toml +5 -0
new file mode 100644
@@ -0,0 +1,5 @@
1+# Pin the toolchain: identical rustc on every platform is part of the pinned
2+# workload. 1.85.0 is the minimum for clap_builder 4.6.x (edition 2024).
3+[toolchain]
4+channel = "1.85.0"
5+profile = "minimal"
new file mode 100644
@@ -0,0 +1,5 @@
1+# Pin the toolchain: identical rustc on every platform is part of the pinned
2+# workload. 1.85.0 is the minimum for clap_builder 4.6.x (edition 2024).
3+[toolchain]
4+channel = "1.85.0"
5+profile = "minimal"
added workloads/rust-build/src/lib.rs +95 -0
new file mode 100644
@@ -0,0 +1,95 @@
1+//! bench-rust-build: the workload behind the ci-bench "rust build" steps.
2+//!
3+//! It deliberately touches every pinned dependency (serde + serde_json, tokio,
4+//! clap, regex, anyhow) so a cold `cargo build` actually compiles the full
5+//! transitive tree, and `cargo test` does real work on top of the warm build.
6+
7+use anyhow::Result;
8+use clap::Parser;
9+use serde::{Deserialize, Serialize};
10+
11+/// Tunable workload parameters (parsing exercises clap's derive machinery).
12+#[derive(Debug, Clone, Serialize, Deserialize, Parser)]
13+#[command(name = "bench-rust-build", about = "ci-bench synthetic workload")]
14+pub struct Config {
15+ /// Number of synthetic work items per round.
16+ #[arg(long, default_value_t = 5000)]
17+ pub items: u32,
18+
19+ /// Rounds to run.
20+ #[arg(long, default_value_t = 4)]
21+ pub rounds: u32,
22+}
23+
24+impl Default for Config {
25+ fn default() -> Self {
26+ Self::parse_from(["bench-rust-build"])
27+ }
28+}
29+
30+/// A synthetic work item; JSON round-tripping exercises serde on a non-trivial
31+/// type (string + vec + float + u64).
32+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
33+pub struct Item {
34+ pub id: u64,
35+ pub label: String,
36+ pub tags: Vec<String>,
37+ pub score: f64,
38+}
39+
40+/// The label pattern that marks an item "interesting" (exercises regex).
41+pub const LABEL_PATTERN: &str = r"^(cold|warm)-(cache|boot)-[0-9]{4}$";
42+
43+/// Matches labels against [`LABEL_PATTERN`].
44+pub fn interesting(label: &str) -> bool {
45+ // Compiled per call on purpose: the regex crate's compile path is part of
46+ // the exercised cost, and the workload is small enough that it stays cheap.
47+ let re = regex::Regex::new(LABEL_PATTERN).expect("static pattern compiles");
48+ re.is_match(label)
49+}
50+
51+/// Builds one synthetic item.
52+pub fn make_item(round: u32, i: u32) -> Item {
53+ let kind = if i % 2 == 0 { "cold-cache" } else { "warm-boot" };
54+ Item {
55+ id: (u64::from(round) << 32) | u64::from(i),
56+ label: format!("{kind}-{:04}", i % 10_000),
57+ tags: vec!["bench".to_string(), "ci-race".to_string()],
58+ score: f64::from(i) * 0.5,
59+ }
60+}
61+
62+/// One round of synthetic work: build items, JSON round-trip each, select on a
63+/// regex. Returns the number of selected items.
64+pub fn run_round(cfg: &Config, round: u32) -> Result<usize> {
65+ let mut selected = 0usize;
66+ for i in 0..cfg.items {
67+ let item = make_item(round, i);
68+ let json = serde_json::to_string(&item)?;
69+ let back: Item = serde_json::from_str(&json)?;
70+ debug_assert_eq!(back, item);
71+ if interesting(&back.label) {
72+ selected += 1;
73+ }
74+ }
75+ Ok(selected)
76+}
77+
78+/// Tokio entry point: rounds on a multi-threaded runtime, briefly yielding
79+/// between rounds so the scheduler paths are exercised too.
80+pub async fn drive(cfg: Config) -> Result<usize> {
81+ let mut total = 0usize;
82+ for r in 0..cfg.rounds {
83+ total += run_round(&cfg, r)?;
84+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;
85+ }
86+ Ok(total)
87+}
88+
89+/// Synchronous entry point for the binary.
90+pub fn run(cfg: Config) -> Result<usize> {
91+ let rt = tokio::runtime::Builder::new_multi_thread()
92+ .worker_threads(2)
93+ .build()?;
94+ rt.block_on(drive(cfg))
95+}
new file mode 100644
@@ -0,0 +1,95 @@
1+//! bench-rust-build: the workload behind the ci-bench "rust build" steps.
2+//!
3+//! It deliberately touches every pinned dependency (serde + serde_json, tokio,
4+//! clap, regex, anyhow) so a cold `cargo build` actually compiles the full
5+//! transitive tree, and `cargo test` does real work on top of the warm build.
6+
7+use anyhow::Result;
8+use clap::Parser;
9+use serde::{Deserialize, Serialize};
10+
11+/// Tunable workload parameters (parsing exercises clap's derive machinery).
12+#[derive(Debug, Clone, Serialize, Deserialize, Parser)]
13+#[command(name = "bench-rust-build", about = "ci-bench synthetic workload")]
14+pub struct Config {
15+ /// Number of synthetic work items per round.
16+ #[arg(long, default_value_t = 5000)]
17+ pub items: u32,
18+
19+ /// Rounds to run.
20+ #[arg(long, default_value_t = 4)]
21+ pub rounds: u32,
22+}
23+
24+impl Default for Config {
25+ fn default() -> Self {
26+ Self::parse_from(["bench-rust-build"])
27+ }
28+}
29+
30+/// A synthetic work item; JSON round-tripping exercises serde on a non-trivial
31+/// type (string + vec + float + u64).
32+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
33+pub struct Item {
34+ pub id: u64,
35+ pub label: String,
36+ pub tags: Vec<String>,
37+ pub score: f64,
38+}
39+
40+/// The label pattern that marks an item "interesting" (exercises regex).
41+pub const LABEL_PATTERN: &str = r"^(cold|warm)-(cache|boot)-[0-9]{4}$";
42+
43+/// Matches labels against [`LABEL_PATTERN`].
44+pub fn interesting(label: &str) -> bool {
45+ // Compiled per call on purpose: the regex crate's compile path is part of
46+ // the exercised cost, and the workload is small enough that it stays cheap.
47+ let re = regex::Regex::new(LABEL_PATTERN).expect("static pattern compiles");
48+ re.is_match(label)
49+}
50+
51+/// Builds one synthetic item.
52+pub fn make_item(round: u32, i: u32) -> Item {
53+ let kind = if i % 2 == 0 { "cold-cache" } else { "warm-boot" };
54+ Item {
55+ id: (u64::from(round) << 32) | u64::from(i),
56+ label: format!("{kind}-{:04}", i % 10_000),
57+ tags: vec!["bench".to_string(), "ci-race".to_string()],
58+ score: f64::from(i) * 0.5,
59+ }
60+}
61+
62+/// One round of synthetic work: build items, JSON round-trip each, select on a
63+/// regex. Returns the number of selected items.
64+pub fn run_round(cfg: &Config, round: u32) -> Result<usize> {
65+ let mut selected = 0usize;
66+ for i in 0..cfg.items {
67+ let item = make_item(round, i);
68+ let json = serde_json::to_string(&item)?;
69+ let back: Item = serde_json::from_str(&json)?;
70+ debug_assert_eq!(back, item);
71+ if interesting(&back.label) {
72+ selected += 1;
73+ }
74+ }
75+ Ok(selected)
76+}
77+
78+/// Tokio entry point: rounds on a multi-threaded runtime, briefly yielding
79+/// between rounds so the scheduler paths are exercised too.
80+pub async fn drive(cfg: Config) -> Result<usize> {
81+ let mut total = 0usize;
82+ for r in 0..cfg.rounds {
83+ total += run_round(&cfg, r)?;
84+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;
85+ }
86+ Ok(total)
87+}
88+
89+/// Synchronous entry point for the binary.
90+pub fn run(cfg: Config) -> Result<usize> {
91+ let rt = tokio::runtime::Builder::new_multi_thread()
92+ .worker_threads(2)
93+ .build()?;
94+ rt.block_on(drive(cfg))
95+}
added workloads/rust-build/src/main.rs +15 -0
new file mode 100644
@@ -0,0 +1,15 @@
1+//! Thin CLI wrapper so the workload is runnable as a binary, not just a lib.
2+
3+use clap::Parser;
4+
5+use bench_rust_build::{run, Config};
6+
7+fn main() -> anyhow::Result<()> {
8+ let cfg = Config::parse();
9+ let selected = run(cfg.clone())?;
10+ println!(
11+ "items/round={} rounds={} selected={}",
12+ cfg.items, cfg.rounds, selected
13+ );
14+ Ok(())
15+}
new file mode 100644
@@ -0,0 +1,15 @@
1+//! Thin CLI wrapper so the workload is runnable as a binary, not just a lib.
2+
3+use clap::Parser;
4+
5+use bench_rust_build::{run, Config};
6+
7+fn main() -> anyhow::Result<()> {
8+ let cfg = Config::parse();
9+ let selected = run(cfg.clone())?;
10+ println!(
11+ "items/round={} rounds={} selected={}",
12+ cfg.items, cfg.rounds, selected
13+ );
14+ Ok(())
15+}
added workloads/rust-build/tests/workload.rs +51 -0
new file mode 100644
@@ -0,0 +1,51 @@
1+//! Integration tests: real work over the warm build, so `cargo test` is a
2+//! meaningful, separately-timed step (not a no-op).
3+
4+use bench_rust_build::{drive, interesting, make_item, run_round, Config};
5+
6+fn small() -> Config {
7+ Config {
8+ items: 200,
9+ rounds: 2,
10+ }
11+}
12+
13+#[test]
14+fn labels_match_the_documented_pattern() {
15+ assert!(interesting("cold-cache-0001"));
16+ assert!(interesting("warm-boot-9999"));
17+ assert!(!interesting("hot-cache-0001"));
18+ assert!(!interesting("cold-cache-1"));
19+ assert!(!interesting(""));
20+}
21+
22+#[test]
23+fn items_round_trip_through_json() {
24+ let item = make_item(3, 42);
25+ let json = serde_json::to_string(&item).unwrap();
26+ let back: bench_rust_build::Item = serde_json::from_str(&json).unwrap();
27+ assert_eq!(item, back);
28+}
29+
30+#[test]
31+fn a_round_selects_exactly_the_cold_items() {
32+ // items are alternating cold-cache/warm-boot, all matching the pattern;
33+ // "selected" counts matches, so every item is selected.
34+ let cfg = small();
35+ let selected = run_round(&cfg, 0).unwrap();
36+ assert_eq!(selected, cfg.items as usize);
37+}
38+
39+#[tokio::test]
40+async fn the_tokio_runtime_drives_all_rounds() {
41+ let cfg = small();
42+ let selected = drive(cfg.clone()).await.unwrap();
43+ assert_eq!(selected, cfg.items as usize * cfg.rounds as usize);
44+}
45+
46+#[test]
47+fn config_defaults_are_sane() {
48+ let cfg = Config::default();
49+ assert!(cfg.items > 0);
50+ assert!(cfg.rounds > 0);
51+}
new file mode 100644
@@ -0,0 +1,51 @@
1+//! Integration tests: real work over the warm build, so `cargo test` is a
2+//! meaningful, separately-timed step (not a no-op).
3+
4+use bench_rust_build::{drive, interesting, make_item, run_round, Config};
5+
6+fn small() -> Config {
7+ Config {
8+ items: 200,
9+ rounds: 2,
10+ }
11+}
12+
13+#[test]
14+fn labels_match_the_documented_pattern() {
15+ assert!(interesting("cold-cache-0001"));
16+ assert!(interesting("warm-boot-9999"));
17+ assert!(!interesting("hot-cache-0001"));
18+ assert!(!interesting("cold-cache-1"));
19+ assert!(!interesting(""));
20+}
21+
22+#[test]
23+fn items_round_trip_through_json() {
24+ let item = make_item(3, 42);
25+ let json = serde_json::to_string(&item).unwrap();
26+ let back: bench_rust_build::Item = serde_json::from_str(&json).unwrap();
27+ assert_eq!(item, back);
28+}
29+
30+#[test]
31+fn a_round_selects_exactly_the_cold_items() {
32+ // items are alternating cold-cache/warm-boot, all matching the pattern;
33+ // "selected" counts matches, so every item is selected.
34+ let cfg = small();
35+ let selected = run_round(&cfg, 0).unwrap();
36+ assert_eq!(selected, cfg.items as usize);
37+}
38+
39+#[tokio::test]
40+async fn the_tokio_runtime_drives_all_rounds() {
41+ let cfg = small();
42+ let selected = drive(cfg.clone()).await.unwrap();
43+ assert_eq!(selected, cfg.items as usize * cfg.rounds as usize);
44+}
45+
46+#[test]
47+fn config_defaults_are_sane() {
48+ let cfg = Config::default();
49+ assert!(cfg.items > 0);
50+ assert!(cfg.rounds > 0);
51+}