| ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions) 0baf736 Olivier Girardot yesterday | 1 | # ci-bench — cross-platform CI race benchmark (rickub CI vs GitHub Actions). |
| 2 | # bash + python3(stdlib) only; everything degrades gracefully. |
| 3 | |
| 4 | SHELL := /usr/bin/env bash |
| 5 | |
| 6 | PLATFORM ?= local |
| 7 | RUN_ID ?= local-$(shell date -u +%Y%m%dT%H%M%SZ) |
| 8 | |
| 9 | .PHONY: bootstrap run-local collect compare clean |
| 10 | |
| 11 | ## bootstrap — make scripts executable and create results/ (safe to re-run) |
| 12 | bootstrap: |
| 13 | chmod +x scripts/*.sh workloads/probe/probe.sh |
| 14 | mkdir -p results |
| 15 | |
| 16 | ## run-local — run the benchmark step sequence on THIS machine (smoke-test / |
| 17 | ## local baseline). Emits ./results.jsonl, mirroring the CI step order. |
| 18 | run-local: bootstrap |
| 19 | rm -f results.jsonl |
| 20 | set -euo pipefail; \ |
| 21 | export BENCH_RUN_ID="$(RUN_ID)"; \ |
| 22 | source scripts/emit_timing.sh; \ |
| 23 | bench_mark job-start; \ |
| 24 | if command -v cargo >/dev/null 2>&1; then \ |
| 25 | bench_step rust-cold-build bash -ec \ |
| 26 | 'rm -rf workloads/rust-build/target && cargo build --manifest-path workloads/rust-build/Cargo.toml'; \ |
| 27 | bench_step rust-warm-build cargo build --manifest-path workloads/rust-build/Cargo.toml; \ |
| 28 | bench_step rust-test cargo test --manifest-path workloads/rust-build/Cargo.toml; \ |
| 29 | else \ |
| 30 | bench_skip rust-cold-build "cargo not found"; \ |
| 31 | bench_skip rust-warm-build "cargo not found"; \ |
| 32 | bench_skip rust-test "cargo not found"; \ |
| 33 | fi; \ |
| 34 | if command -v docker >/dev/null 2>&1; then \ |
| 35 | bench_step docker-build bash -ec \ |
| 36 | 'docker build -t ci-bench:docker workloads/docker-build && docker run --rm ci-bench:docker'; \ |
| 37 | else \ |
| 38 | bench_skip docker-build "docker not found"; \ |
| 39 | fi; \ |
| 40 | bash workloads/probe/probe.sh; \ |
| 41 | cat results.jsonl |
| 42 | |
| 43 | ## collect — fold a results.jsonl into the results tree: |
| 44 | ## make collect PLATFORM=github RUN_ID=<id> SRC=/path/to/results.jsonl |
| 45 | collect: |
| 46 | @test -n "$(SRC)" || { echo "usage: make collect PLATFORM=<p> RUN_ID=<id> SRC=<results.jsonl>"; exit 2; } |
| 47 | scripts/collect.sh "$(PLATFORM)" "$(RUN_ID)" "$(SRC)" |
| 48 | |
| 49 | ## compare — median/p95 table from everything collected so far |
| 50 | compare: |
| 51 | python3 scripts/compare.py |
| 52 | |
| 53 | ## clean — drop local run outputs (not the collected results/ tree) |
| 54 | clean: |
| 55 | rm -f results.jsonl |
| 56 | rm -rf workloads/rust-build/target |