rickub
Sign in Get started
rickub/ci-benchpublic⑂ Fork 0
⑂ a4751cd
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/rickub/ci-bench.git
git clone ssh://git@rickub.com/rickub/ci-bench.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

bench: real-world tier — ripgrep 15.2.0, sqlite 3.45.1, typescript 5.9.3, guava 33.7.1 (pinned, guarded, both twins)Unverified

Olivier Girardot committed 2026-09-13T11:09:09+02:00 Browse files
a4751cd parent: 7764675
modified .github/workflows/bench.yml +50 -1
@@ -28,7 +28,7 @@ env:
2828 jobs:
2929 bench:
3030 runs-on: ubuntu-latest
31- timeout-minutes: 30
31+ timeout-minutes: 45
3232 steps:
3333 # Pin comment policy mirrors rickub's own ci.yml: SHA-pinned actions.
3434 - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -76,6 +76,55 @@ jobs:
7676 bench_skip docker-build "docker not available on runner"
7777 fi
7878
79+ # --- real-world tier: notorious OSS at pinned tags ----------------------
80+ # Each step clones/downloads its project INSIDE the timed window (network
81+ # is part of a real build) and builds cold — every run is a fresh VM by
82+ # construction. Toolchains are guarded: a runner without the tool records
83+ # "skipped", never fails the race. ClickHouse itself is deliberately not
84+ # here: a full build needs dozens of cores and ~100 GB, which no shared-
85+ # runner tier provides — SQLite's amalgamation stands in for C/C++ (see
86+ # README).
87+
88+ - name: rust real build (ripgrep 15.2.0)
89+ run: |
90+ set -euo pipefail
91+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
92+ if command -v cargo >/dev/null 2>&1; then
93+ bench_step rust-ripgrep bash -ec \
94+ 'rm -rf /tmp/real-ripgrep && git clone -q --depth 1 --branch 15.2.0 https://github.com/BurntSushi/ripgrep /tmp/real-ripgrep && cd /tmp/real-ripgrep && cargo build --release'
95+ else
96+ bench_skip rust-ripgrep "cargo not available on runner"
97+ fi
98+
99+ - name: c/c++ real build (sqlite 3.45.1 amalgamation)
100+ run: |
101+ set -euo pipefail
102+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
103+ bench_step cpp-sqlite bash -ec \
104+ 'rm -rf /tmp/real-sqlite && mkdir -p /tmp/real-sqlite && cd /tmp/real-sqlite && curl -fsSL https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz | tar xz --strip-components=1 && ./configure --disable-tcl && make -j"$(nproc)"'
105+
106+ - name: node real build (typescript v5.9.3)
107+ run: |
108+ set -euo pipefail
109+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
110+ if command -v node >/dev/null 2>&1; then
111+ bench_step node-typescript bash -ec \
112+ 'rm -rf /tmp/real-ts && git clone -q --depth 1 --branch v5.9.3 https://github.com/microsoft/TypeScript /tmp/real-ts && cd /tmp/real-ts && npm ci --no-audit --no-fund && npm run build'
113+ else
114+ bench_skip node-typescript "node not available on runner"
115+ fi
116+
117+ - name: java real build (guava v33.7.1)
118+ run: |
119+ set -euo pipefail
120+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
121+ if command -v mvn >/dev/null 2>&1; then
122+ bench_step java-guava bash -ec \
123+ 'rm -rf /tmp/real-guava && git clone -q --depth 1 --branch v33.7.1 https://github.com/google/guava /tmp/real-guava && cd /tmp/real-guava && mvn -B -q -DskipTests compile'
124+ else
125+ bench_skip java-guava "maven not available on runner"
126+ fi
127+
79128 - name: probe runner
80129 run: bash workloads/probe/probe.sh
81130
@@ -28,7 +28,7 @@ env:
28 jobs:28 jobs:
29 bench:29 bench:
30 runs-on: ubuntu-latest30 runs-on: ubuntu-latest
31- timeout-minutes: 3031+ timeout-minutes: 45
32 steps:32 steps:
33 # Pin comment policy mirrors rickub's own ci.yml: SHA-pinned actions.33 # Pin comment policy mirrors rickub's own ci.yml: SHA-pinned actions.
34 - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v434 - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -76,6 +76,55 @@ jobs:
76 bench_skip docker-build "docker not available on runner"76 bench_skip docker-build "docker not available on runner"
77 fi77 fi
78 78
79+ # --- real-world tier: notorious OSS at pinned tags ----------------------
80+ # Each step clones/downloads its project INSIDE the timed window (network
81+ # is part of a real build) and builds cold — every run is a fresh VM by
82+ # construction. Toolchains are guarded: a runner without the tool records
83+ # "skipped", never fails the race. ClickHouse itself is deliberately not
84+ # here: a full build needs dozens of cores and ~100 GB, which no shared-
85+ # runner tier provides — SQLite's amalgamation stands in for C/C++ (see
86+ # README).
87+
88+ - name: rust real build (ripgrep 15.2.0)
89+ run: |
90+ set -euo pipefail
91+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
92+ if command -v cargo >/dev/null 2>&1; then
93+ bench_step rust-ripgrep bash -ec \
94+ 'rm -rf /tmp/real-ripgrep && git clone -q --depth 1 --branch 15.2.0 https://github.com/BurntSushi/ripgrep /tmp/real-ripgrep && cd /tmp/real-ripgrep && cargo build --release'
95+ else
96+ bench_skip rust-ripgrep "cargo not available on runner"
97+ fi
98+
99+ - name: c/c++ real build (sqlite 3.45.1 amalgamation)
100+ run: |
101+ set -euo pipefail
102+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
103+ bench_step cpp-sqlite bash -ec \
104+ 'rm -rf /tmp/real-sqlite && mkdir -p /tmp/real-sqlite && cd /tmp/real-sqlite && curl -fsSL https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz | tar xz --strip-components=1 && ./configure --disable-tcl && make -j"$(nproc)"'
105+
106+ - name: node real build (typescript v5.9.3)
107+ run: |
108+ set -euo pipefail
109+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
110+ if command -v node >/dev/null 2>&1; then
111+ bench_step node-typescript bash -ec \
112+ 'rm -rf /tmp/real-ts && git clone -q --depth 1 --branch v5.9.3 https://github.com/microsoft/TypeScript /tmp/real-ts && cd /tmp/real-ts && npm ci --no-audit --no-fund && npm run build'
113+ else
114+ bench_skip node-typescript "node not available on runner"
115+ fi
116+
117+ - name: java real build (guava v33.7.1)
118+ run: |
119+ set -euo pipefail
120+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
121+ if command -v mvn >/dev/null 2>&1; then
122+ bench_step java-guava bash -ec \
123+ 'rm -rf /tmp/real-guava && git clone -q --depth 1 --branch v33.7.1 https://github.com/google/guava /tmp/real-guava && cd /tmp/real-guava && mvn -B -q -DskipTests compile'
124+ else
125+ bench_skip java-guava "maven not available on runner"
126+ fi
127+
79 - name: probe runner128 - name: probe runner
80 run: bash workloads/probe/probe.sh129 run: bash workloads/probe/probe.sh
81 130
modified .rickub/workflows/bench.yml +50 -1
@@ -34,7 +34,7 @@ env:
3434 jobs:
3535 bench:
3636 runs-on: large
37- timeout-minutes: 30
37+ timeout-minutes: 45
3838 steps:
3939 - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
4040
@@ -85,6 +85,55 @@ jobs:
8585 bench_skip docker-build "docker not available on runner"
8686 fi
8787
88+ # --- real-world tier: notorious OSS at pinned tags ----------------------
89+ # Each step clones/downloads its project INSIDE the timed window (network
90+ # is part of a real build) and builds cold — every run is a fresh VM by
91+ # construction. Toolchains are guarded: a runner without the tool records
92+ # "skipped", never fails the race. ClickHouse itself is deliberately not
93+ # here: a full build needs dozens of cores and ~100 GB, which no shared-
94+ # runner tier provides — SQLite's amalgamation stands in for C/C++ (see
95+ # README).
96+
97+ - name: rust real build (ripgrep 15.2.0)
98+ run: |
99+ set -euo pipefail
100+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
101+ if command -v cargo >/dev/null 2>&1; then
102+ bench_step rust-ripgrep bash -ec \
103+ 'rm -rf /tmp/real-ripgrep && git clone -q --depth 1 --branch 15.2.0 https://github.com/BurntSushi/ripgrep /tmp/real-ripgrep && cd /tmp/real-ripgrep && cargo build --release'
104+ else
105+ bench_skip rust-ripgrep "cargo not available on runner"
106+ fi
107+
108+ - name: c/c++ real build (sqlite 3.45.1 amalgamation)
109+ run: |
110+ set -euo pipefail
111+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
112+ bench_step cpp-sqlite bash -ec \
113+ 'rm -rf /tmp/real-sqlite && mkdir -p /tmp/real-sqlite && cd /tmp/real-sqlite && curl -fsSL https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz | tar xz --strip-components=1 && ./configure --disable-tcl && make -j"$(nproc)"'
114+
115+ - name: node real build (typescript v5.9.3)
116+ run: |
117+ set -euo pipefail
118+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
119+ if command -v node >/dev/null 2>&1; then
120+ bench_step node-typescript bash -ec \
121+ 'rm -rf /tmp/real-ts && git clone -q --depth 1 --branch v5.9.3 https://github.com/microsoft/TypeScript /tmp/real-ts && cd /tmp/real-ts && npm ci --no-audit --no-fund && npm run build'
122+ else
123+ bench_skip node-typescript "node not available on runner"
124+ fi
125+
126+ - name: java real build (guava v33.7.1)
127+ run: |
128+ set -euo pipefail
129+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
130+ if command -v mvn >/dev/null 2>&1; then
131+ bench_step java-guava bash -ec \
132+ 'rm -rf /tmp/real-guava && git clone -q --depth 1 --branch v33.7.1 https://github.com/google/guava /tmp/real-guava && cd /tmp/real-guava && mvn -B -q -DskipTests compile'
133+ else
134+ bench_skip java-guava "maven not available on runner"
135+ fi
136+
88137 - name: probe runner
89138 run: bash workloads/probe/probe.sh
90139
@@ -34,7 +34,7 @@ env:
34 jobs:34 jobs:
35 bench:35 bench:
36 runs-on: large36 runs-on: large
37- timeout-minutes: 3037+ timeout-minutes: 45
38 steps:38 steps:
39 - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v439 - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
40 40
@@ -85,6 +85,55 @@ jobs:
85 bench_skip docker-build "docker not available on runner"85 bench_skip docker-build "docker not available on runner"
86 fi86 fi
87 87
88+ # --- real-world tier: notorious OSS at pinned tags ----------------------
89+ # Each step clones/downloads its project INSIDE the timed window (network
90+ # is part of a real build) and builds cold — every run is a fresh VM by
91+ # construction. Toolchains are guarded: a runner without the tool records
92+ # "skipped", never fails the race. ClickHouse itself is deliberately not
93+ # here: a full build needs dozens of cores and ~100 GB, which no shared-
94+ # runner tier provides — SQLite's amalgamation stands in for C/C++ (see
95+ # README).
96+
97+ - name: rust real build (ripgrep 15.2.0)
98+ run: |
99+ set -euo pipefail
100+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
101+ if command -v cargo >/dev/null 2>&1; then
102+ bench_step rust-ripgrep bash -ec \
103+ 'rm -rf /tmp/real-ripgrep && git clone -q --depth 1 --branch 15.2.0 https://github.com/BurntSushi/ripgrep /tmp/real-ripgrep && cd /tmp/real-ripgrep && cargo build --release'
104+ else
105+ bench_skip rust-ripgrep "cargo not available on runner"
106+ fi
107+
108+ - name: c/c++ real build (sqlite 3.45.1 amalgamation)
109+ run: |
110+ set -euo pipefail
111+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
112+ bench_step cpp-sqlite bash -ec \
113+ 'rm -rf /tmp/real-sqlite && mkdir -p /tmp/real-sqlite && cd /tmp/real-sqlite && curl -fsSL https://sqlite.org/2024/sqlite-autoconf-3450100.tar.gz | tar xz --strip-components=1 && ./configure --disable-tcl && make -j"$(nproc)"'
114+
115+ - name: node real build (typescript v5.9.3)
116+ run: |
117+ set -euo pipefail
118+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
119+ if command -v node >/dev/null 2>&1; then
120+ bench_step node-typescript bash -ec \
121+ 'rm -rf /tmp/real-ts && git clone -q --depth 1 --branch v5.9.3 https://github.com/microsoft/TypeScript /tmp/real-ts && cd /tmp/real-ts && npm ci --no-audit --no-fund && npm run build'
122+ else
123+ bench_skip node-typescript "node not available on runner"
124+ fi
125+
126+ - name: java real build (guava v33.7.1)
127+ run: |
128+ set -euo pipefail
129+ source "${GITHUB_WORKSPACE:-$PWD}/scripts/emit_timing.sh"
130+ if command -v mvn >/dev/null 2>&1; then
131+ bench_step java-guava bash -ec \
132+ 'rm -rf /tmp/real-guava && git clone -q --depth 1 --branch v33.7.1 https://github.com/google/guava /tmp/real-guava && cd /tmp/real-guava && mvn -B -q -DskipTests compile'
133+ else
134+ bench_skip java-guava "maven not available on runner"
135+ fi
136+
88 - name: probe runner137 - name: probe runner
89 run: bash workloads/probe/probe.sh138 run: bash workloads/probe/probe.sh
90 139
added push-times.json +22 -0
new file mode 100644
@@ -0,0 +1,22 @@
1+{
2+ "github/34747021476-1-0baf736": "2026-09-13T08:10:23Z",
3+ "github/34747558752-1-9990e0f": "2026-09-13T08:23:19.000Z",
4+ "github/34747135670-1-1914eed": "2026-09-13T08:13:05.000Z",
5+ "github/34747186494-1-09e7984": "2026-09-13T08:14:22.000Z",
6+ "github/34747770747-1-35efc9c": "2026-09-13T08:28:27.000Z",
7+ "github/34747992384-1-66a47c3": "2026-09-13T08:33:34.000Z",
8+ "github/34747395033-1-8421335": "2026-09-13T08:19:29.000Z",
9+ "github/34747825179-1-39ba5db": "2026-09-13T08:29:45.000Z",
10+ "github/34747612020-1-fba04eb": "2026-09-13T08:24:37.000Z",
11+ "github/34747344847-1-da2d24b": "2026-09-13T08:18:12.000Z",
12+ "rickub/3-1-09e7984": "2026-09-13T08:15:40.000Z",
13+ "rickub/2-1-1914eed": "2026-09-13T08:11:49.000Z",
14+ "rickub/5-1-8421335": "2026-09-13T08:20:47.000Z",
15+ "rickub/9-1-39ba5db": "2026-09-13T08:31:02.000Z",
16+ "rickub/8-1-35efc9c": "2026-09-13T08:27:10.000Z",
17+ "rickub/4-1-da2d24b": "2026-09-13T08:16:56.000Z",
18+ "rickub/7-1-fba04eb": "2026-09-13T08:25:54.000Z",
19+ "rickub/10-1-66a47c3": "2026-09-13T08:32:18.000Z",
20+ "rickub/6-1-9990e0f": "2026-09-13T08:22:03.000Z",
21+ "rickub/1-1-0baf736": "2026-09-13T08:10:15Z"
22+}
\ No newline at end of file
new file mode 100644
@@ -0,0 +1,22 @@
1+{
2+ "github/34747021476-1-0baf736": "2026-09-13T08:10:23Z",
3+ "github/34747558752-1-9990e0f": "2026-09-13T08:23:19.000Z",
4+ "github/34747135670-1-1914eed": "2026-09-13T08:13:05.000Z",
5+ "github/34747186494-1-09e7984": "2026-09-13T08:14:22.000Z",
6+ "github/34747770747-1-35efc9c": "2026-09-13T08:28:27.000Z",
7+ "github/34747992384-1-66a47c3": "2026-09-13T08:33:34.000Z",
8+ "github/34747395033-1-8421335": "2026-09-13T08:19:29.000Z",
9+ "github/34747825179-1-39ba5db": "2026-09-13T08:29:45.000Z",
10+ "github/34747612020-1-fba04eb": "2026-09-13T08:24:37.000Z",
11+ "github/34747344847-1-da2d24b": "2026-09-13T08:18:12.000Z",
12+ "rickub/3-1-09e7984": "2026-09-13T08:15:40.000Z",
13+ "rickub/2-1-1914eed": "2026-09-13T08:11:49.000Z",
14+ "rickub/5-1-8421335": "2026-09-13T08:20:47.000Z",
15+ "rickub/9-1-39ba5db": "2026-09-13T08:31:02.000Z",
16+ "rickub/8-1-35efc9c": "2026-09-13T08:27:10.000Z",
17+ "rickub/4-1-da2d24b": "2026-09-13T08:16:56.000Z",
18+ "rickub/7-1-fba04eb": "2026-09-13T08:25:54.000Z",
19+ "rickub/10-1-66a47c3": "2026-09-13T08:32:18.000Z",
20+ "rickub/6-1-9990e0f": "2026-09-13T08:22:03.000Z",
21+ "rickub/1-1-0baf736": "2026-09-13T08:10:15Z"
22+}
\ No newline at end of file\ No newline at end of file