rickub/ci-bench · compare.py

⑂ 09e7984e3695d820dceab05c61327b2e486c5019

compare.py · 197 lines · 6.9 KBPython    Blame HistoryRaw
ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions)  Olivier Girardot 1#!/usr/bin/env python3
2"""compare.py — median/p95 table per step per platform, from collected runs.
4Reads results/<platform>/<run-id>.json files (produced by scripts/collect.sh).
5python3 stdlib only.
  scripts/compare.py [--push-times FILE]
  - per step x platform: n, median, p95 of duration_ms (status ok only;
    "fail"/"skipped" counted separately and shown)
  - for metric-bearing steps (value field: probe-*), median of the value
  - with --push-times: push -> job-start latency per platform, where push
    timestamps come from FILE: {"<platform>/<run-id>": "<iso8601>", ...}
    (see README "push->job-start" for where each timestamp comes from)
18Percentiles: linear interpolation between order statistics (the method numpy
19percentile uses), reported at p50 (median) and p95.
22import argparse
23import json
24
25from pathlib import Path
27RESULTS_DIR = Path(__file__).resolve().parent.parent / "results"
28JOB_START_STEP = "job-start"
29DURATION_UNITS = {"duration_ms": "ms"}
32def percentile(sorted_vals, q):
  """Linear-interpolation percentile. sorted_vals non-empty, q in [0,100]."""
  if len(sorted_vals) == 1:
      return sorted_vals[0]
  pos = (len(sorted_vals) - 1) * q / 100.0
  lo = int(pos)
  hi = min(lo + 1, len(sorted_vals) - 1)
  frac = pos - lo
  return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac
43def fmt_ms(v):
  return f"{v / 1000.0:8.2f}s" if v >= 1000 else f"{v:8.0f}ms"
47def fmt_v(v):
  if v is None:
      return "    null"
  return f"{v:8.1f}"
53def load_platforms():
  platforms = {}
  if not RESULTS_DIR.is_dir():
      return platforms
  for pdir in sorted(RESULTS_DIR.iterdir()):
      if not pdir.is_dir():
          continue
      runs = []
      for rfile in sorted(pdir.glob("*.json")):
          try:
              with open(rfile, encoding="utf-8") as fh:
                  runs.append(json.load(fh))
          except (json.JSONDecodeError, OSError) as exc:
              print(f"compare: skipping unreadable {rfile}: {exc}", file=sys.stderr)
      if runs:
          platforms[pdir.name] = runs
  return platforms
72def parse_iso(ts):
  from datetime import datetime, timezone
  ts = ts.strip()
  if ts.endswith("Z"):
      ts = ts[:-1] + "+00:00"
  dt = datetime.fromisoformat(ts)
  if dt.tzinfo is None:
      dt = dt.replace(tzinfo=timezone.utc)
  return dt
84def main():
  ap = argparse.ArgumentParser()
  ap.add_argument("--push-times", metavar="FILE",
                  help="JSON map of '<platform>/<run-id>' -> push ISO timestamp")
  args = ap.parse_args()
  platforms = load_platforms()
  if not platforms:
      print("compare: no collected results under results/ — run scripts/collect.sh first")
      return 1
  # step -> platform -> list of (record); plus counters
  steps = {}
  for platform, runs in platforms.items():
      for run in runs:
          for rec in run:
              step = rec.get("step", "?")
              steps.setdefault(step, {}).setdefault(platform, []).append(rec)
  step_names = sorted(steps)
  plat_names = sorted(platforms)
  header = f"{'step':<22}" + "".join(f"{p:>26}" for p in plat_names)
  sub = f"{'':<22}" + "".join(f"{'n / median / p95':>26}" for p in plat_names)
  print(header)
  print(sub)
  print("-" * len(header))
  for step in step_names:
      row = f"{step:<22}"
      for plat in plat_names:
          recs = steps[step].get(plat, [])
          durs = sorted(r["duration_ms"] for r in recs
                        if r.get("status") == "ok" and isinstance(r.get("duration_ms"), (int, float)))
          n_skipped = sum(1 for r in recs if r.get("status") == "skipped")
          n_fail = sum(1 for r in recs if r.get("status") == "fail")
          cell = f"{len(durs):>3} /"
          if durs:
              cell += f" {fmt_ms(percentile(durs, 50))} / {fmt_ms(percentile(durs, 95))}"
          else:
              cell += "        - /        -"
          if n_skipped:
              cell += f" (+{n_skipped}skip)"
          if n_fail:
              cell += f" (+{n_fail}FAIL)"
          row += f"{cell:>26}"
      print(row)
  # Metric-bearing steps: median of value.
  metric_steps = [s for s in step_names
                  if any("value" in r for recs in steps[s].values() for r in recs)]
  if metric_steps:
      print()
      print(f"{'metric (median value)':<22}" + "".join(f"{p:>26}" for p in plat_names))
      print("-" * len(header))
      for step in metric_steps:
          row = f"{step:<22}"
          unit = ""
          for plat in plat_names:
              recs = [r for r in steps[step].get(plat, []) if "value" in r]
              nums = sorted(r["value"] for r in recs
                            if isinstance(r.get("value"), (int, float)))
              strs = [r["value"] for r in recs if isinstance(r.get("value"), str)]
              if nums:
                  unit = next((r.get("unit", "") for r in recs if r.get("unit")), "")
                  cell = f"{percentile(nums, 50):.1f} {unit}"
              elif strs:
                  # categorical metric (e.g. fs type): show the most common value
                  cell = max(set(strs), key=strs.count)[:16]
              else:
                  cell = "null"
              row += f"{cell:>26}"
          print(row)
  # push -> job-start latency.
  if args.push_times:
      print()
      try:
          with open(args.push_times, encoding="utf-8") as fh:
              push_times = json.load(fh)
      except (json.JSONDecodeError, OSError) as exc:
          print(f"compare: cannot read --push-times: {exc}", file=sys.stderr)
          return 1
      row = f"{'push->job-start':<22}"
      for plat in plat_names:
          lat = []
          for run in platforms[plat]:
              run_id = next((r.get("run_id") for r in run if r.get("run_id")), None)
              key = f"{plat}/{run_id}"
              if key not in push_times:
                  continue
              mark = next((r for r in run if r.get("step") == JOB_START_STEP), None)
              if not mark:
                  continue
                  delta = (parse_iso(mark["start"]) - parse_iso(push_times[key])).total_seconds() * 1000
              except (ValueError, KeyError):
                  continue
              if delta >= 0:
                  lat.append(delta)
          cell = f"{len(lat):>3} /"
          cell += (f" {fmt_ms(percentile(sorted(lat), 50))} / {fmt_ms(percentile(sorted(lat), 95))}"
                   if lat else "        - /        -")
          row += f"{cell:>26}"
      print(row + "   (cross-clock; see README caveats)")
  print()
  print("durations = emitted end-start per step; percentiles linear-interpolated;")
  print("report n>=10 per platform before trusting any comparison (README: variance).")
  return 0
196if __name__ == "__main__":
  sys.exit(main())

ci-bench: cross-platform CI race scaffold (rickub CI vs GitHub Actions) 0baf736 Olivier Girardot yesterday	1	#!/usr/bin/env python3
	2	"""compare.py — median/p95 table per step per platform, from collected runs.
	3
	4	Reads results/<platform>/<run-id>.json files (produced by scripts/collect.sh).
	5	python3 stdlib only.
	6
	7	Usage:
	8	scripts/compare.py [--push-times FILE]
	9
	10	Output:
	11	- per step x platform: n, median, p95 of duration_ms (status ok only;
	12	"fail"/"skipped" counted separately and shown)
	13	- for metric-bearing steps (value field: probe-*), median of the value
	14	- with --push-times: push -> job-start latency per platform, where push
	15	timestamps come from FILE: {"<platform>/<run-id>": "<iso8601>", ...}
	16	(see README "push->job-start" for where each timestamp comes from)
	17
	18	Percentiles: linear interpolation between order statistics (the method numpy
	19	percentile uses), reported at p50 (median) and p95.
	20	"""
	21
	22	import argparse
	23	import json
	24	import sys
	25	from pathlib import Path
	26
	27	RESULTS_DIR = Path(__file__).resolve().parent.parent / "results"
	28	JOB_START_STEP = "job-start"
	29	DURATION_UNITS = {"duration_ms": "ms"}
	30
	31
	32	def percentile(sorted_vals, q):
	33	"""Linear-interpolation percentile. sorted_vals non-empty, q in [0,100]."""
	34	if len(sorted_vals) == 1:
	35	return sorted_vals[0]
	36	pos = (len(sorted_vals) - 1) * q / 100.0
	37	lo = int(pos)
	38	hi = min(lo + 1, len(sorted_vals) - 1)
	39	frac = pos - lo
	40	return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * frac
	41
	42
	43	def fmt_ms(v):
	44	return f"{v / 1000.0:8.2f}s" if v >= 1000 else f"{v:8.0f}ms"
	45
	46
	47	def fmt_v(v):
	48	if v is None:
	49	return " null"
	50	return f"{v:8.1f}"
	51
	52
	53	def load_platforms():
	54	platforms = {}
	55	if not RESULTS_DIR.is_dir():
	56	return platforms
	57	for pdir in sorted(RESULTS_DIR.iterdir()):
	58	if not pdir.is_dir():
	59	continue
	60	runs = []
	61	for rfile in sorted(pdir.glob("*.json")):
	62	try:
	63	with open(rfile, encoding="utf-8") as fh:
	64	runs.append(json.load(fh))
65	except (json.JSONDecodeError, OSError) as exc:
66	print(f"compare: skipping unreadable {rfile}: {exc}", file=sys.stderr)
67	if runs:
68	platforms[pdir.name] = runs
69	return platforms
70
71
72	def parse_iso(ts):
73	from datetime import datetime, timezone
74
75	ts = ts.strip()
76	if ts.endswith("Z"):
77	ts = ts[:-1] + "+00:00"
78	dt = datetime.fromisoformat(ts)
79	if dt.tzinfo is None:
80	dt = dt.replace(tzinfo=timezone.utc)
81	return dt
82
83
84	def main():
85	ap = argparse.ArgumentParser()
86	ap.add_argument("--push-times", metavar="FILE",
87	help="JSON map of '<platform>/<run-id>' -> push ISO timestamp")
88	args = ap.parse_args()
89
90	platforms = load_platforms()
91	if not platforms:
92	print("compare: no collected results under results/ — run scripts/collect.sh first")
93	return 1
94
95	# step -> platform -> list of (record); plus counters
96	steps = {}
97	for platform, runs in platforms.items():
98	for run in runs:
99	for rec in run:
100	step = rec.get("step", "?")
101	steps.setdefault(step, {}).setdefault(platform, []).append(rec)
102
103	step_names = sorted(steps)
104	plat_names = sorted(platforms)
105
106	header = f"{'step':<22}" + "".join(f"{p:>26}" for p in plat_names)
107	sub = f"{'':<22}" + "".join(f"{'n / median / p95':>26}" for p in plat_names)
108	print(header)
109	print(sub)
110	print("-" * len(header))
111
112	for step in step_names:
113	row = f"{step:<22}"
114	for plat in plat_names:
115	recs = steps[step].get(plat, [])
116	durs = sorted(r["duration_ms"] for r in recs
117	if r.get("status") == "ok" and isinstance(r.get("duration_ms"), (int, float)))
118	n_skipped = sum(1 for r in recs if r.get("status") == "skipped")
119	n_fail = sum(1 for r in recs if r.get("status") == "fail")
120	cell = f"{len(durs):>3} /"
121	if durs:
122	cell += f" {fmt_ms(percentile(durs, 50))} / {fmt_ms(percentile(durs, 95))}"
123	else:
124	cell += " - / -"
125	if n_skipped:
126	cell += f" (+{n_skipped}skip)"
127	if n_fail:
128	cell += f" (+{n_fail}FAIL)"
129	row += f"{cell:>26}"
130	print(row)
131
132	# Metric-bearing steps: median of value.
133	metric_steps = [s for s in step_names
134	if any("value" in r for recs in steps[s].values() for r in recs)]
135	if metric_steps:
136	print()
137	print(f"{'metric (median value)':<22}" + "".join(f"{p:>26}" for p in plat_names))
138	print("-" * len(header))
139	for step in metric_steps:
140	row = f"{step:<22}"
141	unit = ""
142	for plat in plat_names:
143	recs = [r for r in steps[step].get(plat, []) if "value" in r]
144	nums = sorted(r["value"] for r in recs
145	if isinstance(r.get("value"), (int, float)))
146	strs = [r["value"] for r in recs if isinstance(r.get("value"), str)]
147	if nums:
148	unit = next((r.get("unit", "") for r in recs if r.get("unit")), "")
149	cell = f"{percentile(nums, 50):.1f} {unit}"
150	elif strs:
151	# categorical metric (e.g. fs type): show the most common value
152	cell = max(set(strs), key=strs.count)[:16]
153	else:
154	cell = "null"
155	row += f"{cell:>26}"
156	print(row)
157
158	# push -> job-start latency.
159	if args.push_times:
160	print()
161	try:
162	with open(args.push_times, encoding="utf-8") as fh:
163	push_times = json.load(fh)
164	except (json.JSONDecodeError, OSError) as exc:
165	print(f"compare: cannot read --push-times: {exc}", file=sys.stderr)
166	return 1
167	row = f"{'push->job-start':<22}"
168	for plat in plat_names:
169	lat = []
170	for run in platforms[plat]:
171	run_id = next((r.get("run_id") for r in run if r.get("run_id")), None)
172	key = f"{plat}/{run_id}"
173	if key not in push_times:
174	continue
175	mark = next((r for r in run if r.get("step") == JOB_START_STEP), None)
176	if not mark:
177	continue
178	try:
179	delta = (parse_iso(mark["start"]) - parse_iso(push_times[key])).total_seconds() * 1000
180	except (ValueError, KeyError):
181	continue
182	if delta >= 0:
183	lat.append(delta)
184	cell = f"{len(lat):>3} /"
185	cell += (f" {fmt_ms(percentile(sorted(lat), 50))} / {fmt_ms(percentile(sorted(lat), 95))}"
186	if lat else " - / -")
187	row += f"{cell:>26}"
188	print(row + " (cross-clock; see README caveats)")
189
190	print()
191	print("durations = emitted end-start per step; percentiles linear-interpolated;")
192	print("report n>=10 per platform before trusting any comparison (README: variance).")
193	return 0
194
195
196	if __name__ == "__main__":
197	sys.exit(main())