856f865cf98d
perf.rs
//! `cargo xtask perf`: run the release benches, merge their `--json` reports into one result keyed
//! by scenario id, and check it against `perf/budgets.toml` and `perf/baseline.json` (#221).
//!
//! ```text
//! cargo xtask perf [--quick] [--update-baseline [--allow-contended]] [--threshold PCT] [--bench NAME]... [--skip-build] [--reuse]
//! cargo xtask perf --update-baseline --from results.json [--from results.json]... [--allow-contended]
//! ```
//!
//! - `--quick`: only the benches with `quick_args` (small synthetic documents; minutes, not tens
//! of minutes). Quick numbers are compared only with a quick baseline, and budgets don't apply
//! (they are set for the full-size documents).
//! - `--update-baseline`: write this run into `perf/baseline.json` (the only way it changes).
//! A contended run (1-minute load at the start above `settings.max_load_per_core` per core) is
//! refused; `--allow-contended` records it, labelled `"contended": true`.
//! - `--from FILE`: with `--update-baseline`, build the baseline from saved `results.json` files
//! (at least `settings.baseline_min_runs`, one machine class and mode, clean trees) instead of
//! running: per scenario the median p50 and the highest (the noise ceiling regressions are
//! judged against). Prints which budgets could be enforced and which enforced ones are flaky.
//! - `--threshold PCT`: the regression threshold (default `settings.regression_pct`, 15 %).
//! - `--bench NAME`: run only these benches (repeatable).
//! - `--skip-build`: don't run `cargo build` first. `--reuse`: don't run the benches either;
//! re-evaluate the reports already in `target/perf/`.
//!
//! Output: `target/perf/results.json` (everything), `target/perf/summary.md` (the table the
//! nightly job posts as its summary) and one `target/perf/<bench>.json` per bench.
//!
//! Exit status: non-zero when an enforced budget breaks (full mode), a scenario's p50 regresses
//! more than the threshold against a baseline from the same machine class and mode (against its
//! noise ceiling when it was built from several runs), or a bench
//! fails. Runs from different machine classes are never compared.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Instant;
use serde::Deserialize;
use serde_json::{Value, json};
pub const BUDGETS: &str = "perf/budgets.toml";
pub const BASELINE: &str = "perf/baseline.json";
const MISSING_PREFIX: &str = "**missing**: ";
// ---- budgets.toml ----------------------------------------------------------------------------
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Budgets {
#[serde(default)]
pub settings: Settings,
#[serde(default)]
pub bench: Vec<BenchSpec>,
#[serde(default)]
pub scenario: Vec<Scenario>,
}
#[derive(Debug, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Settings {
/// A scenario regresses when its p50 grows by more than this share of the baseline…
pub regression_pct: f64,
/// …and by more than this many milliseconds (timer noise on sub-millisecond rows).
pub regression_min_ms: f64,
/// A run is contended when its 1-minute load at the start exceeds this many per core: its
/// numbers measure the machine's load, so `--update-baseline` refuses it.
pub max_load_per_core: f64,
/// `--update-baseline --from` needs at least this many saved runs.
pub baseline_min_runs: usize,
/// A budget could be enforced when its metric stays within this share of the budget in every run.
pub enforce_margin: f64,
}
impl Default for Settings {
fn default() -> Self {
Self { regression_pct: 15.0, regression_min_ms: 1.0, max_load_per_core: 1.0, baseline_min_runs: 5, enforce_margin: 0.6 }
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BenchSpec {
pub name: String,
pub package: String,
/// Example name; defaults to `name`.
pub example: Option<String>,
/// Arguments of a full run (`--json <file>` is appended).
#[serde(default)]
pub args: Vec<String>,
/// Arguments of a `--quick` run; absent = the bench is skipped in quick mode.
pub quick_args: Option<Vec<String>>,
/// Source file that must exist for the bench to run (a bench that lives on another branch):
/// skipped, not failed, while it is missing.
pub requires: Option<String>,
#[serde(default)]
pub note: String,
}
impl BenchSpec {
pub fn example(&self) -> &str {
self.example.as_deref().unwrap_or(&self.name)
}
}
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum Metric {
P50,
#[default]
P95,
Max,
}
impl Metric {
fn label(self) -> &'static str {
match self {
Metric::P50 => "p50",
Metric::P95 => "p95",
Metric::Max => "max",
}
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Scenario {
pub id: String,
pub title: String,
pub issue: Option<u32>,
/// Bench and row name that measure it; both absent when `not_measurable` is set.
pub bench: Option<String>,
pub row: Option<String>,
#[serde(default)]
pub metric: Metric,
pub budget_ms: Option<f64>,
/// Free-form target when it isn't a plain time budget.
pub target: Option<String>,
/// A broken budget fails the run. Off for targets the code doesn't meet yet: they are
/// reported ("over budget") but only regressions fail.
#[serde(default = "yes")]
pub enforce: bool,
/// Why this can't be measured yet (the code doesn't support it).
pub not_measurable: Option<String>,
}
fn yes() -> bool {
true
}
impl Scenario {
pub fn target_text(&self) -> String {
if let Some(t) = &self.target {
return t.clone();
}
match self.budget_ms {
Some(b) if b.fract() == 0.0 => format!("≤ {b:.0} ms {}", self.metric.label()),
Some(b) => format!("≤ {b} ms {}", self.metric.label()),
None => "track only".into(),
}
}
}
pub fn parse_budgets(text: &str) -> Result<Budgets, String> {
let b: Budgets = toml::from_str(text).map_err(|e| format!("{BUDGETS}: {e}"))?;
let mut ids = std::collections::BTreeSet::new();
for s in &b.scenario {
if !ids.insert(s.id.as_str()) {
return Err(format!("{BUDGETS}: duplicate scenario id {}", s.id));
}
match (&s.bench, &s.row, &s.not_measurable) {
(Some(bench), Some(_), None) => {
if !b.bench.iter().any(|x| &x.name == bench) {
return Err(format!("{BUDGETS}: scenario {} uses undefined bench `{bench}`", s.id));
}
}
(None, None, Some(_)) => {}
_ => return Err(format!("{BUDGETS}: scenario {} needs either bench + row, or not_measurable", s.id)),
}
if s.budget_ms.is_some_and(|v| !(v.is_finite() && v > 0.0)) {
return Err(format!("{BUDGETS}: scenario {} has an invalid budget_ms", s.id));
}
}
if !(b.settings.regression_pct.is_finite() && b.settings.regression_pct > 0.0) {
return Err(format!("{BUDGETS}: settings.regression_pct must be > 0"));
}
if !(b.settings.regression_min_ms.is_finite() && b.settings.regression_min_ms >= 0.0) {
return Err(format!("{BUDGETS}: settings.regression_min_ms must be finite and >= 0"));
}
if !(b.settings.max_load_per_core.is_finite() && b.settings.max_load_per_core > 0.0) {
return Err(format!("{BUDGETS}: settings.max_load_per_core must be > 0"));
}
if b.settings.baseline_min_runs == 0 {
return Err(format!("{BUDGETS}: settings.baseline_min_runs must be >= 1"));
}
if !(b.settings.enforce_margin.is_finite() && b.settings.enforce_margin > 0.0 && b.settings.enforce_margin <= 1.0) {
return Err(format!("{BUDGETS}: settings.enforce_margin must be in (0, 1]"));
}
Ok(b)
}
// ---- measurements ----------------------------------------------------------------------------
/// One scenario's numbers.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Stats {
pub n: usize,
pub p50: Option<f64>,
pub p95: Option<f64>,
pub max: Option<f64>,
pub peak_rss: Option<u64>,
pub gpu_bytes: Option<u64>,
/// A baseline built from several runs: the highest p50 among them (its noise ceiling).
pub p50_max: Option<f64>,
}
impl Stats {
pub fn metric(&self, m: Metric) -> Option<f64> {
match m {
Metric::P50 => self.p50,
Metric::P95 => self.p95,
Metric::Max => self.max,
}
}
pub fn to_json(&self) -> Value {
let mut v =
json!({"n": self.n, "p50_ms": self.p50, "p95_ms": self.p95, "max_ms": self.max, "peak_rss_bytes": self.peak_rss, "gpu_bytes": self.gpu_bytes});
if let (Some(m), Some(o)) = (self.p50_max, v.as_object_mut()) {
o.insert("p50_max_ms".into(), json!(m));
}
v
}
pub fn from_json(v: &Value) -> Self {
Self {
n: v.get("n").and_then(Value::as_u64).unwrap_or(0) as usize,
p50: v.get("p50_ms").and_then(Value::as_f64),
p95: v.get("p95_ms").and_then(Value::as_f64),
max: v.get("max_ms").and_then(Value::as_f64),
peak_rss: v.get("peak_rss_bytes").and_then(Value::as_u64),
gpu_bytes: v.get("gpu_bytes").and_then(Value::as_u64),
p50_max: v.get("p50_max_ms").and_then(Value::as_f64),
}
}
}
/// Nearest-rank percentile of sorted, non-empty `v`.
fn rank(v: &[f64], q: f64) -> Option<f64> {
let n = v.len();
if n == 0 {
return None;
}
let k = ((q * n as f64).ceil() as usize).clamp(1, n);
v.get(k - 1).copied()
}
/// A bench row → stats. Rows carry `samples_ms` (preferred: percentiles are recomputed here),
/// or `p50_ms`/`p95_ms`/`max_ms`, or the older `median_ms`/`max_ms`. A p95 is never invented:
/// with neither samples nor a reported p95 it stays empty.
pub fn row_stats(row: &Value) -> Stats {
let mut samples: Vec<f64> =
row.get("samples_ms").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect()).unwrap_or_default();
samples.sort_by(f64::total_cmp);
let f = |k: &str| row.get(k).and_then(Value::as_f64);
let mut s = Stats {
n: samples.len(),
p50: f("p50_ms").or_else(|| f("median_ms")),
p95: f("p95_ms"),
max: f("max_ms"),
peak_rss: row.get("peak_rss_bytes").and_then(Value::as_u64),
gpu_bytes: row.get("gpu_bytes").and_then(Value::as_u64),
p50_max: None,
};
if !samples.is_empty() {
s.p50 = rank(&samples, 0.5);
s.p95 = rank(&samples, 0.95);
s.max = samples.last().copied();
} else if let Some(n) = row.get("n").and_then(Value::as_u64) {
s.n = n as usize;
}
s
}
/// The row named `name` in a bench report (`{"rows": [{"name": …}, …]}`).
pub fn find_row<'a>(report: &'a Value, name: &str) -> Option<&'a Value> {
report.get("rows")?.as_array()?.iter().find(|r| r.get("name").and_then(Value::as_str) == Some(name))
}
/// `cur` regressed against `base` by more than `pct` % and more than `min_ms`.
pub fn regressed(cur: f64, base: f64, pct: f64, min_ms: f64) -> bool {
base > 0.0 && cur.is_finite() && cur > base * (1.0 + pct / 100.0) && cur - base > min_ms
}
pub fn change_pct(cur: f64, base: f64) -> Option<f64> {
(base > 0.0 && cur.is_finite()).then(|| (cur / base - 1.0) * 100.0)
}
#[derive(Clone, Debug, PartialEq)]
pub enum Status {
Pass,
/// Over budget, not enforced (target not met yet).
Over,
/// Over an enforced budget.
Broken,
Regressed,
/// Measured, no budget, no regression.
Tracked,
/// The bench ran but the row is missing or errored.
Missing(String),
NotRun,
NotMeasurable(String),
}
impl Status {
pub fn label(&self) -> String {
match self {
Status::Pass => "pass".into(),
Status::Over => "over budget (target not met yet)".into(),
Status::Broken => "**BUDGET BROKEN**".into(),
Status::Regressed => "**REGRESSED**".into(),
Status::Tracked => "tracked".into(),
Status::Missing(e) => format!("{MISSING_PREFIX}{e}"),
Status::NotRun => "not run".into(),
Status::NotMeasurable(_) => "not measurable yet".into(),
}
}
pub fn fails(&self) -> bool {
matches!(self, Status::Broken | Status::Regressed | Status::Missing(_))
}
}
/// One evaluated scenario.
#[derive(Clone, Debug)]
pub struct Outcome {
pub id: String,
pub title: String,
pub issue: Option<u32>,
pub target: String,
pub stats: Option<Stats>,
pub baseline_p50: Option<f64>,
pub change_pct: Option<f64>,
pub status: Status,
}
/// Budget and regression verdict for one scenario. `budgets_apply` is false in quick mode.
/// A regression is judged against the baseline's noise ceiling (the highest p50 of the runs it was
/// built from), or its p50 when it came from one run.
pub fn verdict(sc: &Scenario, stats: &Stats, base: Option<&Stats>, settings: &Settings, threshold: f64, budgets_apply: bool) -> Status {
if let (Some(cur), Some(b)) = (stats.p50, base.and_then(|b| b.p50_max.or(b.p50)))
&& regressed(cur, b, threshold, settings.regression_min_ms)
{
return Status::Regressed;
}
match (budgets_apply, sc.budget_ms, stats.metric(sc.metric)) {
(true, Some(budget), Some(v)) if v > budget => {
if sc.enforce {
Status::Broken
} else {
Status::Over
}
}
(true, Some(_), Some(_)) => Status::Pass,
(true, Some(_), None) => Status::Missing(format!("no {} in the bench row", sc.metric.label())),
_ => Status::Tracked,
}
}
/// Evaluates every scenario against the bench reports (`bench name → report`, `None` when the
/// bench didn't run in this mode) and the baseline's scenarios for this mode.
pub fn evaluate(
budgets: &Budgets,
reports: &BTreeMap<String, Result<Value, String>>,
baseline: Option<&BTreeMap<String, Stats>>,
threshold: f64,
budgets_apply: bool,
) -> Vec<Outcome> {
budgets
.scenario
.iter()
.map(|sc| {
let base = baseline.and_then(|b| b.get(&sc.id));
let mut o = Outcome {
id: sc.id.clone(),
title: sc.title.clone(),
issue: sc.issue,
target: sc.target_text(),
stats: None,
baseline_p50: base.and_then(|b| b.p50),
change_pct: None,
status: Status::NotRun,
};
if let Some(why) = &sc.not_measurable {
o.status = Status::NotMeasurable(why.clone());
return o;
}
let (Some(bench), Some(row)) = (&sc.bench, &sc.row) else { return o };
match reports.get(bench) {
None => {}
Some(Err(e)) => o.status = Status::Missing(format!("bench `{bench}` failed: {e}")),
Some(Ok(rep)) => match find_row(rep, row) {
None => {
let err = rep
.get("context")
.and_then(|c| c.get("errors"))
.and_then(Value::as_array)
.and_then(|a| a.iter().find(|e| e.get("name").and_then(Value::as_str) == Some(row.as_str())))
.and_then(|e| e.get("error"))
.and_then(Value::as_str)
.map_or_else(|| format!("row `{row}` not in `{bench}`"), str::to_string);
o.status = Status::Missing(err);
}
Some(r) => {
let st = row_stats(r);
o.change_pct = st.p50.zip(o.baseline_p50).and_then(|(c, b)| change_pct(c, b));
o.status = verdict(sc, &st, base, &budgets.settings, threshold, budgets_apply);
o.stats = Some(st);
}
},
}
o
})
.collect()
}
// ---- formatting ------------------------------------------------------------------------------
pub fn fmt_num(v: f64) -> String {
if v >= 100.0 {
format!("{v:.0}")
} else if v >= 10.0 {
format!("{v:.1}")
} else {
format!("{v:.2}")
}
}
fn fmt_ms(v: Option<f64>) -> String {
v.map_or_else(|| "–".into(), fmt_num)
}
pub fn fmt_bytes(v: Option<u64>) -> String {
match v {
None | Some(0) => "–".into(),
Some(b) if b >= 1 << 30 => format!("{:.1} GB", b as f64 / (1u64 << 30) as f64),
Some(b) => format!("{} MB", b >> 20),
}
}
pub fn issue_link(n: Option<u32>) -> String {
n.map_or_else(String::new, |n| format!("[#{n}](https://github.com/storytold/photocraft/issues/{n})"))
}
/// The scenario table (Markdown); `with_baseline` adds the change against the baseline.
/// Not-measurable rows go to a separate list after it.
pub fn render_table(outcomes: &[Outcome], with_baseline: bool) -> String {
let mut s = if with_baseline {
String::from(
"| Id | Scenario | Target | p50 ms | p95 ms | max ms | Peak RSS | GPU | vs baseline | Status | Issue |\n|---|---|---|---:|---:|---:|---:|---:|---:|---|---|\n",
)
} else {
String::from(
"| Id | Scenario | Target | p50 ms | p95 ms | max ms | Peak RSS | GPU | Status | Issue |\n|---|---|---|---:|---:|---:|---:|---:|---|---|\n",
)
};
for o in outcomes.iter().filter(|o| !matches!(o.status, Status::NotMeasurable(_))) {
let st = o.stats.clone().unwrap_or_default();
let vs = if with_baseline { format!(" {} |", o.change_pct.map_or_else(|| "–".into(), |c| format!("{c:+.0} %"))) } else { String::new() };
s.push_str(&format!(
"| {} | {} | {} | {} | {} | {} | {} | {} |{} {} | {} |\n",
o.id,
o.title,
o.target,
fmt_ms(st.p50),
fmt_ms(st.p95),
fmt_ms(st.max),
fmt_bytes(st.peak_rss),
fmt_bytes(st.gpu_bytes),
vs,
o.status.label(),
issue_link(o.issue),
));
}
let nm: Vec<&Outcome> = outcomes.iter().filter(|o| matches!(o.status, Status::NotMeasurable(_))).collect();
if !nm.is_empty() {
s.push_str(
"\n**Not measurable yet** (the code can't do it yet; no number is recorded):\n\n| Id | Scenario | Target | Why | Issue |\n|---|---|---|---|---|\n",
);
for o in nm {
let why = match &o.status {
Status::NotMeasurable(w) => w.as_str(),
_ => "",
};
s.push_str(&format!("| {} | {} | {} | {} | {} |\n", o.id, o.title, o.target, why, issue_link(o.issue)));
}
}
s
}
/// Counts for the scorecard summary: (within budget, over budget or failing, tracked only, not measurable).
/// Not-run scenarios are in none of them; the scorecard counts those separately.
pub fn counts(outcomes: &[Outcome]) -> (usize, usize, usize, usize) {
let mut c = (0, 0, 0, 0);
for o in outcomes {
match o.status {
Status::Pass => c.0 += 1,
Status::Over | Status::Broken | Status::Regressed | Status::Missing(_) => c.1 += 1,
Status::Tracked => c.2 += 1,
Status::NotMeasurable(_) => c.3 += 1,
Status::NotRun => {}
}
}
c
}
// ---- machine ---------------------------------------------------------------------------------
/// Lower-case, `-`-separated, alphanumeric only.
pub fn slug(s: &str) -> String {
let mut out = String::new();
for ch in s.chars() {
if ch.is_ascii_alphanumeric() {
out.push(ch.to_ascii_lowercase());
} else if !out.ends_with('-') && !out.is_empty() {
out.push('-');
}
}
out.trim_end_matches('-').to_string()
}
/// Runs are compared only within one class: OS, architecture, CPU model and GPU adapter.
pub fn machine_class(os: &str, arch: &str, cpu: &str, gpu: &str) -> String {
[os, arch, cpu, gpu].iter().map(|p| slug(p)).filter(|p| !p.is_empty()).collect::<Vec<_>>().join("-")
}
/// The first three numbers in `/proc/loadavg` or `sysctl -n vm.loadavg` output.
pub fn parse_loadavg(s: &str) -> Option<[f64; 3]> {
let v: Vec<f64> = s.split_whitespace().filter_map(|t| t.trim_matches(|c| c == '{' || c == '}').parse().ok()).take(3).collect();
Some([*v.first()?, *v.get(1)?, *v.get(2)?])
}
fn output(cmd: &str, args: &[&str]) -> Option<String> {
let o = Command::new(cmd).args(args).output().ok()?;
o.status.success().then(|| String::from_utf8_lossy(&o.stdout).trim().to_string()).filter(|s| !s.is_empty())
}
pub fn load_avg() -> Option<[f64; 3]> {
if let Ok(s) = std::fs::read_to_string("/proc/loadavg") {
return parse_loadavg(&s);
}
output("sysctl", &["-n", "vm.loadavg"]).and_then(|s| parse_loadavg(&s))
}
fn proc_field(file: &str, key: &str) -> Option<String> {
let s = std::fs::read_to_string(file).ok()?;
s.lines().find(|l| l.starts_with(key)).and_then(|l| l.split_once(':').or_else(|| l.split_once('='))).map(|(_, v)| v.trim().trim_matches('"').to_string())
}
pub fn machine() -> Value {
let (os, arch) = (std::env::consts::OS, std::env::consts::ARCH);
let cores = std::thread::available_parallelism().map(|n| n.get()).ok();
let (cpu, ram, os_version) = match os {
"macos" => (
output("sysctl", &["-n", "machdep.cpu.brand_string"]),
output("sysctl", &["-n", "hw.memsize"]).and_then(|s| s.parse::<u64>().ok()),
output("sw_vers", &["-productVersion"]).map(|v| format!("macOS {v}")),
),
"linux" => (
proc_field("/proc/cpuinfo", "model name").or_else(|| proc_field("/proc/cpuinfo", "CPU part").map(|p| format!("ARM part {p}"))),
proc_field("/proc/meminfo", "MemTotal")
.and_then(|v| v.split_whitespace().next().and_then(|k| k.parse::<u64>().ok()))
.and_then(|k| k.checked_mul(1024)),
proc_field("/etc/os-release", "PRETTY_NAME"),
),
_ => (std::env::var("PROCESSOR_IDENTIFIER").ok(), None, output("cmd", &["/c", "ver"])),
};
json!({"os": os, "os_version": os_version, "arch": arch, "cpu": cpu, "cores": cores, "ram_bytes": ram})
}
/// `YYYY-MM-DDTHH:MM:SSZ` for a Unix time (proleptic Gregorian, UTC). A time past chrono's year
/// range still converts (the calendar repeats every 400 years, so the instant is moved into range
/// by whole cycles and only the year is carried, matching the old hand-rolled `iso_utc`), never a
/// silent reset to the epoch. Accepts the full `u64` range the old code did.
pub fn iso_utc(secs: u64) -> String {
use chrono::Datelike;
// `secs / 86_400` is at most 213_503_982_334_601, so `days` always fits in `i64` (old code too).
let (days, sod) = ((secs / 86_400) as i64, (secs % 86_400) as i64);
let (cycles, rest) = (days.div_euclid(146_097), days.rem_euclid(146_097));
// `rest + 719_163` (1970-01-01 is CE day 719_163) is always a valid CE day, and `sod` < 86_400.
let date = i32::try_from(rest + 719_163).ok().and_then(chrono::NaiveDate::from_num_days_from_ce_opt);
let time = u32::try_from(sod).ok().and_then(|s| chrono::NaiveTime::from_num_seconds_from_midnight_opt(s, 0));
match (date, time) {
(Some(d), Some(t)) => {
let y = i128::from(d.year()) + i128::from(cycles) * 400;
let y = i64::try_from(y).unwrap_or(if y < 0 { i64::MIN } else { i64::MAX });
// `{:04}` rather than `%Y`, which writes a `+` before years past 9999.
format!("{y:04}-{}", d.and_time(t).format("%m-%dT%H:%M:%SZ"))
}
_ => "1970-01-01T00:00:00Z".to_string(), // unreachable
}
}
fn now_iso() -> String {
iso_utc(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0))
}
fn git(root: &Path, args: &[&str]) -> Option<String> {
let o = Command::new("git").current_dir(root).args(args).output().ok()?;
o.status.success().then(|| String::from_utf8_lossy(&o.stdout).trim().to_string())
}
// ---- baseline --------------------------------------------------------------------------------
/// The baseline's scenarios for `mode`, if it was recorded on `class`. `Err` explains a skip.
pub fn baseline_for(baseline: &Value, class: &str, mode: &str) -> Result<BTreeMap<String, Stats>, String> {
let bc = baseline.get("machine_class").and_then(Value::as_str).unwrap_or("");
if bc != class {
return Err(format!("baseline is from machine class `{bc}`, this is `{class}`: regression check skipped"));
}
let sc = baseline
.get("modes")
.and_then(|m| m.get(mode))
.and_then(|m| m.get("scenarios"))
.and_then(Value::as_object)
.ok_or_else(|| format!("baseline has no `{mode}` run: regression check skipped"))?;
Ok(sc.iter().map(|(k, v)| (k.clone(), Stats::from_json(v))).collect())
}
/// A run is contended when its 1-minute load at the start exceeds `max_per_core` per core:
/// its numbers measure the machine's load, not the code.
/// Without a load (Windows) or a core count, nothing can be judged: not contended.
pub fn contended(load: Option<[f64; 3]>, cores: Option<usize>, max_per_core: f64) -> bool {
match (load, cores) {
(Some([one, _, _]), Some(c)) if c > 0 => one > c as f64 * max_per_core,
_ => false,
}
}
fn load_of(v: Option<&Value>) -> Option<[f64; 3]> {
let a = v?.as_array()?;
Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2)?.as_f64()?])
}
fn cores_of(machine: Option<&Value>) -> Option<usize> {
machine?.get("cores")?.as_u64().map(|c| c as usize)
}
/// Whether a `results.json` (or a baseline mode plus its machine) was contended at the start.
pub fn run_contended(load_avg_start: Option<&Value>, machine: Option<&Value>, max_per_core: f64) -> bool {
contended(load_of(load_avg_start), cores_of(machine), max_per_core)
}
/// What `baseline_from_runs` accepts.
pub struct BaselineRules {
/// Fewer runs are refused.
pub min_runs: usize,
pub max_load_per_core: f64,
/// Accept contended runs (the baseline then records `"contended": true`).
pub allow_contended: bool,
}
fn median(v: &mut [f64]) -> Option<f64> {
v.sort_by(f64::total_cmp);
let n = v.len();
match n {
0 => None,
_ if n % 2 == 1 => v.get(n / 2).copied(),
_ => Some((v.get(n / 2 - 1)? + v.get(n / 2)?) / 2.0),
}
}
/// `old` with one mode replaced by a baseline built from `runs` (saved `results.json` values of
/// one machine class and mode), or a fresh baseline when the machine class changed. Per scenario:
/// `p50_ms` is the median of the runs' p50 and, from two runs on, `p50_max_ms` their maximum (the
/// noise ceiling regressions are judged against); p95 is the median, max and memory the maximum.
/// One run is the plain `--update-baseline` case. Refused: too few runs, mixed machine classes or
/// modes, dirty trees when there are several runs, and contended runs unless allowed.
pub fn baseline_from_runs(old: Option<&Value>, runs: &[Value], rules: &BaselineRules) -> Result<Value, String> {
if runs.len() < rules.min_runs.max(1) {
return Err(format!("a baseline needs at least {} run(s), got {}", rules.min_runs.max(1), runs.len()));
}
let class_of = |r: &Value| r.get("machine").and_then(|m| m.get("class")).cloned().unwrap_or(Value::Null);
let mode_of = |r: &Value| r.get("mode").and_then(Value::as_str).unwrap_or("full").to_string();
let mut runs: Vec<&Value> = runs.iter().collect();
runs.sort_by_key(|r| r.get("date").and_then(Value::as_str).unwrap_or("").to_string());
let Some(last) = runs.last().copied() else { return Err("no runs".into()) };
let (class, mode) = (class_of(last), mode_of(last));
if runs.iter().any(|r| class_of(r) != class) {
return Err("the runs are from different machine classes: a baseline needs one".into());
}
if runs.iter().any(|r| mode_of(r) != mode) {
return Err("the runs mix full and quick modes: a baseline needs one".into());
}
if runs.len() > 1
&& let Some(r) = runs.iter().find(|r| r.get("dirty").and_then(Value::as_bool) == Some(true))
{
return Err(format!("run {} was taken on a dirty tree: a baseline needs committed code", r.get("date").and_then(Value::as_str).unwrap_or("?")));
}
let is_contended = |r: &Value| run_contended(r.get("load_avg_start"), r.get("machine"), rules.max_load_per_core);
let any_contended = runs.iter().any(|r| is_contended(r));
if any_contended && !rules.allow_contended {
let msgs: Vec<String> = runs
.iter()
.filter(|r| is_contended(r))
.map(|r| {
format!(
"load {} on {} cores at the start of {}",
load_of(r.get("load_avg_start")).map_or_else(|| "?".into(), |l| format!("{:.2}", l[0])),
cores_of(r.get("machine")).map_or_else(|| "?".into(), |c| c.to_string()),
r.get("date").and_then(Value::as_str).unwrap_or("the run")
)
})
.collect();
return Err(format!(
"contended run, baseline not written: {} (above {} per core). Its numbers measure machine load; rerun on an idle machine, or pass --allow-contended to record it labelled as contended",
msgs.join("; "),
rules.max_load_per_core
));
}
let mut per: BTreeMap<String, Vec<Stats>> = BTreeMap::new();
let mut failed = serde_json::Map::new();
for r in &runs {
for (k, v) in r.get("scenarios").and_then(Value::as_object).into_iter().flatten() {
if v.get("p50_ms").is_some_and(|x| !x.is_null()) {
per.entry(k.clone()).or_default().push(Stats::from_json(v));
} else if let Some(e) = v.get("status").and_then(Value::as_str).and_then(|s| s.strip_prefix(MISSING_PREFIX)) {
// A crashed row (a crash the bench found): kept so the scorecard says so.
failed.insert(k.clone(), json!(e));
}
}
}
let mut scenarios = serde_json::Map::new();
for (k, st) in &per {
// A row that crashed in any run stays failed, and one missing from some runs is left
// out: no partial aggregate under a full run count.
if failed.contains_key(k) || st.len() != runs.len() {
continue;
}
let mut p50: Vec<f64> = st.iter().filter_map(|s| s.p50).collect();
let mut p95: Vec<f64> = st.iter().filter_map(|s| s.p95).collect();
let fmax = |f: fn(&Stats) -> Option<f64>| st.iter().filter_map(f).reduce(f64::max);
let umax = |f: fn(&Stats) -> Option<u64>| st.iter().filter_map(f).max();
let s = Stats {
n: st.iter().map(|s| s.n).sum(),
p50_max: (runs.len() > 1).then(|| fmax(|s| s.p50)).flatten(),
p50: median(&mut p50),
p95: median(&mut p95),
max: fmax(|s| s.max),
peak_rss: umax(|s| s.peak_rss),
gpu_bytes: umax(|s| s.gpu_bytes),
};
scenarios.insert(k.clone(), s.to_json());
}
let mut modes = old.filter(|o| o.get("machine_class") == Some(&class)).and_then(|o| o.get("modes")).and_then(Value::as_object).cloned().unwrap_or_default();
let mut entry = json!({
"date": last.get("date"),
"commit": last.get("commit"),
"load_avg_start": last.get("load_avg_start"),
"load_avg_end": last.get("load_avg_end"),
"cores": cores_of(last.get("machine")),
"runs": runs.len(),
"scenarios": scenarios,
"failed": failed,
});
if let Some(o) = entry.as_object_mut() {
if any_contended {
o.insert("contended".into(), json!(true));
}
if runs.len() > 1 {
let each: Vec<Value> = runs
.iter()
.map(|r| json!({"date": r.get("date"), "commit": r.get("commit"), "load_avg_start": r.get("load_avg_start"), "contended": is_contended(r)}))
.collect();
o.insert("run_list".into(), json!(each));
}
}
modes.insert(mode, entry);
Ok(json!({"schema": 1, "machine_class": class, "machine": last.get("machine"), "modes": modes}))
}
/// From saved runs: budgets that could be enforced (not enforced, and within `enforce_margin` of
/// the budget in every run) and enforced budgets that went over in some run (flaky).
pub fn enforce_report(budgets: &Budgets, runs: &[Value]) -> (Vec<String>, Vec<String>) {
let (mut could, mut flaky) = (Vec::new(), Vec::new());
let margin = budgets.settings.enforce_margin;
for sc in &budgets.scenario {
let (Some(budget), None) = (sc.budget_ms, &sc.not_measurable) else { continue };
let key = format!("{}_ms", sc.metric.label());
let vals: Vec<f64> = runs.iter().filter_map(|r| r.get("scenarios")?.get(&sc.id)?.get(&key)?.as_f64()).collect();
let Some(worst) = vals.iter().copied().reduce(f64::max) else { continue };
let m = sc.metric.label();
if sc.enforce {
let over = vals.iter().filter(|v| **v > budget).count();
if over > 0 {
flaky.push(format!("{} {}: over {} ms in {over} of {} runs (worst {m} {})", sc.id, sc.title, fmt_num(budget), runs.len(), fmt_num(worst)));
}
} else if vals.len() == runs.len() && worst <= margin * budget {
could.push(format!("{} {}: worst {m} {} ms ≤ {margin} × {} ms in all {} runs", sc.id, sc.title, fmt_num(worst), fmt_num(budget), runs.len()));
}
}
(could, flaky)
}
// ---- running ---------------------------------------------------------------------------------
struct Opts {
quick: bool,
update_baseline: bool,
threshold: Option<f64>,
benches: Vec<String>,
skip_build: bool,
reuse: bool,
/// `--update-baseline --from`: build the baseline from these saved `results.json` files.
from: Vec<PathBuf>,
allow_contended: bool,
}
fn parse_opts(rest: &[&str]) -> Result<Opts, String> {
let mut o = Opts {
quick: false,
update_baseline: false,
threshold: None,
benches: Vec::new(),
skip_build: false,
reuse: false,
from: Vec::new(),
allow_contended: false,
};
let mut it = rest.iter();
while let Some(a) = it.next() {
match *a {
"--quick" => o.quick = true,
"--update-baseline" => o.update_baseline = true,
"--skip-build" => o.skip_build = true,
"--reuse" => {
o.reuse = true;
o.skip_build = true;
}
"--threshold" => {
let v = it.next().and_then(|v| v.parse::<f64>().ok()).filter(|v| v.is_finite() && *v > 0.0).ok_or("--threshold needs a percentage > 0")?;
o.threshold = Some(v);
}
"--bench" => o.benches.push(it.next().ok_or("--bench needs a name")?.to_string()),
"--from" => o.from.push(PathBuf::from(it.next().ok_or("--from needs a results.json path")?)),
"--allow-contended" => o.allow_contended = true,
other => return Err(format!("perf: unknown option `{other}`")),
}
}
if !o.from.is_empty() && !o.update_baseline {
return Err("--from builds a baseline: use it with --update-baseline".into());
}
Ok(o)
}
fn target_dir() -> Result<PathBuf, String> {
let m = crate::metadata()?;
m.get("target_directory").and_then(Value::as_str).map(PathBuf::from).ok_or_else(|| "cargo metadata: no target_directory".into())
}
fn write(path: &Path, text: &str) -> Result<(), String> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir).map_err(|e| format!("create {}: {e}", dir.display()))?;
}
std::fs::write(path, text).map_err(|e| format!("write {}: {e}", path.display()))
}
pub fn run(root: &Path, rest: &[&str]) -> Result<(), String> {
let o = parse_opts(rest)?;
let text = std::fs::read_to_string(root.join(BUDGETS)).map_err(|e| format!("read {BUDGETS}: {e}"))?;
let budgets = parse_budgets(&text)?;
for name in &o.benches {
if !budgets.bench.iter().any(|b| &b.name == name) {
return Err(format!("perf: unknown bench `{name}` (see {BUDGETS})"));
}
}
let baseline_value: Option<Value> = std::fs::read_to_string(root.join(BASELINE)).ok().and_then(|t| serde_json::from_str(&t).ok());
if !o.from.is_empty() {
return baseline_from_files(root, &budgets, baseline_value.as_ref(), &o);
}
let mode = if o.quick { "quick" } else { "full" };
let threshold = o.threshold.unwrap_or(budgets.settings.regression_pct);
let target = target_dir()?;
let out_dir = target.join("perf");
// Which benches run in this mode.
let mut plan: Vec<(&BenchSpec, Vec<String>)> = Vec::new();
let mut skipped: Vec<(String, String)> = Vec::new();
for b in &budgets.bench {
if !o.benches.is_empty() && !o.benches.contains(&b.name) {
continue;
}
let args = if o.quick {
match &b.quick_args {
Some(a) => a.clone(),
None => continue,
}
} else {
b.args.clone()
};
if let Some(req) = &b.requires
&& !root.join(req).exists()
{
skipped.push((b.name.clone(), format!("{req} not on this branch yet")));
continue;
}
plan.push((b, args));
}
// Sampled before the build: the harness's own compile isn't contention. The load after the
// build is still recorded, as the first bench's `load_avg_before`.
let load_start = load_avg();
if !o.skip_build {
let mut by_pkg: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
for (b, _) in &plan {
by_pkg.entry(b.package.as_str()).or_default().push(b.example());
}
for (pkg, examples) in by_pkg {
let mut c = crate::cargo();
c.args(["build", "--release", "-p", pkg]);
for e in &examples {
c.args(["--example", e]);
}
crate::run(c, &format!("cargo build --release -p {pkg} --example {}", examples.join(" --example ")))?;
}
}
let mut reports: BTreeMap<String, Result<Value, String>> = BTreeMap::new();
let mut bench_meta = serde_json::Map::new();
for (b, args) in &plan {
let json_path = out_dir.join(format!("{}.json", b.name));
let before = load_avg();
let t = Instant::now();
let result = if o.reuse {
std::fs::read_to_string(&json_path).map_err(|e| format!("--reuse: {}: {e}", json_path.display()))
} else {
let _ = std::fs::remove_file(&json_path);
if let Some(dir) = json_path.parent() {
std::fs::create_dir_all(dir).map_err(|e| format!("create {}: {e}", dir.display()))?;
}
let exe = target.join("release").join("examples").join(format!("{}{}", b.example(), std::env::consts::EXE_SUFFIX));
let mut c = Command::new(&exe);
c.current_dir(root).args(args).arg("--json").arg(&json_path);
let shown = format!("{} {} --json {}", b.example(), args.join(" "), json_path.display());
crate::run(c, &shown).and_then(|()| std::fs::read_to_string(&json_path).map_err(|e| format!("{}: no JSON report ({e})", b.name)))
};
let parsed = result.and_then(|t| serde_json::from_str::<Value>(&t).map_err(|e| format!("{}: bad JSON: {e}", b.name)));
let after = load_avg();
bench_meta.insert(
b.name.clone(),
json!({
"status": if parsed.is_ok() { "ok" } else { "error" },
"note": b.note,
"error": parsed.as_ref().err(),
"seconds": t.elapsed().as_secs_f64(),
"args": args,
"load_avg_before": before,
"load_avg_after": after,
"context": parsed.as_ref().ok().and_then(|r| r.get("context")).cloned(),
"process_peak_rss_bytes": parsed.as_ref().ok().and_then(|r| r.get("process_peak_rss_bytes")).cloned(),
// Every row, mapped to a scenario or not (new benches show up here first).
"rows": parsed.as_ref().ok().and_then(|r| r.get("rows")).and_then(Value::as_array).map(|rows| {
rows.iter()
.map(|r| {
let mut v = row_stats(r).to_json();
if let Some(o) = v.as_object_mut() {
o.insert("name".into(), r.get("name").cloned().unwrap_or(Value::Null));
}
v
})
.collect::<Vec<_>>()
}),
}),
);
reports.insert(b.name.clone(), parsed);
}
for (name, why) in &skipped {
bench_meta.insert(name.clone(), json!({"status": "skipped", "error": why}));
}
let load_end = load_avg();
let mut m = machine();
let gpu = reports
.values()
.filter_map(|r| r.as_ref().ok())
.find_map(|r| r.get("context").and_then(|c| c.get("gpu_adapter")).and_then(Value::as_str).map(str::to_string));
let gpu_name = gpu.clone().unwrap_or_else(|| "cpu-canvas".into());
let class = machine_class(
m.get("os").and_then(Value::as_str).unwrap_or(""),
m.get("arch").and_then(Value::as_str).unwrap_or(""),
m.get("cpu").and_then(Value::as_str).unwrap_or(""),
&gpu_name,
);
if let Some(obj) = m.as_object_mut() {
obj.insert("gpu_adapter".into(), json!(gpu));
obj.insert("class".into(), json!(class));
}
let (baseline, baseline_note) = match baseline_value.as_ref().map(|b| baseline_for(b, &class, mode)) {
Some(Ok(b)) => (Some(b), None),
Some(Err(e)) => (None, Some(e)),
None => (None, Some(format!("no {BASELINE}: regression check skipped"))),
};
let outcomes = evaluate(&budgets, &reports, baseline.as_ref(), threshold, !o.quick);
let scenarios: serde_json::Map<String, Value> = budgets
.scenario
.iter()
.zip(&outcomes)
.map(|(sc, oc)| {
let mut v = oc.stats.as_ref().map(Stats::to_json).unwrap_or_else(|| json!({}));
if let Some(obj) = v.as_object_mut() {
obj.insert("title".into(), json!(sc.title));
obj.insert("issue".into(), json!(sc.issue));
obj.insert("bench".into(), json!(sc.bench));
obj.insert("row".into(), json!(sc.row));
obj.insert("metric".into(), json!(sc.metric.label()));
obj.insert("budget_ms".into(), json!(sc.budget_ms));
obj.insert("target".into(), json!(sc.target_text()));
obj.insert("enforce".into(), json!(sc.enforce));
obj.insert("baseline_p50_ms".into(), json!(oc.baseline_p50));
obj.insert("change_pct".into(), json!(oc.change_pct));
obj.insert("status".into(), json!(oc.status.label()));
if let Status::NotMeasurable(w) = &oc.status {
obj.insert("not_measurable".into(), json!(w));
}
}
(sc.id.clone(), v)
})
.collect();
let failures: Vec<String> = outcomes.iter().filter(|o| o.status.fails()).map(|o| format!("{} {}: {}", o.id, o.title, o.status.label())).collect();
let bench_failures: Vec<String> = reports.iter().filter_map(|(n, r)| r.as_ref().err().map(|e| format!("bench {n}: {e}"))).collect();
let commit = git(root, &["rev-parse", "HEAD"]);
let dirty = git(root, &["status", "--porcelain"]).is_some_and(|s| !s.is_empty());
let is_contended = contended(load_start, m.get("cores").and_then(Value::as_u64).map(|c| c as usize), budgets.settings.max_load_per_core);
let results = json!({
"schema": 1,
"mode": mode,
"date": now_iso(),
"commit": commit,
"dirty": dirty,
"machine": m,
"load_avg_start": load_start,
"load_avg_end": load_end,
"contended": is_contended,
"threshold_pct": threshold,
"baseline_note": baseline_note,
"benches": bench_meta,
"scenarios": scenarios,
"failures": failures.iter().chain(&bench_failures).collect::<Vec<_>>(),
});
let text = serde_json::to_string_pretty(&results).map_err(|e| format!("encode results: {e}"))?;
write(&out_dir.join("results.json"), &text)?;
let summary = render_summary(&results, &outcomes, baseline_note.as_deref());
write(&out_dir.join("summary.md"), &summary)?;
println!("\n{summary}");
println!("wrote {}", out_dir.join("results.json").display());
let mut errors: Vec<String> = Vec::new();
if o.update_baseline {
// The results are written either way; a contended run never silently becomes the baseline.
let rules = BaselineRules { min_runs: 1, max_load_per_core: budgets.settings.max_load_per_core, allow_contended: o.allow_contended };
match baseline_from_runs(baseline_value.as_ref(), std::slice::from_ref(&results), &rules) {
Ok(nb) => {
write_baseline(root, &nb)?;
println!("updated {BASELINE} ({mode}, machine class {class}{})", if is_contended { ", labelled contended" } else { "" });
}
// A refused baseline still reports this run's perf failures below.
Err(e) => errors.push(e),
}
}
let all: Vec<&String> = failures.iter().chain(&bench_failures).collect();
if !all.is_empty() {
errors.push(format!("{} perf failure(s):\n {}", all.len(), all.iter().map(|s| s.as_str()).collect::<Vec<_>>().join("\n ")));
}
if errors.is_empty() { Ok(()) } else { Err(errors.join("\n")) }
}
fn write_baseline(root: &Path, baseline: &Value) -> Result<(), String> {
let text = serde_json::to_string_pretty(baseline).map_err(|e| format!("encode baseline: {e}"))?;
write(&root.join(BASELINE), &format!("{text}\n"))
}
/// `--update-baseline --from <results.json>...`: no benches run. Prints the enforcement report.
fn baseline_from_files(root: &Path, budgets: &Budgets, old: Option<&Value>, o: &Opts) -> Result<(), String> {
let runs = o
.from
.iter()
.map(|p| {
let t = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?;
serde_json::from_str::<Value>(&t).map_err(|e| format!("{}: {e}", p.display()))
})
.collect::<Result<Vec<_>, _>>()?;
let (could, flaky) = enforce_report(budgets, &runs);
let list = |v: &[String]| if v.is_empty() { " none".to_string() } else { v.iter().map(|l| format!(" {l}")).collect::<Vec<_>>().join("\n") };
println!("Budgets that could be enforced (within {} of the budget in every run):\n{}", budgets.settings.enforce_margin, list(&could));
println!("Enforced budgets that went over in some run (flaky):\n{}", list(&flaky));
let rules = BaselineRules {
min_runs: budgets.settings.baseline_min_runs,
max_load_per_core: budgets.settings.max_load_per_core,
allow_contended: o.allow_contended,
};
let nb = baseline_from_runs(old, &runs, &rules)?;
write_baseline(root, &nb)?;
println!("updated {BASELINE} from {} runs (machine class {})", runs.len(), nb.get("machine_class").and_then(Value::as_str).unwrap_or("?"));
Ok(())
}
/// The committed baseline's full-mode numbers judged against the budgets (what the scorecard
/// shows: deterministic, no run needed). Scenarios without a baseline number are "not run".
pub fn outcomes_from_baseline(budgets: &Budgets, baseline: Option<&Value>) -> Vec<Outcome> {
let mode = baseline.and_then(|b| b.get("modes")?.get("full"));
let full: Option<BTreeMap<String, Stats>> =
mode.and_then(|m| m.get("scenarios")?.as_object().cloned()).map(|m| m.iter().map(|(k, v)| (k.clone(), Stats::from_json(v))).collect());
let failed = |id: &str| mode.and_then(|m| m.get("failed")?.get(id)?.as_str()).map(str::to_string);
budgets
.scenario
.iter()
.map(|sc| {
let stats = full.as_ref().and_then(|f| f.get(&sc.id)).cloned();
let status = match (&sc.not_measurable, &stats) {
(Some(w), _) => Status::NotMeasurable(w.clone()),
(None, Some(st)) => verdict(sc, st, None, &budgets.settings, budgets.settings.regression_pct, true),
(None, None) => failed(&sc.id).map_or(Status::NotRun, Status::Missing),
};
Outcome {
id: sc.id.clone(),
title: sc.title.clone(),
issue: sc.issue,
target: sc.target_text(),
stats,
baseline_p50: None,
change_pct: None,
status,
}
})
.collect()
}
pub fn fmt_load(v: Option<&Value>) -> String {
v.and_then(Value::as_array).map_or_else(|| "n/a".into(), |a| a.iter().filter_map(Value::as_f64).map(|x| format!("{x:.2}")).collect::<Vec<_>>().join(" / "))
}
/// The Markdown summary of one run: context lines and the scenario table.
pub fn render_summary(results: &Value, outcomes: &[Outcome], baseline_note: Option<&str>) -> String {
let m = results.get("machine").cloned().unwrap_or(Value::Null);
let s = |v: Option<&Value>| v.and_then(Value::as_str).unwrap_or("unknown").to_string();
let ram = m.get("ram_bytes").and_then(Value::as_u64).map_or_else(|| "?".into(), |b| format!("{} GB", b >> 30));
let mut out = format!(
"## Performance ({} run)\n\n- Machine: {} ({} cores, {ram}), GPU {}, {} {}; class `{}`\n- Commit {}{}, {}\n- Load average (1/5/15 min): {} at start, {} at end\n",
s(results.get("mode")),
s(m.get("cpu")),
m.get("cores").and_then(Value::as_u64).unwrap_or(0),
s(m.get("gpu_adapter")),
s(m.get("os_version")),
s(m.get("arch")),
s(m.get("class")),
s(results.get("commit")),
if results.get("dirty").and_then(Value::as_bool) == Some(true) { " (dirty)" } else { "" },
s(results.get("date")),
fmt_load(results.get("load_avg_start")),
fmt_load(results.get("load_avg_end")),
);
if results.get("contended").and_then(Value::as_bool) == Some(true) {
out.push_str("- **contended run**: numbers measure machine load (the 1-minute load at the start was above `max_load_per_core` per core)\n");
}
if let Some(n) = baseline_note {
out.push_str(&format!("- {n}\n"));
}
if results.get("mode").and_then(Value::as_str) == Some("quick") {
out.push_str("- Quick mode: small documents, budgets not applied (they are set for full-size documents).\n");
}
if let Some(b) = results.get("benches").and_then(Value::as_object) {
for (name, v) in b {
let st = v.get("status").and_then(Value::as_str).unwrap_or("?");
let secs = v.get("seconds").and_then(Value::as_f64).map_or_else(String::new, |x| format!(", {x:.0} s"));
let err = v.get("error").and_then(Value::as_str).map_or_else(String::new, |e| format!(": {e}"));
out.push_str(&format!("- Bench `{name}`: {st}{secs}, load {}{err}\n", fmt_load(v.get("load_avg_before"))));
}
}
out.push('\n');
out.push_str(&render_table(outcomes, true));
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn iso_utc_handles_boundaries() {
assert_eq!(iso_utc(0), "1970-01-01T00:00:00Z");
assert_eq!(iso_utc(1_791_446_357), "2026-10-08T07:59:17Z");
// Years past 9999 keep the plain zero-padded year (no `+`).
assert_eq!(iso_utc(253_402_300_800), "10000-01-01T00:00:00Z");
// A value past chrono's range still converts via the 400-year cycle (the old hand-rolled
// `iso_utc` did too): pin that output, never the epoch.
assert_eq!(iso_utc(10_000_000_000_000), "318857-05-20T17:46:40Z");
assert_eq!(iso_utc(u64::MAX), "584554051223-11-09T07:00:15Z");
}
const TOML: &str = r#"
[settings]
regression_pct = 15.0
regression_min_ms = 0.5
[[bench]]
name = "b"
package = "p"
quick_args = ["--quick"]
[[scenario]]
id = "P1"
title = "move"
issue = 209
bench = "b"
row = "move row"
budget_ms = 25
[[scenario]]
id = "P2"
title = "opacity"
bench = "b"
row = "opacity row"
budget_ms = 20
enforce = false
[[scenario]]
id = "P3"
title = "zoom"
target = "≤ 16 ms per frame"
not_measurable = "needs a frame harness"
"#;
fn report(rows: Value) -> Value {
json!({"bench": "b", "rows": rows})
}
#[test]
fn budgets_parse_and_validate() {
let b = parse_budgets(TOML).unwrap();
assert_eq!(b.scenario.len(), 3);
assert_eq!(b.scenario[0].metric, Metric::P95);
assert!(b.scenario[0].enforce);
assert!(!b.scenario[1].enforce);
assert_eq!(b.scenario[0].target_text(), "≤ 25 ms p95");
assert_eq!(b.scenario[2].target_text(), "≤ 16 ms per frame");
assert_eq!(b.settings.regression_min_ms, 0.5);
// Duplicate ids, undefined benches, half-specified rows and bad budgets are rejected.
assert!(parse_budgets(&format!("{TOML}\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nnot_measurable = \"y\"\n")).is_err());
assert!(parse_budgets("[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"nope\"\nrow = \"r\"\n").is_err());
assert!(parse_budgets("[[bench]]\nname = \"b\"\npackage = \"p\"\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"b\"\n").is_err());
assert!(
parse_budgets("[[bench]]\nname = \"b\"\npackage = \"p\"\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"b\"\nrow = \"r\"\nbudget_ms = -1\n")
.is_err()
);
assert!(parse_budgets("[settings]\nregression_pct = 0\n").is_err());
assert!(parse_budgets("[settings]\nregression_min_ms = -1\n").is_err());
assert!(parse_budgets("[settings]\nregression_min_ms = 0\n").is_ok());
assert!(parse_budgets("bogus = 1\n").is_err());
assert!(parse_budgets("not toml [").is_err());
}
#[test]
fn row_stats_prefers_samples_and_never_invents_p95() {
let s = row_stats(&json!({"name": "x", "samples_ms": [5.0, 1.0, 3.0, 2.0, 4.0], "p50_ms": 99.0}));
assert_eq!((s.n, s.p50, s.p95, s.max), (5, Some(3.0), Some(5.0), Some(5.0)));
// Older rows: median + max only. p95 stays empty.
let s = row_stats(&json!({"name": "x", "median_ms": 4.0, "min_ms": 1.0, "max_ms": 9.0}));
assert_eq!((s.p50, s.p95, s.max), (Some(4.0), None, Some(9.0)));
let s = row_stats(&json!({"name": "x", "samples_ms": ["bad", null]}));
assert_eq!(s.p50, None);
}
#[test]
fn regression_detection() {
assert!(!regressed(11.4, 10.0, 15.0, 0.5));
assert!(regressed(11.6, 10.0, 15.0, 0.5));
// Below the absolute noise floor: no regression.
assert!(!regressed(0.30, 0.20, 15.0, 0.5));
assert!(!regressed(5.0, 0.0, 15.0, 0.5));
assert!(!regressed(f64::NAN, 1.0, 15.0, 0.0));
assert_eq!(change_pct(12.0, 10.0).map(|v| v.round()), Some(20.0));
assert_eq!(change_pct(1.0, 0.0), None);
}
#[test]
fn evaluate_budgets_regressions_and_missing_rows() {
let b = parse_budgets(TOML).unwrap();
let mut reports = BTreeMap::new();
reports.insert(
"b".to_string(),
Ok(report(json!([{"name": "move row", "samples_ms": [10.0, 12.0, 30.0]}, {"name": "opacity row", "samples_ms": [40.0]}]))),
);
let out = evaluate(&b, &reports, None, 15.0, true);
assert_eq!(out[0].status, Status::Broken, "p95 30 > 25 and enforced");
assert_eq!(out[1].status, Status::Over, "not enforced: reported, doesn't fail");
assert!(matches!(out[2].status, Status::NotMeasurable(_)));
assert!(out[0].status.fails() && !out[1].status.fails() && !out[2].status.fails());
// Quick mode: no budgets, only regressions.
let mut base = BTreeMap::new();
base.insert("P1".to_string(), Stats { p50: Some(10.0), ..Default::default() });
base.insert("P2".to_string(), Stats { p50: Some(30.0), ..Default::default() });
let out = evaluate(&b, &reports, Some(&base), 15.0, false);
assert_eq!(out[0].status, Status::Regressed, "p50 12 vs 10 is +20 %");
assert_eq!(out[1].status, Status::Regressed, "p50 40 vs 30");
let out = evaluate(&b, &reports, Some(&base), 50.0, false);
assert_eq!(out[0].status, Status::Tracked, "threshold is configurable");
// A row the bench didn't produce fails, with the bench's own error when it has one.
let mut reports = BTreeMap::new();
reports.insert("b".to_string(), Ok(json!({"rows": [], "context": {"errors": [{"name": "move row", "error": "boom"}]}})));
let out = evaluate(&b, &reports, None, 15.0, true);
assert_eq!(out[0].status, Status::Missing("boom".into()));
assert!(matches!(&out[1].status, Status::Missing(e) if e.contains("not in")));
// A bench that didn't run in this mode: not run, not a failure.
let out = evaluate(&b, &BTreeMap::new(), None, 15.0, true);
assert_eq!(out[0].status, Status::NotRun);
assert!(!out[0].status.fails());
assert_eq!(counts(&out), (0, 0, 0, 1), "not-run scenarios are not tracked; the scorecard counts them separately");
}
fn rules(min_runs: usize, allow_contended: bool) -> BaselineRules {
BaselineRules { min_runs, max_load_per_core: 1.0, allow_contended }
}
/// The plain `--update-baseline` case: one run.
fn one(old: Option<&Value>, results: &Value) -> Value {
baseline_from_runs(old, std::slice::from_ref(results), &rules(1, false)).unwrap()
}
/// A saved run of class `c` with one p50 (and p95) per scenario.
fn run_of(date: &str, load: f64, rows: &[(&str, f64)]) -> Value {
let sc: serde_json::Map<String, Value> =
rows.iter().map(|(k, v)| (k.to_string(), json!({"n": 5, "p50_ms": v, "p95_ms": v * 1.5, "max_ms": v * 2.0}))).collect();
json!({"mode": "full", "date": date, "commit": format!("c-{date}"), "dirty": false, "machine": {"class": "c", "cores": 3}, "load_avg_start": [load, load, load], "scenarios": sc})
}
#[test]
fn contention_is_judged_on_start_load_per_core() {
assert!(contended(Some([224.9, 226.31, 196.03]), Some(14), 1.0), "the committed 2026-10-05 baseline");
assert!(!contended(Some([0.5, 3.0, 9.0]), Some(12), 1.0), "only the 1-minute load counts");
assert!(!contended(None, Some(12), 1.0), "no load (Windows): can't judge");
assert!(!contended(Some([50.0, 0.0, 0.0]), None, 1.0));
assert!(!contended(Some([9.5, 0.0, 0.0]), Some(3), 4.0), "the threshold is configurable");
assert!(contended(Some([9.5, 0.0, 0.0]), Some(3), 3.0));
assert!(run_contended(Some(&json!([224.9, 226.31, 196.03])), Some(&json!({"cores": 14})), 1.0));
}
#[test]
fn update_baseline_refuses_contended_run_unless_allowed() {
let mut r = run_of("2026-10-05", 224.9, &[("P1", 3.0)]);
r["machine"]["cores"] = json!(14);
let e = baseline_from_runs(None, std::slice::from_ref(&r), &rules(1, false)).unwrap_err();
assert!(e.contains("224.90") && e.contains("14 cores") && e.contains("--allow-contended"), "{e}");
let b = baseline_from_runs(None, std::slice::from_ref(&r), &rules(1, true)).unwrap();
let m = &b["modes"]["full"];
assert_eq!(m["contended"], json!(true));
assert_eq!(m["cores"], json!(14));
assert_eq!(m["runs"], json!(1));
// A quiet run isn't labelled, and one run has no noise ceiling.
let b = one(None, &run_of("d", 0.5, &[("P1", 3.0)]));
assert!(b["modes"]["full"].get("contended").is_none());
assert!(b["modes"]["full"]["scenarios"]["P1"].get("p50_max_ms").is_none());
}
#[test]
fn baseline_from_runs_takes_median_and_noise_ceiling() {
let runs: Vec<Value> = [(1, 2174.0, 30.5), (2, 2868.0, 45.3), (3, 2000.0, 28.0), (4, 2500.0, 31.0), (5, 2300.0, 40.0)]
.iter()
.map(|(d, a, b)| run_of(&format!("2026-10-2{d}"), 1.0, &[("P44", *a), ("P46", *b)]))
.collect();
let b = baseline_from_runs(None, &runs, &rules(5, false)).unwrap();
let m = &b["modes"]["full"];
assert_eq!(m["runs"], json!(5));
assert_eq!(m["date"], json!("2026-10-25"), "the latest run dates the baseline");
assert_eq!(m["run_list"].as_array().map(Vec::len), Some(5));
assert_eq!(m["run_list"][0]["load_avg_start"], json!([1.0, 1.0, 1.0]));
assert_eq!(m["run_list"][0]["contended"], json!(false));
let got = baseline_for(&b, "c", "full").unwrap();
let p44 = &got["P44"];
assert_eq!((p44.p50, p44.p50_max, p44.n), (Some(2300.0), Some(2868.0), 25));
assert_eq!(got["P46"].p50, Some(31.0));
assert_eq!(median(&mut [4.0, 1.0, 3.0, 2.0]), Some(2.5));
}
#[test]
fn baseline_from_runs_keeps_a_row_failed_in_any_run() {
let mut runs: Vec<Value> = (1..=3).map(|d| run_of(&format!("2026-10-2{d}"), 1.0, &[("P1", 3.0), ("P2", 5.0)])).collect();
runs[1]["scenarios"]["P1"] = json!({"status": format!("{MISSING_PREFIX}bench crashed")});
let b = baseline_from_runs(None, &runs, &rules(3, false)).unwrap();
let m = &b["modes"]["full"];
assert!(m["scenarios"].get("P1").is_none(), "no partial aggregate: {m}");
assert_eq!(m["failed"]["P1"], json!("bench crashed"));
assert_eq!(m["scenarios"]["P2"]["p50_ms"], json!(5.0));
assert!(m["failed"].get("P2").is_none());
}
#[test]
fn baseline_from_runs_leaves_out_a_row_missing_from_some_runs() {
let mut runs: Vec<Value> = (1..=3).map(|d| run_of(&format!("2026-10-2{d}"), 1.0, &[("P1", 3.0), ("P2", 5.0)])).collect();
runs[0]["scenarios"].as_object_mut().unwrap().remove("P1");
let b = baseline_from_runs(None, &runs, &rules(3, false)).unwrap();
let m = &b["modes"]["full"];
assert!(m["scenarios"].get("P1").is_none(), "2 of 3 runs is no baseline: {m}");
assert_eq!(m["scenarios"]["P2"]["p50_ms"], json!(5.0));
assert_eq!(m["runs"], json!(3));
}
#[test]
fn baseline_from_runs_rejects_mixed_classes_modes_dirty_and_too_few() {
let runs: Vec<Value> = (1..=5).map(|d| run_of(&format!("2026-10-2{d}"), 1.0, &[("P1", 3.0)])).collect();
assert!(baseline_from_runs(None, &runs, &rules(5, false)).is_ok());
assert!(baseline_from_runs(None, &runs[..4], &rules(5, false)).unwrap_err().contains("at least 5"));
let mut m = runs.clone();
m[2]["machine"]["class"] = json!("other");
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("machine classes"));
let mut m = runs.clone();
m[2]["mode"] = json!("quick");
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("modes"));
let mut m = runs.clone();
m[2]["dirty"] = json!(true);
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("dirty"));
let mut m = runs.clone();
m[2]["load_avg_start"] = json!([9.5, 9.0, 9.0]);
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("contended"));
assert_eq!(baseline_from_runs(None, &m, &rules(5, true)).unwrap()["modes"]["full"]["run_list"][2]["contended"], json!(true));
}
#[test]
fn regression_is_judged_against_the_noise_ceiling() {
// Runner numbers (macos-15, 7 nightlies): median and highest p50 per row.
let b = parse_budgets(TOML).unwrap();
let s = &b.settings;
let sc = &b.scenario[1]; // no enforced budget in play: opacity, budget 20, not enforced
let base = |p50: f64, max: f64| Stats { p50: Some(p50), p50_max: Some(max), ..Default::default() };
let cur = |p50: f64| Stats { p50: Some(p50), ..Default::default() };
assert_eq!(verdict(sc, &cur(3935.0), Some(&base(2174.0, 2868.0)), s, 15.0, false), Status::Regressed, "P44 +81 % (#5694)");
assert_eq!(verdict(sc, &cur(76.9), Some(&base(30.5, 45.3)), s, 15.0, false), Status::Regressed, "P46 +152 % (#5697)");
assert_eq!(verdict(sc, &cur(387.0), Some(&base(258.0, 448.0)), s, 15.0, false), Status::Tracked, "P11 +50 % (#5695) is within runner noise");
// A single-run baseline falls back to its p50.
let single = Stats { p50: Some(258.0), ..Default::default() };
assert_eq!(verdict(sc, &cur(387.0), Some(&single), s, 15.0, false), Status::Regressed);
}
#[test]
fn enforce_report_needs_margin_in_every_run() {
let toml = r#"
[[bench]]
name = "b"
package = "p"
[[scenario]]
id = "P16"
title = "dab"
bench = "b"
row = "r16"
budget_ms = 100
enforce = false
[[scenario]]
id = "P29"
title = "wand"
bench = "b"
row = "r29"
budget_ms = 100
enforce = false
[[scenario]]
id = "P30"
title = "liquify"
bench = "b"
row = "r30"
budget_ms = 100
enforce = false
[[scenario]]
id = "P25"
title = "cancel"
bench = "b"
row = "r25"
budget_ms = 200
[[scenario]]
id = "P99"
title = "track"
bench = "b"
row = "r99"
"#;
let b = parse_budgets(toml).unwrap();
// run_of stores p95 = 1.5 × p50 (the default metric is p95).
let runs: Vec<Value> = [(30.0, 20.0, 30.0, 100.0), (35.2, 22.0, 45.0, 110.0), (20.0, 22.0, 30.0, 199.4)]
.iter()
.enumerate()
.map(|(i, (a, c, d, e))| run_of(&format!("d{i}"), 1.0, &[("P16", *a), ("P29", *c), ("P30", *d), ("P25", *e), ("P99", 1.0)]))
.collect();
let (could, flaky) = enforce_report(&b, &runs);
assert_eq!(could.len(), 2, "{could:?}");
assert!(could[0].starts_with("P16 ") && could[0].contains("52.8"), "{could:?}");
assert!(could[1].starts_with("P29 "));
assert!(!could.iter().any(|l| l.starts_with("P30")), "67.5 > 60 once: not suggested");
assert_eq!(flaky.len(), 1, "{flaky:?}");
assert!(flaky[0].starts_with("P25 ") && flaky[0].contains("in 1 of 3 runs"), "{flaky:?}");
// A row missing from one run can't be vouched for.
let mut gap = runs.clone();
if let Some(o) = gap[1]["scenarios"].as_object_mut() {
o.remove("P29");
}
assert!(!enforce_report(&b, &gap).0.iter().any(|l| l.starts_with("P29")));
}
#[test]
fn baselines_never_cross_machine_classes() {
let results = json!({"mode": "full", "date": "d", "commit": "c", "machine": {"class": "macos-aarch64-m4"}, "scenarios": {"P1": {"p50_ms": 3.0, "p95_ms": 4.0, "max_ms": 5.0, "n": 3}, "P3": {"status": "not measurable yet"}}});
let b = one(None, &results);
let got = baseline_for(&b, "macos-aarch64-m4", "full").unwrap();
assert_eq!(got.get("P1").and_then(|s| s.p50), Some(3.0));
assert!(!got.contains_key("P3"), "rows without numbers are not baselined");
assert!(baseline_for(&b, "linux-x86-64-epyc", "full").is_err());
assert!(baseline_for(&b, "macos-aarch64-m4", "quick").is_err());
// Same class: the other mode is kept. New class: replaced.
let quick = json!({"mode": "quick", "machine": {"class": "macos-aarch64-m4"}, "scenarios": {"P1": {"p50_ms": 1.0}}});
let b2 = one(Some(&b), &quick);
assert!(baseline_for(&b2, "macos-aarch64-m4", "full").is_ok());
assert!(baseline_for(&b2, "macos-aarch64-m4", "quick").is_ok());
// A crashed row is kept as a failure, and the scorecard shows it.
let crashed = json!({"mode": "full", "machine": {"class": "c"}, "scenarios": {"P1": {"status": "**missing**: panicked: boom"}}});
let bc = one(None, &crashed);
let b = parse_budgets(TOML).unwrap();
let o = outcomes_from_baseline(&b, Some(&bc));
assert_eq!(o[0].status, Status::Missing("panicked: boom".into()));
assert_eq!(o[1].status, Status::NotRun);
let other = json!({"mode": "quick", "machine": {"class": "other"}, "scenarios": {}});
let b3 = one(Some(&b2), &other);
assert!(baseline_for(&b3, "other", "full").is_err());
}
#[test]
fn machine_helpers() {
assert_eq!(slug("Apple M4 Pro (Metal)"), "apple-m4-pro-metal");
assert_eq!(machine_class("macos", "aarch64", "Apple M4 Pro", "Apple M4 Pro"), "macos-aarch64-apple-m4-pro-apple-m4-pro");
assert_eq!(parse_loadavg("{ 2.10 1.50 0.75 }"), Some([2.1, 1.5, 0.75]));
assert_eq!(parse_loadavg("0.52 0.58 0.59 1/467 12345\n"), Some([0.52, 0.58, 0.59]));
assert_eq!(parse_loadavg("garbage"), None);
assert_eq!(iso_utc(0), "1970-01-01T00:00:00Z");
assert_eq!(iso_utc(1_791_158_400), "2026-10-05T00:00:00Z");
assert_eq!(iso_utc(951_782_400), "2000-02-29T00:00:00Z");
}
#[test]
fn table_lists_not_measurable_rows_separately() {
let b = parse_budgets(TOML).unwrap();
let mut reports = BTreeMap::new();
reports.insert("b".to_string(), Ok(report(json!([{"name": "move row", "samples_ms": [10.0], "peak_rss_bytes": 3u64 << 30}]))));
let t = render_table(&evaluate(&b, &reports, None, 15.0, true), true);
assert!(t.contains("| P1 | move | ≤ 25 ms p95 | 10.0 | 10.0 | 10.0 | 3.0 GB |"));
assert!(t.contains("**Not measurable yet**"));
assert!(t.contains("| P3 | zoom | ≤ 16 ms per frame | needs a frame harness | |"));
assert_eq!(counts(&evaluate(&b, &reports, None, 15.0, true)), (1, 1, 0, 1), "the row the bench lacks counts as failing");
}
#[test]
fn unknown_bench_is_rejected_before_updating_the_baseline() {
let root = std::env::temp_dir().join(format!("photocraft-perf-unknown-bench-{}", std::process::id()));
std::fs::create_dir(&root).unwrap();
std::fs::create_dir(root.join("perf")).unwrap();
std::fs::write(root.join(BUDGETS), "[[bench]]\nname = \"known\"\npackage = \"unused\"\nrequires = \"missing.rs\"\n").unwrap();
let baseline = "{\"schema\":1,\"sentinel\":true}";
std::fs::write(root.join(BASELINE), baseline).unwrap();
let results = [
run(&root, &["--bench", "unknown", "--reuse", "--update-baseline"]),
run(&root, &["--bench", "known", "--bench", "unknown", "--quick", "--reuse", "--update-baseline"]),
];
let after = std::fs::read_to_string(root.join(BASELINE)).unwrap();
std::fs::remove_dir_all(root).unwrap();
for result in results {
assert_eq!(result, Err(format!("perf: unknown bench `unknown` (see {BUDGETS})")));
}
assert_eq!(after, baseline, "an unknown bench must not update the baseline");
}
#[test]
fn options() {
assert!(parse_opts(&["--bogus"]).is_err());
assert!(parse_opts(&["--threshold", "x"]).is_err());
assert!(parse_opts(&["--threshold", "-3"]).is_err());
let o = parse_opts(&["--quick", "--threshold", "20", "--bench", "a", "--reuse"]).unwrap();
assert!(o.quick && o.reuse && o.skip_build);
assert_eq!(o.threshold, Some(20.0));
assert_eq!(o.benches, ["a"]);
assert!(!o.allow_contended && o.from.is_empty());
let o = parse_opts(&["--update-baseline", "--from", "a.json", "--from", "b.json", "--allow-contended"]).unwrap();
assert!(o.update_baseline && o.allow_contended);
assert_eq!(o.from, [PathBuf::from("a.json"), PathBuf::from("b.json")]);
assert!(parse_opts(&["--from", "a.json"]).is_err(), "--from needs --update-baseline");
assert!(parse_opts(&["--update-baseline", "--from"]).is_err());
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.