5a24289898fd
bench.rs
//! `cargo xtask bench [FILE] [--strict] [--threshold PCT]`: run the end-to-end render benchmark, record
//! the results in `target/bench/history.jsonl` (one JSON object per run, with git revision and load
//! average) and compare with the previous run.
//!
//! Comparisons use **process CPU time**, which reflects the work done even on a busy shared machine;
//! wall-clock is recorded for reference. `--strict` exits non-zero when a CPU metric regressed by more
//! than the threshold (default 20 %).
use std::collections::BTreeMap;
use std::path::Path;
use std::process::Command;
/// Parse `render_bench` output lines such as
/// `loupe draft 1152ร768 cold: cpu: 379.5 ms wall, 139.5 ms cpu | gpu: 19.9 ms wall, 17.2 ms cpu`
/// or `export JPEG encode: 63.9 ms wall, 278.0 ms cpu` into `name/backend/{wall,cpu}` โ ms.
pub fn parse(output: &str) -> BTreeMap<String, f64> {
let mut out = BTreeMap::new();
for line in output.lines() {
let Some((name, rest)) = line.split_once(':') else { continue };
if !rest.contains(" ms ") {
continue;
}
let name = name.trim();
for seg in rest.split('|') {
let seg = seg.trim();
let (backend, body) = match seg.split_once(':') {
Some((b, body)) if matches!(b.trim(), "cpu" | "gpu") => (b.trim(), body),
_ => ("cpu", seg),
};
for part in body.split(',') {
let toks: Vec<&str> = part.split_whitespace().collect();
if let [v, "ms", kind] = toks[..]
&& let Ok(v) = v.parse::<f64>()
&& matches!(kind, "wall" | "cpu")
{
out.insert(format!("{name}/{backend}/{kind}"), v);
}
}
}
}
out
}
/// CPU-time metrics that got slower than `threshold` (fraction) vs `prev`: (key, prev, now).
pub fn regressions(prev: &BTreeMap<String, f64>, now: &BTreeMap<String, f64>, threshold: f64) -> Vec<(String, f64, f64)> {
now.iter()
.filter(|(k, _)| k.ends_with("/cpu"))
.filter_map(|(k, &v)| prev.get(k).map(|&p| (k.clone(), p, v)))
.filter(|(_, p, v)| *p > 1.0 && *v > p * (1.0 + threshold))
.collect()
}
fn load_avg() -> Option<f64> {
let out = Command::new("uptime").output().ok()?;
let s = String::from_utf8_lossy(&out.stdout);
let tail = s.rsplit_once("load average")?.1;
tail.trim_start_matches(['s', ':', ' ']).split([',', ' ']).find(|t| !t.is_empty())?.parse().ok()
}
fn git_rev(root: &Path) -> String {
Command::new("git")
.args(["rev-parse", "--short", "HEAD"])
.current_dir(root)
.output()
.ok()
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.unwrap_or_default()
}
pub fn run(root: &Path, args: &[&str]) -> Result<(), String> {
let strict = args.contains(&"--strict");
let threshold =
args.iter().position(|a| *a == "--threshold").and_then(|i| args.get(i + 1)).and_then(|v| v.parse::<f64>().ok()).unwrap_or(20.0) / 100.0;
let file = args.iter().find(|a| !a.starts_with("--") && a.parse::<f64>().is_err()).map(|s| s.to_string());
let file = file.or_else(|| {
let default = root.join("corpus/raw/arw-sony-a7m3-compressed.arw");
default.exists().then(|| default.to_string_lossy().to_string())
});
let mut cmd = crate::cargo();
cmd.args(["run", "--release", "-q", "-p", "lightcraft-engine", "--example", "render_bench", "--"]);
if let Some(f) = &file {
cmd.arg(f);
}
eprintln!("$ cargo run --release -p lightcraft-engine --example render_bench -- {}", file.as_deref().unwrap_or("(procedural 24 MP source)"));
let out = cmd.output().map_err(|e| format!("render_bench: {e}"))?;
let text = String::from_utf8_lossy(&out.stdout).to_string();
print!("{text}");
if !out.status.success() {
return Err(format!("render_bench failed:\n{}", String::from_utf8_lossy(&out.stderr)));
}
let metrics = parse(&text);
if metrics.is_empty() {
return Err("no metrics parsed from render_bench output".into());
}
let dir = root.join("target/bench");
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
let hist = dir.join("history.jsonl");
let prev: Option<BTreeMap<String, f64>> = std::fs::read_to_string(&hist)
.ok()
.and_then(|s| s.lines().rev().find(|l| !l.trim().is_empty()).map(str::to_string))
.and_then(|l| serde_json::from_str::<serde_json::Value>(&l).ok())
.and_then(|v| serde_json::from_value(v["metrics"].clone()).ok());
let rec = serde_json::json!({
"rev": git_rev(root),
"time": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0),
"load": load_avg(),
"input": file,
"metrics": metrics,
});
let mut line = rec.to_string();
line.push('\n');
use std::io::Write;
std::fs::OpenOptions::new().create(true).append(true).open(&hist).and_then(|mut f| f.write_all(line.as_bytes())).map_err(|e| e.to_string())?;
eprintln!("bench: {} metrics recorded in {}", metrics.len(), hist.display());
let Some(prev) = prev else {
eprintln!("bench: first run โ nothing to compare");
return Ok(());
};
let reg = regressions(&prev, &metrics, threshold);
if reg.is_empty() {
eprintln!("bench: no CPU-time regressions > {:.0}% vs previous run", threshold * 100.0);
return Ok(());
}
for (k, p, v) in ® {
eprintln!("bench: REGRESSION {k}: {p:.1} โ {v:.1} ms (+{:.0}%)", (v / p - 1.0) * 100.0);
}
if strict { Err(format!("{} CPU-time regression(s)", reg.len())) } else { Ok(()) }
}
#[cfg(test)]
mod tests {
use super::*;
const OUT: &str = "gpu: Apple M4 Pro (Metal) (device + kernels: 322 ms)
gpu vs cpu, loupe 1920ร1280 typical: max 1 LSB, mean 0.0001 LSB
loupe draft 1152ร768 cold: cpu: 379.5 ms wall, 139.5 ms cpu | gpu: 19.9 ms wall, 17.2 ms cpu
export JPEG encode: 63.9 ms wall, 278.0 ms cpu
window 3072ร2304 at (3328, 2048) of 9984ร6656
window pan (new window, warm src): cpu: 912.0 ms wall, 2210.5 ms cpu | gpu: 37.5 ms wall, 8.7 ms cpu
";
#[test]
fn parses_cpu_gpu_and_plain_lines() {
let m = parse(OUT);
assert_eq!(m["loupe draft 1152ร768 cold/cpu/wall"], 379.5);
assert_eq!(m["loupe draft 1152ร768 cold/gpu/cpu"], 17.2);
assert_eq!(m["export JPEG encode/cpu/cpu"], 278.0);
// (a comma in the name is no column separator)
assert_eq!(m["window pan (new window, warm src)/cpu/cpu"], 2210.5);
assert_eq!(m["window pan (new window, warm src)/gpu/wall"], 37.5);
assert_eq!(m.len(), 10, "{m:?}");
}
#[test]
fn flags_only_cpu_time_regressions_over_threshold() {
let prev = parse(OUT);
let mut now = prev.clone();
*now.get_mut("export JPEG encode/cpu/cpu").unwrap() = 400.0; // +44 %
*now.get_mut("export JPEG encode/cpu/wall").unwrap() = 400.0; // wall ignored
*now.get_mut("loupe draft 1152ร768 cold/gpu/cpu").unwrap() = 18.0; // +5 %
let r = regressions(&prev, &now, 0.2);
assert_eq!(r.len(), 1);
assert_eq!(r[0].0, "export JPEG encode/cpu/cpu");
}
}
No file is open. Choose one in the tree, or press Ctrl+O (โO on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.