642d5a256791
self_check.rs
//! The builder tests its own work while it builds: `rebuild_check` (in [`super::tools`]) asks the
//! run through `checks/asks/` (the inbox's folder protocol, in a folder only the run reads), and
//! the run answers during the builder's turn with the same checks it runs after the turn, as
//! round 0 (`checks/round-0/`), not counted as a round.
use super::*;
/// The round a builder's own check is run as; real rounds start at 1.
pub const SELF_CHECK_ROUND: u32 = 0;
/// How long one of the builder's own checks may take. The build waits for it, so a check still
/// going after this is stopped and reported. Longer than a build's own steps need.
pub const SELF_CHECK_LIMIT: Duration = Duration::from_secs(30 * 60);
/// Where `rebuild_check` asks a project's run for a check.
pub fn asks_dir(project: &Path) -> PathBuf {
project.join("checks").join("asks")
}
/// What a builder's check gets when the build is stopped while it runs.
pub const STOPPED_ANSWER: &str = "The build was stopped before this check finished, so there is nothing to report.";
/// The mark a run leaves beside a check request it has taken, so `rebuild_check` can tell a
/// check that is still going from a request no run took; `rebuild_check` removes it. None for an
/// id that isn't an inbox id (a request's file can carry any `id`): no mark outside the folder.
pub fn taken_mark(project: &Path, id: &str) -> Option<PathBuf> {
let valid = !id.is_empty() && id.len() <= 64 && id.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
valid.then(|| asks_dir(project).join(format!("{id}.taken")))
}
/// Projects whose build runs in this process. A builder in this process (an API backend) calls
/// tools with the engine locked, so its `rebuild_check` could never be answered by this run.
static BUILDING_HERE: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
/// Marks a project as building in this process until dropped.
pub(super) struct BuildingHere(PathBuf);
impl BuildingHere {
pub(super) fn new(root: &Path) -> BuildingHere {
BUILDING_HERE.lock().unwrap_or_else(PoisonError::into_inner).push(root.to_path_buf());
BuildingHere(root.to_path_buf())
}
}
impl Drop for BuildingHere {
fn drop(&mut self) {
let mut v = BUILDING_HERE.lock().unwrap_or_else(PoisonError::into_inner);
if let Some(i) = v.iter().position(|p| *p == self.0) {
v.remove(i);
}
}
}
/// The project builds in this process.
pub fn building_here(root: &Path) -> bool {
BUILDING_HERE.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|p| p == root)
}
/// The questions the builder recorded (`core_unknown_add`, kept in the project's investigation
/// folder) that aren't resolved yet, most severe first. Read from disk: the builder's tool server
/// writes them while this run works.
pub fn open_questions(ws: &Workspace) -> Vec<craft_anything_evidence::Unknown> {
let mut open: Vec<_> = Project::open(&ws.investigation_dir())
.and_then(|p| p.load_unknowns())
.unwrap_or_default()
.into_iter()
.filter(|u| u.status != craft_anything_evidence::UnknownStatus::Resolved)
.collect();
open.sort_by(|a, b| b.severity.cmp(&a.severity).then_with(|| a.question.cmp(&b.question)));
open
}
/// One plain line on how a round of checks went: how alike when compared, else how many checks
/// pass, and how many of their claims hold ("about 93% like the original, 5 of 5 claims hold";
/// "all 4 checks pass, 5 of 5 claims hold").
pub fn summary(checks: &[CheckResult], good_enough: f64) -> String {
let claims: Vec<_> = checks.iter().flat_map(|c| &c.claims).collect();
let held = claims.iter().filter(|c| c.passed).count();
let claims_part = if claims.is_empty() { String::new() } else { format!(", {held} of {} claims hold", claims.len()) };
let good = checks.iter().filter(|c| c.good(good_enough)).count();
let mut text = match checks.iter().find_map(|c| c.similarity) {
Some(s) => format!("about {s:.0}% like the original{claims_part}"),
None if good == checks.len() => format!("all {} checks pass{claims_part}", checks.len()),
None => format!("{good} of {} checks pass{claims_part}", checks.len()),
};
let unverified: usize = checks.iter().map(|c| c.unknowns.len()).sum();
match unverified {
0 => {}
1 => text.push_str("; 1 comparison remains unverified"),
n => text.push_str(&format!("; {n} comparisons remain unverified")),
}
text
}
/// What the builder is told about its own check: each result with its likeness and the bar,
/// what's still unknown (the checks' and its own open questions), where its pictures are, and
/// whether it would pass.
pub fn report(checks: &[CheckResult], good_enough: f64, open: &[craft_anything_evidence::Unknown], can_refine: impl Fn(&CheckResult) -> bool) -> String {
if checks.is_empty() {
return "There was nothing to check yet.".into();
}
let mut lines = Vec::new();
for c in checks {
let mut line = format!("{}: {}", c.check, if c.passed { "passed" } else { "failed" });
if !c.good(good_enough) && !can_refine(c) {
line.push_str(" (incomplete; another builder turn cannot resolve this)");
}
if let Some(s) = c.similarity {
line.push_str(&format!(", {s:.0}% alike (needs {:.0}%)", c.good_enough.unwrap_or(good_enough)));
}
line.push_str(&format!(" — {}", c.note));
lines.push(line);
}
let failed: Vec<String> = checks.iter().flat_map(|c| c.failed_claims()).map(|c| format!("{} ({})", c.title, c.detail)).collect();
if !failed.is_empty() {
lines.push(format!("These must hold before it can pass: {}.", failed.join("; ")));
}
let unknowns: Vec<&str> = checks.iter().flat_map(|c| &c.unknowns).map(String::as_str).collect();
if !unknowns.is_empty() {
lines.push(format!(
"Not compared, so it can't pass yet: {}. Compare suitable reference inputs and observed results where possible; keep the rest unverified.",
unknowns.join(", ")
));
}
if !open.is_empty() {
let shown: Vec<String> = open.iter().take(8).map(|u| format!("{} ({})", u.question, u.id)).collect();
lines.push(format!(
"Your open questions: {}. Settle what one more look can and close it with core_unknown_edit (status resolved, the resolution and the evidence ids); say what's left in your summary.",
shown.join("; ")
));
}
let pictures: Vec<String> = checks.iter().flat_map(|c| &c.artifacts).map(|p| p.display().to_string().replace('\\', "/")).collect();
if !pictures.is_empty() {
lines.push(format!("Pictures of your build (compare each with the original's in capture/ or reference/): {}", pictures.join(", ")));
}
lines.push(if checks.iter().all(|c| c.good(good_enough)) {
"This would pass. Finish what you're doing, then reply.".to_string()
} else if checks.iter().any(|c| !c.good(good_enough) && can_refine(c)) {
"Not good enough yet: fix what it lists, build again, and check again with rebuild_check.".to_string()
} else {
"These checks remain incomplete; another builder turn cannot resolve them. Finish what you're doing and state the unverified comparisons in your reply."
.to_string()
});
lines.join("\n")
}
impl Worker {
/// Answer the builder's own check requests during its turn (once the build loop has started).
pub(super) fn answer_self_checks(&mut self, ws: &mut Workspace) -> Result<(), ToolError> {
let Some((kind, good_enough)) = self.self_check.clone() else { return Ok(()) };
let asks = Inbox::new(asks_dir(&ws.root));
for r in asks.pending() {
let mark = taken_mark(&ws.root, &r.id);
if let Some(mark) = &mark
&& let Err(e) = std::fs::write(mark, b"")
{
log::warn!("couldn't mark the builder's check as taken ({}): {e}", mark.display());
}
let checking = if ws.manifest.target == TargetRef::None {
"The builder is checking its work…"
} else {
"The builder is checking its work against the original…"
};
// Once taken, every way out answers the request: a stop seen by this step too.
let checked = self.step(ws, Step::Build, Status::Building, checking).and_then(|()| {
self.checks_unless_unchanged(ws, kind.as_ref(), SELF_CHECK_ROUND, || {
self.run_checks_within(ws, kind.as_ref(), SELF_CHECK_ROUND, Some(SELF_CHECK_LIMIT))
})
});
let (text, line) = match checked {
Ok(checks) => {
let line = format!("The builder checked its work: {}.", summary(&checks, good_enough));
let kind_checks = kind.checks(ws);
(report(&checks, good_enough, &open_questions(ws), |c| kinds::can_refine(&kind_checks, c)), line)
}
Err(e) if e.code == ErrorCode::Cancelled => {
// A stopped build says so: the builder isn't left waiting on a check that
// nothing runs any more.
let stopped = Answer { text: Some(STOPPED_ANSWER.to_string()), ..Default::default() };
if let Err(a) = asks.answer(&r.id, &stopped) {
log::warn!("couldn't tell the builder its check was stopped: {}", a.message);
}
return Err(e);
}
Err(e) => (format!("The check couldn't run: {}", e.message), "The builder's check couldn't run.".to_string()),
};
// The mark stays: the asker reads it after its wait, then removes it, so an answer
// that lands as the wait ends can't make a taken check look untaken.
asks.answer(&r.id, &Answer { text: Some(text), ..Default::default() })?;
self.step(ws, Step::Build, Status::Building, &line)?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn report(checks: &[CheckResult], good_enough: f64, open: &[craft_anything_evidence::Unknown]) -> String {
super::report(checks, good_enough, open, |_| true)
}
#[test]
fn the_report_says_what_differs_and_whether_it_would_pass() {
let web = CheckResult {
similarity: Some(82.4),
artifacts: vec![PathBuf::from("checks/round-0/screenshot-390.png")],
..CheckResult::new(CheckResult::WEB_VISUAL, SELF_CHECK_ROUND, true, "at 390 px: the page scrolls sideways (512 px wide)")
};
let r = report(std::slice::from_ref(&web), 90.0, &[]);
assert!(r.contains("82% alike (needs 90%) — at 390 px: the page scrolls sideways") && r.contains("checks/round-0/screenshot-390.png"), "{r}");
assert!(r.ends_with("check again with rebuild_check."), "{r}");
let good = CheckResult { similarity: Some(93.0), ..web };
assert!(report(&[good], 90.0, &[]).ends_with("This would pass. Finish what you're doing, then reply."));
let build = CheckResult::new(CheckResult::BUILD, SELF_CHECK_ROUND, false, "Build failed: error[E0425]");
let r = report(&[build], 90.0, &[]);
assert!(r.starts_with("build: failed — Build failed") && r.contains("Not good enough yet"), "{r}");
assert_eq!(report(&[], 90.0, &[]), "There was nothing to check yet.");
let claim = |passed| craft_anything_engine::workspace::Claim::new("In play, the player moves", passed, "it doesn't");
let game = [
CheckResult::new(CheckResult::BUILD, 1, true, "Build succeeded."),
CheckResult { claims: vec![claim(true), claim(true)], ..CheckResult::new("game-self-test", 1, true, "") },
CheckResult::new("mechanics", 1, true, "app/mechanics.json has 15 values"),
];
assert_eq!(summary(&game, 90.0), "all 3 checks pass, 2 of 2 claims hold");
let failing = [game[0].clone(), CheckResult { claims: vec![claim(true), claim(false)], ..CheckResult::new("game-self-test", 1, false, "") }];
assert_eq!(summary(&failing, 90.0), "1 of 2 checks pass, 1 of 2 claims hold");
let site = [CheckResult { similarity: Some(93.4), ..CheckResult::new(CheckResult::WEB_VISUAL, 1, true, "") }];
assert_eq!(summary(&site, 90.0), "about 93% like the original");
let alike_but_broken = CheckResult {
similarity: Some(96.0),
claims: vec![craft_anything_engine::workspace::Claim::new("The start page loads every file it asks for", false, "/logo.png (404)")],
..CheckResult::new(CheckResult::WEB_VISUAL, 0, true, "")
};
let r = report(&[alike_but_broken], 90.0, &[]);
assert!(
r.contains("These must hold before it can pass: The start page loads every file it asks for (/logo.png (404))")
&& r.contains("Not good enough yet"),
"{r}"
);
let blind = CheckResult {
similarity: Some(95.0),
unknowns: vec!["/about at 768 px couldn't be compared".into()],
..CheckResult::new(CheckResult::WEB_VISUAL, 0, true, "")
};
let r = report(std::slice::from_ref(&blind), 90.0, &[]);
assert!(r.contains("Not compared, so it can't pass yet: /about at 768 px") && r.contains("Not good enough yet"), "an unknown never passes: {r}");
assert_eq!(summary(std::slice::from_ref(&blind), 90.0), "about 95% like the original; 1 comparison remains unverified");
let unknowns = CheckResult { unknowns: vec!["first".into(), "second".into()], ..CheckResult::new("mechanics", 1, true, "readable") };
assert_eq!(summary(&[unknowns], 90.0), "0 of 1 checks pass; 2 comparisons remain unverified");
}
#[test]
fn an_unavailable_mechanics_oracle_does_not_ask_the_builder_to_retry() {
let c = CheckResult {
unknowns: vec!["Mechanics behavior has not been measured.".into()],
..CheckResult::new("mechanics", 0, true, "The sheet is readable; behavior is unverified.")
};
let kind_checks: Vec<Box<dyn Check>> = vec![Box::new(MechanicsCheck), Box::new(BuildCheck)];
let text = super::report(std::slice::from_ref(&c), 90.0, &[], |r| kinds::can_refine(&kind_checks, r));
assert!(text.contains("another builder turn cannot resolve"), "{text}");
assert!(!text.contains("fix what it lists"), "{text}");
assert!(!text.contains("This would pass"), "uncertainty remains: {text}");
let failed = CheckResult::new("build", 0, false, "Build failed.");
let text = super::report(&[c, failed], 90.0, &[], |r| kinds::can_refine(&kind_checks, r));
assert!(text.ends_with("check again with rebuild_check."), "a refinable failure still needs work: {text}");
assert!(
text.contains("mechanics: passed (incomplete; another builder turn cannot resolve this)"),
"a mixed report identifies the unrefinable comparison: {text}"
);
assert!(text.contains("build: failed") && text.contains("Build failed."), "the actionable failure remains clear: {text}");
assert!(kinds::can_refine(&kind_checks, &CheckResult::new("plugin-check", 0, false, "failed")));
}
#[test]
fn the_builders_open_questions_are_read_from_the_project_and_reported() {
use craft_anything_evidence::{Severity, Unknown, UnknownStatus};
let d = tempfile::tempdir().unwrap();
let ws = Workspace::create(d.path(), "Arena", "Recreate this game", TargetRef::None).unwrap();
assert!(open_questions(&ws).is_empty(), "no investigation folder yet");
let project = Project::create(&ws.investigation_dir(), "Arena", Purpose::default()).unwrap();
project.store_unknown(&Unknown::new("Jump height?", "game", Severity::Low)).unwrap();
project.store_unknown(&Unknown::new("Which font is the menu set in?", "ui", Severity::High)).unwrap();
let mut done = Unknown::new("Map size?", "game", Severity::Critical);
done.status = UnknownStatus::Resolved;
project.store_unknown(&done).unwrap();
let open = open_questions(&ws);
let questions: Vec<&str> = open.iter().map(|u| u.question.as_str()).collect();
assert_eq!(questions, ["Which font is the menu set in?", "Jump height?"], "unresolved only, most severe first");
let r = report(&[CheckResult::new(CheckResult::BUILD, 0, true, "Build succeeded.")], 90.0, &open);
assert!(r.contains("Your open questions: Which font is the menu set in? (unk_") && r.contains("close it with core_unknown_edit"), "{r}");
}
#[test]
fn a_project_is_marked_as_building_here_only_while_its_run_lasts() {
let root = PathBuf::from("/projects/marked-here");
assert!(!building_here(&root));
let mark = BuildingHere::new(&root);
assert!(building_here(&root));
drop(mark);
assert!(!building_here(&root));
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.