665bfac0b339
ui_accessibility_audit.rs
//! Accessibility audit (docs/gaps.md gap 25): renders panels headless and walks the AccessKit
//! tree of the frame. Every interactive node (button, check box, text input, combo box, slider,
//! spin button, tree or list item, tab, link, ...) must have a non-empty name, a role that says
//! what it is, and the state that role needs (checked, selected, a value).
//!
//! * The panels that had their pass (`FIXED`: Effects & Presets, Settings, Render Queue with its
//! Templates dialog, Project, Info, Flowchart, Script Console and the 3D Camera Tracker block
//! of Effect Controls) must have **zero** findings.
//! * Every other panel is counted and printed (run with `--nocapture`), and checked against
//! [`BASELINE`], a ratchet that can only shrink: fixing a panel lowers its number, and a number
//! must never go up. A count that falls does not fail the test; lower the entry when a panel improves.
//!
//! A panel that only paints (Info) has no controls; there the audit counts what it *describes*: a
//! graphic (`Role::Image`, which must have a name in a fixed panel), a read-out (`Role::Label`
//! with a name) or a log. A fixed panel must show a minimum number of controls and described
//! items, so zero findings cannot mean the panel was not looked at.
//!
//! What this does not measure: how a real screen reader speaks the tree (none was run), colour
//! contrast, focus order or keyboard operation (the `ui_*_keyboard.rs` tests cover those), or
//! widgets that are scrolled out of view and not drawn.
use std::collections::HashSet;
use effectcraft_engine::Session;
use effectcraft_ui_egui::EffectcraftApp;
use effectcraft_ui_egui::dock::PanelKind;
use egui::accesskit::{Action, Role, Toggled};
use egui::{Rect, pos2};
use egui_kittest::Harness;
use egui_kittest::kittest::NodeT;
use serde_json::json;
/// Groups that must be clean. A group is the first part of a scenario name (before the `:`).
const FIXED: &[&str] = &["EffectsPresets", "Settings", "RenderQueue", "Project", "Info", "Flowchart", "ScriptConsole", "CameraTracker"];
/// Panels audited in a scenario of their own (a fresh window, the panel maximized, so it has its
/// full size; in the shared loop below each panel opened squeezes the next one). The loop still
/// opens them, so the layout the other panels are measured in does not change.
const DEDICATED: &[PanelKind] = &[PanelKind::Info, PanelKind::Flowchart, PanelKind::ScriptConsole];
/// (scenario, findings, of them unnamed controls) per not-yet-fixed scenario, as measured in the
/// shared docking layout (Effect Controls with one Gaussian Blur on the layer, every other panel
/// as it opens). A ratchet: a number may only go down, never up (a lower count does not fail);
/// when a pass over a panel lowers it, lower its entry in the same change, and remove the entry (and add the panel to `FIXED`)
/// when it reaches zero. A scenario that is not listed is held to zero.
const BASELINE: &[(&str, usize, usize)] = &[
("EffectControls:default", 28, 14),
("Composition:default", 4, 2),
("Layer:default", 2, 1),
("Timeline:default", 72, 25),
("Audio:default", 4, 2),
("Properties:default", 31, 15),
("Character:default", 46, 22),
("Paragraph:default", 26, 13),
("Align:default", 12, 0),
("Tracker:default", 11, 3),
("Wiggler:default", 4, 2),
("Smoother:default", 2, 1),
("MotionSketch:default", 8, 4),
("Paint:default", 38, 18),
("Brushes:default", 34, 6),
("History:default", 10, 5),
("MaskInterpolation:default", 16, 8),
("EssentialGraphics:default", 1, 1),
("LumetriScopes:default", 2, 1),
("MediaBrowser:default", 20, 10),
("Metadata:default", 2, 2),
("ContentAwareFill:default", 2, 1),
];
/// How many controls (and, in the panels that paint, described read-outs, graphics and logs) the fixed scenarios must at least show, so that "zero findings" is never
/// the result of looking at nothing. Any scenario not listed needs 2.
const MIN_CONTROLS: &[(&str, usize)] = &[
("EffectsPresets:default", 10),
("EffectsPresets:open", 10),
("EffectsPresets:search", 5),
("EffectsPresets:menu", 8),
("Settings:general", 12),
("Settings:appearance", 15),
("Settings:labels", 30),
("RenderQueue:queued", 8),
("RenderQueue:expanded", 12),
("RenderQueue:menu", 20),
("RenderQueue:templates:renderSettings", 8),
("RenderQueue:templates:outputModule", 8),
// Info, Flowchart and Script Console count read-outs, graphics and logs as well as controls.
("Info:default", 6),
("Info:layer", 8),
("Flowchart:default", 6),
("Flowchart:layers", 8),
("ScriptConsole:default", 4),
("ScriptConsole:log", 4),
("CameraTracker:editor", 3),
];
/// The roles that are controls: anything with one of these (or with a click or focus action) is
/// held to the rules below.
const CONTROL_ROLES: &[Role] = &[
Role::Button,
Role::CheckBox,
Role::ColorWell,
Role::ComboBox,
Role::Link,
Role::ListBoxOption,
Role::ListItem,
Role::MenuItem,
Role::MenuItemCheckBox,
Role::MenuItemRadio,
Role::MultilineTextInput,
Role::RadioButton,
Role::SearchInput,
Role::Slider,
Role::SpinButton,
Role::Switch,
Role::Tab,
Role::TextInput,
Role::TreeItem,
];
#[derive(Debug, Clone)]
struct Finding {
scenario: String,
problem: &'static str,
role: Role,
name: String,
rect: [i32; 4],
}
impl std::fmt::Display for Finding {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "[{}] {}: {:?} {:?} at {:?}", self.scenario, self.problem, self.role, self.name, self.rect)
}
}
fn app_session() -> Session {
let mut s = Session::default();
s.execute("comp.new", json!({"name": "Main", "width": 320, "height": 180, "duration": 4})).unwrap();
s.execute("layer.newSolid", json!({"name": "Red", "color": "#c04040"})).unwrap();
s.execute("layer.newSolid", json!({"name": "Blue", "color": "#4040c0"})).unwrap();
s.execute("effect.apply", json!({"effect": "Gaussian Blur"})).ok();
s
}
fn harness() -> Harness<'static, EffectcraftApp> {
let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| EffectcraftApp::new(app_session()));
// The default workspace gives Preview a short frame at the top of the right column; its
// controls below that frame stay in the tree and would be counted against the panel under it.
// The per-panel pass opens Preview again for its own scenario.
h.state_mut().close_panel(PanelKind::Preview);
h.run_steps(4);
h
}
fn invoke(h: &mut Harness<'_, EffectcraftApp>, id: &str, params: serde_json::Value) {
let ctx = h.ctx.clone();
effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, id, params).unwrap();
h.run_steps(4);
}
fn panel_rect(h: &Harness<'_, EffectcraftApp>, p: PanelKind) -> Option<Rect> {
let e = h.state().auto.find(&format!("panel.{}", p.id()))?;
Some(Rect::from_min_size(pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])))
}
fn dialog_rect(h: &Harness<'_, EffectcraftApp>, title: &str) -> Option<Rect> {
h.ctx.memory(|m| m.area_rect(egui::Id::unique(("modal", title))))
}
/// The ids of every node now, so a later audit can look at what appeared since (a popup).
fn node_ids(h: &Harness<'_, EffectcraftApp>) -> HashSet<String> {
h.root().children_recursive().map(|n| format!("{:?}", n.accesskit_node().id())).collect()
}
/// Walk the tree and check every control whose centre is inside `scope` (and, when `since` is
/// given, that was not in the tree before).
fn audit(h: &Harness<'_, EffectcraftApp>, scenario: &str, scope: Rect, since: Option<&HashSet<String>>) -> (Vec<Finding>, usize) {
let mut out = Vec::new();
let mut controls = 0;
// The rules for graphics apply where the panel has had its pass.
let strict = FIXED.contains(&group(scenario));
for n in h.root().children_recursive() {
let a = n.accesskit_node();
if a.is_hidden() {
continue;
}
if since.is_some_and(|s| s.contains(&format!("{:?}", a.id()))) {
continue;
}
let Some(b) = a.bounding_box() else {
// No bounds: assistive technology cannot place it. Only a control is a finding (and
// it is one wherever it is: there is no position to scope it by).
if CONTROL_ROLES.contains(&a.role()) {
out.push(Finding { scenario: scenario.to_string(), problem: "no bounds", role: a.role(), name: a.label().unwrap_or_default(), rect: [0; 4] });
}
continue;
};
if !scope.contains(pos2(((b.x0 + b.x1) / 2.0) as f32, ((b.y0 + b.y1) / 2.0) as f32)) {
continue;
}
let role = a.role();
// What a panel describes without being operable: a graphic (a name is required in a fixed
// panel), a read-out with a name, a log.
let described = match role {
Role::Image | Role::Canvas | Role::Log => true,
Role::Label => a.label().is_some_and(|l| !l.trim().is_empty()),
_ => false,
};
if described {
if strict && matches!(role, Role::Image | Role::Canvas | Role::Log) && a.label().is_none_or(|l| l.trim().is_empty()) {
out.push(Finding {
scenario: scenario.to_string(),
problem: "graphic without a name",
role,
name: String::new(),
rect: [b.x0 as i32, b.y0 as i32, b.x1 as i32, b.y1 as i32],
});
}
controls += 1;
continue;
}
let actionable = a.data().supports_action(Action::Click) || a.data().supports_action(Action::Focus);
let control = CONTROL_ROLES.contains(&role);
if !(control || (actionable && matches!(role, Role::Unknown | Role::GenericContainer))) {
continue;
}
let name = a.label().unwrap_or_default();
controls += 1;
let finding = |problem: &'static str| Finding {
scenario: scenario.to_string(),
problem,
role,
name: name.clone(),
rect: [b.x0 as i32, b.y0 as i32, b.x1 as i32, b.y1 as i32],
};
if !control && matches!(role, Role::Unknown | Role::GenericContainer) {
// Something you can click or focus, that no role describes.
out.push(finding("clickable or focusable without a control role"));
}
if name.trim().is_empty() {
out.push(finding("no name"));
}
let state_missing = match role {
Role::CheckBox | Role::RadioButton | Role::Switch | Role::MenuItemCheckBox | Role::MenuItemRadio => {
a.toggled().is_none() || a.toggled() == Some(Toggled::Mixed)
}
Role::Slider | Role::SpinButton => a.numeric_value().is_none(),
Role::ComboBox => a.value().is_none_or(|v| v.trim().is_empty()),
Role::TreeItem | Role::ListItem | Role::ListBoxOption | Role::Tab => a.is_selected().is_none(),
_ => false,
};
if state_missing {
out.push(finding("no state or value"));
}
}
(out, controls)
}
fn count_unnamed(f: &[Finding]) -> usize {
f.iter().filter(|f| f.problem == "no name").count()
}
struct Report {
findings: Vec<Finding>,
/// (scenario, findings, of them unnamed, controls looked at)
rows: Vec<(String, usize, usize, usize)>,
}
impl Report {
fn add(&mut self, scenario: &str, (f, controls): (Vec<Finding>, usize)) {
self.rows.push((scenario.to_string(), f.len(), count_unnamed(&f), controls));
self.findings.extend(f);
}
}
fn group(scenario: &str) -> &str {
scenario.split(':').next().unwrap_or(scenario)
}
fn collect() -> Report {
let mut r = Report { findings: vec![], rows: vec![] };
// ---- Effects & Presets: closed, everything open, searching, the contents menu.
let mut h = harness();
h.state_mut().show_panel(PanelKind::EffectsPresets);
h.run_steps(4);
let rect = panel_rect(&h, PanelKind::EffectsPresets).expect("Effects & Presets is drawn");
r.add("EffectsPresets:default", audit(&h, "EffectsPresets:default", rect, None));
{
let s = h.state_mut();
s.session.prefs.toggle_effect_favorite("ec.blur.gaussian");
s.ui.effects_open.insert("favorites".into());
s.ui.effects_open.insert("presets".into());
s.ui.effects_open.insert("presets/Text".into());
s.ui.effects_open.insert("Blur & Sharpen".into());
}
h.run_steps(4);
r.add("EffectsPresets:open", audit(&h, "EffectsPresets:open", rect, None));
h.state_mut().ui.effects_search = "blur".into();
h.run_steps(4);
r.add("EffectsPresets:search", audit(&h, "EffectsPresets:search", rect, None));
let before = node_ids(&h);
h.ctx.data_mut(|d| d.insert_temp(egui::Id::unique("fx-contents-menu").with("open"), true));
h.run_steps(4);
let screen = Rect::from_min_size(pos2(0.0, 0.0), egui::vec2(1600.0, 1000.0));
r.add("EffectsPresets:menu", audit(&h, "EffectsPresets:menu", screen, Some(&before)));
// ---- Settings: every page.
let mut h = harness();
let before = node_ids(&h);
for page in effectcraft_engine::prefs::pages() {
invoke(&mut h, "app.settings", json!({"page": page.id}));
let name = format!("Settings:{}", page.id);
let rect = dialog_rect(&h, "Settings").expect("the Settings dialog is open");
r.add(&name, audit(&h, &name, rect, Some(&before)));
}
// ---- Render Queue: empty, queued and expanded, a menu open, the Templates dialogs.
let mut h = harness();
h.state_mut().show_panel(PanelKind::RenderQueue);
h.run_steps(4);
let rect = panel_rect(&h, PanelKind::RenderQueue).expect("the Render Queue is drawn");
r.add("RenderQueue:empty", audit(&h, "RenderQueue:empty", rect, None));
let comp = h.state().session.active_comp_id().expect("a comp").0;
let mut items = vec![];
for _ in 0..2 {
let out = h.state_mut().session.execute("renderQueue.add", json!({"comp": comp})).unwrap();
items.push(out["item"].as_u64().or_else(|| out["items"][0].as_u64()).unwrap_or(0));
}
h.run_steps(4);
r.add("RenderQueue:queued", audit(&h, "RenderQueue:queued", rect, None));
let first = h.state().session.project.render_queue.first().map(|i| i.id).expect("a queue item");
h.ctx.data_mut(|d| d.insert_temp(egui::Id::unique("rq-ui"), (vec![first], Some(first))));
h.run_steps(4);
r.add("RenderQueue:expanded", audit(&h, "RenderQueue:expanded", rect, None));
let before = node_ids(&h);
h.ctx.data_mut(|d| d.insert_temp(egui::Id::unique(("rq-dd", first, 0)).with("open"), true));
h.run_steps(4);
r.add("RenderQueue:menu", audit(&h, "RenderQueue:menu", screen, Some(&before)));
h.ctx.data_mut(|d| d.insert_temp(egui::Id::unique(("rq-dd", first, 0)).with("open"), false));
let before = node_ids(&h);
for (kind, title) in [("renderSettings", "Render Settings Templates"), ("outputModule", "Output Module Templates")] {
invoke(&mut h, "app.templates", json!({"kind": kind}));
let name = format!("RenderQueue:templates:{kind}");
let rect = dialog_rect(&h, title).expect("the Templates dialog is open");
r.add(&name, audit(&h, &name, rect, Some(&before)));
}
let _ = items;
// ---- Everything else, informational: each panel on its own.
let mut h = harness();
for p in PanelKind::ALL {
if matches!(p, PanelKind::EffectsPresets | PanelKind::RenderQueue) {
continue;
}
h.state_mut().show_panel(p);
h.run_steps(3);
if DEDICATED.contains(&p) {
continue;
}
let name = format!("{}:default", p.id());
let Some(rect) = panel_rect(&h, p) else {
// A panel with a baseline entry must be looked at, or its allowance would pass unchecked.
assert!(!BASELINE.iter().any(|(k, ..)| *k == name), "{name} has a BASELINE entry but was not drawn");
r.rows.push((format!("{name} (not drawn)"), 0, 0, 0));
continue;
};
r.add(&name, audit(&h, &name, rect, None));
if p == PanelKind::Preview {
// Closed again so it is not counted against the panels after it (see `harness`).
h.state_mut().close_panel(PanelKind::Preview);
h.run_steps(3);
}
}
// ---- The panels with their own scenarios: alone in a window, maximized.
let alone = |p: PanelKind| {
let mut h = harness();
h.state_mut().show_panel(p);
h.state_mut().ui.maximized = Some(p);
h.run_steps(4);
h
};
// Info: nothing selected, then a layer selected (its Duration, In and Out), and a key.
let mut h = alone(PanelKind::Info);
h.state_mut().session.execute("layer.select", json!({"layers": []})).ok();
h.run_steps(4);
let rect = panel_rect(&h, PanelKind::Info).expect("Info is drawn");
r.add("Info:default", audit(&h, "Info:default", rect, None));
h.state_mut().session.execute("layer.select", json!({"layers": ["#1"]})).ok();
h.run_steps(4);
r.add("Info:layer", audit(&h, "Info:layer", rect, None));
// Flowchart: the comp and what it uses, then with layers and effects, then top to bottom.
let mut h = alone(PanelKind::Flowchart);
let rect = panel_rect(&h, PanelKind::Flowchart).expect("the Flowchart is drawn");
r.add("Flowchart:default", audit(&h, "Flowchart:default", rect, None));
{
let f = &mut h.state_mut().ui.flowchart;
f.layers = true;
f.effects = true;
f.vertical = true;
}
h.run_steps(4);
r.add("Flowchart:layers", audit(&h, "Flowchart:layers", rect, None));
// Script Console: empty, then with a run in its log.
let mut h = alone(PanelKind::ScriptConsole);
let rect = panel_rect(&h, PanelKind::ScriptConsole).expect("the Script Console is drawn");
r.add("ScriptConsole:default", audit(&h, "ScriptConsole:default", rect, None));
h.state_mut().ui.script_console.input = "app.project.numItems".into();
{
let app = h.state_mut();
let con = &mut app.ui.script_console;
con.record("app.project.numItems", &Ok(json!({"result": 3})));
}
h.run_steps(4);
r.add("ScriptConsole:log", audit(&h, "ScriptConsole:log", rect, None));
// 3D Camera Tracker block of Effect Controls (a separate scenario: the block, not the panel).
let mut h = harness();
h.state_mut().session.execute("effect.apply", json!({"effect": effectcraft_engine::effects::camera_tracker::ID})).ok();
h.state_mut().show_panel(PanelKind::EffectControls);
h.run_steps(4);
let block: Vec<Rect> = h
.state()
.auto
.query("effectControls.cameraTracker.")
.into_iter()
.map(|e| Rect::from_min_size(pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])))
.collect();
if let Some(first) = block.first() {
let area = block.iter().fold(*first, |a, b| a.union(*b));
let area = Rect::from_min_max(area.min - egui::vec2(8.0, 4.0), area.max + egui::vec2(8.0, 40.0));
r.add("CameraTracker:editor", audit(&h, "CameraTracker:editor", area, None));
} else {
r.rows.push(("CameraTracker:editor (not drawn)".to_string(), 0, 0, 0));
}
r
}
#[test]
fn fixed_panels_have_no_unnamed_or_roleless_controls_and_the_others_do_not_get_worse() {
let report = collect();
println!("\nAccessibility audit: findings per scenario (unnamed in brackets)");
let mut by_group: Vec<(String, usize, usize, usize)> = vec![];
for (name, n, unnamed, controls) in &report.rows {
println!(" {name:<44} {n:>4} findings ({unnamed} unnamed) of {controls} controls");
let g = group(name).to_string();
match by_group.iter_mut().find(|(k, ..)| *k == g) {
Some(e) => {
e.1 += n;
e.2 += unnamed;
e.3 += controls;
}
None => by_group.push((g, *n, *unnamed, *controls)),
}
}
println!("\nBy panel (findings, unnamed controls, controls looked at):");
for (g, n, unnamed, controls) in &by_group {
let state = if FIXED.contains(&g.as_str()) { "fixed: must be 0" } else { "not yet audited" };
println!(" {g:<28} {n:>4} findings, {unnamed:>3} unnamed, {controls:>4} controls {state}");
}
let total_unnamed: usize = by_group.iter().map(|(_, _, u, _)| u).sum();
println!(" all panels: {total_unnamed} unnamed controls\n");
let mut failures = vec![];
for f in &report.findings {
if FIXED.contains(&group(&f.scenario)) {
failures.push(f.to_string());
}
}
assert!(failures.is_empty(), "{} accessibility findings in the fixed panels:\n{}", failures.len(), failures.join("\n"));
// A clean panel must not be clean because nothing was looked at.
for (name, _, _, controls) in &report.rows {
if FIXED.contains(&group(name)) {
let min = MIN_CONTROLS.iter().find(|(k, _)| k == name).map_or(2, |(_, v)| *v);
assert!(*controls >= min, "{name}: only {controls} controls found in the tree (expected {min}); the scenario is not showing the panel");
}
}
// The ratchet: a count above its entry is a regression. A count below it passes, so an
// unrelated change (or a machine with fewer files in the Media Browser) never has to edit it;
// lower the entry when a pass over a panel improves it.
let mut off = vec![];
for (name, n, unnamed, _) in &report.rows {
if FIXED.contains(&group(name)) {
continue;
}
let (allowed, allowed_unnamed) = BASELINE.iter().find(|(k, ..)| k == name).map_or((0, 0), |(_, v, u)| (*v, *u));
if *n > allowed || *unnamed > allowed_unnamed {
off.push(format!("{name}: {n} findings ({unnamed} unnamed), the baseline allows {allowed} ({allowed_unnamed}); a panel must not get worse"));
}
}
assert!(off.is_empty(), "{} scenarios do not match BASELINE:\n{}", off.len(), off.join("\n"));
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.