GitCorner

Code

ossartcraftphotocraft🌐
856f865cf98d
//! GPU health in the shell (#243, #4): when the wgpu device is lost or reports an error, the
//! GPU canvas is dropped and every document keeps drawing through the CPU compositor and egui
//! textures, with a recovery warning. After an error Retry GPU builds a new GPU canvas on the same
//! device (#6577); after a loss the CPU canvas stays for the rest of the session. Also runs the desktop app's
//! "started" hook once the first frames have rendered (the crash-safe startup marker), and
//! provides the Help › System Info text (including the monitor profile in use).

use craft::ui::dialog::{PromptAnswer, PromptChoice, StandardPrompt};
use serde_json::json;

use crate::PhotocraftApp;

/// Notice title when the device was lost.
pub const LOST_MESSAGE: &str = "GPU device was lost; using the CPU renderer.";
/// Notice title when the device reported an error (out of memory, validation, internal).
pub const ERROR_MESSAGE: &str = "GPU error; using the CPU renderer.";
/// Frames after which startup counts as done even if a document never drew.
const STARTED_MAX_FRAMES: u64 = 120;

/// Hook run once the app has rendered its first frames (see [`PhotocraftApp::on_started`]).
pub type StartedHook = Box<dyn FnOnce(&mut PhotocraftApp)>;

/// Out-of-memory errors after which the GPU canvas is rebuilt by itself; past them the CPU
/// renderer stays until the user chooses Retry GPU, so a document that never fits doesn't rebuild
/// the canvas on every frame.
pub const OOM_AUTO_RETRIES: u32 = 3;

/// Per-frame check: switch to the CPU canvas when the GPU faulted, and run the started hook. An
/// out-of-memory error on a device that still works is not the end of the GPU for the session
/// (#6564): the dropped canvas freed its memory, so a new one is built at once and the views
/// redraw through it, up to [`OOM_AUTO_RETRIES`] times.
pub fn check(app: &mut PhotocraftApp, ctx: &egui::Context) {
    if let Some(fault) = app.gpu.as_ref().and_then(|g| g.fault()) {
        drop_canvas(app, &fault);
        let mut rebuilt = false;
        if fault.is_out_of_memory() && app.gpu_oom_retries < OOM_AUTO_RETRIES && can_retry_now(app) {
            app.gpu_oom_retries += 1;
            if retry_now(app) {
                app.retry_compositor(ctx);
                log::warn!("{fault}; rebuilt the GPU canvas (automatic retry {} of {OOM_AUTO_RETRIES})", app.gpu_oom_retries);
                rebuilt = true;
            }
        }
        // The prompt opens only when the CPU path stays, so nothing has to close it.
        if !rebuilt {
            queue_fallback_notice(app, fault.to_string());
        }
        ctx.request_repaint();
    }
    if app.started.is_some() {
        // Frames 1–2 set up fonts; by frame 3 the window has presented, and a document open at
        // launch has drawn its canvas (or the GPU path was abandoned).
        let drawn = app.session.documents().is_empty() || !app.perf.last_refresh.is_empty() || app.gpu.is_none();
        if app.frame >= 3 && (drawn || app.frame >= STARTED_MAX_FRAMES) {
            if let Some(hook) = app.started.take() {
                hook(app);
            }
        } else {
            ctx.request_repaint();
        }
    }
}

/// End-of-frame check of the device the window itself draws with (#3438). Once wgpu reports it
/// lost (a driver reset, for example after the PC slept), egui-wgpu can't draw another frame: one
/// with shapes panics creating its staging buffers (#1215), and one without leaves a frozen window
/// whose close button opens an unsaved-changes prompt nobody can see. So this frame and every later
/// one draw nothing; the first writes every unsaved document to crash recovery, waits for the
/// writes, and closes the app without asking once all of them are saved. The next launch reopens
/// the documents. If a write fails (a full disk, say) the app stays open with the documents in
/// memory and tries again every [`RECOVERY_RETRY_SECS`]: closing would lose them.
pub fn check_window(app: &mut PhotocraftApp, ctx: &egui::Context) {
    let Some((device, health)) = app.window_device.as_ref() else { return };
    // A loss while the app sat idle or the PC slept is noticed here, before the next frame.
    health.probe(device);
    if !health.device_lost() {
        return;
    }
    ctx.graphics_mut(|g| *g = Default::default());
    let now = ctx.input(|i| i.time);
    if !app.window_lost {
        app.window_lost = true;
        let reason = health.fault().map_or_else(|| "GPU device lost".to_string(), |f| f.to_string());
        log::error!("{reason}: the window can't be drawn any more; saving unsaved documents for recovery and closing");
    } else {
        match app.window_recovery_retry {
            // Saved, and the close is on its way.
            None => return,
            Some(at) if now < at => {
                ctx.request_repaint_after(std::time::Duration::from_secs_f64((at - now).max(0.0)));
                return;
            }
            Some(_) => {}
        }
    }
    match save_for_recovery(app) {
        Ok(()) => {
            app.window_recovery_retry = None;
            app.allow_close = true;
            ctx.send_viewport_cmd(egui::ViewportCommand::Close);
        }
        Err(e) => {
            log::error!("{e}; keeping the documents open and trying again in {RECOVERY_RETRY_SECS} s");
            app.window_recovery_retry = Some(now + RECOVERY_RETRY_SECS);
            ctx.request_repaint_after(std::time::Duration::from_secs_f64(RECOVERY_RETRY_SECS));
        }
    }
}

/// Seconds between attempts to save unsaved work for recovery after the window's device was lost.
pub const RECOVERY_RETRY_SECS: f64 = 2.0;

/// Write every document with unsaved changes to crash recovery, whatever the autosave timer and
/// preference, and wait until the writes finished (see `Services::flush_autosaves`). `Ok` only when
/// each of them is known to be on disk.
fn save_for_recovery(app: &mut PhotocraftApp) -> Result<(), String> {
    let jobs: Vec<_> = app.session.documents().iter().filter(|d| d.is_dirty()).map(|d| (d.doc.clone(), d.revision, d.path.clone())).collect();
    if jobs.is_empty() {
        return Ok(());
    }
    // (document id, why) for each document not known to be saved.
    let mut failed: Vec<(u64, String)> = Vec::new();
    let Some(save) = app.services.autosave.as_mut() else { return Err("no crash-recovery autosave".into()) };
    for (doc, revision, path) in &jobs {
        if let Err(e) = save(doc, *revision, path.as_deref()) {
            failed.push((doc.id.0, format!("{}: {e}", doc.name)));
        }
    }
    // Without a flush, only a service whose `autosave` reports the finished write can confirm it.
    let outcomes = match app.services.flush_autosaves.as_mut() {
        Some(flush) => flush(),
        None if app.services.autosave_results.is_none() => Vec::new(),
        None => return Err("crash-recovery autosaves can't be waited for".into()),
    };
    let confirm = app.services.flush_autosaves.is_some();
    for (doc, revision, _) in &jobs {
        if failed.iter().any(|(id, _)| *id == doc.id.0) {
            continue;
        }
        // The writer skips to the newest queued snapshot, so a later revision confirms this one.
        let mine: Vec<_> = outcomes.iter().filter(|(id, written, _)| *id == doc.id.0 && written >= revision).collect();
        if let Some((.., Err(e))) = mine.iter().find(|(.., outcome)| outcome.is_err()) {
            failed.push((doc.id.0, format!("{}: {e}", doc.name)));
        } else if confirm && mine.is_empty() {
            failed.push((doc.id.0, format!("{}: the write wasn't confirmed", doc.name)));
        } else {
            log::info!("{} (revision {revision}) saved for recovery", doc.name);
        }
    }
    let failed: Vec<_> = failed.into_iter().map(|(_, why)| why).collect();
    if failed.is_empty() { Ok(()) } else { Err(format!("couldn't save for recovery: {}", failed.join("; "))) }
}

/// Drop the GPU canvas after `fault`: free its resources, send every view through the CPU
/// path, record why, and tell the user. Documents are untouched.
pub fn fall_back(app: &mut PhotocraftApp, fault: &photocraft_gpu::Fault) {
    if drop_canvas(app, fault) {
        queue_fallback_notice(app, fault.to_string());
    }
}

/// Everything [`fall_back`] does except telling the user, so [`check`] can rebuild the canvas
/// without showing a prompt. Returns whether there was a GPU canvas to drop.
fn drop_canvas(app: &mut PhotocraftApp, fault: &photocraft_gpu::Fault) -> bool {
    let Some(gpu) = app.gpu.take() else { return false };
    // The GPU canvas is gone (that's this error); the switch to the CPU is `render_path`'s line.
    log::error!("{fault}; GPU canvas dropped");
    gpu.release();
    app.gpu_fault = Some(fault.clone());
    // Canvas caches pointed at GPU textures: rebuild them as egui textures.
    app.canvases.clear();
    // Viewport pages lived on the GPU too: their work stops, and the CPU canvas draws instead.
    app.streams.clear();
    app.proxy_uploaded = None;
    app.prefs_rt.gpu_style = None;
    app.perf.gpu = false;
    crate::render_path::note_app(app, Some(crate::render_path::CpuReason::from_fault(fault)));
    // The CPU canvas has no GPU memory budget: GPU Memory Full no longer applies.
    app.perf.gpu_layer_fit = None;
    app.perf.gpu_info.canvas = "cpu".into();
    app.perf.gpu_info.lost = Some(fault.to_string());
    let title = if fault.is_lost() { LOST_MESSAGE } else { ERROR_MESSAGE };
    app.ui.status = tl!(title).to_string();
    app.ui.status_error = true;
    true
}

/// English diagnostics for issue reports (Copy and the control API), independent of UI language.
pub fn system_info(app: &PhotocraftApp) -> Vec<String> {
    crate::i18n::with_language(crate::i18n::Lang::EN, || system_info_ui(app))
}

/// Help › System Info's on-screen text: version, platform and the graphics state.
pub fn system_info_ui(app: &PhotocraftApp) -> Vec<String> {
    let mut v = vec![format!("PhotoCraft {}", photocraft_engine::build_info::long_version()), {
        let template = "Platform: {os} {arch}";
        crate::i18n::fmt(tl!(template), &[("os", std::env::consts::OS), ("arch", std::env::consts::ARCH)])
    }];
    v.extend(app.perf.gpu_info.lines());
    v.extend(crate::monitor_status::summary_lines(app));
    v
}

/// The `help.systemInfo` result: the same facts as JSON.
pub fn system_info_json(app: &PhotocraftApp) -> serde_json::Value {
    json!({
        "version": photocraft_engine::build_info::long_version(),
        "os": std::env::consts::OS,
        "arch": std::env::consts::ARCH,
        "gpu": app.perf.gpu_info,
        "monitor": app.session.color.monitor_status(),
        "displays": app.session.color.display_statuses(),
        "lines": system_info(app),
    })
}

/// Marks the Switched to CPU prompt, so [`answer`] knows its result.
struct GpuFallback;

/// The Switched to CPU prompt for `reason`. Escape keeps the CPU.
fn prompt(app: &PhotocraftApp, reason: &str) -> StandardPrompt {
    let retry_line =
        if can_retry_now(app) { tl!("Retry GPU tries the graphics processor again now.") } else { tl!("Retry GPU requires a restart. Save your work first.") };
    StandardPrompt::custom(tl!("GPU acceleration could not continue. Your documents are unchanged.").to_string())
        .title(tl!("Switched to CPU rendering").to_string())
        .paragraph(tl!("PhotoCraft is using the CPU image compositor. The window may still use your graphics adapter.").to_string())
        .details(tl!("Details").to_string(), reason.to_owned())
        .paragraph(retry_line.to_string())
        .button(PromptChoice::Proceed, tl!("Keep Using CPU").to_string())
        .button(PromptChoice::Alternate, tl!("Retry GPU").to_string())
        .escape(PromptChoice::Proceed)
        .tag(GpuFallback)
}

/// Record the recovery warning for startup and runtime failures and open its prompt.
pub fn queue_fallback_notice(app: &mut PhotocraftApp, reason: impl Into<String>) {
    let reason = reason.into();
    let prompt = prompt(app, &reason);
    // A prompt of another kind already open keeps the input; `gpu_fallback_notice` stays set for the inspector.
    let _ = app.framework.dialogs.open_unless_open(prompt);
    app.ui.gpu_fallback_notice = Some(reason);
}

/// A `PROMPTS` row: the answer of the Switched to CPU prompt. Keep Using CPU (and Escape) stores
/// the CPU mode; Retry GPU stores the GPU mode and rebuilds the canvas when the device allows. A
/// failed save reports the error and opens the prompt again, since the reason is still set.
pub fn answer(app: &mut PhotocraftApp, ctx: &egui::Context, answer: &mut PromptAnswer) -> bool {
    if answer.take_tag::<GpuFallback>().is_none() {
        return false;
    }
    let retry = answer.choice == PromptChoice::Alternate;
    match choose_recovery(app, retry) {
        Ok(()) if retry => app.retry_compositor(ctx),
        Ok(()) => (),
        Err(error) => {
            crate::notices::error(app, error);
            if let Some(reason) = app.ui.gpu_fallback_notice.clone() {
                queue_fallback_notice(app, reason);
            }
        }
    }
    true
}

/// Whether Retry GPU can build a new GPU canvas now: the canvas was dropped after an error (out
/// of memory, validation) on a device that still works. After a loss, or a GPU start that
/// failed, only a restart can try the GPU again.
fn can_retry_now(app: &PhotocraftApp) -> bool {
    app.gpu_render_state.is_some() && app.gpu_fault.as_ref().is_some_and(|f| !f.is_lost())
}

/// Build a new GPU canvas on the device the dropped one used, with a fresh health flag, and send
/// the views back through it (#6577). Returns whether the canvas is active again.
fn retry_now(app: &mut PhotocraftApp) -> bool {
    let Some(rs) = app.gpu_render_state.clone() else { return false };
    app.set_wgpu(rs);
    if !app.gpu_active() {
        return false;
    }
    log::info!("GPU canvas rebuilt after {}", app.gpu_fault.as_ref().map_or_else(String::new, ToString::to_string));
    app.gpu_fault = None;
    // The CPU canvas's egui textures: rebuild every view on the GPU.
    app.canvases.clear();
    app.proxy_uploaded = None;
    crate::render_path::note_app(app, None);
    app.perf.gpu_info.lost = None;
    app.ui.status = tl!("GPU acceleration resumed.").to_string();
    app.ui.status_error = false;
    true
}

/// Store the next-launch choice without restarting or risking unsaved documents. Retry GPU after
/// an error on a working device also builds a new GPU canvas at once (see [`retry_now`]).
fn choose_recovery(app: &mut PhotocraftApp, retry: bool) -> Result<(), String> {
    app.run(
        "prefs.set",
        json!({"values": {
            "performance.renderingMode": if retry { "gpu" } else { "cpu" },
            "performance.useGpu": retry,
            "performance.gpuBackend": "auto"
        }}),
    )?;
    crate::prefs_ui::save_preferences(app)?;
    app.ui.gpu_fallback_notice = None;
    if retry && can_retry_now(app) && retry_now(app) {
        return Ok(());
    }
    if retry {
        crate::notices::post(
            app,
            "GPU retry scheduled",
            vec!["Save your work and restart PhotoCraft to retry GPU acceleration. CPU rendering remains active for this session.".into()],
            craft::theme::variant::Tone::Neutral,
            None,
        );
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn system_info_copy_and_api_stay_english_in_czech_ui() {
        if !crate::i18n::needs_catalogs() {
            return;
        }
        let mut app = PhotocraftApp::new(photocraft_engine::Session::new(), crate::Services::default());
        app.perf.gpu_info.adapter = "Test GPU".into();
        app.perf.gpu_info.preference = "auto".into();
        let english = crate::i18n::with_language(crate::i18n::Lang::EN, || system_info(&app));
        crate::i18n::with_language(crate::i18n::Lang::from_code("cs").unwrap(), || {
            assert_eq!(system_info(&app), english, "clipboard diagnostics include English monitor profiles too");
            let app_ctx = app.ctx().clone();
            let result = crate::menus::invoke(&mut app, &app_ctx, "help.systemInfo", json!({})).unwrap();
            assert_eq!(result["info"]["lines"], json!(english));
            assert_eq!(crate::i18n::current().code(), "cs", "diagnostics restore the drawing language");
            let screen = system_info_ui(&app);
            let cs = crate::i18n::current();
            let adapter = crate::i18n::fmt(crate::i18n::tr(cs, "Graphics adapter: {adapter}"), &[("adapter", "Test GPU")]);
            assert!(screen.contains(&adapter), "{screen:?}");
            let profile = crate::i18n::tr(cs, "Monitor profile: {profile}").split('{').next().unwrap_or_default();
            assert!(screen.iter().any(|line| line.starts_with(profile)), "{screen:?}");
            assert!(english.iter().any(|line| line == "GPU backend preference: auto"));
        });
    }

    #[test]
    fn system_info_translates_empty_and_stored_auto_only_for_display() {
        if !crate::i18n::needs_catalogs() {
            return;
        }
        let mut app = PhotocraftApp::new(photocraft_engine::Session::new(), crate::Services::default());
        // Hardware names are opaque even when they happen to match a lookup key.
        app.perf.gpu_info.adapter = "auto".into();
        crate::i18n::with_language(crate::i18n::Lang::from_code("cs").unwrap(), || {
            let empty = system_info_ui(&app);
            app.perf.gpu_info.preference = "auto".into();
            app.perf.gpu_info.selected = "auto".into();
            assert_eq!(system_info_ui(&app), empty);
            let cs = crate::i18n::current();
            let adapter = crate::i18n::fmt(crate::i18n::tr(cs, "Graphics adapter: {adapter}"), &[("adapter", "auto")]);
            assert!(empty.contains(&adapter), "{empty:?}");
            let preference = crate::i18n::fmt(crate::i18n::tr(cs, "GPU backend preference: {backend}"), &[("backend", crate::i18n::tr(cs, "auto"))]);
            assert!(empty.contains(&preference), "{empty:?}");
            assert_eq!(app.perf.gpu_info.preference, "auto");
            assert_eq!(app.perf.gpu_info.selected, "auto");
        });
    }

    /// An app whose settings live in `files` (a scratch location), so a second launch reads what
    /// the first one stored.
    fn launch(files: &std::sync::Arc<craft::settings::testing::MemoryFiles>) -> Box<PhotocraftApp> {
        let scratch = std::env::temp_dir().join("photocraft-gpu-status");
        let options = craft::settings::OpenOptions::new(craft::settings::Location::Dir(scratch)).files(files.clone());
        let services = crate::Services { settings: Some(craft::settings::SettingsStore::open_with(options)), ..Default::default() };
        PhotocraftApp::new(photocraft_engine::Session::new(), services)
    }

    #[test]
    fn failed_preference_save_keeps_recovery_warning_open() {
        let files = std::sync::Arc::new(craft::settings::testing::MemoryFiles::new());
        let mut app = launch(&files);
        files.fail_writes(true);
        queue_fallback_notice(&mut app, "GPU fault");
        assert!(app.framework.dialogs.is_open::<StandardPrompt>(), "the prompt opens with the notice");
        assert!(choose_recovery(&mut app, true).is_err());
        assert_eq!(app.ui.gpu_fallback_notice.as_deref(), Some("GPU fault"));
        assert!(app.ui.notices.is_empty());
    }

    #[test]
    fn recovery_choices_save_next_launch_mode_without_restarting() {
        let files = std::sync::Arc::new(craft::settings::testing::MemoryFiles::new());
        let mut app = launch(&files);
        queue_fallback_notice(&mut app, "device unavailable");
        choose_recovery(&mut app, false).unwrap();
        assert!(!app.session.prefs().performance.use_gpu);
        assert_eq!(app.session.prefs().get("performance.renderingMode"), Some(json!("cpu")));
        assert!(app.ui.gpu_fallback_notice.is_none());
        assert!(!launch(&files).session.prefs().performance.use_gpu, "the next launch starts on the CPU");
        choose_recovery(&mut app, true).unwrap();
        assert!(app.session.prefs().performance.use_gpu);
        assert_eq!(app.session.prefs().get("performance.renderingMode"), Some(json!("gpu")));
        assert!(app.gpu.is_none());
        assert!(launch(&files).session.prefs().performance.use_gpu, "the next launch retries the GPU");
    }

    mod gpu_tests {
        use super::*;

        /// #6564: an out-of-memory error on a device that still works rebuilds the GPU canvas by itself,
        /// with no notice and the documents untouched, a few times a session; then the CPU renderer
        /// stays until Retry GPU. Other errors and a loss still fall back at once.
        #[test]
        fn out_of_memory_rebuilds_the_gpu_canvas_by_itself() {
            let _gpu = photocraft_testkit::gpu_lock();
            let rs = std::panic::catch_unwind(|| egui_kittest::wgpu::create_render_state(crate::gpu_canvas::wgpu_setup(), Default::default()));
            let Ok(rs) = rs else {
                eprintln!("skipping: no GPU adapter");
                return;
            };
            let mut app = PhotocraftApp::new(photocraft_engine::Session::new(), Default::default());
            app.set_wgpu(rs);
            app.run("file.new", json!({"width": 64, "height": 48})).unwrap();
            let ctx = app.ctx().clone();
            let oom = || photocraft_gpu::Fault::Error("Out of Memory".into());
            for n in 1..=OOM_AUTO_RETRIES {
                app.gpu_health().unwrap().mark(oom());
                check(&mut app, &ctx);
                assert!(app.gpu_active(), "retry {n}: the GPU canvas is back");
                assert!(app.gpu_health().unwrap().is_ok(), "retry {n}: with a fresh health flag");
                assert_eq!(app.perf.gpu_info.canvas, "gpu");
                assert!(app.ui.gpu_fallback_notice.is_none(), "retry {n}: no recovery dialog");
                assert_eq!(app.session.documents().len(), 1);
            }
            // Past the limit the CPU renderer stays and the dialog offers Retry GPU.
            app.gpu_health().unwrap().mark(oom());
            check(&mut app, &ctx);
            assert!(!app.gpu_active() && app.ui.gpu_fallback_notice.is_some());
            assert_eq!(app.perf.gpu_info.canvas, "cpu");

            // A validation error is not retried by itself.
            let rs = egui_kittest::wgpu::create_render_state(crate::gpu_canvas::wgpu_setup(), Default::default());
            let mut app = PhotocraftApp::new(photocraft_engine::Session::new(), Default::default());
            app.set_wgpu(rs);
            app.run("file.new", json!({"width": 64, "height": 48})).unwrap();
            app.gpu_health().unwrap().mark(photocraft_gpu::Fault::Error("Validation Error".into()));
            check(&mut app, &ctx);
            assert!(!app.gpu_active() && app.ui.gpu_fallback_notice.is_some());
        }

        /// #6577: after a GPU error on a device that still works (out of memory, validation), Retry
        /// GPU builds a new GPU canvas at once, without a restart or closing documents. After a loss
        /// the device can't be used again, so the retry still waits for the next launch.
        #[test]
        fn retry_gpu_after_an_error_rebuilds_the_canvas_without_a_restart() {
            let _gpu = photocraft_testkit::gpu_lock();
            let rs = std::panic::catch_unwind(|| egui_kittest::wgpu::create_render_state(crate::gpu_canvas::wgpu_setup(), Default::default()));
            let Ok(rs) = rs else {
                eprintln!("skipping: no GPU adapter");
                return;
            };
            let mut app = launch(&std::sync::Arc::new(craft::settings::testing::MemoryFiles::new()));
            app.set_wgpu(rs);
            app.run("file.new", json!({"width": 64, "height": 48})).unwrap();
            let ctx = app.ctx().clone();
            // An error on a live device (one not retried by itself, see #6564): Retry GPU rebuilds
            // the canvas now.
            app.gpu_health().unwrap().mark(photocraft_gpu::Fault::Error("Validation Error".into()));
            check(&mut app, &ctx);
            assert!(!app.gpu_active() && app.ui.gpu_fallback_notice.is_some());
            assert!(can_retry_now(&app));
            choose_recovery(&mut app, true).unwrap();
            assert!(app.gpu_active(), "Retry GPU rebuilt the GPU canvas");
            assert!(app.gpu_health().unwrap().is_ok(), "with a fresh health flag");
            assert_eq!(app.perf.gpu_info.canvas, "gpu");
            assert_eq!((app.perf.gpu_info.lost.as_deref(), app.perf.gpu_fallback.as_deref()), (None, None));
            assert_eq!(app.ui.status, "GPU acceleration resumed.");
            assert!(app.ui.gpu_fallback_notice.is_none() && app.ui.notices.is_empty(), "no restart notice");
            assert_eq!(app.session.documents().len(), 1, "the document stayed open");
            // A loss: the device can't be reused, so the retry is scheduled for the next launch.
            app.gpu_health().unwrap().mark(photocraft_gpu::Fault::Lost("simulated".into()));
            check(&mut app, &ctx);
            assert!(!can_retry_now(&app));
            choose_recovery(&mut app, true).unwrap();
            assert!(!app.gpu_active());
            assert_eq!(app.ui.notices.len(), 1, "the restart notice");
        }
    }
}