856f865cf98d
health.rs
//! Device health (#243): notices a lost wgpu device or an uncaptured wgpu error so callers stop
//! issuing GPU work and fall back to the CPU compositor instead of panicking.
//!
//! wgpu's default uncaptured-error handler panics ("Buffer 'pc_compose_uniforms' is invalid"
//! after the OS killed a huge submission), and `Device::poll` panics on a lost device whatever
//! the handler. [`DeviceHealth::watch`] replaces the handler with one that records the fault,
//! registers the device-lost callback, and [`DeviceHealth::wait`] polls only while the device is
//! healthy (catching the poll's panic as a last resort). Every GPU entry point checks
//! [`DeviceHealth::fault`] first.
//!
//! Builds for the web too: the callbacks only touch atomics and a mutex.
use std::fmt;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
#[cfg(not(target_arch = "wasm32"))]
use std::time::{Duration, Instant};
/// Outcome of a bounded wait for submitted GPU work.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum WaitOutcome {
/// The submitted work finished before the deadline.
Finished,
/// The deadline passed (or the device faulted) before the work finished.
TimedOut,
/// The caller cancelled the wait before the work finished.
Cancelled,
}
/// Why the GPU can't be used any more.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Fault {
/// The device was lost (driver reset, the OS killed a submission, `Device::destroy`).
Lost(String),
/// An uncaptured wgpu error (out of memory, validation, internal): the GPU output can't be
/// trusted, and on most drivers the device is about to be lost anyway.
Error(String),
}
impl Fault {
/// Whether the device itself was lost (rather than an error reported).
pub fn is_lost(&self) -> bool {
matches!(self, Fault::Lost(_))
}
/// An out-of-memory error on a device that still works: freeing memory and trying again can
/// succeed, unlike after a loss or a validation error (#6564). Only wgpu's own out-of-memory
/// error is recorded with exactly [`OUT_OF_MEMORY`]; a validation error that merely mentions
/// memory is not one.
pub fn is_out_of_memory(&self) -> bool {
matches!(self, Fault::Error(s) if s == OUT_OF_MEMORY)
}
/// The driver's or wgpu's message.
pub fn detail(&self) -> &str {
match self {
Fault::Lost(s) | Fault::Error(s) => s,
}
}
}
impl fmt::Display for Fault {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Fault::Lost(s) if s.is_empty() => f.write_str("GPU device lost"),
Fault::Lost(s) => write!(f, "GPU device lost: {s}"),
Fault::Error(s) => write!(f, "GPU error: {s}"),
}
}
}
/// The message an out-of-memory [`Fault::Error`] carries (wgpu's `Error::OutOfMemory`).
pub const OUT_OF_MEMORY: &str = "Out of Memory";
#[derive(Default)]
struct Inner {
faulted: AtomicBool,
fault: Mutex<Option<Fault>>,
errors: AtomicU64,
/// wgpu itself reported the device lost (see [`DeviceHealth::device_lost`]).
device_lost: AtomicBool,
}
/// Shared health flag of one wgpu device. Cheap to clone; every clone sees the same state.
#[derive(Clone, Default)]
pub struct DeviceHealth(Arc<Inner>);
impl fmt::Debug for DeviceHealth {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("DeviceHealth").field("fault", &self.fault()).field("errors", &self.errors()).finish()
}
}
impl DeviceHealth {
/// A healthy flag not attached to a device (tests, CPU-only callers).
pub fn new() -> Self {
Self::default()
}
/// Watch `device`: its device-lost callback and uncaptured errors mark the returned flag
/// instead of panicking. Replaces any handler set before (wgpu keeps one of each).
pub fn watch(device: &wgpu::Device) -> Self {
let health = Self::new();
let lost = health.clone();
// Called by wgpu with its own locks held: only record, never call back into wgpu.
device.set_device_lost_callback(move |reason, message| {
let detail = match (reason, message.is_empty()) {
(wgpu::DeviceLostReason::Destroyed, true) => "device destroyed".to_string(),
(_, true) => "lost by the driver".to_string(),
(_, false) => message,
};
log::error!("wgpu device lost ({reason:?}): {detail}");
lost.0.device_lost.store(true, Ordering::Release);
lost.mark(Fault::Lost(detail));
});
let errors = health.clone();
device.on_uncaptured_error(Arc::new(move |e: wgpu::Error| errors.uncaptured(&e)));
health
}
/// Record an uncaptured wgpu error.
fn uncaptured(&self, e: &wgpu::Error) {
self.0.errors.fetch_add(1, Ordering::Relaxed);
let text = e.to_string();
log::error!("uncaptured wgpu error: {text}");
self.mark(fault_of(e));
}
/// Record an error an error scope caught.
///
/// [`DeviceHealth::watch`]'s uncaptured-error handler never sees a scoped error, so without
/// this a watched device would keep being handed to a GPU that has already failed. Polls
/// first: a lost device is a [`Fault::Lost`], and its callback may not have run yet (it fires
/// on a poll, and `Device::poll` panics on a lost device).
///
/// The poll is a single non-blocking check, never a fence wait. This runs on the UI thread
/// immediately after a failed frame, so `PollType::Wait` with no timeout would stall the app
/// until every in-flight submission drained (or forever, if the device is wedged). A
/// non-blocking poll loses nothing here: wgpu still reads the fence and fires the
/// device-lost callback for a lost device, it just never blocks waiting for one.
///
/// A device that only recorded an error is still polled and marked, so a loss or an
/// out-of-memory error can upgrade that fault as [`DeviceHealth::mark`] allows (#6564).
pub fn caught(&self, device: &wgpu::Device, error: &wgpu::Error) {
if self.lost() {
return;
}
#[cfg(not(target_arch = "wasm32"))]
self.poll(device);
#[cfg(target_arch = "wasm32")]
let _ = device;
if self.lost() {
return;
}
self.mark(fault_of(error));
}
fn lost(&self) -> bool {
self.fault().is_some_and(|f| f.is_lost())
}
/// Mark the device unusable. The first fault is kept; a later device loss upgrades an error,
/// and a later out-of-memory error upgrades another one, so the canvas can still recover by
/// itself when memory ran out after a validation error in the same frame (#6564).
pub fn mark(&self, fault: Fault) {
let mut slot = self.0.fault.lock().unwrap_or_else(PoisonError::into_inner);
let replace = match slot.as_ref() {
None => true,
Some(old) => !old.is_lost() && (fault.is_lost() || (fault.is_out_of_memory() && !old.is_out_of_memory())),
};
if replace {
*slot = Some(fault);
}
self.0.faulted.store(true, Ordering::Release);
}
/// Whether the device may still be used.
pub fn is_ok(&self) -> bool {
!self.0.faulted.load(Ordering::Acquire)
}
/// Why the device can't be used, if it can't.
pub fn fault(&self) -> Option<Fault> {
if self.is_ok() {
return None;
}
self.0.fault.lock().unwrap_or_else(PoisonError::into_inner).clone()
}
/// Uncaptured wgpu errors seen so far.
pub fn errors(&self) -> u64 {
self.0.errors.load(Ordering::Relaxed)
}
/// Whether wgpu itself reported the device lost: its device-lost callback ran, or polling it
/// panicked. Unlike a fault recorded with [`mark`](Self::mark) (a simulated loss, or an
/// uncaptured error that names one), nothing drawn with the device reaches the screen any
/// more, the window's own frames included (#3438).
pub fn device_lost(&self) -> bool {
self.0.device_lost.load(Ordering::Acquire)
}
/// Non-blocking poll that runs even after a fault was recorded, so a device that is lost
/// later (a driver reset while the app sat idle or the PC slept) is noticed before the next
/// frame is drawn with it. Skipped once [`device_lost`](Self::device_lost).
pub fn probe(&self, device: &wgpu::Device) {
if self.device_lost() {
return;
}
if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| device.poll(wgpu::PollType::Poll))).is_err() {
self.lost_by_poll();
}
}
/// `Device::poll` panicked: wgpu found the device lost.
fn lost_by_poll(&self) {
self.0.device_lost.store(true, Ordering::Release);
self.mark(Fault::Lost("the device stopped responding".into()));
}
/// Block until `index` (or all submitted work) finished, unless the device is unusable.
/// Returns whether the device is still healthy afterwards. `Device::poll` panics on a lost
/// device, so a panic is caught and recorded as a loss (native only; the web has no blocking
/// wait).
#[cfg(not(target_arch = "wasm32"))]
pub fn wait(&self, device: &wgpu::Device, index: Option<wgpu::SubmissionIndex>) -> bool {
self.wait_idle(device, index);
self.is_ok()
}
/// Like [`Self::wait`], but returns whether the wait completed: `false` also when the poll
/// itself failed, so the work may still be running.
#[cfg(not(target_arch = "wasm32"))]
pub fn wait_idle(&self, device: &wgpu::Device, index: Option<wgpu::SubmissionIndex>) -> bool {
if !self.is_ok() {
return false;
}
let poll = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| device.poll(wgpu::PollType::Wait { submission_index: index, timeout: None })));
match poll {
Ok(Ok(_)) => self.is_ok(),
Ok(Err(e)) => {
log::warn!("wgpu poll: {e}");
false
}
Err(_) => {
self.lost_by_poll();
false
}
}
}
/// Non-blocking poll (reclaims staging memory), skipped once the device is unusable.
pub fn poll(&self, device: &wgpu::Device) {
if !self.is_ok() {
return;
}
if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| device.poll(wgpu::PollType::Poll))).is_err() {
self.lost_by_poll();
}
}
/// Like [`Self::wait`], but bounded: returns whether the submitted work finished within
/// `timeout`. A timeout (or a poll that fails without reporting a loss) marks the device
/// faulted, so later exports stop submitting work to a wedged GPU instead of each waiting the
/// full timeout again. A device that was already faulted returns `false` untouched.
#[cfg(not(target_arch = "wasm32"))]
pub fn wait_timeout(&self, device: &wgpu::Device, index: Option<wgpu::SubmissionIndex>, timeout: Duration) -> bool {
self.wait_timeout_cancellable(device, index, timeout, &|| false) == WaitOutcome::Finished
}
/// [`Self::wait_timeout`] that polls in short slices and checks `cancelled` between them, so a
/// cancelled headless export stops waiting promptly while keeping the same overall deadline.
/// A slice timeout means "not finished yet"; only the overall deadline (or a failed poll)
/// marks the device faulted.
#[cfg(not(target_arch = "wasm32"))]
pub fn wait_timeout_cancellable(
&self,
device: &wgpu::Device,
index: Option<wgpu::SubmissionIndex>,
timeout: Duration,
cancelled: &(dyn Fn() -> bool + Sync),
) -> WaitOutcome {
if !self.is_ok() {
return WaitOutcome::TimedOut;
}
let mut failure = None;
let outcome = wait_slices(timeout, cancelled, |slice| {
let poll = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
device.poll(wgpu::PollType::Wait { submission_index: index.clone(), timeout: Some(slice) })
}));
match poll {
Ok(Ok(_)) => {
if self.is_ok() {
SlicePoll::Done
} else {
SlicePoll::Failed
}
}
Ok(Err(wgpu::PollError::Timeout)) => SlicePoll::Pending,
Ok(Err(e)) => {
failure = Some(format!("GPU poll failed: {e}"));
SlicePoll::Failed
}
Err(_) => {
self.lost_by_poll();
SlicePoll::Failed
}
}
});
if outcome == WaitOutcome::TimedOut && self.is_ok() {
let fault = failure.map_or_else(|| Fault::Lost(format!("GPU wait timed out after {:.1} s", timeout.as_secs_f64())), Fault::Error);
self.mark(fault);
}
outcome
}
}
/// What one bounded poll reported.
#[cfg(not(target_arch = "wasm32"))]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum SlicePoll {
Done,
Pending,
Failed,
}
/// Wait for `total`, polling in slices no longer than 100 ms and checking `cancelled` between
/// them. A slice that times out is pending, not a failure; only the overall deadline or a failed
/// poll ends the wait early.
#[cfg(not(target_arch = "wasm32"))]
fn wait_slices(total: Duration, cancelled: &(dyn Fn() -> bool + Sync), mut poll: impl FnMut(Duration) -> SlicePoll) -> WaitOutcome {
const SLICE: Duration = Duration::from_millis(100);
let start = Instant::now();
loop {
if cancelled() {
return WaitOutcome::Cancelled;
}
let remaining = total.saturating_sub(start.elapsed());
if remaining.is_zero() {
return WaitOutcome::TimedOut;
}
match poll(remaining.min(SLICE)) {
SlicePoll::Done => return WaitOutcome::Finished,
SlicePoll::Pending => {}
SlicePoll::Failed => return WaitOutcome::TimedOut,
}
}
}
/// The fault a wgpu error stands for. Errors about invalid or lost objects follow a device loss
/// whose callback may not have run yet (it fires on the next poll), so they count as a loss
/// rather than as an error. Only wgpu's own out-of-memory error is [`OUT_OF_MEMORY`].
fn fault_of(e: &wgpu::Error) -> Fault {
let text = e.to_string();
let lower = text.to_ascii_lowercase();
if matches!(e, wgpu::Error::OutOfMemory { .. }) {
Fault::Error(OUT_OF_MEMORY.into())
} else if lower.contains("lost") || lower.contains("destroyed") {
Fault::Lost(first_line(&text))
} else {
Fault::Error(first_line(&text))
}
}
fn first_line(s: &str) -> String {
let line = s.lines().map(str::trim).find(|l| !l.is_empty()).unwrap_or("");
let mut out: String = line.chars().take(200).collect();
if line.chars().count() > 200 {
out.push('…');
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn first_fault_wins_but_loss_upgrades_an_error() {
let h = DeviceHealth::new();
assert!(h.is_ok());
assert_eq!(h.fault(), None);
h.mark(Fault::Error("out of memory".into()));
assert!(!h.is_ok());
let c = h.clone();
c.mark(Fault::Error("second".into()));
assert_eq!(h.fault(), Some(Fault::Error("out of memory".into())));
c.mark(Fault::Lost("reset".into()));
assert_eq!(h.fault(), Some(Fault::Lost("reset".into())));
c.mark(Fault::Lost("later".into()));
assert_eq!(h.fault().map(|f| f.to_string()).as_deref(), Some("GPU device lost: reset"));
// A recorded loss is not wgpu reporting the device lost (#3438).
assert!(!h.device_lost());
}
#[test]
#[cfg(not(target_arch = "wasm32"))]
fn wait_slices_cancel_timeout_and_completion() {
let never = || false;
assert_eq!(wait_slices(Duration::ZERO, &never, |_| SlicePoll::Done), WaitOutcome::TimedOut);
assert_eq!(wait_slices(Duration::from_secs(1), &never, |_| SlicePoll::Done), WaitOutcome::Finished);
assert_eq!(wait_slices(Duration::from_secs(1), &never, |_| SlicePoll::Failed), WaitOutcome::TimedOut);
let mut polls = 0;
let outcome = wait_slices(Duration::from_secs(1), &never, |_| {
polls += 1;
if polls < 3 { SlicePoll::Pending } else { SlicePoll::Done }
});
assert_eq!(outcome, WaitOutcome::Finished);
assert_eq!(polls, 3, "a timed-out slice must be retried, not treated as failure");
let cancelled = || true;
let mut called = false;
assert_eq!(
wait_slices(Duration::from_secs(1), &cancelled, |_| {
called = true;
SlicePoll::Done
}),
WaitOutcome::Cancelled
);
assert!(!called, "cancellation must be checked before polling");
}
#[test]
#[cfg(not(target_arch = "wasm32"))]
fn wait_slices_observes_cancel_while_pending() {
let flag = AtomicBool::new(false);
let cancelled = || flag.load(Ordering::SeqCst);
let mut polls = 0;
let outcome = wait_slices(Duration::from_secs(1), &cancelled, |_| {
polls += 1;
flag.store(true, Ordering::SeqCst);
SlicePoll::Pending
});
assert_eq!(outcome, WaitOutcome::Cancelled);
assert_eq!(polls, 1, "cancellation set during a pending poll must end the wait");
}
#[test]
fn fault_text() {
assert_eq!(Fault::Lost(String::new()).to_string(), "GPU device lost");
assert_eq!(Fault::Error("x".into()).to_string(), "GPU error: x");
assert_eq!(first_line("\n Buffer 'pc' is invalid\nmore"), "Buffer 'pc' is invalid");
assert_eq!(first_line(&"a".repeat(300)).chars().count(), 201);
assert!(Fault::Error("Out of Memory".into()).is_out_of_memory());
assert!(!Fault::Error("Validation Error".into()).is_out_of_memory());
assert!(!Fault::Error("Validation Error: buffer allocation would run out of memory".into()).is_out_of_memory());
// A later out-of-memory error replaces another error, but never a loss.
let h = DeviceHealth::new();
h.mark(Fault::Error("Validation Error".into()));
h.mark(Fault::Error(OUT_OF_MEMORY.into()));
assert!(h.fault().is_some_and(|f| f.is_out_of_memory()));
h.mark(Fault::Lost("reset".into()));
h.mark(Fault::Error(OUT_OF_MEMORY.into()));
assert!(h.fault().is_some_and(|f| f.is_lost()));
assert!(!Fault::Lost("out of memory".into()).is_out_of_memory());
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.