5a24289898fd
ctx.rs
//! The wgpu device, the compiled kernels, buffers, dispatch and readback.
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
/// A kernel module: WGSL source, its storage bindings (name, writable, element type) after the
/// parameter block `P` (binding 0), and its entry points.
struct Module {
src: &'static str,
bindings: &'static [(&'static str, bool, &'static str)],
entries: &'static [&'static str],
}
const MODULES: &[Module] = &[
Module {
src: include_str!("wgsl/finish.wgsl"),
bindings: &[
("img", false, "f32"),
("log_l", false, "f32"),
("base_p", false, "f32"),
("clar", false, "f32"),
("tex", false, "f32"),
("dark", false, "f32"),
("masks", false, "f32"),
("aux", false, "f32"),
("out", true, "u32"),
],
entries: &["main"],
},
Module {
src: include_str!("wgsl/blur.wgsl"),
bindings: &[("src", false, "f32"), ("dst", true, "f32")],
entries: &["box_h", "box_v", "minmax", "median_h", "median_v"],
},
Module {
src: include_str!("wgsl/gauss.wgsl"),
bindings: &[("src", false, "f32"), ("taps", false, "f32"), ("dst", true, "f32")],
entries: &["gauss_h", "gauss_v"],
},
Module {
src: include_str!("wgsl/resize.wgsl"),
bindings: &[("src", false, "f32"), ("table", false, "u32"), ("dst", true, "f32")],
entries: &["resize_h", "resize_v"],
},
Module {
src: include_str!("wgsl/map.wgsl"),
bindings: &[("a", false, "f32"), ("b", false, "f32"), ("c", false, "f32"), ("dst", true, "f32")],
entries: &[
"log_lum_k",
"dark_k",
"guided_pre",
"guided_ab",
"guided_apply",
"clamp_between_k",
"wb_k",
"nr_lum",
"chroma_k",
"nr_col",
"subsample",
"redeye_k",
"xguided_pre",
"xguided_ab",
"xguided_apply",
"fringe_lab",
"fringe_apply",
],
},
Module {
src: include_str!("wgsl/nrcj.wgsl"),
bindings: &[("a", false, "f32"), ("b", false, "f32"), ("dst", true, "f32")],
entries: &[
"nrcj_yuv",
"nrcj_uv",
"nrcj_samples",
"nrcj_speck",
"nrcj_pilot",
"nrcj_links1",
"nrcj_paths",
"nrcj_links",
"nrcj_dh",
"nrcj_dv",
"nrcj_recon",
],
},
Module {
src: include_str!("wgsl/spot.wgsl"),
bindings: &[("img", true, "f32"), ("pts", false, "f32"), ("al", true, "f32"), ("sp", true, "f32"), ("df", true, "f32"), ("wt", true, "f32")],
entries: &["spot_prep", "spot_apply", "spot_score"],
},
Module { src: include_str!("wgsl/hist.wgsl"), bindings: &[("px", false, "u32"), ("bins", true, "atomic<u32>")], entries: &["histogram"] },
Module {
src: include_str!("wgsl/mask.wgsl"),
bindings: &[("img", false, "f32"), ("log_l", false, "f32"), ("aux", false, "f32"), ("c", true, "f32"), ("alpha", true, "f32")],
entries: &["shape", "combine", "finalize"],
},
Module {
src: include_str!("wgsl/present.wgsl"),
bindings: &[("px", false, "u32"), ("dst", true, "u32"), ("count", true, "atomic<u32>")],
entries: &["unwritten", "pad_rows"],
},
Module {
src: include_str!("wgsl/geom.wgsl"),
bindings: &[("src", false, "f32"), ("dst", true, "f32"), ("coverage", false, "u32")],
entries: &["orient", "sample_affine", "sample_warp"],
},
];
struct Kernel {
pipeline: wgpu::ComputePipeline,
layout: wgpu::BindGroupLayout,
nbuf: usize,
}
/// A module's shader and layouts, shared by the kernels of its entry points.
struct Prepared {
module: wgpu::ShaderModule,
layout: wgpu::BindGroupLayout,
pipeline_layout: wgpu::PipelineLayout,
nbuf: usize,
}
/// Validation and internal error scopes. They belong to the thread that pushed them.
fn push_scopes(device: &wgpu::Device) -> [wgpu::ErrorScopeGuard; 2] {
[device.push_error_scope(wgpu::ErrorFilter::Validation), device.push_error_scope(wgpu::ErrorFilter::Internal)]
}
/// The error the scopes caught, if any (popped innermost first).
fn pop_scopes(scopes: [wgpu::ErrorScopeGuard; 2]) -> Option<String> {
let [validation, internal] = scopes;
let (internal, validation) = (pollster::block_on(internal.pop()), pollster::block_on(validation.pop()));
internal.or(validation).map(|e| e.to_string())
}
/// Build every kernel. A kernel the driver's compiler refuses is a validation or an internal error,
/// and then there is no GPU, not a half-built one.
///
/// Issue #2936: built one after another, the 27 pipelines took ~2.2 s on an Intel iGPU under DX12
/// (device start-up 1.7โ4.4 s in all, 7 s on the first run after an install), where FXC (issue #471)
/// compiles each one, single-threaded, inside `create_compute_pipeline`. The
/// device takes pipeline creation from any thread, so the entry points are shared out to worker
/// threads. This thread works too: every kernel is built even when no thread can be started.
fn build_kernels(device: &wgpu::Device) -> Result<HashMap<&'static str, Kernel>, String> {
let scopes = push_scopes(device);
let consts = constants();
let prepared: Vec<Prepared> = MODULES
.iter()
.map(|m| {
let src = module_source(m, &consts);
let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: None, source: wgpu::ShaderSource::Wgsl(src.into()) });
let mut entries = vec![storage_entry(0, true)];
entries.extend(m.bindings.iter().enumerate().map(|(i, (_, w, _))| storage_entry(i as u32 + 1, !w)));
let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: None, entries: &entries });
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: None,
bind_group_layouts: &[Some(&layout)],
immediate_size: 0,
});
Prepared { module, layout, pipeline_layout, nbuf: m.bindings.len() }
})
.collect();
if let Some(e) = pop_scopes(scopes) {
return Err(e);
}
let jobs: Vec<(&Prepared, &'static str)> = MODULES.iter().zip(&prepared).flat_map(|(m, p)| m.entries.iter().map(move |e| (p, *e))).collect();
let next = AtomicUsize::new(0);
let built = std::sync::Mutex::new(HashMap::with_capacity(jobs.len()));
let workers = std::thread::available_parallelism().map_or(1, |n| n.get()).min(jobs.len()).saturating_sub(1);
let errors: Vec<String> = std::thread::scope(|s| {
let work = || build_jobs(device, &jobs, &next, &built);
let threads: Vec<_> =
(0..workers).filter_map(|_| std::thread::Builder::new().name("lc-gpu-kernels".into()).spawn_scoped(s, work).ok()).collect();
let mut errors: Vec<String> = work().into_iter().collect();
for t in threads {
match t.join() {
Ok(e) => errors.extend(e),
Err(_) => errors.push("a kernel build thread panicked".into()),
}
}
errors
});
if let Some(e) = errors.into_iter().next() {
return Err(e);
}
Ok(built.into_inner().unwrap_or_else(std::sync::PoisonError::into_inner))
}
/// One worker of [`build_kernels`]: build kernels until none is left; the first error its scopes caught.
fn build_jobs(
device: &wgpu::Device,
jobs: &[(&Prepared, &'static str)],
next: &AtomicUsize,
built: &std::sync::Mutex<HashMap<&'static str, Kernel>>,
) -> Option<String> {
let scopes = push_scopes(device);
while let Some(&(p, entry)) = jobs.get(next.fetch_add(1, Ordering::Relaxed)) {
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some(entry),
layout: Some(&p.pipeline_layout),
module: &p.module,
entry_point: Some(entry),
// Only the median kernels (`blur.wgsl`, `med_tile`) use workgroup memory, and they
// load every element they read before the barrier, so wgpu's default zeroing is not
// needed. On DX12 it is written out as one assignment per element, which made FXC take
// about 87 s over these kernels at every start of a release build.
compilation_options: wgpu::PipelineCompilationOptions { zero_initialize_workgroup_memory: false, ..Default::default() },
cache: None,
});
let kernel = Kernel { pipeline, layout: p.layout.clone(), nbuf: p.nbuf };
built.lock().unwrap_or_else(std::sync::PoisonError::into_inner).insert(entry, kernel);
}
pop_scopes(scopes)
}
/// A device buffer of `len` 32-bit values.
///
/// Dropped buffers are recycled: they first wait in a per-thread list (commands recorded on this
/// thread may still use them) and become reusable once this thread submits ([`Gpu::submitted`]),
/// as the queue then runs every earlier use before any later one. Reuse matters: allocating (and
/// zero-filling) a fresh 100โ300 MB buffer per pass costs as much as the pass itself.
pub struct Buf {
buf: Option<Tracked>,
pub len: usize,
}
impl Buf {
pub fn raw(&self) -> &wgpu::Buffer {
// `buf` is `Some` from construction (`Gpu::buffer`/`upload`) until `Drop` takes it back
// for the pool, so a live `Buf` always has it.
#[allow(clippy::expect_used)]
&self.buf.as_ref().expect("live buffer").0
}
}
impl Drop for Buf {
fn drop(&mut self) {
let Some(b) = self.buf.take() else { return };
// Outside a render this thread has no commands recorded (only renders record), so the
// buffer is reusable at once; without this, buffers dropped on threads that rarely
// submit (the UI thread clearing a view's stages) would wait indefinitely.
let Ok(in_render) = IN_RENDER.try_with(|c| c.get() > 0) else {
// thread teardown: wgpu's thread-locals may be gone too, and dropping (or trimming
// the pool, which drops) could abort โ leak it (the device owns its memory)
std::mem::forget(b);
return;
};
if !in_render && let Some(g) = crate::existing_device() {
g.pool([b]);
return;
}
// during thread teardown the list may already be gone: then the buffer leaks
let size = b.0.size();
let mut b = Some(b);
let _ = RETIRED.try_with(|r| r.0.borrow_mut().extend(b.take()));
match b {
// Leaking is deliberate during thread teardown โ dropping a wgpu buffer whose device
// thread-locals have already gone can abort the process. The device owns it.
Some(b) => std::mem::forget(b), // NOSONAR
None => {
RETIRED_BYTES.fetch_add(size, Ordering::Relaxed);
}
}
}
}
/// Buffers dropped on a thread since its last submit.
struct Retired(std::cell::RefCell<Vec<Tracked>>);
impl Drop for Retired {
/// At thread exit wgpu's own thread-locals may already be destroyed, and dropping a buffer
/// then aborts the process: leak the few buffers left instead (the device owns their memory).
fn drop(&mut self) {
for b in self.0.get_mut().drain(..) {
RETIRED_BYTES.fetch_sub(b.0.size(), Ordering::Relaxed);
std::mem::forget(b);
}
}
}
/// This thread's retired buffers (empty during thread teardown).
fn take_retired() -> Vec<Tracked> {
let retired = RETIRED.try_with(|r| std::mem::take(&mut *r.0.borrow_mut())).unwrap_or_default();
RETIRED_BYTES.fetch_sub(retired.iter().map(|b| b.0.size()).sum(), Ordering::Relaxed);
retired
}
thread_local! {
/// Renders running on this thread (they may hold recorded, unsubmitted commands).
static IN_RENDER: std::cell::Cell<u32> = const { std::cell::Cell::new(0) };
}
/// Marks a render on this thread; when it ends, the buffers it released join the pool (everything
/// it recorded has been submitted by then).
pub(crate) struct RenderScope<'a>(&'a Gpu);
impl<'a> RenderScope<'a> {
pub fn new(g: &'a Gpu) -> RenderScope<'a> {
IN_RENDER.with(|c| c.set(c.get() + 1));
RenderScope(g)
}
}
impl Drop for RenderScope<'_> {
fn drop(&mut self) {
let outer = IN_RENDER.with(|c| {
c.set(c.get().saturating_sub(1));
c.get() == 0
});
if outer {
self.0.pool(take_retired());
}
}
}
/// A device buffer, counted in [`ALLOCATED`] while it exists.
pub(crate) struct Tracked(wgpu::Buffer);
impl Tracked {
fn new(b: wgpu::Buffer) -> Tracked {
let now = ALLOCATED.fetch_add(b.size(), Ordering::Relaxed) + b.size();
PEAK.fetch_max(now, Ordering::Relaxed);
Tracked(b)
}
}
impl Drop for Tracked {
fn drop(&mut self) {
ALLOCATED.fetch_sub(self.0.size(), Ordering::Relaxed);
}
}
#[cfg(test)]
thread_local! {
/// Polls on this thread panic as wgpu's do once a Vulkan or DX12 device is lost (#1858).
static SIMULATED_LOST_POLL: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
}
thread_local! {
static RETIRED: Retired = const { Retired(std::cell::RefCell::new(Vec::new())) };
}
/// Bytes of device buffers that exist (in use, retired or pooled).
static ALLOCATED: AtomicU64 = AtomicU64::new(0);
/// Most bytes allocated since [`take_peak`] (profiling).
static PEAK: AtomicU64 = AtomicU64::new(0);
/// The most bytes allocated since the last call; starts over from what is allocated now.
pub(crate) fn take_peak() -> (u64, u64) {
let now = ALLOCATED.load(Ordering::Relaxed);
(PEAK.swap(now, Ordering::Relaxed), now)
}
/// Bytes waiting in per-thread retired lists.
static RETIRED_BYTES: AtomicU64 = AtomicU64::new(0);
/// The render device is lost: wgpu's `poll` panics from then on, so [`Gpu::poll`] stops calling it.
/// Other failures that retire the GPU leave the device usable for releasing its buffers.
static DEVICE_LOST: AtomicBool = AtomicBool::new(false);
/// Bytes in the free pool.
static POOLED: AtomicU64 = AtomicU64::new(0);
/// Most bytes kept in the free pool (see [`crate::set_pool_limit`]).
pub(crate) static POOL_LIMIT: AtomicU64 = AtomicU64::new(DEFAULT_POOL_BYTES);
/// Default for the most bytes kept in the free pool (apps derive it from their memory budget).
pub(crate) const DEFAULT_POOL_BYTES: u64 = 256 << 20;
/// Device buffers held by the renderer (bytes).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuMemory {
/// Every buffer that exists (in use + retired + pooled).
pub allocated: u64,
/// Of which recycled buffers waiting in the free pool.
pub pooled: u64,
/// Of which buffers dropped since their thread's last submit (pooled at its next submit).
pub retired: u64,
}
pub(crate) fn memory() -> GpuMemory {
GpuMemory { allocated: ALLOCATED.load(Ordering::Relaxed), pooled: POOLED.load(Ordering::Relaxed), retired: RETIRED_BYTES.load(Ordering::Relaxed) }
}
pub struct Gpu {
pub device: wgpu::Device,
pub queue: wgpu::Queue,
pub info: wgpu::AdapterInfo,
kernels: HashMap<&'static str, Kernel>,
dummy: wgpu::Buffer,
/// Recycled buffers (see [`Buf`]).
free: std::sync::Mutex<Vec<Tracked>>,
/// Largest storage buffer the device accepts (bytes).
pub max_buffer: u64,
/// The crystal grain tables of the last grain pattern, resident: a render (each Amount step
/// of a drag) copies them on the GPU instead of uploading their 12 MB again ([`Gpu::grain_tables`]).
/// Shared with each render that recorded a copy from them until it submits: replacing them
/// while another thread's copy is unsubmitted must not hand the buffer back to the pool.
grain: std::sync::Mutex<Option<(std::sync::Arc<lightcraft_pipeline::crystal::CrystalTile>, std::sync::Arc<Buf>)>>,
/// The device is the window's (eframe's): renders can hand it textures to draw directly.
pub window: bool,
}
/// WGSL constants shared by every module (generated from the CPU pipeline's values).
fn constants() -> String {
use lightcraft_pipeline::finish::{GRAIN_HASH, MASK_SUMS, MASK_TERMS, SRGB_LUT_N};
use lightcraft_pipeline::tone::{CHROMA_N, LUT_MAX_EV, LUT_MIN_EV, LUT_N};
let mut s = String::new();
let [to_lms, from_lms, to_lab, from_lab] = lightcraft_color::perceptual::oklab_matrices();
let (to_pro, from_pro) = lightcraft_pipeline::light_v3::prophoto_matrices();
for (name, m) in [
("OK_TO_LMS", to_lms),
("OK_FROM_LMS", from_lms),
("OK_TO_LAB", to_lab),
("OK_FROM_LAB", from_lab),
("PRO_FROM_2020", to_pro),
("PRO_TO_2020", from_pro),
] {
let rows: Vec<String> = m.iter().map(|r| format!("vec3<f32>({:?}, {:?}, {:?})", r[0], r[1], r[2])).collect();
s += &format!("const {name} = array<vec3<f32>, 3>({});\n", rows.join(", "));
}
s += &format!(
"const TONE_MIN_EV: f32 = {LUT_MIN_EV:?};\nconst TONE_MAX_EV: f32 = {LUT_MAX_EV:?};\nconst TONE_N: u32 = {LUT_N}u;\nconst CHROMA_N: u32 = {CHROMA_N}u;\n"
);
s += &format!("const TONE_MIN_GAIN: f32 = {:?};\n", 2f32.powf(LUT_MIN_EV));
let b = lightcraft_pipeline::geometry::BLANK;
s += &format!("const BLANK_R: f32 = {:?};\nconst BLANK_G: f32 = {:?};\nconst BLANK_B: f32 = {:?};\n", b[0], b[1], b[2]);
s += &format!(
"const SRGB_N: u32 = {SRGB_LUT_N}u;\nconst CURVE_N: u32 = {}u;\nconst MASK_TERMS: u32 = {MASK_TERMS}u;\nconst MASK_SUMS: u32 = {MASK_SUMS}u;\n",
crate::params::CURVE_N
);
s += &format!("const POINT_WORDS: u32 = {}u;\n", lightcraft_pipeline::colorops::POINT_WORDS);
let y = lightcraft_pipeline::light_v3::PRO_Y;
s += &format!("const PRO_Y = vec3<f32>({:?}, {:?}, {:?});\nconst LIGHT_N: u32 = {}u;\n", y[0], y[1], y[2], lightcraft_pipeline::light_v3::LUT_N);
s += &format!("const EYE_WORDS: u32 = {}u;\n", lightcraft_pipeline::redeye::EYE_WORDS);
use lightcraft_pipeline::masks::{AUTO_TOL_CHROMA, AUTO_TOL_EV};
s += &format!("const AUTO_TOL_EV: f32 = {AUTO_TOL_EV:?};\nconst AUTO_TOL_CHROMA: f32 = {AUTO_TOL_CHROMA:?};\n");
s += &format!("const SHADOW_TINT_K: f32 = {:?};\n", lightcraft_pipeline::colorops::SHADOW_TINT);
s += &format!("const GAMUT_WARN: f32 = {:?};\n", lightcraft_pipeline::finish::GAMUT_WARN);
s += &format!("const NCAND: u32 = {}u;\n", lightcraft_pipeline::spots::candidates(1.0).len());
use lightcraft_pipeline::output::{PROOF_DEST_WARNING, PROOF_DISPLAY_WARNING};
for (name, c) in [("PROOF_DEST", PROOF_DEST_WARNING), ("PROOF_DISP", PROOF_DISPLAY_WARNING)] {
s += &format!("const {name}_R: f32 = {:?};\nconst {name}_G: f32 = {:?};\nconst {name}_B: f32 = {:?};\n", c[0], c[1], c[2]);
}
for (i, h) in GRAIN_HASH.iter().enumerate() {
s += &format!("const GRAIN_H{i}: u32 = {h}u;\n");
}
use lightcraft_pipeline::crystal::{TILE_PER_LONG, TORI};
s += &format!(
"const CRYSTAL_T0: u32 = {}u;\nconst CRYSTAL_T1: u32 = {}u;\nconst CRYSTAL_TPL: f32 = {:?};\n",
TORI[0], TORI[1], TILE_PER_LONG as f32
);
s += &format!("const DEEP_WRITTEN: u32 = {}u;\n", crate::params::DEEP_WRITTEN);
for (name, idx) in crate::params::finish_fields() {
s += &format!("const F_{name}: u32 = {idx}u;\n");
}
s
}
fn module_source(m: &Module, consts: &str) -> String {
let mut s = String::from(consts);
s += "@group(0) @binding(0) var<storage, read> P: array<u32>;\n";
for (i, (name, write, ty)) in m.bindings.iter().enumerate() {
let access = if *write { "read_write" } else { "read" };
s += &format!("@group(0) @binding({}) var<storage, {access}> {name}: array<{ty}>;\n", i + 1);
}
s += include_str!("wgsl/common.wgsl");
s += m.src;
s
}
impl Gpu {
/// Create a device on the best available adapter (no software adapter unless
/// `LIGHTCRAFT_GPU_ALLOW_SOFTWARE`), or why there is none.
pub fn new(backends: wgpu::Backends) -> Result<Gpu, String> {
let mut desc = wgpu::InstanceDescriptor::new_without_display_handle();
// only these drivers are loaded (`crate::backend`: never Vulkan on Windows unless asked)
desc.backends = backends;
// DX12 shaders compile with FXC (issue #471)
desc.backend_options = crate::backend::backend_options();
// no indirect-call validation pipeline: Intel Metal fails to compile it (#250)
desc.flags = crate::backend::instance_flags_from_env(desc.flags);
let instance = wgpu::Instance::new(desc);
// the card LIGHTCRAFT_GPU_ADAPTER names, else the high-performance one
let adapter = crate::backend::pick_adapter(&instance, backends)?;
let info = adapter.get_info();
if refuse_adapter(info.device_type, software_allowed()) {
log::info!("gpu: only a software adapter ({}), using the CPU pipeline", info.name);
return Err(format!("software adapter ({}) skipped: the CPU pipeline is faster", info.name));
}
if info.device_type == wgpu::DeviceType::Cpu {
log::info!("gpu: LIGHTCRAFT_GPU_ALLOW_SOFTWARE: using the software adapter {}", info.name);
}
let mut limits = adapter.limits();
if limits.max_storage_buffers_per_shader_stage < 10 {
log::info!("gpu: {} has too few storage buffers per stage", info.name);
return Err(format!("{} has too few storage buffers per shader stage ({} < 10)", info.name, limits.max_storage_buffers_per_shader_stage));
}
limits = capped(limits, limits_override());
let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {
label: Some("lightcraft"),
required_limits: limits.clone(),
..Default::default()
}))
.map_err(|e| format!("{}: device creation failed ({e})", info.name))?;
Gpu::with_device(device, queue, info, limits, false)
}
/// The renderer on the window's device ([`crate::adopt_window_device`]), within its limits
/// lowered by `LIGHTCRAFT_GPU_LIMITS` as a device of its own would be.
pub fn adopt(device: wgpu::Device, queue: wgpu::Queue, info: wgpu::AdapterInfo) -> Result<Gpu, String> {
Gpu::adopt_capped(device, queue, info, limits_override())
}
/// [`Gpu::adopt`] with the limits override given (`LIGHTCRAFT_GPU_LIMITS`, parsed).
fn adopt_capped(device: wgpu::Device, queue: wgpu::Queue, info: wgpu::AdapterInfo, cap: Option<(u64, u64)>) -> Result<Gpu, String> {
let limits = capped(device.limits(), cap);
Gpu::with_device(device, queue, info, limits, true)
}
/// The renderer on an existing device: the window's (`window`, see [`crate::adopt_window_device`])
/// or one [`Gpu::new`] created with `limits`. Builds the kernels (~0.4โ2 s: not on the UI thread).
pub fn with_device(device: wgpu::Device, queue: wgpu::Queue, info: wgpu::AdapterInfo, limits: wgpu::Limits, window: bool) -> Result<Gpu, String> {
if limits.max_storage_buffers_per_shader_stage < 10 {
return Err(format!("{} has too few storage buffers per shader stage ({} < 10)", info.name, limits.max_storage_buffers_per_shader_stage));
}
let started = std::time::Instant::now();
let kernels = match build_kernels(&device) {
Ok(kernels) => kernels,
Err(e) => {
log::error!("gpu: kernels failed to build on {}: {e}", info.name);
return Err(format!("{}: kernels failed to build ({e})", info.name));
}
};
let build_ms = started.elapsed().as_millis();
// Only a device the renderer keeps gets these: a window's device declined above keeps the
// window's own handler, so a later error of the window's can't stop the renderer's device
// (review of PR #1065).
// Errors outside a render's error scopes (see `ErrorScopes`): log, and stop using the GPU.
device.on_uncaptured_error(std::sync::Arc::new(|e| {
log::error!("gpu: {e}");
crate::mark_broken(&format!("device error: {e}"));
}));
// wgpu reports a lost device (driver reset, GPU hang, watchdog) only here โ never as an
// error โ so without this callback a lost device went unnoticed.
device.set_device_lost_callback(|reason, msg| {
DEVICE_LOST.store(true, Ordering::Relaxed);
log::error!("gpu: device lost ({reason:?}): {msg}");
crate::mark_broken(&format!("device lost ({reason:?}): {msg}"));
});
let dummy = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("dummy"),
size: 16,
usage: wgpu::BufferUsages::STORAGE,
mapped_at_creation: false,
});
let max_buffer = limits.max_storage_buffer_binding_size.min(limits.max_buffer_size);
log::info!(
"gpu: {} ({:?}), {} kernels in {build_ms} ms, buffers โค {} MiB{}",
info.name,
info.backend,
kernels.len(),
max_buffer >> 20,
if window { ", the window's device" } else { "" }
);
Ok(Gpu { device, queue, info, kernels, dummy, free: Default::default(), max_buffer, window, grain: Default::default() })
}
/// A buffer of `len` 32-bit values with undefined contents (a recycled one when possible).
///
/// A buffer over the device limit fails the render ([`FailKind::Limit`]) instead of raising a
/// validation error: the caller gets a placeholder and the render is redone on the CPU.
pub fn buffer(&self, len: usize) -> Buf {
let size = (len.max(1) * 4) as u64;
if size > self.limit() {
self.over_limit(size);
return Buf { buf: Some(Tracked::new(self.placeholder())), len };
}
let recycled = {
let mut f = self.free.lock().unwrap_or_else(|e| e.into_inner());
// the smallest pooled buffer that holds `size` with at most 25 % to spare: photos of
// slightly different sizes reuse each other's buffers instead of piling up new ones
let b = f
.iter()
.enumerate()
.filter(|(_, b)| b.0.size() >= size && b.0.size() <= size + size / 4)
.min_by_key(|(_, b)| b.0.size())
.map(|(i, _)| i)
.map(|i| f.swap_remove(i));
if let Some(b) = &b {
POOLED.fetch_sub(b.0.size(), Ordering::Relaxed);
}
b
};
let buf = recycled.unwrap_or_else(|| {
Tracked::new(self.device.create_buffer(&wgpu::BufferDescriptor {
label: None,
size,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
}))
});
Buf { buf: Some(buf), len }
}
/// Submit command buffers, then make the buffers this thread dropped before reusable.
/// Nothing is submitted once this render has failed: the render is discarded and recomputed
/// on the CPU, and the failure contract says no further GPU work runs after a fatal failure.
pub fn submit(&self, cmds: impl IntoIterator<Item = wgpu::CommandBuffer>) {
if !failed() {
self.queue.submit(cmds);
}
let retired = take_retired();
self.pool(retired);
}
/// Put reusable buffers into the free pool (trimmed to its limit).
fn pool(&self, bufs: impl IntoIterator<Item = Tracked>) {
let mut added = false;
{
let mut f = self.free.lock().unwrap_or_else(|e| e.into_inner());
for b in bufs {
POOLED.fetch_add(b.0.size(), Ordering::Relaxed);
f.push(b);
added = true;
}
}
if added {
self.trim(POOL_LIMIT.load(Ordering::Relaxed));
}
}
/// Free pooled buffers (oldest first) until at most `keep` bytes stay pooled.
pub fn trim(&self, keep: u64) {
let mut f = self.free.lock().unwrap_or_else(|e| e.into_inner());
let mut total: u64 = f.iter().map(|b| b.0.size()).sum();
let mut dropped = Vec::new();
while total > keep && !f.is_empty() {
let b = f.remove(0);
total -= b.0.size();
POOLED.fetch_sub(b.0.size(), Ordering::Relaxed);
dropped.push(b);
}
drop(f);
if !dropped.is_empty() {
drop(dropped);
// let wgpu release the memory now rather than at its next maintenance
let _ = self.poll(wgpu::PollType::Poll);
}
}
/// Process finished GPU work: `device.poll`, with its error as text. Never panics (#1858): once a
/// device is lost, wgpu 30 panics in `poll` rather than returning an error (a device error is
/// not a `PollError`), and the buffer pool polls from destructors ([`Gpu::trim`]), where a
/// second panic while a failed render unwinds aborts the process. A lost device is not polled
/// again; a poll that panics counts as one and stops the GPU.
pub(crate) fn poll(&self, t: wgpu::PollType) -> Result<wgpu::PollStatus, String> {
poll_device(&self.device, t)
}
/// Upload 32-bit values.
pub fn upload<T: bytemuck::Pod>(&self, data: &[T]) -> Buf {
let bytes: &[u8] = bytemuck::cast_slice(data);
let len = bytes.len() / 4;
// A failed render runs no further GPU work (`Gpu::submit` skips the submission and the
// render is redone on the CPU): staging the bytes would still queue a copy for the next
// submission, so the caller gets the placeholder with the requested length instead.
if failed() {
return Buf { buf: Some(Tracked::new(self.placeholder())), len };
}
if bytes.is_empty() {
return self.buffer(1);
}
if bytes.len() as u64 > self.limit() {
self.over_limit(bytes.len() as u64);
return Buf { buf: Some(Tracked::new(self.placeholder())), len };
}
let usage = wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST;
if let Some(size) = wgpu::BufferSize::new(bytes.len() as u64).filter(|s| s.get() % wgpu::COPY_BUFFER_ALIGNMENT == 0) {
// through the queue: the bytes are copied once, into a staging buffer the next submission copies from, in
// parallel parts (`create_buffer_init` zeroes a staging buffer first and copies over that, two passes on
// one thread over a full-size photo's ~290 MB)
let buf = self.device.create_buffer(&wgpu::BufferDescriptor { label: None, size: size.get(), usage, mapped_at_creation: false });
if let Some(mut view) = self.queue.write_buffer_with(&buf, 0, size) {
const PART: usize = 1 << 20;
{
let (whole, mut tail) = view.slice(..).into_chunks::<PART>();
let (src, src_tail) = bytes.as_chunks::<PART>();
let mut parts: Vec<_> = whole.into_iter().zip(src).map(Some).collect();
lightcraft_raster::par_rows(&mut parts, 1, |_, part| {
for (d, s) in part.iter_mut().filter_map(Option::take) {
wgpu::WriteOnly::<[u8]>::from(d).copy_from_slice(s);
}
});
tail.copy_from_slice(src_tail);
}
// (dropping the view schedules the copy)
drop(view);
} else {
// recorded now, not only when the error scope pops: the zeroed buffer must not be
// shared before then (`render::shared_source`)
fail(FailKind::OutOfMemory, "out of device memory: the upload's staging buffer couldn't be allocated".into());
}
// no view: the buffer couldn't be allocated (out of device memory). The error scope has recorded why and
// the render fails over to the CPU; allocating it again would only fail again (issue #1764)
return Buf { buf: Some(Tracked::new(buf)), len };
}
let buf = self.init_buffer(bytes, usage);
Buf { buf: Some(Tracked::new(buf)), len }
}
/// A buffer holding `contents` (zero-padded to the copy alignment), written through the queue. Not
/// `create_buffer_init`: it maps the new buffer, and mapping one that couldn't be allocated (out of device
/// memory) panics inside wgpu instead of raising an error the render's scopes catch (issue #1764).
fn init_buffer(&self, contents: &[u8], usage: wgpu::BufferUsages) -> wgpu::Buffer {
let align = wgpu::COPY_BUFFER_ALIGNMENT as usize;
let size = contents.len().max(align).next_multiple_of(align);
let buf = self.device.create_buffer(&wgpu::BufferDescriptor {
label: None,
size: size as u64,
usage: usage | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
if size == contents.len() {
self.queue.write_buffer(&buf, 0, contents);
} else {
let mut padded = contents.to_vec();
padded.resize(size, 0);
self.queue.write_buffer(&buf, 0, &padded);
}
buf
}
fn over_limit(&self, size: u64) {
fail(
FailKind::Limit,
format!("a {} MiB buffer exceeds the device's {} MiB storage-buffer limit", size.div_ceil(1 << 20), self.limit() >> 20),
);
}
/// Record into `enc` a copy of `tile`'s summed-area tables (both tori, back to back) into
/// `dst` at value offset `at`, uploading them first unless they are the resident ones.
/// Returns the buffer copied from, which the caller keeps until it has submitted `enc` (another
/// render may replace the resident tables meanwhile). None when `dst` can't hold them (an
/// over-limit placeholder: the render has failed already).
pub fn grain_tables(
&self,
enc: &mut wgpu::CommandEncoder,
tile: &std::sync::Arc<lightcraft_pipeline::crystal::CrystalTile>,
dst: &Buf,
at: usize,
) -> Option<std::sync::Arc<Buf>> {
let len: usize = tile.sats.iter().map(Vec::len).sum();
let (start, bytes) = ((at as u64) * 4, (len as u64) * 4);
if len == 0 || dst.raw().size() < start.saturating_add(bytes) {
return None;
}
let mut resident = self.grain.lock().unwrap_or_else(|e| e.into_inner());
if !resident.as_ref().is_some_and(|(t, b)| std::sync::Arc::ptr_eq(t, tile) && b.raw().size() >= bytes) {
// the old tables go back to the pool first (once no unsubmitted copy holds them): one
// pattern's tables are resident at a time
*resident = None;
let buf = self.buffer(len);
if buf.raw().size() < bytes {
return None;
}
let mut off = 0u64;
for sat in &tile.sats {
self.queue.write_buffer(buf.raw(), off, bytemuck::cast_slice(sat));
off += (sat.len() as u64) * 4;
}
*resident = Some((tile.clone(), std::sync::Arc::new(buf)));
}
let (_, src) = resident.as_ref()?;
enc.copy_buffer_to_buffer(src.raw(), 0, dst.raw(), start, bytes);
Some(src.clone())
}
/// Stands in for a buffer that could not be created (the render has failed already).
fn placeholder(&self) -> wgpu::Buffer {
self.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("placeholder"),
size: 16,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
})
}
/// Whether a buffer of `len` 32-bit values fits the device limits.
pub fn fits(&self, len: usize) -> bool {
(len as u64) * 4 <= self.limit()
}
/// Largest buffer a render may use (bytes): the device limit, or a lower one set for the
/// render on this thread ([`LimitOverride`]).
pub fn limit(&self) -> u64 {
LIMIT.try_with(|l| l.get()).ok().flatten().map_or(self.max_buffer, |l| l.min(self.max_buffer))
}
pub fn encoder(&self) -> wgpu::CommandEncoder {
self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None })
}
/// Record kernel `name` with parameters `p` over buffers `bufs` (None = unused binding).
pub fn run(&self, enc: &mut wgpu::CommandEncoder, name: &str, p: &[u32], bufs: &[Option<&Buf>], groups: [u32; 3]) {
if failed() {
return; // the render is redone on the CPU: don't spend the GPU on it
}
let Some(k) = self.kernels.get(name) else {
// a kernel name typo: fail this render over to the CPU instead of crashing
fail(FailKind::Fatal, format!("unknown kernel {name}"));
return;
};
if bufs.len() != k.nbuf {
// a caller/kernel mismatch would bind the wrong buffers: fail this render over to the
// CPU instead of crashing
fail(FailKind::Fatal, format!("{name}: binding count {} != {}", bufs.len(), k.nbuf));
return;
}
let params = self.init_buffer(bytemuck::cast_slice(if p.is_empty() { &[0u32] } else { p }), wgpu::BufferUsages::STORAGE);
let mut entries = vec![wgpu::BindGroupEntry { binding: 0, resource: params.as_entire_binding() }];
for (i, b) in bufs.iter().enumerate() {
let buf = b.map(|b| b.raw()).unwrap_or(&self.dummy);
entries.push(wgpu::BindGroupEntry { binding: i as u32 + 1, resource: buf.as_entire_binding() });
}
let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor { label: None, layout: &k.layout, entries: &entries });
let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some(name), timestamp_writes: None });
pass.set_pipeline(&k.pipeline);
pass.set_bind_group(0, &bg, &[]);
pass.dispatch_workgroups(groups[0], groups[1], groups[2]);
}
/// Submit `enc`, then read back the first `len` 32-bit values of `src`.
///
/// A failed readback (device lost, map error, timeout) or a `len` past the buffer's end fails
/// the render ([`FailKind::Fatal`]) and returns zeros, which the caller discards; after an
/// earlier failure in the same render nothing more is submitted. `T` is always a 32-bit type
/// at the call sites (the copy and the readback tail are defined in 32-bit words).
pub fn finish_and_read<T: bytemuck::Pod>(&self, mut enc: wgpu::CommandEncoder, src: &Buf, len: usize) -> Vec<T> {
let zeros = || vec![<T as bytemuck::Zeroable>::zeroed(); len];
if failed() {
return zeros();
}
// a zero-length readback would issue a `copy_buffer_to_buffer` of size 0, which wgpu
// rejects as an invalid copy: submit the (possibly staged) work untouched and return
// zeros instead
if len == 0 {
self.submit([enc.finish()]);
return zeros();
}
if len > src.len {
// a caller bug would read past the buffer: fail this render over to the CPU
fail(FailKind::Fatal, format!("readback of {len} values from a buffer of {}", src.len));
return zeros();
}
let size = (len * 4) as u64;
let staging = self.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("readback"),
size: size.max(4),
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
enc.copy_buffer_to_buffer(src.raw(), 0, &staging, 0, size);
self.submit([enc.finish()]);
let slice = staging.slice(..size);
let (tx, rx) = std::sync::mpsc::channel();
slice.map_async(wgpu::MapMode::Read, move |r| {
let _ = tx.send(r);
});
let failed = |why: String| {
fail(FailKind::Fatal, format!("readback failed: {why}"));
zeros()
};
// A lost device makes the wait fail ([`Gpu::poll`] turns wgpu's panic into the error) and
// the map callback never run: checking both keeps this from blocking forever (or handing
// back an unwritten buffer as the image).
if let Err(e) = self.poll(wgpu::PollType::Wait { submission_index: None, timeout: Some(READBACK_TIMEOUT) }) {
return failed(e);
}
// another thread's poll may be the one that runs the callback: allow it a moment
match rx.recv_timeout(std::time::Duration::from_secs(10)) {
Ok(Ok(())) => {}
Ok(Err(e)) => return failed(format!("{e}")),
Err(e) => return failed(format!("map callback: {e}")),
}
let data = match slice.get_mapped_range() {
Ok(d) => d,
Err(e) => return failed(format!("{e:?}")),
};
// copied out in parallel parts: one thread reads mapped memory slowly (~2 GB/s, so ~45 ms for a 24 MP
// export's pixels), and the new vector's pages are first touched there too
let src: &[u8] = &data;
// (`len` counts 32-bit words: as many `T` as they fill)
let mut out = vec![<T as bytemuck::Zeroable>::zeroed(); src.len() / std::mem::size_of::<T>().max(1)];
const PART: usize = 1 << 20;
lightcraft_raster::par_rows(bytemuck::cast_slice_mut::<T, u8>(&mut out), PART, |k, part| {
if let Some(s) = src.get(k * PART..k * PART + part.len()) {
part.copy_from_slice(s);
}
});
drop(data);
staging.unmap();
out
}
}
/// Longest wait for a render's GPU work: beyond it the render fails over to the CPU (drivers
/// normally reset a hung GPU much sooner).
const READBACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
/// Whether to turn down an adapter of `device_type`: a software rasterizer (llvmpipe / lavapipe,
/// WARP) is slower than the CPU pipeline, so it is refused unless `allow_software`.
fn refuse_adapter(device_type: wgpu::DeviceType, allow_software: bool) -> bool {
device_type == wgpu::DeviceType::Cpu && !allow_software
}
/// `LIGHTCRAFT_GPU_ALLOW_SOFTWARE=1`: accept a software adapter. For tests on machines without a
/// GPU (Mesa lavapipe on Linux, WARP on Windows), so the CPU โ GPU equivalence tests run there
/// instead of skipping; the app never sets it (software rendering is slower than the CPU pipeline).
fn software_allowed() -> bool {
std::env::var("LIGHTCRAFT_GPU_ALLOW_SOFTWARE").is_ok_and(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on"))
}
/// `LIGHTCRAFT_GPU_LIMITS`: lower the device limits, e.g. to reproduce a smaller adapter.
/// `webgpu` / `downlevel` = 128 MiB storage bindings and 256 MiB buffers (the WebGPU defaults),
/// `<n>` = n MiB storage bindings and 2n MiB buffers. Returns (binding bytes, buffer bytes).
fn limits_override() -> Option<(u64, u64)> {
parse_limits(&std::env::var("LIGHTCRAFT_GPU_LIMITS").ok()?)
}
/// `limits` lowered to `cap` ([`limits_override`]: storage binding bytes, buffer bytes).
fn capped(mut limits: wgpu::Limits, cap: Option<(u64, u64)>) -> wgpu::Limits {
if let Some((binding, buffer)) = cap {
limits.max_storage_buffer_binding_size = limits.max_storage_buffer_binding_size.min(binding);
limits.max_buffer_size = limits.max_buffer_size.min(buffer);
log::info!("gpu: LIGHTCRAFT_GPU_LIMITS: storage bindings โค {} MiB, buffers โค {} MiB", binding >> 20, buffer >> 20);
}
limits
}
/// `device.poll` that never panics: see [`Gpu::poll`]. [`read_texture`] uses it too: its device
/// (the window's, when adopted) shares the device-lost flag.
fn poll_device(device: &wgpu::Device, t: wgpu::PollType) -> Result<wgpu::PollStatus, String> {
if DEVICE_LOST.load(Ordering::Relaxed) {
return Err("the device is lost".into());
}
let polled = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
#[cfg(test)]
if SIMULATED_LOST_POLL.with(std::cell::Cell::get) {
panic!("Error in Device::poll: Validation Error\n\nCaused by:\n Parent device is lost (simulated)");
}
device.poll(t)
}));
match polled {
Ok(r) => r.map_err(|e| e.to_string()),
Err(_) => {
DEVICE_LOST.store(true, Ordering::Relaxed);
crate::mark_broken("device lost: Device::poll failed");
Err("the device is lost (Device::poll failed)".into())
}
}
}
/// See [`crate::read_texture`].
pub(crate) fn read_texture(
device: &wgpu::Device,
queue: &wgpu::Queue,
texture: &wgpu::Texture,
width: usize,
height: usize,
) -> Result<lightcraft_raster::Rgba8, String> {
if DEVICE_LOST.load(Ordering::Relaxed) {
return Err("the device is lost".into());
}
if width == 0 || height == 0 {
return Err("an empty texture".into());
}
let (Ok(w32), Ok(h32)) = (u32::try_from(width), u32::try_from(height)) else { return Err("texture too large".into()) };
if w32 > texture.width() || h32 > texture.height() || texture.format() != wgpu::TextureFormat::Rgba8Unorm {
return Err("not an 8-bit view texture of that size".into());
}
// texture copies take rows of a multiple of 256 bytes
let stride = width.checked_mul(4).ok_or("texture too large")?.next_multiple_of(256);
let size = stride.checked_mul(height).ok_or("texture too large")?;
let (Ok(stride32), Ok(size64)) = (u32::try_from(stride), u64::try_from(size)) else { return Err("texture too large".into()) };
let scope = device.push_error_scope(wgpu::ErrorFilter::Validation);
let staging = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("view readback"),
size: size64,
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("view readback") });
enc.copy_texture_to_buffer(
wgpu::TexelCopyTextureInfo { texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
wgpu::TexelCopyBufferInfo {
buffer: &staging,
layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(stride32), rows_per_image: Some(h32) },
},
wgpu::Extent3d { width: w32, height: h32, depth_or_array_layers: 1 },
);
queue.submit([enc.finish()]);
if let Some(e) = pollster::block_on(scope.pop()) {
return Err(format!("texture copy failed: {e}"));
}
let slice = staging.slice(..);
let (tx, rx) = std::sync::mpsc::channel();
slice.map_async(wgpu::MapMode::Read, move |r| {
let _ = tx.send(r);
});
// (the render path's wait: queued work ahead of the copy can take that long too)
let wait = READBACK_TIMEOUT;
poll_device(device, wgpu::PollType::Wait { submission_index: None, timeout: Some(wait) })?;
match rx.recv_timeout(wait) {
Ok(Ok(())) => {}
Ok(Err(e)) => return Err(format!("{e}")),
Err(e) => return Err(format!("map callback: {e}")),
}
let bytes = slice.get_mapped_range().map_err(|e| format!("{e:?}"))?;
let mut data = Vec::with_capacity(width * height);
for row in bytes.chunks(stride).take(height) {
let row = row.get(..width * 4).ok_or("short readback row")?;
data.extend(row.as_chunks::<4>().0.iter().copied());
}
drop(bytes);
staging.unmap();
if data.len() != width * height {
return Err("incomplete readback".into());
}
Ok(lightcraft_raster::Rgba8 { width, height, data })
}
fn parse_limits(v: &str) -> Option<(u64, u64)> {
let v = v.trim().to_ascii_lowercase();
match v.as_str() {
"" => None,
"webgpu" | "downlevel" | "default" => {
let d = wgpu::Limits::default();
Some((d.max_storage_buffer_binding_size, d.max_buffer_size))
}
n => {
let mb: u64 = n.trim_end_matches("mib").trim_end_matches("mb").trim().parse().ok()?;
let b = mb.max(1).checked_mul(1 << 20)?;
Some((b, b.checked_mul(2)?))
}
}
}
/// How a GPU render failed (it is then redone on the CPU).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum FailKind {
/// A buffer over the device limits: this render only.
Limit,
/// The device ran out of memory: this render only (the buffer pool is emptied).
OutOfMemory,
/// The image came back entirely black: redone on the CPU in case the GPU dropped work.
Blank,
/// Device error or loss, readback failure, incomplete image: the GPU is not used again.
Fatal,
}
#[derive(Clone, Debug)]
pub(crate) struct Failure {
pub kind: FailKind,
pub reason: String,
}
thread_local! {
/// A lower buffer limit for the render on this thread (fault injection in tests).
static LIMIT: std::cell::Cell<Option<u64>> = const { std::cell::Cell::new(None) };
}
/// Lowers [`Gpu::limit`] on this thread while it lives.
pub(crate) struct LimitOverride;
impl LimitOverride {
pub fn new(bytes: u64) -> LimitOverride {
LIMIT.with(|l| l.set(Some(bytes)));
LimitOverride
}
}
impl Drop for LimitOverride {
fn drop(&mut self) {
let _ = LIMIT.try_with(|l| l.set(None));
}
}
thread_local! {
/// The first failure of the render running on this thread.
static FAILURE: std::cell::RefCell<Option<Failure>> = const { std::cell::RefCell::new(None) };
}
/// Record a failure of the render running on this thread (the first one is kept).
pub(crate) fn fail(kind: FailKind, reason: String) {
let _ = FAILURE.try_with(|f| {
let mut f = f.borrow_mut();
if f.is_none() {
*f = Some(Failure { kind, reason });
}
});
}
/// Has the render on this thread failed?
pub(crate) fn failed() -> bool {
FAILURE.try_with(|f| f.borrow().is_some()).unwrap_or(false)
}
/// The render's failure, clearing it.
pub(crate) fn take_failure() -> Option<Failure> {
FAILURE.try_with(|f| f.borrow_mut().take()).ok().flatten()
}
/// wgpu error scopes around a render: out-of-memory, validation and internal errors raised by
/// this thread's calls are captured (instead of reaching the uncaptured-error handler) and fail
/// the render.
pub(crate) struct ErrorScopes([wgpu::ErrorScopeGuard; 3]);
impl ErrorScopes {
pub fn push(g: &Gpu) -> ErrorScopes {
ErrorScopes([
g.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory),
g.device.push_error_scope(wgpu::ErrorFilter::Validation),
g.device.push_error_scope(wgpu::ErrorFilter::Internal),
])
}
/// Pop the scopes: the most telling error captured (out of memory first โ an allocation
/// failure makes later commands invalid too โ then internal, then validation).
pub fn pop(self) -> Option<Failure> {
let [oom, validation, internal] = self.0;
// scopes pop in reverse order of their creation
let internal = pollster::block_on(internal.pop());
let validation = pollster::block_on(validation.pop());
let oom = pollster::block_on(oom.pop());
if let Some(e) = oom {
return Some(Failure { kind: FailKind::OutOfMemory, reason: format!("out of device memory: {e}") });
}
internal.or(validation).map(|e| Failure { kind: FailKind::Fatal, reason: format!("device error: {e}") })
}
}
fn storage_entry(binding: u32, read_only: bool) -> wgpu::BindGroupLayoutEntry {
wgpu::BindGroupLayoutEntry {
binding,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Storage { read_only }, has_dynamic_offset: false, min_binding_size: None },
count: None,
}
}
/// Workgroups covering a `w ร h` grid with `wg`-sized groups.
pub fn groups2(w: usize, h: usize, wg: [usize; 2]) -> [u32; 3] {
[w.div_ceil(wg[0]).max(1) as u32, h.div_ceil(wg[1]).max(1) as u32, 1]
}
/// Workgroups for a 1-D kernel of `n` threads (256 per group) on a 2-D grid (see `lin_index`).
pub fn groups1(n: usize) -> [u32; 3] {
let g = n.div_ceil(256).max(1);
let x = g.min(32768);
[x as u32, g.div_ceil(x) as u32, 1]
}
#[cfg(test)]
mod tests {
/// Every kernel module, as it is handed to the device (the generated constants, the bindings, `common.wgsl`, the
/// module), parses and validates without a GPU.
#[test]
fn every_kernel_module_validates_without_a_gpu() {
let consts = super::constants();
for m in super::MODULES {
let name = m.entries.first().copied().unwrap_or("?");
assert_eq!(crate::validate_wgsl(name, &super::module_source(m, &consts)), Ok(()));
}
}
#[test]
fn limits_override_parses() {
assert_eq!(super::parse_limits("webgpu"), Some((128 << 20, 256 << 20)));
assert_eq!(super::parse_limits(" Downlevel "), Some((128 << 20, 256 << 20)));
assert_eq!(super::parse_limits("64"), Some((64 << 20, 128 << 20)));
assert_eq!(super::parse_limits("512MiB"), Some((512 << 20, 1024 << 20)));
assert_eq!(super::parse_limits(""), None);
assert_eq!(super::parse_limits("lots"), None);
assert_eq!(super::parse_limits("9223372036854775807"), None);
assert_eq!(super::parse_limits("17592186044416"), None);
}
/// `LIGHTCRAFT_GPU_LIMITS` caps the window's device as it caps a device of its own (review of
/// PR #1065: adopting the window's device skipped it, so a 1024 ร 1024 RGB float buffer โ 12 MiB โ
/// was taken under an 8 MiB cap).
#[test]
fn the_limits_override_caps_an_adopted_device() {
let big = wgpu::Limits { max_storage_buffer_binding_size: 1 << 30, max_buffer_size: 1 << 31, ..wgpu::Limits::default() };
let c = super::capped(big.clone(), super::parse_limits("8"));
assert_eq!((c.max_storage_buffer_binding_size, c.max_buffer_size), (8 << 20, 16 << 20));
let c = super::capped(big.clone(), None);
assert_eq!((c.max_storage_buffer_binding_size, c.max_buffer_size), (1 << 30, 1 << 31));
let _lock = crate::DEVICE_TEST_LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let Some((device, queue, info)) = test_device() else {
lightcraft_testkit::skip!(Gpu, "no GPU adapter");
return;
};
let full = device.limits().max_storage_buffer_binding_size.min(device.limits().max_buffer_size);
if full <= 12 << 20 {
lightcraft_testkit::skip!(Gpu, "the adapter's buffers are already โค 12 MiB");
return;
}
let g = super::Gpu::adopt_capped(device, queue, info, super::parse_limits("8")).expect("kernels build");
assert!(g.window);
assert_eq!(g.limit(), 8 << 20, "the cap applies to the adopted device");
assert!(1024 * 1024 * 3 * 4 > g.limit(), "a 12 MiB buffer is over it");
}
/// A device as eframe makes the window's (the adapter's own limits).
fn test_device() -> Option<(wgpu::Device, wgpu::Queue, wgpu::AdapterInfo)> {
let backends = crate::backend::compute_backends()?;
let mut desc = wgpu::InstanceDescriptor::new_without_display_handle();
desc.backends = backends;
desc.backend_options = crate::backend::backend_options();
let instance = wgpu::Instance::new(desc);
let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?;
let info = adapter.get_info();
if info.device_type == wgpu::DeviceType::Cpu {
return None;
}
let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {
label: Some("test window"),
required_limits: adapter.limits(),
..Default::default()
}))
.ok()?;
Some((device, queue, info))
}
/// Issue #2936: the kernels are built on several threads; every entry point of every module has one.
#[test]
fn every_entry_point_has_a_kernel() {
let Some(g) = crate::device() else {
lightcraft_testkit::skip!(Gpu, "no GPU device");
return;
};
let entries: Vec<&str> = super::MODULES.iter().flat_map(|m| m.entries.iter().copied()).collect();
assert_eq!(g.kernels.len(), entries.len());
for e in entries {
assert!(g.kernels.contains_key(e), "no kernel for {e}");
}
}
/// Run `child` (an ignored test in this module) in its own process: it loses the device, which
/// no other test running beside it may see. Returns its stdout, or `None` without a GPU.
fn run_child(child: &str) -> Option<String> {
let exe = std::env::current_exe().unwrap();
// output goes to files: a pipe nobody reads while we wait could fill and stall the child
let log = |stream: &str| std::env::temp_dir().join(format!("lc-gpu-{child}-{}.{stream}", std::process::id()));
let (out_path, err_path) = (log("out"), log("err"));
let mut proc = std::process::Command::new(exe)
.args(["--exact", &format!("ctx::tests::{child}"), "--ignored", "--nocapture", "--test-threads=1"])
.stdout(std::fs::File::create(&out_path).unwrap())
.stderr(std::fs::File::create(&err_path).unwrap())
.spawn()
.unwrap();
// a driver that stalls must fail the test, not hang CI
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120);
let status = loop {
if let Some(status) = proc.try_wait().unwrap() {
break status;
}
if std::time::Instant::now() > deadline {
let _ = proc.kill();
let _ = proc.wait();
panic!("{child}: still running after 120 s");
}
std::thread::sleep(std::time::Duration::from_millis(20));
};
let stdout = std::fs::read_to_string(&out_path).unwrap_or_default();
let stderr = std::fs::read_to_string(&err_path).unwrap_or_default();
let _ = (std::fs::remove_file(&out_path), std::fs::remove_file(&err_path));
if stdout.contains("CHILD: no GPU") {
return None;
}
// an abort (SIGABRT, "panic in a destructor during cleanup") has no exit code
assert!(status.success(), "{child}: {status:?}\n{stdout}\n{stderr}");
assert!(stdout.contains("CHILD: survived"), "{child}: {stdout}");
Some(stdout)
}
/// An upload reaches the device whole: it is copied into staging memory in parallel parts, the last one
/// shorter than the others (and a small upload is that last part alone).
#[test]
fn uploads_read_back_exactly() {
let Some(g) = crate::device() else {
lightcraft_testkit::skip!(Gpu, "no GPU device");
return;
};
for len in [1usize, 1000, (3 << 20) / 4 + 777] {
let data: Vec<u32> = (0..len as u32).map(|i| i.wrapping_mul(2_654_435_761)).collect();
let b = g.upload(&data);
let back: Vec<u32> = g.finish_and_read(g.encoder(), &b, len);
assert!(back == data, "{len} values");
}
// read back as wider values (`len` counts 32-bit words), as `read_rgb` does
let data: Vec<u32> = (0..3 << 20).collect();
let b = g.upload(&data);
let back: Vec<[u32; 3]> = g.finish_and_read(g.encoder(), &b, data.len());
assert!(back.len() == 1 << 20 && back.as_flattened() == data.as_slice());
}
/// An upload after this render has failed keeps the failure contract (`Gpu::submit` skips the
/// submission, so nothing further runs on the GPU): the caller gets the placeholder with the
/// requested length instead of a staging buffer holding a copy of the data for a submission
/// that will never happen.
#[test]
fn an_upload_after_a_failure_returns_the_placeholder() {
let Some(g) = crate::device() else {
lightcraft_testkit::skip!(Gpu, "no GPU device");
return;
};
super::fail(super::FailKind::Fatal, "simulated failure before an upload".into());
assert!(super::failed());
let b = g.upload(&[7u32; 1024]);
assert_eq!(b.buf.as_ref().map(|t| t.0.size()), Some(16), "the placeholder, not a staged 4 KiB buffer");
assert_eq!(b.len, 1024, "the requested length is preserved");
// leave this thread's failure for nobody: later tests on the thread start unfailed
assert_eq!(super::take_failure().map(|f| f.kind), Some(super::FailKind::Fatal));
}
/// Issue #1858: once a Vulkan or DX12 device is lost, wgpu 30 panics in `Device::poll` instead of
/// returning an error. A render's readback panicked; while it unwound, a buffer's way back to
/// the pool polls again from a destructor, and that second panic aborted the app.
#[test]
fn a_poll_on_a_lost_device_neither_panics_nor_aborts() {
if run_child("lost_poll_child").is_none() {
lightcraft_testkit::skip!(Gpu, "no GPU device");
}
}
/// Review of PR #1065: a view readback (headless screenshot) whose poll finds the device lost
/// returns an error instead of panicking, stops the GPU, and doesn't poll the device again.
#[test]
fn a_view_readback_on_a_lost_device_fails_without_a_panic() {
if run_child("lost_readback_child").is_none() {
lightcraft_testkit::skip!(Gpu, "no GPU device");
}
}
#[test]
#[ignore = "run in its own process by a_view_readback_on_a_lost_device_fails_without_a_panic"]
fn lost_readback_child() {
let Some(g) = crate::device() else {
println!("CHILD: no GPU");
return;
};
let texture = g.device.create_texture(&wgpu::TextureDescriptor {
label: Some("view"),
size: wgpu::Extent3d { width: 8, height: 8, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::TEXTURE_BINDING,
view_formats: &[],
});
assert!(super::read_texture(&g.device, &g.queue, &texture, 8, 8).is_ok(), "a live device reads back");
super::SIMULATED_LOST_POLL.with(|c| c.set(true));
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| super::read_texture(&g.device, &g.queue, &texture, 8, 8)));
assert!(matches!(r, Ok(Err(_))), "the lost device fails the readback without a panic");
assert!(!crate::available(), "the GPU is not used again");
super::SIMULATED_LOST_POLL.with(|c| c.set(false));
assert!(super::read_texture(&g.device, &g.queue, &texture, 8, 8).is_err(), "a lost device is not polled again");
println!("CHILD: survived");
}
/// A Metal device really lost (its command buffers exhausted) is never polled again.
#[test]
fn a_really_lost_device_is_not_polled_again() {
if run_child("really_lost_child").is_none() {
lightcraft_testkit::skip!(Gpu, "no GPU device, or one that can't be lost on purpose here");
}
}
/// Resident grain tables replaced by another pattern stay out of the pool while a render
/// that recorded a copy from them hasn't submitted it: reusing them then would hand that
/// render another tile's tables.
#[test]
fn replaced_grain_tables_stay_alive_until_their_copy_is_submitted() {
use lightcraft_pipeline::crystal::{CrystalLook, CrystalTile};
let Some(g) = crate::device() else {
lightcraft_testkit::skip!(Gpu, "no GPU device");
return;
};
let look = CrystalLook::new(25.0, 50.0);
let (Some(a), Some(b)) = (CrystalTile::new(look, 0x7ab1_0001), CrystalTile::new(look, 0x7ab1_0002)) else { panic!("tiles") };
let (a, b) = (std::sync::Arc::new(a), std::sync::Arc::new(b));
let len: usize = a.sats.iter().map(Vec::len).sum();
let (dst_a, dst_b) = (g.buffer(len), g.buffer(len));
let mut first = g.encoder();
let held = g.grain_tables(&mut first, &a, &dst_a, 0).expect("tables copied");
// another render replaces the resident tables before `first` is submitted
let mut second = g.encoder();
assert!(g.grain_tables(&mut second, &b, &dst_b, 0).is_some());
let next = g.buffer(len);
assert!(next.raw() != held.raw(), "the replaced tables went back to the pool before their copy was submitted");
g.submit([first.finish()]);
// each render read its own tile's tables
let tables = |t: &lightcraft_pipeline::crystal::CrystalTile| t.sats.concat();
assert!(g.finish_and_read::<f32>(second, &dst_a, len) == tables(&a), "the first render copied other tables");
assert!(g.finish_and_read::<f32>(g.encoder(), &dst_b, len) == tables(&b), "the second render copied other tables");
}
/// A GPU retired by another failure (a validation error) keeps a live device: its buffers are
/// still released, which takes a poll.
#[test]
fn a_retired_but_live_device_is_still_polled() {
if run_child("retired_child").is_none() {
lightcraft_testkit::skip!(Gpu, "no GPU device");
}
}
#[test]
#[ignore = "run in its own process by a_retired_but_live_device_is_still_polled"]
fn retired_child() {
let Some(g) = crate::device() else {
println!("CHILD: no GPU");
return;
};
crate::mark_broken("device error: simulated validation error");
assert!(!crate::available());
drop(g.upload(&[1u32; 1024]));
g.poll(wgpu::PollType::Poll).unwrap();
println!("CHILD: survived");
}
/// A render that runs out of device memory falls back to the CPU and leaves the GPU in use: the upload
/// whose buffer couldn't be allocated used to try again through `create_buffer_init`, which panicked
/// mapping the second failed buffer, and the panic stopped GPU use for the session (issue #1764).
#[test]
fn running_out_of_device_memory_falls_back_without_stopping_the_gpu() {
if run_child("out_of_memory_child").is_none() {
lightcraft_testkit::skip!(Gpu, "no GPU device, or its memory never ran out");
}
}
#[test]
#[ignore = "run in its own process by running_out_of_device_memory_falls_back_without_stopping_the_gpu"]
fn out_of_memory_child() {
let Some(g) = crate::device() else {
println!("CHILD: no GPU");
return;
};
// hold device memory until an allocation fails: 1 GiB at a time, then smaller, so little is
// left; no larger than the device allows in one buffer, which would be a validation error
let max = g.device.limits().max_buffer_size & !(wgpu::COPY_BUFFER_ALIGNMENT - 1);
let mut sizes: Vec<u64> = [1u64 << 30, 1 << 28, 1 << 26].into_iter().map(|s| s.min(max)).collect();
sizes.dedup();
let mut held = Vec::new();
let mut full = false;
for size in sizes {
for _ in 0..128 {
let validation = g.device.push_error_scope(wgpu::ErrorFilter::Validation);
let oom = g.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory);
let usage = wgpu::BufferUsages::STORAGE;
let buf = g.device.create_buffer(&wgpu::BufferDescriptor { label: None, size, usage, mapped_at_creation: false });
let ran_out = pollster::block_on(oom.pop()).is_some();
if let Some(e) = pollster::block_on(validation.pop()) {
panic!("holding {size} bytes of device memory: {e}");
}
if ran_out {
full = true;
break;
}
held.push(buf);
}
}
if !full {
println!("CHILD: no GPU (its memory never ran out)");
return;
}
// a render that needs more than is left: a larger one when the driver still finds room for the first
let s = lightcraft_develop::DevelopSettings::default();
let mut ran_out = false;
for (w, h) in [(4000, 3000), (8000, 6000)] {
let src = std::sync::Arc::new(lightcraft_raster::Rgb32f { width: w, height: h, data: vec![[0.18, 0.18, 0.18]; w * h] });
let req = lightcraft_pipeline::RenderRequest::fit(w, h);
match crate::try_render(&src, &lightcraft_pipeline::SourceInfo::default(), &s, &req, None) {
Ok(_) => continue,
Err(crate::Fallback::Failed(reason)) => assert!(reason.contains("out of device memory"), "{reason}"),
Err(crate::Fallback::NotTried) => panic!("the GPU wasn't tried"),
}
ran_out = true;
break;
}
assert_eq!(crate::unavailable_reason(), None, "the GPU stays in use");
drop(held);
if !ran_out {
println!("CHILD: no GPU (the renders found room anyway)");
return;
}
println!("CHILD: survived");
}
/// Empties the buffer pool when dropped, as the UI does (`trim_pool(0)`) and as returning a
/// buffer to a full pool does.
struct TrimOnDrop(&'static super::Gpu);
impl Drop for TrimOnDrop {
fn drop(&mut self) {
self.0.trim(0);
}
}
#[test]
#[ignore = "run in its own process by a_poll_on_a_lost_device_neither_panics_nor_aborts"]
fn lost_poll_child() {
let Some(g) = crate::device() else {
println!("CHILD: no GPU");
return;
};
let src = g.upload(&[1u32; 1024]);
drop(g.upload(&[2u32; 1024])); // pooled: trimming it polls
super::SIMULATED_LOST_POLL.with(|c| c.set(true));
// a render (caught as `try_render` catches it) whose readback finds the device lost
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _trim = TrimOnDrop(g);
let v: Vec<u32> = g.finish_and_read(g.encoder(), &src, 1024);
assert!(v.iter().all(|x| *x == 0), "a failed readback hands back zeros");
}));
assert!(r.is_ok(), "the lost device fails the render without a panic");
assert!(super::failed(), "the render failed, so it is redone on the CPU");
assert!(!crate::available(), "the GPU is not used again");
// a render that panics for another reason still unwinds through a destructor that polls
drop(g.upload(&[3u32; 1024]));
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _trim = TrimOnDrop(g);
panic!("a render panics (expected in this test)");
}));
assert!(r.is_err());
println!("CHILD: survived");
}
#[test]
#[ignore = "run in its own process by a_really_lost_device_is_not_polled_again"]
fn really_lost_child() {
let Some(g) = crate::device().filter(|g| g.info.backend == wgpu::Backend::Metal) else {
println!("CHILD: no GPU");
return;
};
// wgpu-hal's Metal backend treats a command buffer past its 4096 unsubmitted as a lost device
let b = g.upload(&[1u32; 16]);
let held: Vec<wgpu::CommandBuffer> = (0..4200)
.map(|_| {
let mut e = g.encoder();
e.clear_buffer(b.raw(), 0, None);
e.finish()
})
.collect();
assert!(crate::unavailable_reason().is_some_and(|r| r.contains("device lost")), "{:?}", crate::unavailable_reason());
let e = g.poll(wgpu::PollType::Poll).unwrap_err();
assert!(e.contains("lost"), "{e}");
drop(held);
println!("CHILD: survived");
}
#[test]
fn software_adapters_need_the_opt_in() {
use wgpu::DeviceType::{Cpu, DiscreteGpu, IntegratedGpu, Other, VirtualGpu};
assert!(super::refuse_adapter(Cpu, false), "llvmpipe / WARP are refused by default");
assert!(!super::refuse_adapter(Cpu, true), "LIGHTCRAFT_GPU_ALLOW_SOFTWARE accepts them");
for t in [DiscreteGpu, IntegratedGpu, VirtualGpu, Other] {
assert!(!super::refuse_adapter(t, false), "{t:?}");
assert!(!super::refuse_adapter(t, true), "{t:?}");
}
}
#[test]
fn dispatches_stay_within_the_workgroup_limit() {
// 1-D kernels over a 100 MP image and 2-D ones over 16k ร 16k stay under 65535 per axis
assert!(super::groups1(100_000_000 * 3).iter().all(|g| *g <= 65535));
assert!(super::groups2(16384, 16384, [16, 16]).iter().all(|g| *g <= 65535));
}
}
No file is open. Choose one in the tree, or press Ctrl+O (โO on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.