5a24289898fd
render.rs
//! The GPU renderer: the CPU pipeline's stages, evaluated on the device where a kernel exists and
//! on the CPU otherwise (per-stage hybrid). Stage results stay on the device and are cached per
//! view exactly like [`lightcraft_pipeline::StageCache`] (same keys, see [`lightcraft_pipeline::Plan`]).
use std::sync::{Arc, Mutex};
use lightcraft_develop::DevelopSettings;
use lightcraft_geom::Orientation;
use lightcraft_pipeline::finish::{FinishParams, FinishParamsInput, MASK_TERMS, mask_terms};
use lightcraft_pipeline::geometry::SampleMode;
use lightcraft_pipeline::local::plane_at;
use lightcraft_pipeline::nrc_joint::JointNoise;
use lightcraft_pipeline::{Plan, RenderRequest, Rendered, SourceInfo, local};
use lightcraft_raster::resample::Filter;
use lightcraft_raster::{Histogram, Plane, Rgb32f, Rgba8};
use crate::ctx::{Buf, FailKind, Gpu, fail, groups1, groups2};
use crate::params::{DEEP_WRITTEN, Present, finish_block};
/// Device-resident stages of one view, kept next to the CPU [`lightcraft_pipeline::StageCache`]
/// (as its extension). Holds the last two output sizes, like the CPU cache.
#[derive(Default)]
pub struct GpuStages {
entries: Mutex<Vec<Entry>>,
/// The uploaded source (a view renders one photo at several sizes).
source: Mutex<Option<(Arc<Rgb32f>, Arc<Buf>)>>,
}
const CAPACITY: usize = 2;
#[derive(Clone)]
struct Entry {
src: Arc<Rgb32f>,
geo: u64,
sampled: Arc<Buf>,
lin: Option<(u64, Arc<Buf>)>,
/// The joint colour NR's noise model of the whole frame, keyed by `Plan::noise_key`.
colour_noise: Option<(u64, JointNoise)>,
planes: Planes,
}
/// Spatial planes of one linear image (`key` = its `lin_key`), each tagged with its radius.
#[derive(Clone, Default)]
struct Planes {
key: u64,
log_l: Option<Arc<Buf>>,
base: Option<(local::BaseKey, Arc<Buf>)>,
clarity: Option<(u32, Arc<Buf>)>,
texture: Option<(u32, Arc<Buf>)>,
sharpen: Option<(local::SharpenKey, Arc<Buf>)>,
dark: Option<(u32, Arc<Buf>, f32)>,
chroma: Option<(u32, Arc<Buf>)>,
}
impl GpuStages {
fn get(&self, src: &Arc<Rgb32f>, geo: u64) -> Option<Entry> {
self.lock().iter().find(|e| e.geo == geo && Arc::ptr_eq(&e.src, src)).cloned()
}
fn put(&self, e: Entry) {
let mut v = self.lock();
v.retain(|o| !(o.geo == e.geo && Arc::ptr_eq(&o.src, &e.src)));
v.push(e);
while v.len() > CAPACITY {
v.remove(0);
}
}
fn source(&self, src: &Arc<Rgb32f>, upload: impl FnOnce() -> Buf) -> Arc<Buf> {
let mut g = self.source.lock().unwrap_or_else(|e| e.into_inner());
match &*g {
Some((s, b)) if Arc::ptr_eq(s, src) => b.clone(),
_ => {
*g = None; // free the previous photo's buffer first
let b = Arc::new(upload());
*g = Some((src.clone(), b.clone()));
b
}
}
}
fn lock(&self) -> std::sync::MutexGuard<'_, Vec<Entry>> {
self.entries.lock().unwrap_or_else(|e| e.into_inner())
}
/// Number of cached output sizes.
pub fn len(&self) -> usize {
self.lock().len()
}
/// Device bytes held (each buffer counted once).
pub fn bytes(&self) -> usize {
let mut seen: Vec<*const Buf> = Vec::new();
let mut total = 0;
let mut add = |b: &Arc<Buf>| {
let p = Arc::as_ptr(b);
if !seen.contains(&p) {
seen.push(p);
total += b.len * 4;
}
};
for e in self.lock().iter() {
add(&e.sampled);
if let Some((_, b)) = &e.lin {
add(b);
}
let p = &e.planes;
p.log_l.iter().for_each(&mut add);
p.base.iter().for_each(|(_, b)| add(b));
p.clarity.iter().for_each(|(_, b)| add(b));
p.texture.iter().for_each(|(_, b)| add(b));
p.sharpen.iter().for_each(|(_, b)| add(b));
p.dark.iter().for_each(|(_, b, _)| add(b));
}
if let Some((_, b)) = &*self.source.lock().unwrap_or_else(|e| e.into_inner()) {
add(b);
}
total
}
/// Drop everything (e.g. when memory is needed).
pub fn clear(&self) {
self.lock().clear();
*self.source.lock().unwrap_or_else(|e| e.into_inner()) = None;
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
/// A render in progress: the device and the commands recorded so far. Reading a result back
/// submits everything recorded before it.
pub(crate) struct Cx<'a> {
pub gpu: &'a Gpu,
enc: Option<wgpu::CommandEncoder>,
/// Kernel invocations recorded since the last submit.
pending: u64,
/// Buffers shared with other renders that recorded commands read from (the resident grain
/// tables): kept until those commands are submitted. Declared after `enc`, so an unsubmitted
/// encoder is dropped before them.
held: Vec<std::sync::Arc<Buf>>,
}
/// Kernel invocations recorded before a render submits them (under one full-size pass over a
/// 24 MP export, several passes over a preview): no single submission runs long.
const FLUSH_INVOCATIONS: u64 = 16 << 20;
/// Pixels per dispatch of the per-pixel stage (the heaviest kernel).
const BAND_PIXELS: usize = 4 << 20;
impl<'a> Cx<'a> {
pub fn new(gpu: &'a Gpu) -> Cx<'a> {
Cx { gpu, enc: Some(gpu.encoder()), pending: 0, held: Vec::new() }
}
/// Record kernel `name` (see [`Gpu::run`]).
pub fn run(&mut self, name: &str, p: &[u32], bufs: &[Option<&Buf>], groups: [u32; 3]) {
let enc = self.enc.get_or_insert_with(|| self.gpu.encoder());
self.gpu.run(enc, name, p, bufs, groups);
// every kernel runs 256 invocations per workgroup
self.pending += groups.iter().map(|g| *g as u64).product::<u64>() * 256;
if self.pending >= FLUSH_INVOCATIONS {
self.flush();
}
}
/// The finish kernel's `aux` buffer: `aux`, then the crystal grain's tables when it has them
/// (copied on the GPU from the resident ones, see [`Gpu::grain_tables`]).
pub fn finish_aux(&mut self, aux: &[f32], grain: Option<&lightcraft_pipeline::finish::GrainModel>) -> Buf {
let Some(lightcraft_pipeline::finish::GrainModel::Crystal(g)) = grain else { return self.gpu.upload(aux) };
let tables: usize = g.tile.sats.iter().map(Vec::len).sum();
let out = self.gpu.buffer(aux.len().saturating_add(tables));
if !aux.is_empty() && out.raw().size() >= (aux.len() as u64) * 4 {
self.gpu.queue.write_buffer(out.raw(), 0, bytemuck::cast_slice(aux));
}
let enc = self.enc.get_or_insert_with(|| self.gpu.encoder());
self.held.extend(self.gpu.grain_tables(enc, &g.tile, &out, aux.len()));
out
}
/// Submit what is recorded and read back `len` values of `b`.
pub fn read<T: bytemuck::Pod>(&mut self, b: &Buf, len: usize) -> Vec<T> {
self.pending = 0;
let enc = self.enc.take().unwrap_or_else(|| self.gpu.encoder());
let out = self.gpu.finish_and_read(enc, b, len);
self.held.clear();
out
}
/// Submit what is recorded so far without waiting. Renders submit stage by stage and every
/// [`FLUSH_INVOCATIONS`]: long submissions are what GPU watchdogs reset (Windows TDR, the
/// i915 hang check; issue #78's iGPU spent seconds on an export recorded as one submission),
/// and buffers dropped by earlier work become reusable by later work (lower peak memory).
pub fn flush(&mut self) {
self.pending = 0;
if let Some(enc) = self.enc.take() {
self.gpu.submit([enc.finish()]);
}
self.held.clear();
}
/// Submit what is recorded and wait for it (profiling only: stage timings).
pub fn sync(&mut self) {
self.pending = 0;
if let Some(enc) = self.enc.take() {
self.gpu.submit([enc.finish()]);
}
self.held.clear();
let _ = self.gpu.poll(wgpu::PollType::wait_indefinitely());
}
/// A zero-filled buffer (the clear is recorded).
pub fn zeroed(&mut self, len: usize) -> Buf {
let out = self.gpu.buffer(len);
let enc = self.enc.get_or_insert_with(|| self.gpu.encoder());
enc.clear_buffer(out.raw(), 0, None);
out
}
/// A copy of `b` (recorded).
pub fn copy(&mut self, b: &Buf) -> Buf {
let out = self.gpu.buffer(b.len);
let enc = self.enc.get_or_insert_with(|| self.gpu.encoder());
enc.copy_buffer_to_buffer(b.raw(), 0, out.raw(), 0, (b.len * 4) as u64);
out
}
/// `len` values of `b` from `offset` (both in 32-bit words), as a new buffer (recorded).
pub fn slice(&mut self, b: &Buf, offset: usize, len: usize) -> Buf {
let out = self.gpu.buffer(len);
let enc = self.enc.get_or_insert_with(|| self.gpu.encoder());
enc.copy_buffer_to_buffer(b.raw(), (offset * 4) as u64, out.raw(), 0, (len * 4) as u64);
out
}
/// Copy all of `src` into `dst` at `offset` (32-bit words; recorded).
pub fn copy_into(&mut self, src: &Buf, dst: &Buf, offset: usize) {
let enc = self.enc.get_or_insert_with(|| self.gpu.encoder());
enc.copy_buffer_to_buffer(src.raw(), 0, dst.raw(), (offset * 4) as u64, (src.len * 4) as u64);
}
/// Copy `w × h` texels from `b` (rows of `stride` 32-bit texels, a multiple of 64) into
/// `texture` (recorded).
pub fn copy_to_texture(&mut self, b: &Buf, stride: usize, texture: &wgpu::Texture, w: usize, h: usize) {
let enc = self.enc.get_or_insert_with(|| self.gpu.encoder());
enc.copy_buffer_to_texture(
wgpu::TexelCopyBufferInfo {
buffer: b.raw(),
layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some((stride * 4) as u32), rows_per_image: Some(h as u32) },
},
wgpu::TexelCopyTextureInfo { texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
wgpu::Extent3d { width: w as u32, height: h as u32, depth_or_array_layers: 1 },
);
}
pub fn read_rgb(&mut self, b: &Buf, w: usize, h: usize) -> Rgb32f {
Rgb32f { width: w, height: h, data: self.read(b, w * h * 3) }
}
pub fn read_plane(&mut self, b: &Buf, w: usize, h: usize) -> Plane {
Plane { width: w, height: h, data: self.read(b, w * h) }
}
}
pub(crate) fn rgb_words(img: &Rgb32f) -> &[f32] {
bytemuck::cast_slice(&img.data)
}
// ---------------------------------------------------------------------------------------------
// Filters (twins of `lightcraft_raster::blur` / `resample` and `lightcraft_pipeline::local`)
/// Rows each `box_v` thread slides its running sum over (at least; longer for large radii, so the
/// priming sum stays a small share of the work).
const CHUNK: usize = 16;
/// One box pass of radius `r` (clamped edges) over a `w × h` image of `nc` interleaved channels,
/// `src → dst`: horizontal or vertical.
fn box_pass(cx: &mut Cx<'_>, horizontal: bool, src: &Buf, dst: &Buf, (w, h): (usize, usize), nc: usize, r: usize) {
let chunk = (2 * r + 1).clamp(CHUNK, 128);
let p = [w as u32, h as u32, nc as u32, r as u32, chunk as u32];
let (k, groups) = if horizontal { ("box_h", groups2(w, h, [64, 4])) } else { ("box_v", groups2(w, h.div_ceil(chunk), [64, 4])) };
cx.run(k, &p, &[Some(src), Some(dst)], groups);
}
/// Gaussian blur of a `w × h` image of `nc` interleaved channels: three box passes each way
/// (`lightcraft_raster::blur::gaussian`).
pub(crate) fn gaussian(cx: &mut Cx<'_>, src: &Buf, w: usize, h: usize, nc: usize, sigma: f32) -> Buf {
let Some(radii) = lightcraft_raster::blur::radii_for(sigma, w, h) else {
return cx.copy(src);
};
let passes: Vec<(bool, usize)> = radii.iter().map(|r| (true, *r)).chain(radii.iter().map(|r| (false, *r))).filter(|p| p.1 > 0).collect();
if passes.is_empty() {
return cx.copy(src);
}
// ping-pong between two buffers
let mut bufs = [cx.gpu.buffer(w * h * nc), cx.gpu.buffer(w * h * nc)];
for (i, (horizontal, r)) in passes.iter().enumerate() {
let (out, prev) = (&bufs[i % 2], &bufs[(i + 1) % 2]);
box_pass(cx, *horizontal, if i == 0 { src } else { prev }, out, (w, h), nc, *r);
}
let last = (passes.len() - 1) % 2;
let [a, b] = std::mem::replace(&mut bufs, [cx.gpu.buffer(0), cx.gpu.buffer(0)]);
if last == 0 { a } else { b }
}
/// Exact Gaussian (`lightcraft_raster::blur::gaussian_taps`) of a one-channel plane.
pub(crate) fn gaussian_fir(cx: &mut Cx<'_>, src: &Buf, w: usize, h: usize, sigma: f32) -> Buf {
let taps = lightcraft_raster::blur::gauss_taps(sigma);
let taps_buf = cx.gpu.upload(&taps);
let r = (taps.len() / 2) as u32;
let (tmp, out) = (cx.gpu.buffer(w * h), cx.gpu.buffer(w * h));
let p = [w as u32, h as u32, r];
cx.run("gauss_h", &p, &[Some(src), Some(&taps_buf), Some(&tmp)], groups2(w, h, [16, 16]));
cx.run("gauss_v", &p, &[Some(&tmp), Some(&taps_buf), Some(&out)], groups2(w, h, [16, 16]));
out
}
/// Resample taps of `lightcraft_raster::resample::resize`, packed for the `resize_*` kernels.
fn taps(src: usize, dst: usize, filter: Filter) -> Vec<u32> {
let w = lightcraft_raster::resample::weights(src, dst, filter);
let mut t = Vec::with_capacity(dst * 3);
let mut weights = Vec::new();
let base = dst * 3;
for (lo, ws) in &w {
t.extend_from_slice(&[*lo as u32, ws.len() as u32, (base + weights.len()) as u32]);
weights.extend(ws.iter().map(|v| v.to_bits()));
}
t.extend(weights);
t
}
/// `lightcraft_raster::resample::resize` of an `nc`-channel image.
pub(crate) fn resize(cx: &mut Cx<'_>, src: &Buf, (sw, sh): (usize, usize), (dw, dh): (usize, usize), nc: usize, filter: Filter) -> Buf {
let (dw, dh) = (dw.max(1), dh.max(1));
if (sw, sh) == (dw, dh) {
return cx.copy(src);
}
let tx = cx.gpu.upload(&taps(sw, dw, filter));
let tmp = cx.gpu.buffer(dw * sh * nc);
cx.run("resize_h", &[sw as u32, sh as u32, dw as u32, nc as u32], &[Some(src), Some(&tx), Some(&tmp)], groups2(dw, sh, [16, 16]));
let ty = cx.gpu.upload(&taps(sh, dh, filter));
let out = cx.gpu.buffer(dw * dh * nc);
cx.run("resize_v", &[dw as u32, sh as u32, dh as u32, nc as u32], &[Some(&tmp), Some(&ty), Some(&out)], groups2(dw, dh, [16, 16]));
out
}
/// An element-wise kernel of the `map` module over `n` items.
pub(crate) fn map(cx: &mut Cx<'_>, k: &str, n: usize, extra: &[u32], ins: [Option<&Buf>; 3], out: &Buf) {
let mut p = vec![n as u32];
p.extend_from_slice(extra);
cx.run(k, &p, &[ins[0], ins[1], ins[2], Some(out)], groups1(n));
}
/// Self-guided filter of a plane (`local::guided`).
fn guided(cx: &mut Cx<'_>, p: &Buf, w: usize, h: usize, sigma: f32, eps: f32) -> Buf {
let ab = guided_coeffs(cx, p, w, h, sigma, eps);
let q = cx.gpu.buffer(w * h);
map(cx, "guided_apply", w * h, &[], [Some(p), Some(&ab), None], &q);
q
}
/// Blurred (a, b) coefficients, interleaved.
fn guided_coeffs(cx: &mut Cx<'_>, p: &Buf, w: usize, h: usize, sigma: f32, eps: f32) -> Buf {
let n = w * h;
let pp = cx.gpu.buffer(2 * n);
map(cx, "guided_pre", n, &[], [Some(p), None, None], &pp);
let m = gaussian(cx, &pp, w, h, 2, sigma);
let ab = cx.gpu.buffer(2 * n);
map(cx, "guided_ab", n, &[eps.to_bits()], [Some(&m), None, None], &ab);
gaussian(cx, &ab, w, h, 2, sigma)
}
/// Pixels per thread along the pass of `median_h` / `median_v` (`MED_K` in `blur.wgsl`).
const MEDIAN_K: usize = 4;
/// Separable median of a plane (`local::median_hv`).
fn median_hv(cx: &mut Cx<'_>, p: &Buf, w: usize, h: usize, r: usize) -> Buf {
if r == 0 || w * h == 0 {
return cx.copy(p);
}
let pr = [w as u32, h as u32, 1, r.min(local::BASE_DETAIL_MAX) as u32];
let tmp = cx.gpu.buffer(w * h);
cx.run("median_h", &pr, &[Some(p), Some(&tmp)], groups2(w.div_ceil(MEDIAN_K), h, [16, 16]));
let out = cx.gpu.buffer(w * h);
cx.run("median_v", &pr, &[Some(&tmp), Some(&out)], groups2(w, h.div_ceil(MEDIAN_K), [16, 16]));
out
}
/// The highlights/shadows base (`local::tone_base`; `detail` 0: V1, no detail removal).
fn tone_base(cx: &mut Cx<'_>, l: &Buf, w: usize, h: usize, sigma: f32, detail: usize, keeps_gaps: bool) -> Buf {
if detail == 0 {
return guided_fast(cx, l, w, h, sigma, local::BASE_EPS);
}
let m = if keeps_gaps { detail_free(cx, l, w, h, detail) } else { median_hv(cx, l, w, h, detail) };
guided_fast(cx, &m, w, h, sigma, local::BASE_EPS)
}
/// Log luminance without detail finer than `r` px (`local::detail_free`): clamped between its
/// median at `r` and the median of a `local::BASE_COARSE_STEP`× downscaled copy, enlarged.
fn detail_free(cx: &mut Cx<'_>, l: &Buf, w: usize, h: usize, r: usize) -> Buf {
if r == 0 || w * h == 0 {
return cx.copy(l);
}
let fine = median_hv(cx, l, w, h, r);
let s = local::BASE_COARSE_STEP;
let (lw, lh) = (w.div_ceil(s).max(1), h.div_ceil(s).max(1));
let lo = resize(cx, l, (w, h), (lw, lh), 1, Filter::Box);
let lo_median = median_hv(cx, &lo, lw, lh, r);
let coarse = resize(cx, &lo_median, (lw, lh), (w, h), 1, Filter::Bilinear);
let out = cx.gpu.buffer(w * h);
map(cx, "clamp_between_k", w * h, &[], [Some(l), Some(&fine), Some(&coarse)], &out);
out
}
/// Fast guided filter (`local::guided_fast`): coefficients on a subsampled grid.
fn guided_fast(cx: &mut Cx<'_>, p: &Buf, w: usize, h: usize, sigma: f32, eps: f32) -> Buf {
let s = local::guided_fast_step(sigma);
if s <= 1 {
return guided(cx, p, w, h, sigma, eps);
}
let (lw, lh) = (w.div_ceil(s).max(1), h.div_ceil(s).max(1));
let lo = resize(cx, p, (w, h), (lw, lh), 1, Filter::Box);
let ab = guided_coeffs(cx, &lo, lw, lh, sigma / s as f32, eps);
let up = resize(cx, &ab, (lw, lh), (w, h), 2, Filter::Bilinear);
let q = cx.gpu.buffer(w * h);
map(cx, "guided_apply", w * h, &[], [Some(p), Some(&up), None], &q);
q
}
// ---------------------------------------------------------------------------------------------
// Geometry (`Frame::sample`)
/// Integer pixel map `(x, y) → (m0·x + m1·y + m2, m3·x + m4·y + m5)` from an oriented image back
/// to the source of `w × h` (`Image::oriented`: optional horizontal flip, then quarter turns).
pub(crate) fn orient_map(o: Orientation, w: usize, h: usize) -> [i32; 6] {
let (flip, turns) = o.to_parts();
let (w, h) = (w as i32, h as i32);
let mut ops: Vec<fn(i32, i32) -> [i32; 6]> = Vec::new();
if flip {
ops.push(|w, _| [-1, 0, w - 1, 0, 1, 0]);
}
let cw: fn(i32, i32) -> [i32; 6] = |_, h| [0, 1, 0, -1, 0, h - 1];
let half: fn(i32, i32) -> [i32; 6] = |w, h| [-1, 0, w - 1, 0, -1, h - 1];
match turns {
1 => ops.push(cw),
2 => ops.push(half),
3 => {
ops.push(half);
ops.push(cw);
}
_ => {}
}
// compose: T ← T ∘ op, tracking each op's input size
let mut t = [1, 0, 0, 0, 1, 0];
let (mut cw_, mut ch_) = (w, h);
for op in ops {
let o = op(cw_, ch_);
t = [
t[0] * o[0] + t[1] * o[3],
t[0] * o[1] + t[1] * o[4],
t[0] * o[2] + t[1] * o[5] + t[2],
t[3] * o[0] + t[4] * o[3],
t[3] * o[1] + t[4] * o[4],
t[3] * o[2] + t[4] * o[5] + t[5],
];
if o[0] == 0 {
(cw_, ch_) = (ch_, cw_);
}
}
t
}
fn affine_bits(a: &lightcraft_geom::Affine) -> [u32; 6] {
a.0.map(|v| (v as f32).to_bits())
}
/// The `sample_warp` kernel's parameters (layout documented in `geom.wgsl`).
fn warp_params(
wp: &lightcraft_pipeline::optics::Warp,
base: (usize, usize),
out: (usize, usize),
o2t: &lightcraft_geom::Affine,
sx: f64,
sy: f64,
) -> Vec<u32> {
let f = |v: f64| (v as f32).to_bits();
let mut p = vec![base.0 as u32, base.1 as u32, out.0 as u32, out.1 as u32];
p.extend(affine_bits(o2t));
p.extend([f(sx), f(sy), f(wp.w), f(wp.h)]);
let persp = wp.persp != lightcraft_geom::Homography::IDENTITY;
p.push(persp as u32);
p.extend(wp.persp_inv.0.map(f));
p.push(f(wp.k1));
p.extend(wp.ca.map(f));
p.push(f(wp.lens_dist));
let warp = wp.lens.and_then(|l| l.warp);
p.push(warp.is_some() as u32);
let w = warp.unwrap_or_default();
p.extend(w.planes.iter().flatten().map(|v| f(*v)));
p.extend([f(w.center.x), f(w.center.y), f(w.radius)]);
p.extend([f(wp.vig_stops), f(wp.vig_power), f(wp.lens_vig)]);
let vig = wp.lens.and_then(|l| l.vignette);
p.push(vig.is_some() as u32);
let v = vig.unwrap_or_default();
p.extend(v.k.map(f));
p.extend([f(v.center.x), f(v.center.y), f(v.radius)]);
p.push(wp.per_channel() as u32);
p.push(wp.has_gain() as u32);
debug_assert_eq!(p.len(), 65);
p
}
/// Output rows per block of [`coverage_mask`]; a block is one mask word (32 px) wide. 16 rows measured
/// ~20 % faster than 8 at 6000 × 4000 (fewer interval evaluations, few more edge pixels).
const COVER_ROWS: usize = 16;
/// The largest source (bytes on the device) a view's stages keep uploaded between renders.
const RETAIN_SOURCE_BYTES: usize = 96 << 20;
/// Big sources uploaded to the shared copy so far (diagnostics and tests).
static SOURCE_UPLOADS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
/// The one big source (the original a zoom window is cut from) kept on the device for all the
/// views of the photo: the Before and After windows and every pan share it, instead of each
/// view's stages holding (or re-uploading) 288 MB of a 24 MP photo. It does not count as a view's
/// stages; it is released with the buffer pool when the app idles.
static SHARED_SOURCE: std::sync::Mutex<Option<(std::sync::Weak<Rgb32f>, Arc<Buf>)>> = std::sync::Mutex::new(None);
fn shared_source(src: &Arc<Rgb32f>, upload: impl FnOnce() -> Buf) -> Arc<Buf> {
let mut g = SHARED_SOURCE.lock().unwrap_or_else(|e| e.into_inner());
if let Some((weak, buf)) = &*g
&& weak.upgrade().is_some_and(|s| Arc::ptr_eq(&s, src))
{
return buf.clone();
}
// the previous photo's copy goes (on a render thread it is retired with the render's other
// buffers, so for a moment both exist)
*g = None;
SOURCE_UPLOADS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let buf = Arc::new(upload());
// a failed upload's buffer holds zeros: this render falls back, and no other may reuse it
if crate::ctx::failed() {
return buf;
}
*g = Some((Arc::downgrade(src), buf.clone()));
buf
}
/// Free the shared big source (see [`shared_source`]).
pub(crate) fn release_shared_source() {
*SHARED_SOURCE.lock().unwrap_or_else(|e| e.into_inner()) = None;
}
/// Device bytes of the shared big source, 0 when there is none.
pub(crate) fn shared_source_bytes() -> usize {
SHARED_SOURCE.lock().unwrap_or_else(|e| e.into_inner()).as_ref().map_or(0, |(_, b)| b.len * 4)
}
pub(crate) fn source_uploads() -> u64 {
SOURCE_UPLOADS.load(std::sync::atomic::Ordering::Relaxed)
}
/// The reference framing decision ([`Warp::covers`](lightcraft_pipeline::optics::Warp::covers)) for every
/// output pixel, one bit per pixel, rows padded to 32-bit words. Blocks of 32 × [`COVER_ROWS`] pixels whose
/// interval bounds place them clearly inside or outside the image are filled at once
/// ([`Warp::block_coverage`](lightcraft_pipeline::optics::Warp::block_coverage), the same formulas evaluated
/// with outward-rounded intervals); only blocks near the image edge evaluate each pixel. Same bits as
/// evaluating every pixel.
fn coverage_mask(wp: &lightcraft_pipeline::optics::Warp, o2t: &lightcraft_geom::Affine, w: usize, h: usize) -> Vec<u32> {
let words = w.div_ceil(32);
let mut coverage = vec![0u32; words * h];
if words == 0 {
return coverage;
}
lightcraft_raster::par_rows(&mut coverage, words * COVER_ROWS, |band, rows| {
let y0 = band * COVER_ROWS;
let y1 = y0 + rows.len() / words;
for word in 0..words {
let (x0, x1) = (word * 32, (word * 32 + 32).min(w));
let full = if x1 - x0 == 32 { u32::MAX } else { (1u32 << (x1 - x0)) - 1 };
let block = wp.block_coverage(o2t, x0, x1, y0, y1);
for (y, row) in (y0..y1).zip(rows.chunks_exact_mut(words)) {
let Some(bits) = row.get_mut(word) else { continue };
*bits = match block {
Some(true) => full,
Some(false) => 0,
None => (x0..x1).filter(|&x| wp.covers(o2t, x, y)).fold(0, |b, x| b | 1 << (x - x0)),
};
}
}
});
coverage
}
/// Resample the source into the output frame (`Frame::sample`). May return the source buffer
/// itself when the frame is the identity.
fn sample(cx: &mut Cx<'_>, src: &Arc<Rgb32f>, src_buf: Arc<Buf>, plan: &Plan<'_>) -> Arc<Buf> {
let (w, h) = (plan.w, plan.h);
let fr = &plan.frame;
let sp = fr.sample_plan(src.width, src.height, w, h);
let oriented = if fr.orient == Orientation::Normal {
src_buf
} else {
let out = cx.gpu.buffer(sp.ow * sp.oh * 3);
let mut p = vec![src.width as u32, src.height as u32, sp.ow as u32, sp.oh as u32];
p.extend(orient_map(fr.orient, src.width, src.height).map(|v| v as u32));
cx.run("orient", &p, &[Some(&src_buf), Some(&out), None], groups2(sp.ow, sp.oh, [16, 16]));
Arc::new(out)
};
let (base, (bw, bh)) = match sp.prefilter {
Some(d) => (Arc::new(resize(cx, &oriented, (sp.ow, sp.oh), d, 3, Filter::Mitchell)), d),
None => (oriented, (sp.ow, sp.oh)),
};
let mut affine = |xf: &lightcraft_geom::Affine| {
let out = cx.gpu.buffer(w * h * 3);
let mut p = vec![bw as u32, bh as u32, w as u32, h as u32];
p.extend(affine_bits(xf));
cx.run("sample_affine", &p, &[Some(&base), Some(&out), None], groups2(w, h, [16, 16]));
Arc::new(out)
};
match (&sp.mode, fr.warp.as_ref()) {
(SampleMode::Copy, _) => base,
(SampleMode::Affine(xf), _) => affine(xf),
// `sample_plan` plans a warp only when there is one; without it the same mapping is affine
(SampleMode::Warp(o2t), None) => affine(&(lightcraft_geom::Affine::scale(sp.sx, sp.sy) * *o2t)),
(SampleMode::Warp(o2t), Some(wp)) => {
let out = cx.gpu.buffer(w * h * 3);
let p = warp_params(wp, (bw, bh), (w, h), o2t, sp.sx, sp.sy);
// f32 rounding can cross a source edge. Keep the reference f64
// framing decision in a bit mask; color sampling stays on the GPU.
let coverage = cx.gpu.upload(&coverage_mask(wp, o2t, w, h));
cx.run("sample_warp", &p, &[Some(&base), Some(&out), Some(&coverage)], groups2(w, h, [16, 16]));
Arc::new(out)
}
}
}
// ---------------------------------------------------------------------------------------------
// The render
/// CPU copies made along the way (for stages that still run on the CPU).
#[derive(Default)]
struct Host {
sampled: Option<Rgb32f>,
lin: Option<Arc<Rgb32f>>,
log_l: Option<Arc<Plane>>,
}
pub(crate) fn profiling() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var_os("LIGHTCRAFT_PROFILE").is_some())
}
/// A finished render on the window's device, ready to draw: an `Rgba8Unorm` texture (the 8-bit
/// output, sRGB-curve encoded, as egui's own textures) that never went through host memory.
#[derive(Clone, Debug)]
pub struct DisplayFrame {
pub texture: wgpu::Texture,
pub width: usize,
pub height: usize,
}
/// What a render produced: pixels in host memory, or (asked for with `display` and possible) a
/// texture on the window's device plus the histogram.
pub enum Output {
Image(Rendered),
Display { frame: DisplayFrame, histogram: Histogram },
}
/// Whether a render of this plan can end in a texture instead of a readback: the device is the
/// window's, nothing is drawn over the image on the CPU (overlays) or cut from it (a window that
/// spots grew), and the texture fits the device.
fn can_display(gpu: &Gpu, plan: &Plan<'_>, req: &RenderRequest) -> bool {
let max = gpu.device.limits().max_texture_dimension_2d as usize;
gpu.window && req.overlay == lightcraft_pipeline::Overlay::None && plan.keep.is_none() && plan.w <= max && plan.h <= max && plan.w * plan.h > 0
}
/// Render on `gpu`, reusing `stages` (if given); with `display`, into a texture when
/// [`can_display`]. `None` (or a result the caller discards) when the render failed: the reason is
/// recorded with [`crate::ctx::fail`].
#[allow(clippy::too_many_arguments)]
pub fn render(
gpu: &Gpu,
src: &Arc<Rgb32f>,
info: &SourceInfo,
s: &DevelopSettings,
req: &RenderRequest,
stages: Option<&GpuStages>,
fault: Option<crate::Fault>,
display: bool,
) -> Option<Output> {
let _in_flight = InFlight::new();
let mut t = profiling().then(std::time::Instant::now);
// Each stage is submitted on its own; under `LIGHTCRAFT_PROFILE` also waited for, so the
// timings are real.
let lap = |what: &str, t: &mut Option<std::time::Instant>, cx: &mut Cx<'_>| match t {
Some(t) => {
cx.sync();
let (peak, now) = crate::ctx::take_peak();
eprintln!(" gpu {what}: {:.1} ms (device buffers: peak {} MB, now {} MB)", t.elapsed().as_secs_f64() * 1e3, peak >> 20, now >> 20);
*t = std::time::Instant::now();
}
None => cx.flush(),
};
let plan = lightcraft_pipeline::plan(src, info, s, req);
let (w, h) = (plan.w, plan.h);
let Some(n) = w.checked_mul(h).filter(|n| n.checked_mul(12).is_some()) else {
fail(FailKind::Limit, format!("{w}×{h} output is too large"));
return None;
};
let _limit = match fault {
Some(crate::Fault::Limit(bytes)) => Some(crate::ctx::LimitOverride::new(bytes)),
_ => None,
};
if !gpu.fits(n * 3) {
fail(
FailKind::Limit,
format!("{w}×{h} needs {} MiB buffers, over the device's {} MiB storage-buffer limit", (n * 12).div_ceil(1 << 20), gpu.limit() >> 20),
);
return None;
}
let s = &*plan.settings;
let cached = stages.and_then(|c| c.get(src, plan.geo));
let mut host = Host::default();
let mut cx = Cx::new(gpu);
lap("plan", &mut t, &mut cx);
// 1. geometry (on the CPU when the source exceeds the device's buffer limit)
let sampled = match &cached {
Some(e) => e.sampled.clone(),
// (a WarpRectilinear2 chain also goes to the CPU: its degree-14 polynomial needs f64)
None if gpu.fits(src.data.len() * 3) && !plan.frame.warp.as_ref().is_some_and(|w| w.needs_f64()) => {
let upload = || gpu.upload(rgb_words(src));
// (a source too big for a view's stages to keep is the one shared copy: a window is
// cut from the photo's own pixels, and what its stages keep is its own sampled pixels)
let src_buf = match stages {
Some(c) if src.data.len() * 12 <= RETAIN_SOURCE_BYTES => c.source(src, upload),
Some(_) => shared_source(src, upload),
None => Arc::new(upload()),
};
sample(&mut cx, src, src_buf, &plan)
}
None => {
let img = plan.frame.sample(src, w, h);
let b = Arc::new(gpu.upload(rgb_words(&img)));
host.sampled = Some(img);
b
}
};
lap("sample", &mut t, &mut cx);
// 2. white balance, defringe, spots, noise reduction
let mut colour_noise = cached.as_ref().and_then(|e| e.colour_noise).filter(|(k, _)| *k == plan.noise_key);
let lin = match cached.as_ref().and_then(|e| e.lin.clone()).filter(|(k, _)| *k == plan.lin_key) {
Some((_, b)) => b,
None => {
if colour_noise.is_none() {
colour_noise = lightcraft_pipeline::colour_noise(src, info, &plan).map(|n| (plan.noise_key, n));
}
Arc::new(linear(&mut cx, &sampled, info, &plan, &mut host, colour_noise.as_ref().map(|c| &c.1)))
}
};
lap("wb/nr", &mut t, &mut cx);
// 3. spatial planes
let mut planes = match cached.map(|e| e.planes) {
Some(p) if p.key == plan.lin_key => p,
_ => Planes { key: plan.lin_key, ..Default::default() },
};
let mut prep = prepare(&mut cx, &lin, &plan, req, &mut planes);
// a window dehazes with the whole frame's airlight
if let Some(a) = plan.fixed_air {
prep.air = a;
}
lap("planes", &mut t, &mut cx);
if let Some(c) = stages {
c.put(Entry { src: src.clone(), geo: plan.geo, sampled, lin: Some((plan.lin_key, lin.clone())), colour_noise, planes });
}
// 4. masks
let (masks, terms) = masks(&mut cx, &lin, &prep, &plan, &mut host);
lap("masks", &mut t, &mut cx);
// 5. per-pixel stage
let mut fp = FinishParams::new(FinishParamsInput {
settings: s,
frame: &plan.frame,
info,
image_size: (w, h),
px_per_long: plan.px_per_long,
air_pre: prep.air,
space: req.space,
draft: req.quality == lightcraft_pipeline::Quality::Draft,
});
// soft proofing as the CPU's `finish` sets it up (an output-device table never gets here: see `render`)
fp.proof = req.proof.map(|pr| pr.params(req.space));
// a float render is `finish_deep`: its samples are the output space's, whatever the display
let deep = req.depth != lightcraft_pipeline::OutputDepth::U8;
if let Some(d) = req.display.as_ref().filter(|_| !deep) {
fp.for_display(d, req.proof);
}
let present = Present {
clarity: prep.clarity.is_some(),
texture: prep.texture.is_some(),
sharpen: prep.sharpen.is_some(),
dark: prep.dark.is_some(),
chroma: masks.is_some() && prep.chroma.is_some(),
};
// the finish kernel's `tex` binding: the texture plane, then the sharpening plane (one binding
// for both keeps the kernel within the storage-buffer count every supported device has)
let tex_buf: Option<Arc<Buf>> = match (&prep.texture, &prep.sharpen) {
(Some(t), Some(sh)) => {
let both = cx.gpu.buffer(2 * n);
cx.copy_into(t, &both, 0);
cx.copy_into(sh, &both, n);
Some(Arc::new(both))
}
(Some(b), None) | (None, Some(b)) => Some(b.clone()),
(None, None) => None,
};
let (p, aux) = finish_block(&fp, &terms, &present, deep);
if crate::ctx::failed() {
return None; // a malformed parameter block failed the render: the CPU fallback redoes it
}
let aux = cx.finish_aux(&aux, fp.grain.as_ref());
// cleared, so that work which never ran cannot pass for an image (see the check below)
let out = cx.zeroed(if deep { 3 * n } else { n });
if fault == Some(crate::Fault::Validation) {
// a dispatch the device rejects (too many workgroups)
cx.run("box_h", &[0; 5], &[Some(&aux), Some(&out)], [gpu.device.limits().max_compute_workgroups_per_dimension + 1, 1, 1]);
}
// in bands of rows: each dispatch (and submission) stays short on a slow GPU
let rows = (BAND_PIXELS / w.max(1) / 16 * 16).max(16); // whole workgroups: bands never overlap
let y0_at = crate::params::index("Y0").unwrap_or(0);
let mut p = p;
for y0 in (0..h).step_by(rows).filter(|_| fault != Some(crate::Fault::DropWork)) {
p[y0_at] = y0 as u32;
cx.run(
"main",
&p,
&[
Some(&lin),
Some(&prep.log_l),
Some(&prep.base),
prep.clarity.as_deref(),
tex_buf.as_deref(),
prep.dark.as_deref(),
masks.as_ref(),
Some(&aux),
Some(&out),
],
groups2(w, rows.min(h - y0), [16, 16]),
);
}
if deep {
let rendered = read_deep(&mut cx, &out, &plan, req);
lap("finish + readback (deep)", &mut t, &mut cx);
return rendered.map(Output::Image);
}
if display && can_display(gpu, &plan, req) {
return to_texture(&mut cx, &out, w, h).map(|(frame, histogram)| Output::Display { frame, histogram });
}
// the alpha a mask overlay shows: copied out before the readback below submits
let overlay_mask = req.overlay.mask(s).map(|m| {
let list = s.masks.iter().filter(|m| m.visible && !m.components.is_empty());
match (list.map(|m| m.id).position(|id| id == m.id), masks.as_ref()) {
(Some(mi), Some(all)) => Ok(cx.slice(all, mi * n, n)),
_ => Err(m),
}
});
// every visible mask's alpha, for `Overlay::AllMasks` (one tint over all masks at once).
// The masks buffer holds each visible mask's alpha back-to-back in the same `list` order the
// single-mask overlay uses, so read the whole buffer back in ONE GPU->CPU transfer and split it
// on the CPU, instead of one synchronous readback per mask (which stalls the pipeline N times).
let all_overlay_alphas: Option<Vec<Plane>> = if matches!(req.overlay, lightcraft_pipeline::Overlay::AllMasks { .. }) {
masks.as_ref().map(|all| {
let list: Vec<_> = s.masks.iter().filter(|m| m.visible && !m.components.is_empty()).collect();
if list.is_empty() {
return Vec::new();
}
let raw: Vec<f32> = cx.read(all, n * list.len());
let mut alphas = Vec::with_capacity(list.len());
for mi in 0..list.len() {
let s = mi * n;
alphas.push(Plane { width: w, height: h, data: raw[s..s + n].to_vec() });
}
alphas
})
} else {
None
};
let data: Vec<[u8; 4]> = cx.read(&out, n);
let image = Rgba8 { width: w, height: h, data };
lap("finish + readback", &mut t, &mut cx);
if crate::ctx::failed() {
return None;
}
// The per-pixel kernel writes alpha 255 everywhere and `out` was cleared: a pixel with any
// other alpha means work silently did not run (a driver that reset or dropped a submission
// without reporting it — issue #78's all-black exports).
let unwritten = unwritten(&image.data);
if unwritten > 0 {
fail(FailKind::Fatal, format!("the GPU returned an incomplete image ({unwritten} of {n} pixels unwritten)"));
return None;
}
let region = plan.keep.map_or((0, 0, w, h), |k| (k.x, k.y, k.w, k.h));
let histogram = histogram(&mut cx, &out, w, region);
lap("histogram", &mut t, &mut cx);
// An entirely black result (upstream work that did not run would give that too): redo it on
// the CPU, which costs time only for the rare photo that really is black.
if n >= 4096 && [&histogram.r, &histogram.g, &histogram.b].iter().all(|c| c.first() == Some(&histogram.total)) {
fail(FailKind::Blank, "the GPU result is entirely black; rendering it on the CPU to be sure".into());
return None;
}
let mut image = image;
let overlay_mask = overlay_mask.map(|r| match r {
Ok(b) => cx.read_plane(&b, w, h),
// a hidden mask: evaluate it on the CPU
Err(m) => {
let img = host.lin.get_or_insert_with(|| Arc::new(cx.read_rgb(&lin, w, h))).clone();
let l = host.log_l.get_or_insert_with(|| Arc::new(cx.read_plane(&prep.log_l, w, h))).clone();
lightcraft_pipeline::masks::evaluate_one(m, &plan.frame, w, h, &img, &l, s.light.exposure as f32, plan.mattes.as_deref())
}
});
// a sharpening view: the detail planes this render sharpened with
let overlay_mask = overlay_mask.or_else(|| {
req.overlay.reads_detail().then(|| {
let l = host.log_l.get_or_insert_with(|| Arc::new(cx.read_plane(&prep.log_l, w, h))).clone();
let band = if local::pixel_sharpen(&plan.settings) { &prep.sharpen } else { &prep.texture };
let blur = band.as_deref().map(|b| cx.read_plane(b, w, h));
lightcraft_pipeline::visualize::sharpen_plane(req.overlay, &plan.settings, &l, blur.as_ref(), plan.px_per_long, plan.native_long as f64)
})?
});
lightcraft_pipeline::visualize::apply(&mut image, req.overlay, &plan, overlay_mask.as_ref(), all_overlay_alphas.as_deref());
// spots grew the window the render worked on: cut it back to the request
if let Some(k) = plan.keep {
image = image.crop(k.x, k.y, k.w, k.h);
}
Some(Output::Image(Rendered { image, histogram, deep: None }))
}
/// The finished float output `out` (three words per pixel) read back as the CPU's float render
/// returns it, with the same checks as an 8-bit readback: every pixel written, not entirely black.
fn read_deep(cx: &mut Cx<'_>, out: &Buf, plan: &Plan<'_>, req: &RenderRequest) -> Option<Rendered> {
use lightcraft_pipeline::{DeepImage, DeepSamples};
let (w, h) = (plan.w, plan.h);
let n = w * h;
let words: Vec<u32> = cx.read(out, 3 * n);
if crate::ctx::failed() {
return None;
}
let unwritten = words.as_chunks::<3>().0.iter().filter(|p| p.iter().any(|v| v & DEEP_WRITTEN == 0)).count();
if unwritten > 0 {
fail(FailKind::Fatal, format!("the GPU returned an incomplete image ({unwritten} of {n} pixels unwritten)"));
return None;
}
let samples = words.into_iter().map(|v| f32::from_bits(v & !DEEP_WRITTEN)).collect();
let deep = DeepImage { width: w, height: h, space: req.space, samples: DeepSamples::F32(samples) };
let deep = match plan.keep {
Some(k) => deep.crop(k.x, k.y, k.w, k.h),
None => deep,
};
let image = deep.to_rgba8();
let histogram = Histogram::of_srgb8(&image);
if n >= 4096 && [&histogram.r, &histogram.g, &histogram.b].iter().all(|c| c.first() == Some(&histogram.total)) {
fail(FailKind::Blank, "the GPU result is entirely black; rendering it on the CPU to be sure".into());
return None;
}
Some(Rendered { image, histogram, deep: Some(deep) })
}
/// The finished output `out` (`w × h` packed RGBA8) as a texture on the device, with its
/// histogram; the same checks as a readback (every pixel written, not entirely black), done on
/// the device: only the histogram and a pixel count come back.
fn to_texture(cx: &mut Cx<'_>, out: &Buf, w: usize, h: usize) -> Option<(DisplayFrame, Histogram)> {
let n = w * h;
let unwritten = cx.zeroed(1);
cx.run("unwritten", &[n as u32], &[Some(out), None, Some(&unwritten)], groups1(n));
let histogram = histogram(cx, out, w, (0, 0, w, h));
let left = cx.read::<u32>(&unwritten, 1).first().copied().unwrap_or(0);
if crate::ctx::failed() {
return None;
}
if left > 0 {
fail(FailKind::Fatal, format!("the GPU returned an incomplete image ({left} of {n} pixels unwritten)"));
return None;
}
if n >= 4096 && [&histogram.r, &histogram.g, &histogram.b].iter().all(|c| c.first() == Some(&histogram.total)) {
fail(FailKind::Blank, "the GPU result is entirely black; rendering it on the CPU to be sure".into());
return None;
}
// texture copies take rows of a multiple of 256 bytes: pad them unless they already are
let stride = w.next_multiple_of(64);
let padded = (stride != w).then(|| {
let b = cx.gpu.buffer(stride * h);
cx.run("pad_rows", &[w as u32, h as u32, stride as u32], &[Some(out), Some(&b), None], groups2(w, h, [16, 16]));
b
});
let texture = cx.gpu.device.create_texture(&wgpu::TextureDescriptor {
label: Some("lightcraft view"),
size: wgpu::Extent3d { width: w as u32, height: h as u32, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
// (COPY_SRC: a screenshot or a test can read it back)
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::COPY_SRC,
view_formats: &[],
});
cx.copy_to_texture(padded.as_ref().unwrap_or(out), stride, &texture, w, h);
cx.flush();
if crate::ctx::failed() {
return None;
}
Some((DisplayFrame { texture, width: w, height: h }, histogram))
}
/// Pixels whose alpha is not 255 (cheap: no early exit, so it vectorizes).
fn unwritten(data: &[[u8; 4]]) -> usize {
data.chunks(4096).filter(|c| c.iter().fold(255u8, |a, p| a & p[3]) != 255).map(|c| c.iter().filter(|p| p[3] != 255).count()).sum()
}
/// The white-balanced, retouched, denoised image (`colour_noise`: see `denoise`).
fn linear(cx: &mut Cx<'_>, sampled: &Buf, info: &SourceInfo, plan: &Plan<'_>, host: &mut Host, colour_noise: Option<&JointNoise>) -> Buf {
let (w, h) = (plan.w, plan.h);
let n = w * h;
let s = &*plan.settings;
let img = if lightcraft_pipeline::local_tone::enabled(info, s) {
// the camera's local tone mapping (ProRAW gain table, off by default): the whole
// scene-linear stage on the CPU
let mut img = match host.sampled.take() {
Some(i) => i,
None => cx.read_rgb(sampled, w, h),
};
lightcraft_pipeline::lin_cpu(&mut img, info, plan);
cx.gpu.upload(rgb_words(&img))
} else {
// `lin_cpu`'s order: white balance, defringe, spots, red eye
let m = local::wb_matrix_for(info, s);
let mut p = vec![m.is_some() as u32];
p.extend(m.unwrap_or_default().iter().flatten().map(|v| v.to_bits()));
let out = cx.gpu.buffer(n * 3);
map(cx, "wb_k", n, &p, [Some(sampled), None, None], &out);
let out = defringe(cx, out, plan);
let out = spots(cx, out, plan);
if plan.eyes.is_empty() {
out
} else {
let mut p = vec![w as u32, plan.eyes.len() as u32];
p.extend(plan.eyes.iter().flat_map(|e| e.words()).map(f32::to_bits));
let eyed = cx.gpu.buffer(n * 3);
map(cx, "redeye_k", n, &p, [Some(&out), None, None], &eyed);
eyed
}
};
let img = denoise(cx, img, plan, colour_noise);
if !lightcraft_pipeline::lens_blur::renders(s) {
return img;
}
// Lens Blur (global or a mask's): on the CPU, like the CPU renderer's (`lens_blur::apply`)
let mut host_img = cx.read_rgb(&img, w, h);
lightcraft_pipeline::lens_blur::apply_with(&mut host_img, s, &plan.frame, plan.px_per_long, plan.mattes.as_deref(), plan.lens_subject.as_deref());
cx.gpu.upload(rgb_words(&host_img))
}
/// Defringe (`optics::defringe`) of the white-balanced image.
fn defringe(cx: &mut Cx<'_>, img: Buf, plan: &Plan<'_>) -> Buf {
use lightcraft_pipeline::optics;
let Some(dp) = optics::defringe_params(&plan.settings, plan.px_per_long / optics::DEFRINGE_REF_LONG) else { return img };
let (w, h) = (plan.w, plan.h);
let n = w * h;
let lab = cx.gpu.buffer(n * 3);
map(cx, "fringe_lab", n, &[], [Some(&img), None, None], &lab);
// the lightness channel, then its min and max around each pixel
let l = cx.gpu.buffer(n);
map(cx, "subsample", n, &[3], [Some(&lab), None, None], &l);
// (a radius past the image reads the same pixels as one at its size)
let r = dp.radius.min(w.max(h)) as u32;
let filtered = |cx: &mut Cx<'_>, max: u32| {
let tmp = cx.gpu.buffer(n);
cx.run("minmax", &[w as u32, h as u32, r, 0, max], &[Some(&l), Some(&tmp)], groups2(w, h, [16, 16]));
let out = cx.gpu.buffer(n);
cx.run("minmax", &[w as u32, h as u32, r, 1, max], &[Some(&tmp), Some(&out)], groups2(w, h, [16, 16]));
out
};
let (lo, hi) = (filtered(cx, 0), filtered(cx, 1));
// the kernel writes only the pixels it changes
let out = cx.copy(&img);
let p = [dp.purple.0, dp.purple.1, dp.green.0, dp.green.1, dp.purple_strength, dp.green_strength].map(f32::to_bits);
map(cx, "fringe_apply", n, &p, [Some(&lab), Some(&lo), Some(&hi)], &out);
out
}
/// Spot removal (`spots::apply`), spot by spot on `img` in place: each spot reads the image as
/// the earlier ones left it.
fn spots(cx: &mut Cx<'_>, img: Buf, plan: &Plan<'_>) -> Buf {
use lightcraft_pipeline::spots;
let (w, h) = (plan.w, plan.h);
for spot in &plan.settings.spots {
let Some(l) = spots::layout(spot, &plan.frame, w, h, plan.px_per_long) else { continue };
let Some(&first) = l.pts.first() else { continue };
let offset = match l.offset {
Some(o) => o,
None => auto_source(cx, &img, w, h, first, l.r),
};
let pts = cx.gpu.upload(&l.pts.iter().flat_map(|p| [p.0, p.1]).collect::<Vec<f32>>());
let nb = l.bw * l.bh;
let (al, sp, df, wt) = (cx.gpu.buffer(nb), cx.gpu.buffer(3 * nb), cx.gpu.buffer(3 * nb), cx.gpu.buffer(nb));
let mut p = vec![w as u32, h as u32, l.bx as u32, l.by as u32, l.bw as u32, l.bh as u32, l.pts.len() as u32];
p.extend([l.r, l.inner, l.opacity, offset.0, offset.1].map(f32::to_bits));
p.push(l.heal as u32);
let groups = groups2(l.bw, l.bh, [16, 16]);
cx.run("spot_prep", &p, &[Some(&img), Some(&pts), Some(&al), Some(&sp), Some(&df), Some(&wt)], groups);
// Heal: the tone correction is the masked difference's normalized blur
let blurred = l.heal.then(|| {
let sigma = (l.r * 0.6).max(1.0);
(gaussian(cx, &df, l.bw, l.bh, 3, sigma), gaussian(cx, &wt, l.bw, l.bh, 1, sigma))
});
let (bd, bm) = blurred.as_ref().map_or((&df, &wt), |(d, m)| (d, m));
cx.run("spot_apply", &p, &[Some(&img), Some(&pts), Some(&al), Some(&sp), Some(bd), Some(bm)], groups);
}
img
}
/// `spots::auto_source` on the device: every candidate scored at once, the 48 scores read back.
fn auto_source(cx: &mut Cx<'_>, img: &Buf, w: usize, h: usize, target: (f32, f32), r: f32) -> (f32, f32) {
use lightcraft_pipeline::spots;
let cands = spots::candidates(r);
let mut data: Vec<f32> = cands.iter().flat_map(|d| [d.0, d.1]).collect();
data.extend(spots::ring_dirs().iter().flat_map(|d| [d.0, d.1]));
let pts = cx.gpu.upload(&data);
let [around, within] = spots::score_radii(r);
let mut p = vec![w as u32, h as u32];
p.extend([target.0, target.1].iter().chain(&around).chain(&within).map(|v| v.to_bits()));
let scores = cx.gpu.buffer(cands.len());
cx.run("spot_score", &p, &[Some(img), Some(&pts), Some(&scores), None, None, None], [cands.len().div_ceil(64) as u32, 1, 1]);
let scores: Vec<f32> = cx.read(&scores, cands.len());
spots::rank(&cands, &scores).first().copied().unwrap_or_else(|| spots::fallback_source(r))
}
/// The output's histogram (`Histogram::of_srgb8` of the `region` x, y, w, h of the `w`-wide
/// packed RGBA8 `out`, on the same sampling grid), counted on the device.
fn histogram(cx: &mut Cx<'_>, out: &Buf, w: usize, region: (usize, usize, usize, usize)) -> Histogram {
let (x0, y0, rw, rh) = region;
let step = (((rw * rh) as f64 / 1_000_000.0).sqrt().ceil() as usize).max(1);
let (nx, ny) = (rw.div_ceil(step), rh.div_ceil(step));
let bins = cx.zeroed(4 * Histogram::BINS + 1);
let p = [w, x0, y0, nx, ny, step].map(|v| v as u32);
cx.run("histogram", &p, &[Some(out), Some(&bins)], groups2(nx, ny, [16, 16]));
let v: Vec<u32> = cx.read(&bins, 4 * Histogram::BINS + 1);
let part = |k: usize| v.get(k * Histogram::BINS..(k + 1) * Histogram::BINS).map_or_else(|| vec![0; Histogram::BINS], <[u32]>::to_vec);
// (SDR bins: an HDR edit's histogram is made after the render, `media::develop`)
Histogram { r: part(0), g: part(1), b: part(2), luma: part(3), total: v.get(4 * Histogram::BINS).copied().unwrap_or(0), hdr: None }
}
/// Noise reduction (`local::denoise`). `colour_noise`: the whole frame's noise model for the joint colour
/// NR (`lightcraft_pipeline::colour_noise`); `None`: measured on `img`.
fn denoise(cx: &mut Cx<'_>, img: Buf, plan: &Plan<'_>, colour_noise: Option<&JointNoise>) -> Buf {
let (w, h) = (plan.w, plan.h);
let n = w * h;
let (lum, col) = local::nr_params(&plan.settings, plan.detail_scale);
let mut img = img;
if let Some(nr) = lum {
let l = cx.gpu.buffer(n);
map(cx, "log_lum_k", n, &[], [Some(&img), None, None], &l);
let f = guided(cx, &l, w, h, nr.sigma, nr.eps);
let out = cx.gpu.buffer(n * 3);
map(cx, "nr_lum", n, &[nr.k.to_bits()], [Some(&img), Some(&l), Some(&f)], &out);
img = out;
}
if col.is_some() && local::joint_colour_nr(&plan.settings) {
let Some(p) = local::nrc_params(&plan.settings, plan.detail_scale) else { return img };
let s = &plan.settings.detail;
let (detail, amount) = ((s.nr_color_detail / 100.0) as f32, (s.nr_color / 100.0) as f32);
if let Some(out) = color_nr_joint(cx, &img, w, h, &p, detail, amount, colour_noise) {
return out;
}
// too small for the noise samples, or not even one band fits the device: this step on the CPU
let mut host = cx.read_rgb(&img, w, h);
let mut d = (*plan.settings).clone();
d.detail.nr_luminance = 0.0;
local::denoise(&mut host, &d, plan.detail_scale, colour_noise);
return cx.gpu.upload(rgb_words(&host));
}
if let Some(nr) = col {
let chroma = cx.gpu.buffer(n * 3);
map(cx, "chroma_k", n, &[], [Some(&img), None, None], &chroma);
let b = gaussian(cx, &chroma, w, h, 3, nr.sigma);
let out = cx.gpu.buffer(n * 3);
map(cx, "nr_col", n, &[nr.t.to_bits()], [Some(&img), Some(&chroma), Some(&b)], &out);
img = out;
}
img
}
/// Run a `nrcj.wgsl` kernel on `n` items with inputs `a`, `b` and output `out`.
fn nrcj(cx: &mut Cx<'_>, k: &str, n: usize, p: &[u32], a: &Buf, b: Option<&Buf>, out: &Buf) {
cx.run(k, p, &[Some(a), b, Some(out)], groups1(n));
}
/// Device memory left aside when the joint NR plans its bands from the free memory (the rest of the
/// render, the display, other programs).
const VRAM_RESERVE: u64 = 384 << 20;
/// Renders on the device right now ([`render`]). An export batch renders photos side by side: while one
/// plans its noise reduction bands, each of the others may still allocate the buffers of its other stages.
static RENDERS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
/// Held while a noise reduction plans its bands and runs them: two plans made at once from the same free
/// memory would together ask for more than there is.
static BANDS: Mutex<()> = Mutex::new(());
/// Counts a render in [`RENDERS`] while it lives.
struct InFlight;
impl InFlight {
fn new() -> Self {
RENDERS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
InFlight
}
}
impl Drop for InFlight {
fn drop(&mut self) {
RENDERS.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
}
}
/// The band budget from the device memory free right now and our own pooled buffers (which we reuse), less
/// [`VRAM_RESERVE`] for each render on the device (this one's other stages and every other render's).
fn band_budget_from(free: u64, pooled: u64, renders: usize) -> usize {
let reserve = VRAM_RESERVE.saturating_mul(renders.max(1) as u64);
free.saturating_add(pooled).saturating_sub(reserve) as usize
}
/// The joint colour noise reduction (`lightcraft_pipeline::nrc_joint`) on the GPU, the same steps as the
/// CPU: only the noise samples are read back (the noise model is fitted on the CPU, by the same code).
/// Large images are filtered in horizontal bands whose halo covers every step's reach, so each kept pixel
/// gets the whole-image result; the noise model is the whole image's, or `frame_noise` (the whole frame's,
/// for a window). `None` when the image is too small for the noise samples (the caller falls back to the CPU).
#[allow(clippy::too_many_arguments)]
fn color_nr_joint(
cx: &mut Cx<'_>,
img: &Buf,
w: usize,
h: usize,
p: &local::NrcParams,
detail: f32,
amount: f32,
frame_noise: Option<&JointNoise>,
) -> Option<Buf> {
use lightcraft_pipeline::nrc_joint;
let strength = nrc_joint::strength(amount);
let brake = nrc_joint::speck_brake(amount);
if w < 3 || h < 3 {
return None;
}
if strength <= 0.0 {
return Some(cx.copy(img));
}
let n = w * h;
let started = profiling().then(std::time::Instant::now);
let mut noise = match frame_noise {
Some(noise) => *noise,
None => {
// noise samples of the whole image: same pixels as the CPU, fitted by the same code
let stride = nrc_joint::noise_sample_stride(n);
let count = n.div_ceil(stride);
let smp = cx.gpu.buffer(11 * count);
nrcj(cx, "nrcj_samples", count, &[count as u32, w as u32, h as u32, stride as u32], img, None, &smp);
let raw: Vec<f32> = cx.read(&smp, 11 * count);
drop(smp);
if let Some(t) = started {
eprintln!(" gpu joint samples + read: {:.1} ms", t.elapsed().as_secs_f64() * 1e3);
}
let samples: Vec<nrc_joint::NoiseSample> = raw
.as_chunks::<11>()
.0
.iter()
.filter(|s| s[0] == 1.0 && s.iter().all(|v| v.is_finite()))
.map(|s| (s[1], [[s[2], s[3], s[4]], [s[5], s[6], s[7]], [s[8], s[9], s[10]]]))
.collect();
nrc_joint::noise_from_samples(samples)
}
};
noise.tolerance = nrc_joint::tolerance(detail);
if let Some(t) = started {
eprintln!(" gpu joint noise model: {:.1} ms", t.elapsed().as_secs_f64() * 1e3);
}
let sigma = p.passes.last().map_or(0.0, |q| q.sigma);
let plan = nrc_joint::scale_plan(sigma, w, h);
// Rows a kept pixel depends on: a wrong value at a band edge (treated as an image border) moves
// inwards by delta per vertical sweep, two sweeps per scale; at the farthest row it reaches, a
// scale's weight and path still read up to delta rows further (at most the largest delta); the
// guide's blur and pilot reach a few rows, the speck brake two more. Hence 2·Σdelta + max delta + 10.
let halo = plan.iter().map(|d| 2 * d.0).sum::<usize>() + plan.last().map_or(0, |d| d.0) + 10;
// Memory: the whole output (3 values per pixel) is held besides the input; a band then holds about
// 12 values per pixel (guide 3, two chroma buffers 2 + 2, paths 2, weights 2; up to 12 while the
// speck brake and the guide run). The budget for both together is `band_budget_from` the device memory
// free right now (where the driver can be asked, `crate::vram`); else a fixed 2 GB. One band plan at a
// time (`BANDS`), held until the bands are done. PROTOTYPE: `NRV2_GPU_BAND_MB` overrides it.
// (free memory asked before the lock: the first question for a device waits for the driver)
let free = crate::vram::free_bytes(&cx.gpu.info);
let _bands = BANDS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
let fixed = std::env::var("NRV2_GPU_BAND_MB").ok().and_then(|v| v.parse::<usize>().ok()).map(|mb| mb << 20);
let renders = RENDERS.load(std::sync::atomic::Ordering::Relaxed);
let measured = free.map(|free| band_budget_from(free, crate::ctx::memory().pooled, renders));
let budget = fixed.or(measured).unwrap_or(2048 << 20);
if profiling() {
eprintln!(
" gpu joint budget: {} MB ({}, {renders} render(s) on the device)",
budget >> 20,
if fixed.is_some() {
"NRV2_GPU_BAND_MB"
} else if measured.is_some() {
"free device memory"
} else {
"fixed default"
}
);
}
let out_bytes = 12 * n;
let per_row = 48 * w;
// no band may exceed the device's largest buffer either (3 values per pixel are the largest)
let limit_rows = (cx.gpu.limit() as usize / (12 * w)).max(1);
let rows = (budget.saturating_sub(out_bytes) / per_row).min(limit_rows);
let core = if rows >= h {
h
} else if rows > 2 * halo {
rows - 2 * halo
} else {
// not even one band with its halo fits: this stage runs on the CPU (the rest of the render stays here)
return None;
};
let out = cx.gpu.buffer(3 * n);
let mut y0 = 0;
while y0 < h {
let y1 = (y0 + core).min(h);
let (b0, b1) = if core >= h { (0, h) } else { (y0.saturating_sub(halo), (y1 + halo).min(h)) };
let filtered = joint_band(cx, img, w, b0, b1 - b0, &noise, brake, &plan);
let kept = (y1 - y0) * w;
nrcj(cx, "nrcj_recon", kept, &[kept as u32, strength.to_bits(), (y0 * w) as u32, (b0 * w) as u32], img, Some(&filtered), &out);
drop(filtered);
// wait: the driver frees a dropped buffer only once the GPU is done with it, and a band's
// buffers are large (without waiting, freed memory piled up to the whole 4 GB of a Quadro M2200)
cx.sync();
y0 = y1;
}
Some(out)
}
/// The filtered chroma pairs of rows `b0 .. b0 + bh` of the image, treated as an image of their own.
#[allow(clippy::too_many_arguments)]
fn joint_band(
cx: &mut Cx<'_>,
img: &Buf,
w: usize,
b0: usize,
bh: usize,
noise: &lightcraft_pipeline::nrc_joint::JointNoise,
brake: lightcraft_pipeline::nrc_joint::SpeckBrake,
plan: &[(usize, f32)],
) -> Buf {
let n = w * bh;
let mut np = vec![n as u32, w as u32, bh as u32];
np.extend(noise.model.alpha.iter().chain(&noise.model.beta).chain(&noise.ymax).map(|v| v.to_bits()));
np.push(noise.tolerance.to_bits());
let mut yuv = cx.gpu.buffer(3 * n);
nrcj(cx, "nrcj_yuv", n, &[n as u32, (b0 * w) as u32], img, None, &yuv);
if brake.share > 0.0 {
let mut sp = np.clone();
sp.extend([brake.threshold.to_bits(), brake.share.to_bits()]);
let despecked = cx.gpu.buffer(3 * n);
nrcj(cx, "nrcj_speck", n, &sp, &yuv, None, &despecked);
yuv = despecked;
}
let blurred = gaussian(cx, &yuv, w, bh, 3, 0.8);
let guide = cx.gpu.buffer(3 * n);
nrcj(cx, "nrcj_pilot", n, &np, &yuv, Some(&blurred), &guide);
drop(blurred);
let mut filtered = cx.gpu.buffer(2 * n);
nrcj(cx, "nrcj_uv", n, &[n as u32], &yuv, None, &filtered);
drop(yuv);
// dropped buffers only become reusable (and their memory free) once the GPU is done with them
cx.sync();
// paths (exp(−path), as on the CPU) and pair weights; the next scale's paths go to the scratch buffer,
// free after the diffusion steps, as on the CPU
let mut paths = cx.gpu.buffer(2 * n);
nrcj(cx, "nrcj_links1", n, &np, &guide, None, &paths);
let links = cx.gpu.buffer(2 * n);
let mut lp = np.clone();
lp.push(1);
nrcj(cx, "nrcj_links", n, &lp, &guide, Some(&paths), &links);
let mut scratch = cx.gpu.buffer(2 * n);
for (k, &(delta, rate)) in plan.iter().enumerate() {
let dp = [n as u32, w as u32, bh as u32, delta as u32, rate.to_bits()];
for _ in 0..2 {
nrcj(cx, "nrcj_dh", n, &dp, &filtered, Some(&links), &scratch);
std::mem::swap(&mut filtered, &mut scratch);
nrcj(cx, "nrcj_dv", n, &dp, &filtered, Some(&links), &scratch);
std::mem::swap(&mut filtered, &mut scratch);
}
if let Some(&(next, _)) = plan.get(k + 1) {
nrcj(cx, "nrcj_paths", n, &[n as u32, w as u32, bh as u32, delta as u32], &paths, None, &scratch);
std::mem::swap(&mut paths, &mut scratch);
let mut lp = np.clone();
lp.push(next as u32);
nrcj(cx, "nrcj_links", n, &lp, &guide, Some(&paths), &links);
}
}
drop((scratch, paths, links, guide));
cx.sync();
filtered
}
/// The planes the per-pixel stage reads.
struct Prep {
log_l: Arc<Buf>,
base: Arc<Buf>,
clarity: Option<Arc<Buf>>,
texture: Option<Arc<Buf>>,
sharpen: Option<Arc<Buf>>,
dark: Option<Arc<Buf>>,
/// Blurred chromaticity (local Moiré / Noise).
chroma: Option<Arc<Buf>>,
air: f32,
}
/// The spatial planes (`local::prepare`), reusing those in `planes`.
fn prepare(cx: &mut Cx<'_>, lin: &Buf, plan: &Plan<'_>, req: &RenderRequest, planes: &mut Planes) -> Prep {
let (w, h) = (plan.w, plan.h);
let n = w * h;
let sig = local::plane_sigmas(&plan.settings, plan.px_per_long, plan.native_long as f64, req.quality);
let log_l = match &planes.log_l {
Some(b) => b.clone(),
None => {
let l = cx.gpu.buffer(n);
map(cx, "log_lum_k", n, &[], [Some(lin), None, None], &l);
let b = Arc::new(l);
planes.log_l = Some(b.clone());
b
}
};
let base = match sig.base {
Some(sg) => {
let key = local::base_key(&plan.settings, sg, sig.base_detail);
plane_at(&mut planes.base, key, || tone_base(cx, &log_l, w, h, sg, sig.base_detail, local::base_keeps_gaps(&plan.settings)))
}
None => log_l.clone(),
};
let clarity = sig.clarity.map(|sg| plane_at(&mut planes.clarity, sg.to_bits(), || guided_fast(cx, &log_l, w, h, sg, local::CLARITY_EPS)));
let texture = sig.texture.map(|sg| plane_at(&mut planes.texture, sg.to_bits(), || gaussian(cx, &log_l, w, h, 1, sg)));
let sharpen = sig.sharpen.map(|sg| plane_at(&mut planes.sharpen, local::sharpen_key(&plan.settings, sg), || gaussian_fir(cx, &log_l, w, h, sg)));
let (dark, air) = match sig.dark {
None => (None, 1.0),
Some(sg) => match &planes.dark {
Some((k, d, air)) if *k == sg.to_bits() => (Some(d.clone()), *air),
_ => {
let d0 = cx.gpu.buffer(n);
map(cx, "dark_k", n, &[], [Some(lin), None, None], &d0);
let d = gaussian(cx, &d0, w, h, 1, sg);
// airlight: a percentile of every 7th value, as on the CPU
let m = n.div_ceil(local::AIRLIGHT_STEP);
let sub = cx.gpu.buffer(m);
map(cx, "subsample", m, &[local::AIRLIGHT_STEP as u32], [Some(&d), None, None], &sub);
let air = local::airlight_of(cx.read(&sub, m));
cx.flush();
let b = Arc::new(d);
planes.dark = Some((sg.to_bits(), b.clone(), air));
(Some(b), air)
}
},
};
let chroma = sig.chroma.map(|sg| {
plane_at(&mut planes.chroma, sg.to_bits(), || {
let ch = cx.gpu.buffer(n * 3);
map(cx, "chroma_k", n, &[], [Some(lin), None, None], &ch);
gaussian(cx, &ch, w, h, 3, sg)
})
});
Prep { log_l, base, clarity, texture, sharpen, dark, chroma, air }
}
/// Most brush dabs the mask kernel evaluates per pixel (more: the CPU rasterizes the brush).
const MAX_DABS: usize = 4096;
/// The `shape` kernel's brush data for `dabs`: stroke records (12 words: dab offset, dab count,
/// r, hard, flow, density, erase, bbox x0 y0 x1 y1, auto) then the dab centres.
fn brush_aux(dabs: &[&lightcraft_pipeline::masks::BrushDabs], erase: bool) -> Vec<f32> {
let mut aux = Vec::new();
let mut off = 12 * dabs.len();
for d in dabs {
let (mut x0, mut y0, mut x1, mut y1) = (f64::MAX, f64::MAX, f64::MIN, f64::MIN);
for q in &d.dabs {
(x0, y0, x1, y1) = (x0.min(q.x - d.r), y0.min(q.y - d.r), x1.max(q.x + d.r), y1.max(q.y + d.r));
}
aux.extend([off as f32, d.dabs.len() as f32, d.r as f32, d.hard as f32, d.flow, d.density, (erase && d.erase) as u8 as f32]);
aux.extend([x0, y0, x1, y1].map(|v| v as f32));
aux.push(d.auto as u8 as f32);
off += 2 * d.dabs.len();
}
for d in dabs {
aux.extend(d.dabs.iter().flat_map(|q| [q.x as f32, q.y as f32]));
}
aux
}
/// How [`guided_cross`] combines the filtered plane with its input.
#[derive(Clone, Copy)]
enum Combine {
/// The larger of the two (Auto Mask strokes).
Max,
/// The input moved towards the filtered plane by this much (Refine Edges).
Mix(f32),
}
/// Cross guided filter (`masks::guided_cross`) of `p` steered by `guide`, combined with `p`.
#[allow(clippy::too_many_arguments)]
fn guided_cross(cx: &mut Cx<'_>, guide: &Buf, p: &Buf, w: usize, h: usize, sigma: f32, eps: f32, how: Combine) -> Buf {
let n = w * h;
let (m0, m1) = (cx.gpu.buffer(2 * n), cx.gpu.buffer(2 * n));
map(cx, "xguided_pre", n, &[0], [Some(guide), Some(p), None], &m0);
map(cx, "xguided_pre", n, &[1], [Some(guide), Some(p), None], &m1);
let (b0, b1) = (gaussian(cx, &m0, w, h, 2, sigma), gaussian(cx, &m1, w, h, 2, sigma));
let ab = cx.gpu.buffer(2 * n);
map(cx, "xguided_ab", n, &[eps.to_bits()], [Some(&b0), Some(&b1), None], &ab);
let ab = gaussian(cx, &ab, w, h, 2, sigma);
let q = cx.gpu.buffer(n);
let mode = match how {
Combine::Max => [0, 0],
Combine::Mix(k) => [1, k.to_bits()],
};
map(cx, "xguided_apply", n, &mode, [Some(guide), Some(&ab), Some(p)], &q);
q
}
/// Evaluate the masks (`lightcraft_pipeline::masks::evaluate`): their alpha planes (concatenated,
/// followed by the blurred chromaticity when local Moiré / Noise need it) and their adjustment
/// terms. Shapes without a kernel (Sky, Subject, …) run on the CPU.
fn masks(cx: &mut Cx<'_>, lin: &Buf, prep: &Prep, plan: &Plan<'_>, host: &mut Host) -> (Option<Buf>, Vec<[f32; MASK_TERMS]>) {
use lightcraft_develop::{MaskOp, MaskShape};
let s = &*plan.settings;
let list: Vec<_> = s.masks.iter().filter(|m| m.visible && !m.components.is_empty()).collect();
if list.is_empty() {
return (None, Vec::new());
}
let (w, h) = (plan.w, plan.h);
let n = w * h;
let frame = &plan.frame;
let ev = s.light.exposure as f32;
let long = frame.ow.max(frame.oh);
let [a, b, c, d, e, f] = frame.out_to_norm(w, h).0;
let (kx, ky) = (frame.ow / long, frame.oh / long);
let aff = [a * kx, b * ky, c * kx, d * ky, e * kx, f * ky].map(|v| (v as f32).to_bits());
let alpha = cx.zeroed(n * list.len() + if prep.chroma.is_some() { 3 * n } else { 0 });
if let Some(ch) = &prep.chroma {
cx.copy_into(ch, &alpha, n * list.len());
}
let comp_plane = cx.gpu.buffer(n);
for (mi, m) in list.iter().enumerate() {
// its coordinates on the photo before the warp (`masks::evaluate_one`): through a lens or
// perspective warp the kernels' affine positions don't hold, so its shapes run on the CPU
let mframe = plan.frame.for_mask(m.on_photo);
let frame = &*mframe;
let warped = frame.mask_warp().is_some();
let mut first = true;
for comp in &m.components {
let op = match comp.op {
_ if first => 0u32,
MaskOp::Add => 1,
MaskOp::Subtract => 2,
MaskOp::Intersect => 3,
};
first = false;
let mut p = vec![w as u32, h as u32, 0, comp.invert as u32];
p.extend_from_slice(&aff);
let f32s = |v: &[f64]| v.iter().map(|x| (*x as f32).to_bits()).collect::<Vec<u32>>();
let mut aux: Vec<f32> = Vec::new();
let mut own: Option<Buf> = None;
let kind = match &comp.shape {
MaskShape::Linear { .. } | MaskShape::Radial { .. } if warped => None,
MaskShape::Linear { start, end } => {
let (a, b) = (frame.norm_to_long(*start), frame.norm_to_long(*end));
let d = b - a;
p.extend(f32s(&[a.x, a.y, d.x, d.y, d.dot(d).max(1e-12)]));
Some(0)
}
MaskShape::Radial { center, rx, ry, angle, feather, invert } => {
let c = frame.norm_to_long(*center);
let (sn, co) = (-angle.to_radians()).sin_cos();
p.extend(f32s(&[c.x, c.y, co, sn, rx.max(1e-6), ry.max(1e-6), (*feather / 100.0).clamp(0.0, 1.0)]));
p.push(*invert as u32);
Some(1)
}
MaskShape::LuminanceRange { lo, hi, lo_feather, hi_feather } => {
p.extend(f32s(&[*lo, *hi, lo_feather.max(1e-3), hi_feather.max(1e-3)]));
p.push(ev.to_bits());
Some(2)
}
MaskShape::ColorRange { samples, refine } => {
let tol = lightcraft_pipeline::masks::color_range_tolerance(*refine);
// at most the CPU's samples (`MAX_COLOR_RANGE_SAMPLES`): cost is pixels x samples
let samples = samples.iter().take(lightcraft_develop::MAX_COLOR_RANGE_SAMPLES);
p.extend([tol.to_bits(), ev.exp2().to_bits(), samples.len() as u32]);
aux.extend(samples.flat_map(|s| s.map(|v| v as f32)));
Some(3)
}
MaskShape::Brush { strokes } => {
// The kernel takes at most MAX_DABS across the whole mask, so no more than
// that (plus one, to tell "at it" from "past it") is ever built: each stroke
// may otherwise construct two million dabs before the sum below rejects them
// all (#1520).
let mut room = MAX_DABS + 1;
let dabs: Vec<_> = strokes
.iter()
.map(|st| {
let d = lightcraft_pipeline::masks::brush_dabs_within(st, frame, w, h, room);
room -= d.dabs.len().min(room);
d
})
.collect();
// pen pressure varies radius and flow per dab, which the kernel doesn't take: the CPU rasterizes it
if dabs.iter().map(|d| d.dabs.len()).sum::<usize>() > MAX_DABS || dabs.iter().any(|d| !d.pressure.is_empty()) {
None
} else if dabs.iter().any(|d| d.auto) {
// Auto Mask: stroke by stroke (each one refined before it's combined)
let brush = cx.zeroed(n);
let stroke = cx.gpu.buffer(n);
for d in &dabs {
let mut q = p.clone();
q[2] = 4;
q[3] = 0;
q.push(1);
let aux = cx.gpu.upload(&brush_aux(&[d], false));
cx.run("shape", &q, &[Some(lin), Some(&prep.log_l), Some(&aux), Some(&stroke), Some(&brush)], groups2(w, h, [16, 16]));
let refined = d.auto.then(|| {
let (sigma, eps) = lightcraft_pipeline::masks::auto_refine(d.r);
guided_cross(cx, &prep.log_l, &stroke, w, h, sigma, eps, Combine::Max)
});
let op = if d.erase { 2 } else { 1 };
cx.run(
"combine",
&[n as u32, 0, 0, op],
&[None, None, None, Some(refined.as_ref().unwrap_or(&stroke)), Some(&brush)],
groups1(n),
);
}
if comp.invert {
cx.run("finalize", &[n as u32, 0, 0, 1, 1f32.to_bits()], &[None, None, None, Some(&stroke), Some(&brush)], groups1(n));
}
own = Some(brush);
None
} else {
p.push(dabs.len() as u32);
aux = brush_aux(&dabs.iter().collect::<Vec<_>>(), true);
Some(4)
}
}
_ => None,
};
let plane = match kind {
_ if own.is_some() => own,
Some(k) => {
p[2] = k;
let aux = (!aux.is_empty()).then(|| cx.gpu.upload(&aux));
cx.run("shape", &p, &[Some(lin), Some(&prep.log_l), aux.as_ref(), Some(&comp_plane), Some(&alpha)], groups2(w, h, [16, 16]));
None
}
None => {
// no kernel: evaluate on the CPU
let img = host.lin.get_or_insert_with(|| Arc::new(cx.read_rgb(lin, w, h))).clone();
let l = host.log_l.get_or_insert_with(|| Arc::new(cx.read_plane(&prep.log_l, w, h))).clone();
let mut v = lightcraft_pipeline::masks::shape_alpha(&comp.shape, frame, w, h, &img, &l, ev, plan.mattes.as_deref());
if comp.invert {
v.data.iter_mut().for_each(|x| *x = 1.0 - *x);
}
Some(cx.gpu.upload(&v.data))
}
};
let cbuf = plane.as_ref().unwrap_or(&comp_plane);
let k = [n as u32, 0, mi as u32, op];
cx.run("combine", &k, &[None, None, None, Some(cbuf), Some(&alpha)], groups1(n));
}
if m.refine > 0.0 {
// Refine Edges (`masks::evaluate_one`): the combined alpha follows the photo's edges,
// before the mask's invert and amount
let k = (m.refine / 100.0).clamp(0.0, 1.0) as f32;
let sigma = (0.04 * frame.output_long(w, h) as f32 * k).max(1.0);
let plane = cx.slice(&alpha, mi * n, n);
let refined = guided_cross(cx, &prep.log_l, &plane, w, h, sigma, 0.02, Combine::Mix(k.sqrt()));
cx.copy_into(&refined, &alpha, mi * n);
}
let amt = (m.adjust.amount / 100.0) as f32;
cx.run(
"finalize",
&[n as u32, 0, mi as u32, m.invert as u32, amt.to_bits()],
&[None, None, None, Some(&comp_plane), Some(&alpha)],
groups1(n),
);
}
let terms = list.iter().map(|m| mask_terms(&m.adjust)).collect();
(Some(alpha), terms)
}
#[cfg(test)]
mod tests {
use super::*;
/// Two exports side by side: each band plan leaves room for the other render, so the two together
/// stay within the free memory they both measured.
#[test]
fn band_budget_leaves_room_for_every_render_on_the_device() {
let (free, pooled) = (3000u64 << 20, 200u64 << 20);
let alone = band_budget_from(free, pooled, 1);
assert_eq!(alone, (3200usize << 20) - VRAM_RESERVE as usize);
assert_eq!(band_budget_from(free, pooled, 0), alone, "the render itself is always counted");
let two = band_budget_from(free, pooled, 2);
assert_eq!(two, alone - VRAM_RESERVE as usize);
assert_eq!(band_budget_from(100 << 20, 0, 3), 0, "nothing left: no band fits, the CPU runs the step");
let before = RENDERS.load(std::sync::atomic::Ordering::Relaxed);
{
let _a = InFlight::new();
let _b = InFlight::new();
assert!(RENDERS.load(std::sync::atomic::Ordering::Relaxed) >= before + 2);
}
assert!(RENDERS.load(std::sync::atomic::Ordering::Relaxed) < before + 2, "a finished render no longer counts");
}
#[test]
fn coverage_mask_matches_per_pixel_decision() {
use lightcraft_develop::{EmbeddedLens, EmbeddedWarp};
let lens = EmbeddedLens {
warp: Some(EmbeddedWarp {
planes: [[1.0, -0.03, 0.01, 0.0, 0.001, -0.002]; 3],
center: lightcraft_geom::Point::new(0.52, 0.48),
radius: 0.6,
}),
vignette: None,
stages: Default::default(),
source: Default::default(),
};
let mut s = DevelopSettings::default();
s.optics.lens_profile = true;
s.optics.distortion = -40.0;
s.geometry.horizontal = -15.0;
s.geometry.rotate = 3.0;
for orientation in [Orientation::Normal, Orientation::Rotate270] {
s.orientation = orientation;
let f = lightcraft_pipeline::geometry::Frame::with_lens(900, 600, &s, true, Some(&lens));
let wp = f.warp.as_ref().expect("warp");
// widths that do and don't fill the last word; heights that do and don't fill the last band
for (w, h) in [f.fit(700, 700), (333, 221), (64, 16), (1, 1)] {
let o2t = f.out_to_oriented(w, h);
let mask = coverage_mask(wp, &o2t, w, h);
let words = w.div_ceil(32);
assert_eq!(mask.len(), words * h);
for y in 0..h {
for x in 0..w {
let bit = mask[y * words + x / 32] >> (x % 32) & 1 == 1;
assert_eq!(bit, wp.covers(&o2t, x, y), "{orientation:?} {w}x{h}: pixel {x},{y}");
}
let pad = mask[y * words + words - 1] >> (w % 32);
assert!(w.is_multiple_of(32) || pad == 0, "padding bits set");
}
}
}
}
/// The perspective horizon in view: its denominator is exactly 0 on output row 75 (clamped to 1e-300) and
/// changes sign there, so the blocks around it are undecided (clamp, and division by a range that may hold
/// zero) and are decided per pixel, while blocks clear of it are filled at once.
#[test]
fn coverage_mask_with_the_horizon_in_view() {
use lightcraft_geom::{Affine, Homography};
let (w, h) = (900, 600);
let mut wp = lightcraft_pipeline::optics::Warp::identity(w as f64, h as f64);
// centred coordinates: v = (y − 300) / 450, denominator 2·v + m8, exactly 0 at y = 75.5
let m8 = -((75.5 - 300.0) / 450.0 * 2.0);
wp.persp_inv = Homography([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 2.0, m8]);
wp.persp = wp.persp_inv.inverse().expect("invertible");
let o2t = Affine::IDENTITY;
assert_eq!(wp.block_coverage(&o2t, 0, 32, 64, 80), None, "the block holding the horizon");
assert_eq!(wp.block_coverage(&o2t, 0, 32, 0, 16), Some(false), "beyond the horizon");
assert_eq!(wp.block_coverage(&o2t, 448, 480, 288, 304), Some(true), "the centre");
let mask = coverage_mask(&wp, &o2t, w, h);
let words = w.div_ceil(32);
let mut covered = 0;
for y in 0..h {
for x in 0..w {
let bit = mask[y * words + x / 32] >> (x % 32) & 1 == 1;
assert_eq!(bit, wp.covers(&o2t, x, y), "pixel {x},{y}");
covered += bit as usize;
}
}
assert!(covered > 0 && covered < w * h, "{covered} px covered");
}
/// The blur against `lightcraft_raster::blur::gaussian`: widths that do and don't fill the last
/// workgroup, windows wider than the image, and 1, 2 (as two planes) and 3 channels.
#[test]
fn gaussian_matches_cpu() {
// (creating the device writes the crash sentinel another test swaps out)
let _serial = crate::DEVICE_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner());
use lightcraft_raster::blur;
let Some(gpu) = crate::device() else { return };
let _scope = crate::ctx::RenderScope::new(gpu);
let v = |x: usize, y: usize, c: usize| ((x * 7 + y * 13 + c * 5) % 17) as f32 / 16.0 + if x % 61 < 3 { 2.0 } else { 0.0 };
for (w, h) in [(1000, 5), (257, 3), (5, 4), (1, 9)] {
for sigma in [0.8f32, 2.0, 4.5, 30.0, 250.0, 1000.0] {
let planes: Vec<Plane> = (0..3).map(|c| Plane::from_fn(w, h, |x, y| v(x, y, c))).collect();
let want: Vec<Plane> = planes.iter().map(|p| blur::gaussian(p, sigma)).collect();
for nc in [1, 2, 3] {
let words: Vec<f32> = (0..w * h).flat_map(|i| (0..nc).map(move |c| (i, c))).map(|(i, c)| planes[c].data[i]).collect();
let mut cx = Cx::new(gpu);
let src = gpu.upload(&words);
let out = gaussian(&mut cx, &src, w, h, nc, sigma);
let got: Vec<f32> = cx.read(&out, w * h * nc);
for (i, g) in got.iter().enumerate() {
let want = want[i % nc].data[i / nc];
assert!((g - want).abs() < 1e-5 * (1.0 + sigma / 10.0), "{w}×{h} nc={nc} sigma={sigma}: word {i} {g} vs {want}");
}
}
}
}
}
/// Kernel timings on a 24 MP plane: `cargo test --release -p lightcraft-gpu -- --ignored --nocapture`.
#[test]
#[ignore]
fn bench_kernels() {
// (creating the device writes the crash sentinel another test swaps out)
let _serial = crate::DEVICE_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let Some(gpu) = crate::device() else { return };
let _scope = crate::ctx::RenderScope::new(gpu);
let (w, h) = (6000, 4000);
eprintln!("{} ({:?})", gpu.info.name, gpu.info.backend);
let src = gpu.upload(&vec![0.5f32; w * h * 3]);
let time = |name: &str, f: &mut dyn FnMut(&mut Cx<'_>)| {
let mut best = f64::MAX;
for _ in 0..5 {
let mut cx = Cx::new(gpu);
cx.sync();
let t = std::time::Instant::now();
f(&mut cx);
cx.sync();
best = best.min(t.elapsed().as_secs_f64() * 1e3);
}
eprintln!("{name:<40} {best:.1} ms");
};
for nc in [1, 2, 3] {
let out = gpu.buffer(w * h * nc);
// noise reduction's radii, Texture's and Dehaze's at 6000 px
for r in [2, 11, 120] {
for (k, horizontal) in [("box_h", true), ("box_v", false)] {
time(&format!("{k} nc={nc} r={r}"), &mut |cx| box_pass(cx, horizontal, &src, &out, (w, h), nc, r));
}
}
}
for nc in [1, 2, 3] {
time(&format!("gaussian nc={nc} sigma=2"), &mut |cx| drop(gaussian(cx, &src, w, h, nc, 2.0)));
}
time("gaussian nc=3 sigma=4.5 (colour NR)", &mut |cx| drop(gaussian(cx, &src, w, h, 3, 4.5)));
time("map log_lum", &mut |cx| {
let o = cx.gpu.buffer(w * h);
map(cx, "log_lum_k", w * h, &[], [Some(&src), None, None], &o);
});
time("guided sigma=1.75", &mut |cx| drop(guided(cx, &src, w, h, 1.75, 0.01)));
}
/// The tiled median kernels select exactly what the CPU's sorted window does: edge tiles,
/// planes smaller than a tile or a window, every radius and many ties.
#[test]
fn median_matches_cpu_bit_for_bit() {
// (creating the device writes the crash sentinel another test swaps out)
let _serial = crate::DEVICE_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let Some(gpu) = crate::device() else { return lightcraft_testkit::skip!(Gpu, "no GPU adapter") };
let _scope = crate::ctx::RenderScope::new(gpu);
let mut seed = 0x9e37_79b9_u32;
let mut next = move || {
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
seed
};
for (w, h) in [(1, 1), (2, 3), (5, 70), (70, 5), (63, 65), (130, 67), (200, 129)] {
// a coarse level set gives ties; a smooth ramp and fine noise give distinct values
let mut p = Plane::new(w, h);
for (i, o) in p.data.iter_mut().enumerate() {
let (x, y) = (i % w, i / w);
*o = match (next() % 3, next()) {
(0, v) => (v % 7) as f32 * 0.5 - 2.0,
(1, v) => (x + 2 * y) as f32 * 0.01 + (v % 100) as f32 * 1e-4,
(_, v) => (v % 10_000) as f32 * 1e-3 - 5.0,
};
}
let src = gpu.upload(&p.data);
for r in 1..=local::BASE_DETAIL_MAX {
let mut cx = Cx::new(gpu);
let out = median_hv(&mut cx, &src, w, h, r);
let got: Vec<f32> = cx.read(&out, w * h);
let want = local::median_hv(&p, r);
let bad = got.iter().zip(&want.data).position(|(g, c)| g.to_bits() != c.to_bits());
assert!(bad.is_none(), "{w}×{h} r={r}: pixel {bad:?}");
}
}
}
/// Highlights/shadows base timings on the #3460 frame size:
/// `cargo test --release -p lightcraft-gpu bench_tone_base -- --ignored --nocapture`.
#[test]
#[ignore]
fn bench_tone_base() {
let Some(gpu) = crate::device() else { return eprintln!("no gpu: {:?}", crate::unavailable_reason()) };
let _scope = crate::ctx::RenderScope::new(gpu);
let (w, h) = (6960, 4640);
let r = local::base_detail_radius(w as f64);
eprintln!("{} ({:?}) r={r}", gpu.info.name, gpu.info.backend);
let l: Vec<f32> = (0..w * h).map(|i: usize| ((i.wrapping_mul(2_654_435_761)) % 1000) as f32 / 250.0 - 6.0).collect();
let src = gpu.upload(&l);
let time = |name: &str, f: &mut dyn FnMut(&mut Cx<'_>)| {
let mut ts = Vec::new();
for _ in 0..9 {
let mut cx = Cx::new(gpu);
cx.sync();
let t = std::time::Instant::now();
f(&mut cx);
cx.sync();
ts.push(t.elapsed().as_secs_f64() * 1e3);
}
ts.sort_by(f64::total_cmp);
eprintln!("{name:<40} min {:.1} ms median {:.1} ms", ts[0], ts[4]);
};
let sigma = 0.015 * w as f32;
time("median_hv", &mut |cx| drop(median_hv(cx, &src, w, h, r)));
time("detail_free", &mut |cx| drop(detail_free(cx, &src, w, h, r)));
time("guided_fast", &mut |cx| drop(guided_fast(cx, &src, w, h, sigma, local::BASE_EPS)));
time("tone_base", &mut |cx| drop(tone_base(cx, &src, w, h, sigma, r, true)));
}
/// The device scores Remove's candidate sources like the CPU and picks the same one.
#[test]
fn auto_source_picks_the_cpu_candidate() {
// (creating the device writes the crash sentinel another test swaps out)
let _serial = crate::DEVICE_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let Some(gpu) = crate::device() else {
let why = crate::unavailable_reason().unwrap_or_default();
assert!(!why.contains("kernels failed to build"), "{why}");
return;
};
let _scope = crate::ctx::RenderScope::new(gpu);
let img = lightcraft_scenes::demo_library()[3].render(640, 427);
let buf = gpu.upload(rgb_words(&img));
let mut differ = Vec::new();
for (tx, ty, r) in [(25.6, 213.0, 19.2), (320.0, 192.0, 19.2), (100.0, 50.0, 12.0), (600.0, 400.0, 25.0), (320.0, 213.0, 6.0)] {
let mut cx = Cx::new(gpu);
let g = auto_source(&mut cx, &buf, img.width, img.height, (tx, ty), r);
let c = lightcraft_pipeline::spots::auto_source(&img, (tx, ty), r);
if g != c {
differ.push(((tx, ty, r), g, c));
}
}
assert!(differ.is_empty(), "(target, gpu, cpu): {differ:?}");
}
#[test]
fn orient_map_matches_image_oriented() {
use Orientation::*;
let img = Rgb32f::from_fn(5, 3, |x, y| [x as f32, y as f32, 0.0]);
for o in [Normal, Rotate90, Rotate180, Rotate270, FlipH, Transverse, FlipV, Transpose] {
let r = img.oriented(o);
let m = orient_map(o, 5, 3);
for y in 0..r.height {
for x in 0..r.width {
let (x, y) = (x as i32, y as i32);
let (sx, sy) = (m[0] * x + m[1] * y + m[2], m[3] * x + m[4] * y + m[5]);
assert_eq!(r.get(x as usize, y as usize), img.get(sx as usize, sy as usize), "{o:?} at {x},{y}");
}
}
}
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.