5a24289898fd
hdr.rs
//! HDR merge of exposure brackets.
//!
//! 1. **Order and reference.** Frames are ordered by brightness (EXIF exposure, or median
//! luminance without EXIF; ties by content); the middle one is the reference (geometry, colour
//! and the output's default brightness come from it). Everything below runs in that order, so
//! the result is the same whatever order the frames come in.
//! 2. **Auto align** ([`crate::align`]): each frame is registered to the reference and resampled
//! into its geometry.
//! 3. **Exposure normalisation.** The exposure ratio between brightness-adjacent frames is
//! measured robustly — the median of log ratios over pixels well exposed in both — and chained
//! to the reference; EXIF is the fallback when too few pixels overlap. Radiance
//! `Rᵢ = vᵢ / kᵢ` is expressed in reference units.
//! 4. **Weights.** `wᵢ = kᵢ · (1 − smoothstep(0.80·clip, 0.96·clip, max channel within 1 px))`:
//! longer exposures get more weight (shot-noise optimal for a linear sensor), saturated pixels
//! none (all channels together, so a clipped channel never shifts colour), nor their neighbours,
//! which demosaicing interpolated from clipped photosites. The darkest frame keeps a tiny weight
//! so fully saturated pixels still get a value. Where the reference is at or near its clip, the scene is
//! at least that bright: a frame reading far darker there saw something that moved (a star, the moon, a light) and is
//! left out, so a clipped highlight of the reference is never replaced by content from another moment.
//! 5. **Deghost** (None/Low/Medium/High): on a ≤ 2048 px version, a per-pixel consistency test
//! against the reference: the reference radiance (or, where the reference is clipped or black,
//! the nearest well-exposed frame's) predicts what each frame should have recorded; frames that
//! disagree by more than a threshold (in log units, with a noise floor) are masked there. Masks
//! are dilated and feathered (Gaussian), upsampled, and remove the frame's weight. Their union
//! is the deghost overlay.
use lightcraft_raster::blur::gaussian;
use lightcraft_raster::{Plane, Rgb32f};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use crate::align::{self, Alignment};
use crate::frame::{Frame, FrameColor, consensus_multipliers};
use crate::{MergeError, Progress, Result, check};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Deghost {
#[default]
None,
Low,
Medium,
High,
}
impl Deghost {
/// (log-ratio threshold, dilation/feather radius in deghost-map pixels).
fn params(self) -> Option<(f32, usize)> {
match self {
Deghost::None => None,
Deghost::Low => Some((1.0, 2)),
Deghost::Medium => Some((0.6, 3)),
Deghost::High => Some((0.35, 5)),
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub struct HdrOptions {
pub align: bool,
pub deghost: Deghost,
}
impl Default for HdrOptions {
fn default() -> Self {
HdrOptions { align: true, deghost: Deghost::None }
}
}
/// The merged result, in the reference frame's geometry and units.
#[derive(Clone, Debug)]
pub struct HdrResult {
/// Scene-linear radiance; 1.0 = the reference frame's white level.
pub radiance: Rgb32f,
/// Deghost overlay (0..1, the union of the masks) at the radiance's size; empty without deghosting.
pub ghost: Plane,
/// Index (into the inputs) of the reference frame.
pub reference: usize,
/// Exposure of each input relative to the reference, in EV (as used for the merge).
pub ev: Vec<f64>,
/// The same from EXIF, when every frame had it.
pub ev_exif: Option<Vec<f64>>,
pub alignments: Vec<Alignment>,
pub color: FrameColor,
pub raw: bool,
pub orientation: lightcraft_geom::Orientation,
pub metadata: lightcraft_meta::Metadata,
pub baseline_exposure: f64,
/// The reference frame's lens corrections, for `radiance` ([`Frame::lens`]).
pub lens: Vec<lightcraft_raw::Opcode>,
/// The inputs' as-shot white-balance multipliers they agree on ([`crate::frame::consensus_multipliers`]), already in
/// `color`; kept so merged brackets stitched into a panorama can vote as frames of their own.
pub wb_multipliers: Option<[f32; 3]>,
}
/// Long edge (at most) of the frames the deghost test compares: small moving things (a star at 200 mm) still span a few
/// pixels there.
const DEGHOST_EDGE: usize = 2048;
#[inline]
fn smoothstep(a: f32, b: f32, x: f32) -> f32 {
let t = ((x - a) / (b - a)).clamp(0.0, 1.0);
t * t * (3.0 - 2.0 * t)
}
#[inline]
fn maxc(p: [f32; 3]) -> f32 {
p[0].max(p[1]).max(p[2])
}
/// Merge exposure brackets. Frames must share their pixel size (and orientation).
pub fn merge_hdr(mut frames: Vec<Frame>, opts: &HdrOptions, progress: &Progress) -> Result<HdrResult> {
let n = frames.len();
if n < 2 {
return Err(MergeError::TooFew(n, 2));
}
let (w, h) = (frames[0].width(), frames[0].height());
if frames.iter().any(|f| f.width() != w || f.height() != h) {
return Err(MergeError::Mismatch("HDR merge needs photos of the same size".into()));
}
// every buffer must hold exactly its frame's pixels before anything is sized from the dimensions
if frames.iter().any(|f| w.checked_mul(h) != Some(f.image.data.len())) {
return Err(MergeError::Mismatch("an HDR frame's pixels do not match its size".into()));
}
if frames.iter().any(|f| f.orientation != frames[0].orientation) {
for f in frames.iter_mut() {
f.orient();
}
let (w0, h0) = (frames[0].width(), frames[0].height());
if frames.iter().any(|f| f.width() != w0 || f.height() != h0) {
return Err(MergeError::Mismatch("HDR merge needs photos of the same size".into()));
}
}
let (w, h) = (frames[0].width(), frames[0].height());
check(progress, 0.02, "Analysing exposures")?;
// 1. brightness order and reference
let exif: Option<Vec<f64>> = frames.iter().map(|f| f.exposure).collect();
let medians: Vec<f32> = frames.par_iter().map(|f| align::median_luma(&f.image, f.clip)).collect();
let brightness: Vec<f64> = match &exif {
Some(e) if e.iter().any(|v| (v - e[0]).abs() > 0.1) => e.clone(),
_ => medians.iter().map(|m| (*m as f64).log2()).collect(),
};
// ties (equal brightness) by content, never by position
let fingerprint: Vec<f64> =
frames.par_iter().map(|f| f.image.data.iter().step_by(997).map(|p| p[0] as f64 + 2.0 * p[1] as f64 + 3.0 * p[2] as f64).sum()).collect();
let mut input: Vec<usize> = (0..n).collect();
input.sort_by(|&a, &b| {
brightness[a].total_cmp(&brightness[b]).then(medians[a].total_cmp(&medians[b])).then(fingerprint[a].total_cmp(&fingerprint[b]))
});
// From here on frames are in brightness order (`input[p]` is the caller's index of position `p`): alignment seeds,
// sums and deghost masks no longer depend on the order the caller passed. Per-frame results go back at the end.
let mut slots: Vec<Option<Frame>> = frames.into_iter().map(Some).collect();
let mut frames: Vec<Frame> = input.iter().filter_map(|&i| slots.get_mut(i).and_then(Option::take)).collect();
if frames.len() != n {
return Err(MergeError::Mismatch("HDR merge lost a frame while ordering them".into()));
}
let exif: Option<Vec<f64>> = exif.map(|e| input.iter().map(|&i| e[i]).collect());
let medians: Vec<f32> = input.iter().map(|&i| medians[i]).collect();
let brightness: Vec<f64> = input.iter().map(|&i| brightness[i]).collect();
let order: Vec<usize> = (0..n).collect();
let reference = n / 2;
// 2. alignment
let mut alignments = vec![Alignment::identity(); n];
let mut valid: Vec<Option<Vec<bool>>> = vec![None; n];
if opts.align {
check(progress, 0.05, "Aligning")?;
let g0 = 0.18 / medians[reference].max(1e-6);
let rel: Vec<f32> = (0..n).map(|i| 2f64.powf(brightness[reference] - brightness[i]) as f32).collect();
let rel: Vec<f32> = if exif.is_some() && brightness.iter().any(|b| (b - brightness[0]).abs() > 0.1) {
rel
} else {
(0..n).map(|i| medians[reference] / medians[i].max(1e-6)).collect()
};
let feats: Vec<(crate::features::Features, f64)> = frames
.par_iter()
.enumerate()
.map(|(i, f)| {
let (p, s) = align::work_image(&f.image, g0 * rel[i], align::WORK_EDGE);
(crate::features::detect(&p, 1500), s)
})
.collect();
check(progress, 0.25, "Aligning")?;
let al: Vec<(usize, Alignment)> = (0..n)
.into_par_iter()
.filter(|&i| i != reference)
.map(|i| (i, align::align_pair(&feats[reference].0, &feats[i].0, feats[i].1, w, h, i as u64 + 1)))
.collect();
for (i, a) in al {
alignments[i] = a;
}
check(progress, 0.35, "Resampling")?;
// one frame at a time (the warp is parallel inside): each warped copy replaces its frame before the next is made
for i in (0..n).filter(|&i| i != reference && alignments[i].model != "identity") {
let (img, v) = align::warp(&frames[i].image, &alignments[i].h);
frames[i].image = img;
valid[i] = Some(v);
}
}
check(progress, 0.45, "Normalising exposures")?;
// 3. exposure ratios between brightness neighbours, chained to the reference (ln units)
let mut step_ln = vec![0f64; n.saturating_sub(1)];
for p in 0..n - 1 {
let (a, b) = (order[p], order[p + 1]);
let est = log_ratio(&frames[a], valid[a].as_deref(), &frames[b], valid[b].as_deref());
step_ln[p] = match (est, &exif) {
(Some(r), _) => r,
(None, Some(e)) => (e[b] - e[a]) * std::f64::consts::LN_2,
(None, None) => ((medians[b] / medians[a].max(1e-6)) as f64).max(1e-6).ln(),
};
}
let mut ln_k = vec![0f64; n];
let rpos = n / 2;
for p in (0..rpos).rev() {
ln_k[order[p]] = ln_k[order[p + 1]] - step_ln[p];
}
for p in rpos + 1..n {
ln_k[order[p]] = ln_k[order[p - 1]] + step_ln[p - 1];
}
let k: Vec<f32> = ln_k.iter().map(|v| v.exp() as f32).collect();
let ev: Vec<f64> = ln_k.iter().map(|v| v / std::f64::consts::LN_2).collect();
let ev_exif = exif.as_ref().map(|e| e.iter().map(|v| v - e[reference]).collect());
// 4./5. deghost maps (small scale)
check(progress, 0.55, "Deghosting")?;
let ghost_small: Option<(Vec<Plane>, usize)> = opts.deghost.params().map(|(thr, r)| deghost_maps(&frames, &valid, &k, &order, reference, thr, r));
check(progress, 0.7, "Merging")?;
let darkest = order[0];
let clips: Vec<f32> = frames.iter().map(|f| f.clip).collect();
// how close each pixel of each frame is to saturation: its brightest channel within 1 px (see `saturation`)
let sat: Vec<Vec<f32>> = frames.par_iter().map(|f| saturation(&f.image)).collect();
let mut out = Rgb32f::new(w, h);
let fr = &frames;
let vd = &valid;
let gs = ghost_small.as_ref();
out.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
for x in 0..w {
let i_px = y * w + x;
let mut acc = [0f64; 3];
let mut ws = 0f64;
// the reference at or near its clip: the radiance is at least this
let mr = maxc(fr[reference].image.data[i_px]);
let bound = if mr >= 0.80 * clips[reference] { mr / k[reference] } else { 0.0 };
for i in 0..n {
if vd[i].as_ref().is_some_and(|v| !v[i_px]) {
continue;
}
let v = fr[i].image.data[i_px];
let m = sat[i][i_px];
let mut wt = k[i] * (1.0 - smoothstep(0.80 * clips[i], 0.96 * clips[i], m));
if i == darkest {
wt += 1e-6;
}
if i == reference {
wt += 1e-9;
} else if bound > 0.0 {
// a frame far darker than that saw something that moved there
wt *= smoothstep(0.35, 0.6, m / k[i] / bound);
}
if i != reference
&& let Some((maps, fac)) = gs
{
let g = sample_plane(&maps[i], (x as f32 + 0.5) / *fac as f32, (y as f32 + 0.5) / *fac as f32);
wt *= 1.0 - g;
}
if wt <= 0.0 {
continue;
}
let inv = 1.0 / k[i];
for c in 0..3 {
acc[c] += (wt * v[c] * inv) as f64;
}
ws += wt as f64;
}
if ws > 0.0 {
row[x] = acc.map(|a| (a / ws) as f32);
} else {
row[x] = fr[reference].image.data[i_px];
}
}
});
check(progress, 0.92, "Merging")?;
let ghost = match &ghost_small {
Some((maps, fac)) => {
let fac = *fac as f32;
let mut g = Plane::new(w, h);
g.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
for (x, o) in row.iter_mut().enumerate() {
let (sx, sy) = ((x as f32 + 0.5) / fac, (y as f32 + 0.5) / fac);
*o = (0..n).filter(|&i| i != reference).map(|i| sample_plane(&maps[i], sx, sy)).fold(0.0, f32::max);
}
});
g
}
None => Plane::new(0, 0),
};
// per-frame results in the caller's order
let mut ev_in = vec![0.0; n];
let mut alignments_in = vec![Alignment::identity(); n];
for (p, &i) in input.iter().enumerate() {
ev_in[i] = ev[p];
alignments_in[i] = alignments[p].clone();
}
let ev_exif = ev_exif.map(|e: Vec<f64>| {
let mut v = vec![0.0; n];
for (p, &i) in input.iter().enumerate() {
v[i] = e[p];
}
v
});
let rf = &frames[reference];
// the merge's white balance is the median of its inputs', as a panorama's: a CR2's or NEF's colour tags have none
let wb_multipliers = consensus_multipliers(&frames);
let mut color = rf.color.clone();
if let Some(multipliers) = wb_multipliers {
color.set_as_shot_multipliers(multipliers);
}
Ok(HdrResult {
radiance: out,
ghost,
reference: input[reference],
ev: ev_in,
ev_exif,
alignments: alignments_in,
color,
raw: rf.raw,
orientation: rf.orientation,
metadata: rf.metadata.clone(),
baseline_exposure: rf.baseline_exposure,
lens: rf.lens.clone(),
wb_multipliers,
})
}
/// Bilinear sample of a plane at continuous coordinates (pixel centres at +0.5).
pub(crate) fn sample_plane(p: &Plane, x: f32, y: f32) -> f32 {
if p.width == 0 || p.height == 0 {
return 0.0;
}
crate::features::bilinear(p, x, y)
}
/// Median of ln(b / a) on the green channel over pixels well exposed in both (≥ 500 samples).
fn log_ratio(a: &Frame, va: Option<&[bool]>, b: &Frame, vb: Option<&[bool]>) -> Option<f64> {
let len = a.image.data.len();
let step = (len / 400_000).max(1);
let (ca, cb) = (a.clip, b.clip);
let lo = |f: &Frame| if f.raw { 0.01 } else { 0.04 };
let (la, lb) = (lo(a) * ca, lo(b) * cb);
let mut v: Vec<f32> = (0..len)
.into_par_iter()
.step_by(step)
.filter_map(|i| {
if va.is_some_and(|v| !v[i]) || vb.is_some_and(|v| !v[i]) {
return None;
}
let (pa, pb) = (a.image.data[i], b.image.data[i]);
if maxc(pa) >= 0.8 * ca || maxc(pb) >= 0.8 * cb || pa[1] <= la || pb[1] <= lb {
return None;
}
Some((pb[1] / pa[1]).ln())
})
.collect();
if v.len() < 500 {
return None;
}
v.sort_by(|x, y| x.total_cmp(y));
Some(v[v.len() / 2] as f64)
}
/// Box-downsample by `fac` (with the fraction of valid source pixels).
fn shrink(img: &Rgb32f, valid: Option<&[bool]>, fac: usize) -> (Rgb32f, Vec<f32>) {
let (w, h) = (img.width.div_ceil(fac), img.height.div_ceil(fac));
let mut out = Rgb32f::new(w, h);
let mut vf = vec![0f32; w * h];
out.data.par_chunks_mut(w).zip(vf.par_chunks_mut(w)).enumerate().for_each(|(y, (row, vrow))| {
for x in 0..w {
let mut s = [0f32; 3];
let (mut nv, mut nt) = (0usize, 0usize);
for sy in y * fac..((y + 1) * fac).min(img.height) {
for sx in x * fac..((x + 1) * fac).min(img.width) {
let i = sy * img.width + sx;
nt += 1;
if valid.is_none_or(|v| v[i]) {
let p = img.data[i];
for c in 0..3 {
s[c] += p[c];
}
nv += 1;
}
}
}
if nv > 0 {
row[x] = s.map(|v| v / nv as f32);
}
vrow[x] = nv as f32 / nt.max(1) as f32;
}
});
(out, vf)
}
/// Per-frame ghost masks (0..1) at 1/`fac` scale; the reference's is all zero.
fn deghost_maps(
frames: &[Frame],
valid: &[Option<Vec<bool>>],
k: &[f32],
order: &[usize],
reference: usize,
thr: f32,
r: usize,
) -> (Vec<Plane>, usize) {
let n = frames.len();
let (w, h) = (frames[0].width(), frames[0].height());
let fac = w.max(h).div_ceil(DEGHOST_EDGE).max(1);
// the dilation/feather radius was chosen for a 1024 px map: keep it the same share of the picture
let r = (r as f32 * (w.max(h) as f32 / fac as f32 / 1024.0).max(1.0)).round() as usize;
let small: Vec<(Rgb32f, Vec<f32>)> = (0..n).into_par_iter().map(|i| shrink(&frames[i].image, valid[i].as_deref(), fac)).collect();
let (sw, sh) = (small[0].0.width, small[0].0.height);
let rpos = order.iter().position(|&i| i == reference).unwrap_or(0);
// `dark`: the share of the clip below which a frame's (averaged) pixel is too noisy to predict from
let good = |i: usize, p: usize, dark: f32| -> bool {
let f = &frames[i];
let m = maxc(small[i].0.data[p]);
small[i].1[p] > 0.99 && m < 0.8 * f.clip && m > dark * f.clip
};
let dark = |i: usize| if frames[i].raw { 0.01 } else { 0.04 };
// predicted reference radiance per pixel
let pred: Vec<[f32; 3]> = (0..sw * sh)
.into_par_iter()
.map(|p| {
let rad = |i: usize| small[i].0.data[p].map(|v| v / k[i]);
// the reference predicts down to a tenth of the other frames' floor (the map's averaging tames its noise): in a
// dark sky the brighter frames must not predict themselves, or their ghosts would look consistent
if good(reference, p, dark(reference) * 0.1) {
return rad(reference);
}
let bright = maxc(small[reference].0.data[p]) >= 0.8 * frames[reference].clip;
if bright {
for q in (0..rpos).rev() {
if good(order[q], p, dark(order[q])) {
return rad(order[q]);
}
}
rad(order[0])
} else {
for &i in &order[rpos + 1..] {
if good(i, p, dark(i)) {
return rad(i);
}
}
rad(reference)
}
})
.collect();
let maps: Vec<Plane> = (0..n)
.into_par_iter()
.map(|i| {
if i == reference {
return Plane::new(sw, sh);
}
let f = &frames[i];
let e = if f.raw { 0.01 } else { 0.03 } * f.clip;
let mut m = Plane::new(sw, sh);
for p in 0..sw * sh {
if small[i].1[p] < 0.5 {
continue;
}
let obs = small[i].0.data[p];
let mut d = 0f32;
for c in 0..3 {
let o = obs[c].min(f.clip);
let pr = (pred[p][c] * k[i]).min(f.clip);
d = d.max(((o + e) / (pr + e)).ln().abs());
}
m.data[p] = smoothstep(thr * 0.7, thr, d);
}
// dilate (max filter, square r) then feather
let dil = dilate(&m, r);
let mut g = gaussian(&dil, r as f32);
for v in g.data.iter_mut() {
*v = v.clamp(0.0, 1.0);
}
g
})
.collect();
(maps, fac)
}
fn dilate(p: &Plane, r: usize) -> Plane {
let (w, h) = (p.width, p.height);
let mut tmp = Plane::new(w, h);
for y in 0..h {
for x in 0..w {
let (a, b) = (x.saturating_sub(r), (x + r).min(w - 1));
tmp.data[y * w + x] = p.data[y * w + a..=y * w + b].iter().copied().fold(0.0, f32::max);
}
}
let mut out = Plane::new(w, h);
for y in 0..h {
let (a, b) = (y.saturating_sub(r), (y + r).min(h - 1));
for x in 0..w {
out.data[y * w + x] = (a..=b).map(|yy| tmp.data[yy * w + x]).fold(0.0, f32::max);
}
}
out
}
/// A frame's saturation per pixel: the brightest channel within 1 px (3 × 3). A demosaiced pixel next to clipped
/// photosites is interpolated from them and reads low in the channel that clipped (green, in camera RGB), so it loses
/// its weight with them. On real ARW brackets the brightest frame's green was 3–5 % low within 2 px of clipped pixels
/// (and linear to 1 % elsewhere), and as that frame carries the most weight, bright windows got magenta fringes.
fn saturation(img: &Rgb32f) -> Vec<f32> {
let (w, h) = (img.width, img.height);
let m: Vec<f32> = img.data.iter().map(|p| maxc(*p)).collect();
if w == 0 || h == 0 {
return m;
}
let mut rows = vec![0f32; w * h];
rows.par_chunks_mut(w).zip(m.par_chunks(w)).for_each(|(out, src)| {
for (x, o) in out.iter_mut().enumerate() {
*o = src[x.saturating_sub(1)..=(x + 1).min(w - 1)].iter().copied().fold(0.0, f32::max);
}
});
let mut out = vec![0f32; w * h];
out.par_chunks_mut(w).enumerate().for_each(|(y, row)| {
let (a, b) = (y.saturating_sub(1), (y + 1).min(h - 1));
for (x, v) in row.iter_mut().enumerate() {
*v = (a..=b).map(|yy| rows[yy * w + x]).fold(0.0, f32::max);
}
});
out
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.