5a24289898fd
frame.rs
//! Merge inputs: scene-linear frames decoded from raw files (camera RGB, before white balance) or
//! standard images (linear light in the file's primaries), with their exposure from EXIF.
use lightcraft_color::Mat3;
use lightcraft_geom::Orientation;
use lightcraft_meta::Metadata;
use lightcraft_raster::Rgb32f;
use lightcraft_raster::resample::{Filter, fit};
use lightcraft_raw::{ColorData, Opcode, RawImage};
use crate::{MergeError, Result};
/// What the RGB values of a frame mean.
#[derive(Clone, Debug, PartialEq)]
pub enum FrameColor {
/// Camera RGB of a raw file (not white balanced): the DNG colour tags of the source.
Camera(Box<ColorData>),
/// Linear RGB with these primaries: linear RGB → XYZ (D50-adapted, ICC PCS).
Linear { to_xyz_d50: Mat3 },
}
impl FrameColor {
/// Make `multipliers` (green = 1) the as-shot white balance of camera colour. A DNG says it as `AsShotNeutral`, the
/// reciprocal of the multipliers. The source's `AsShotWhiteXY` goes: a reader takes it before the neutral and the
/// DNG spec has the two exclude each other, so a stale one would hide the white balance set here. Linear colour has
/// no white balance to set, and multipliers that aren't finite and above zero change nothing.
pub fn set_as_shot_multipliers(&mut self, multipliers: [f32; 3]) {
if let FrameColor::Camera(c) = self
&& multipliers.iter().all(|v| v.is_finite() && *v > 0.0)
{
c.as_shot_neutral = Some(multipliers.map(|v| 1.0 / f64::from(v)));
c.as_shot_white_xy = None;
}
}
}
/// One input photo.
#[derive(Clone, Debug)]
pub struct Frame {
/// Scene-linear pixels; 1.0 = the sensor/file white level.
pub image: Rgb32f,
/// Values at or above this (any channel) are saturated.
pub clip: f32,
/// Capture brightness from EXIF, in EV: log2(t · ISO / N²) (higher = brighter frame); `None`
/// when the file doesn't say.
pub exposure: Option<f64>,
pub raw: bool,
pub color: FrameColor,
/// How `image` must be rotated/flipped for display (`Normal` once applied).
pub orientation: Orientation,
pub metadata: Metadata,
/// Raw files: the source's `BaselineExposure` (EV), carried into the output.
pub baseline_exposure: f64,
/// Raw files: the lens corrections the source applies (`WarpRectilinear`, `WarpRectilinear2`, `FixVignetteRadial`,
/// in order), re-expressed for `image` (the default crop, not yet turned), for the merged DNG to carry. Empty for
/// other sources and once [`Frame::orient`] has turned the pixels.
pub lens: Vec<Opcode>,
/// Raw files: the as-shot white-balance multipliers from the vendor's maker notes (green = 1), for formats such
/// as CR2 and NEF that have no `AsShotNeutral` of their own. `None` when the file doesn't say.
pub wb_multipliers: Option<[f32; 3]>,
}
impl Frame {
/// A frame from linear pixels in the develop working space (linear Rec.2020), e.g. a rendered
/// procedural scene. `exposure` as in [`Frame::exposure`].
pub fn from_linear_rec2020(image: Rgb32f, exposure: Option<f64>) -> Frame {
let to_xyz_d50 = lightcraft_color::bradford(lightcraft_color::D65, lightcraft_color::D50).mul(&lightcraft_color::REC2020.to_xyz());
let clip = detect_clip(&image);
Frame {
image,
clip,
exposure,
raw: false,
color: FrameColor::Linear { to_xyz_d50 },
orientation: Orientation::Normal,
metadata: Metadata::default(),
baseline_exposure: 0.0,
lens: Vec::new(),
wb_multipliers: None,
}
}
pub fn width(&self) -> usize {
self.image.width
}
pub fn height(&self) -> usize {
self.image.height
}
/// Apply the display orientation to the pixels. The lens corrections, written for the unturned pixels, are dropped.
pub fn orient(&mut self) {
if self.orientation != Orientation::Normal {
self.image = self.image.oriented(self.orientation);
self.orientation = Orientation::Normal;
self.lens.clear();
}
}
}
/// EV of a capture from its EXIF: log2(t · ISO / N²). ISO defaults to 100 and N to 1 when absent
/// (only ratios between frames matter); `None` without an exposure time.
pub fn exif_exposure(m: &Metadata) -> Option<f64> {
let t = m.exposure_time.filter(|t| *t > 0.0 && t.is_finite())?;
let iso = m.iso.filter(|i| *i > 0).unwrap_or(100) as f64;
let n = m.f_number.filter(|n| *n > 0.0 && n.is_finite()).unwrap_or(1.0);
Some((t * iso / (n * n)).log2())
}
/// The as-shot white balance `frames` agree on, as multipliers with green = 1: the median R/G and the median B/G of
/// the frames that have usable multipliers (all three finite and above zero), so one frame the camera balanced
/// differently doesn't move it. With an even number of voters the median is the mean of the middle two. Frames without
/// usable multipliers don't vote; `None` when none does.
pub fn consensus_multipliers<'a>(frames: impl IntoIterator<Item = &'a Frame>) -> Option<[f32; 3]> {
let (mut r_over_g, mut b_over_g) = (Vec::new(), Vec::new());
for [r, g, b] in frames.into_iter().filter_map(|f| f.wb_multipliers) {
if [r, g, b].iter().all(|v| v.is_finite() && *v > 0.0) {
r_over_g.push(f64::from(r) / f64::from(g));
b_over_g.push(f64::from(b) / f64::from(g));
}
}
Some([median(r_over_g)? as f32, 1.0, median(b_over_g)? as f32])
}
/// The median of `v` (the mean of the middle two when the count is even); `None` when empty.
fn median(mut v: Vec<f64>) -> Option<f64> {
v.sort_by(f64::total_cmp);
let upper = *v.get(v.len() / 2)?;
let lower = *v.get(v.len().checked_sub(1)? / 2)?;
Some((lower + upper) / 2.0)
}
/// The saturation level of a frame: 1.0, or the (lower) level at which values pile up when the
/// file's white level is optimistic. Returned slightly below the level.
pub fn detect_clip(img: &Rgb32f) -> f32 {
let maxv = img.data.iter().map(|p| p[0].max(p[1]).max(p[2])).fold(0.0f32, f32::max);
if maxv.is_nan() || maxv <= 0.0 {
return 1.0;
}
let level = maxv.min(1.0);
// a pile-up: the top 0.5 % of the range is ≥ 4× denser than the 2 % just below it
let (mut near, mut below) = (0usize, 0usize);
for p in &img.data {
let m = p[0].max(p[1]).max(p[2]);
if m >= level * 0.995 {
near += 1;
} else if m >= level * 0.97 && m < level * 0.99 {
below += 1;
}
}
let piled = near as f64 > img.data.len() as f64 * 1e-4 && near >= 4 && near as f64 / 0.005 > 4.0 * below as f64 / 0.02;
if maxv >= 1.0 || piled { level * 0.995 } else { 1.0 }
}
/// The colour tags a raw source starts from when the caller knows better than the file (`None`: the
/// file's own tags). Called with the decoded raw, its samples still there, and the file's bytes.
pub type RawColour<'a> = dyn Fn(&mut lightcraft_raw::RawImage, &[u8]) -> Option<ColorData> + Sync + 'a;
/// Decode a photo (raw or standard format) into a frame no larger than `max_edge` (when given).
/// `orient` applies the EXIF orientation to the pixels.
pub fn load_frame(bytes: &[u8], max_edge: Option<usize>, orient: bool) -> Result<Frame> {
load_frame_with(bytes, max_edge, orient, &|_, _| None)
}
/// [`load_frame`] with a raw's colour tags from `colour`: a raw without colour matrices of its own
/// (CR2, NEF, ARW…) starts from a colour the app works out (issue #965).
pub fn load_frame_with(bytes: &[u8], max_edge: Option<usize>, orient: bool, colour: &RawColour) -> Result<Frame> {
let metadata = lightcraft_meta::extract(bytes);
let exposure = exif_exposure(&metadata);
if lightcraft_raw::probe(bytes).is_some() {
let raw = lightcraft_raw::decode(bytes).map_err(|e| MergeError::Decode(e.to_string()))?;
return raw_frame(raw, bytes, metadata, exposure, max_edge, orient, colour);
}
let opts = match max_edge {
Some(m) => lightcraft_codecs::DecodeOptions::fit(m as u32, m as u32),
None => lightcraft_codecs::DecodeOptions::default(),
};
let d = lightcraft_codecs::decode(bytes, opts).map_err(|e| MergeError::Decode(e.to_string()))?;
let mut image = d.image;
if let Some(m) = max_edge
&& image.width.max(image.height) > m
{
image = fit(&image, m, m, Filter::Box);
}
let clip = detect_clip(&image);
let mut f = Frame {
image,
clip,
exposure,
raw: false,
color: FrameColor::Linear { to_xyz_d50: d.space.to_xyz_d50 },
orientation: Orientation::from_exif(d.orientation),
metadata,
baseline_exposure: 0.0,
lens: Vec::new(),
wb_multipliers: None,
};
if orient {
f.orient();
}
Ok(f)
}
/// A decoded raw (from `bytes`, which `colour` may read) as a frame: developed to camera RGB, with its colour tags,
/// its as-shot white-balance multipliers and its orientation.
fn raw_frame(
mut raw: RawImage,
bytes: &[u8],
metadata: Metadata,
exposure: Option<f64>,
max_edge: Option<usize>,
orient: bool,
colour: &RawColour,
) -> Result<Frame> {
// Lens corrections stay with the develop settings ("profile corrections"): frames are merged uncorrected, and
// the output DNG carries the reference frame's corrections, re-expressed for the default crop `develop` keeps.
// The DNG 1.6 skip rule is applied here, so no `WarpRectilinear2` is left optional.
let (aw, ah) = (raw.active_area.width, raw.active_area.height);
let crop = raw.develop_crop(aw, ah);
let lens = lightcraft_raw::opcodes::applied_ops(&raw.opcodes.list3)
.filter(|op| op.is_lens_correction())
.filter_map(|op| op.cropped(aw, ah, crop))
.map(|op| match op {
Opcode::WarpRectilinear2 { planes, center, reciprocal, .. } => Opcode::WarpRectilinear2 { planes, center, reciprocal, optional: false },
op => op,
})
.collect();
raw.opcodes.list3.retain(|op| !op.is_lens_correction());
if let Some(color) = colour(&mut raw, bytes) {
raw.color = color;
}
let small = max_edge.is_some_and(|m| m <= 1200);
let method = if small { lightcraft_raw::Method::Bilinear } else { lightcraft_raw::Method::Ahd };
let mut image = raw.develop(method).map_err(|e| MergeError::Decode(e.to_string()))?;
for p in image.data.iter_mut() {
*p = p.map(|v| v.max(0.0));
}
let clip = detect_clip(&image);
if let Some(m) = max_edge
&& image.width.max(image.height) > m
{
image = fit(&image, m, m, Filter::Box);
}
let mut f = Frame {
image,
clip,
exposure,
raw: true,
color: FrameColor::Camera(Box::new(raw.color.clone())),
orientation: raw.orientation,
metadata: if raw.metadata == Metadata::default() { metadata } else { raw.metadata.clone() },
baseline_exposure: raw.color.baseline_exposure,
lens,
wb_multipliers: raw.wb_multipliers,
};
if f.exposure.is_none() {
f.exposure = exif_exposure(&f.metadata);
}
if orient {
f.orient();
}
Ok(f)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn exposure_from_exif() {
let m = Metadata { exposure_time: Some(1.0 / 125.0), f_number: Some(8.0), iso: Some(100), ..Default::default() };
let e = exif_exposure(&m).unwrap();
let m2 = Metadata { exposure_time: Some(4.0 / 125.0), ..m.clone() };
assert!((exif_exposure(&m2).unwrap() - e - 2.0).abs() < 1e-9);
assert!(exif_exposure(&Metadata::default()).is_none());
}
#[test]
fn clip_detection() {
let mut img = Rgb32f::from_fn(100, 100, |x, _| [x as f32 / 100.0; 3]);
assert_eq!(detect_clip(&img), 1.0);
for p in img.data.iter_mut().take(500) {
*p = [0.9, 0.9, 0.9];
}
for p in img.data.iter_mut() {
*p = p.map(|v| v.min(0.9));
}
assert!((detect_clip(&img) - 0.9 * 0.995).abs() < 1e-6);
}
/// A 16×12 CR2-style mosaic (no colour tags, as the vendor decoders leave them) with as-shot multipliers.
fn mosaic(wb_multipliers: Option<[f32; 3]>) -> RawImage {
use lightcraft_raw::{BlackLevel, Cfa, OpcodeLists, RawData, RawFormat, Rect};
let (w, h) = (16usize, 12usize);
RawImage {
format: RawFormat::Cr2,
width: w,
height: h,
cpp: 1,
data: RawData::U16((0..w * h).map(|i| (100 + (i * 37) % 900) as u16).collect()),
cfa: Cfa::bayer("RGGB"),
bits: 12,
black: BlackLevel::uniform(64.0),
white: vec![1023.0],
active_area: Rect::new(0, 0, w, h),
crop: Rect::new(0, 0, w, h),
orientation: Orientation::Normal,
color: ColorData::default(),
wb_multipliers,
linearized: false,
opcodes: OpcodeLists::default(),
lens_metadata: Default::default(),
pixel_aspect: 1.0,
metadata: Metadata::default(),
}
}
fn frame_of(raw: RawImage, colour: &RawColour) -> Frame {
raw_frame(raw, &[], Metadata::default(), None, None, true, colour).unwrap()
}
/// Issue #1144: the as-shot multipliers of a CR2, NEF… (its only white balance) were dropped on loading, so
/// nothing downstream could tell a merged DNG what the camera had balanced to.
#[test]
fn a_raw_frame_carries_the_as_shot_multipliers() {
let wb = [2.05, 1.0, 1.62];
let f = frame_of(mosaic(Some(wb)), &|_, _| None);
assert!(f.raw);
assert_eq!(f.wb_multipliers, Some(wb));
// colour tags the caller works out leave them alone
let own = ColorData { as_shot_neutral: Some([0.5, 1.0, 0.6]), ..Default::default() };
let f = frame_of(mosaic(Some(wb)), &|_, _| Some(own.clone()));
assert_eq!(f.wb_multipliers, Some(wb));
assert_eq!(f.color, FrameColor::Camera(Box::new(own)));
// a file that gives none gives none
assert_eq!(frame_of(mosaic(None), &|_, _| None).wb_multipliers, None);
assert_eq!(Frame::from_linear_rec2020(Rgb32f::new(2, 2), None).wb_multipliers, None);
}
fn voter(wb_multipliers: Option<[f32; 3]>) -> Frame {
Frame { wb_multipliers, ..Frame::from_linear_rec2020(Rgb32f::new(2, 2), None) }
}
fn consensus(wb: &[Option<[f32; 3]>]) -> Option<[f32; 3]> {
let frames: Vec<Frame> = wb.iter().map(|m| voter(*m)).collect();
consensus_multipliers(&frames)
}
fn close(a: [f32; 3], b: [f32; 3]) -> bool {
a.iter().zip(b).all(|(x, y)| (x - y).abs() < 1e-6)
}
#[test]
fn the_consensus_is_the_median_of_the_voters() {
// one voter is its own median
assert_eq!(consensus(&[Some([2.0, 1.0, 1.6])]), Some([2.0, 1.0, 1.6]));
// an odd count takes the middle one of each ratio, whichever frame it came from, whatever the order
let odd = [Some([2.4, 1.0, 1.4]), Some([1.8, 1.0, 1.9]), Some([2.0, 1.0, 1.6])];
assert_eq!(consensus(&odd), Some([2.0, 1.0, 1.6]));
// an outlier doesn't pull it
let outlier = [Some([2.0, 1.0, 1.6]), Some([2.1, 1.0, 1.5]), Some([1.9, 1.0, 1.7]), Some([5.0, 1.0, 0.7]), Some([2.0, 1.0, 1.6])];
assert_eq!(consensus(&outlier), Some([2.0, 1.0, 1.6]));
// an even count takes the mean of the middle two
let even = [Some([1.8, 1.0, 1.9]), Some([2.0, 1.0, 1.6]), Some([2.2, 1.0, 1.5]), Some([2.4, 1.0, 1.4])];
assert!(close(consensus(&even).unwrap(), [2.1, 1.0, 1.55]), "{:?}", consensus(&even));
let two = [Some([2.0, 1.0, 1.0]), Some([3.0, 1.0, 2.0])];
assert!(close(consensus(&two).unwrap(), [2.5, 1.0, 1.5]), "{:?}", consensus(&two));
// green is the unit even when a file's multipliers aren't normalised to it
assert!(close(consensus(&[Some([4.0, 2.0, 3.0])]).unwrap(), [2.0, 1.0, 1.5]));
}
#[test]
fn frames_without_usable_multipliers_do_not_vote() {
assert_eq!(consensus(&[]), None);
assert_eq!(consensus(&[None, None]), None);
let bad = |m: [f32; 3]| consensus(&[Some(m)]);
assert_eq!(bad([0.0, 1.0, 1.5]), None);
assert_eq!(bad([2.0, 0.0, 1.5]), None);
assert_eq!(bad([2.0, 1.0, -1.5]), None);
assert_eq!(bad([f32::NAN, 1.0, 1.5]), None);
assert_eq!(bad([2.0, f32::INFINITY, 1.5]), None);
assert_eq!(bad([2.0, 1.0, f32::NEG_INFINITY]), None);
// mixed: the good ones decide, the missing and the unusable ones are not counted as anything
let mixed = [None, Some([2.0, 1.0, 1.6]), Some([f32::NAN, 1.0, 1.0]), Some([2.2, 1.0, 1.4]), Some([0.0, 0.0, 0.0])];
assert!(close(consensus(&mixed).unwrap(), [2.1, 1.0, 1.5]), "{:?}", consensus(&mixed));
assert_eq!(consensus(&[None, Some([2.0, 1.0, 1.6]), None]), Some([2.0, 1.0, 1.6]));
}
#[test]
fn multipliers_become_the_reciprocal_as_shot_neutral() {
use lightcraft_color::{D50, Xy};
let tagged = ColorData { as_shot_neutral: Some([0.9, 1.0, 0.9]), as_shot_white_xy: Some(Xy::new(0.34, 0.35)), ..Default::default() };
let mut color = FrameColor::Camera(Box::new(tagged.clone()));
color.set_as_shot_multipliers([2.0, 1.0, 1.6]);
let FrameColor::Camera(c) = &color else { panic!("still camera colour") };
assert_eq!(c.as_shot_neutral, Some([0.5, 1.0, 1.0 / f64::from(1.6f32)]));
assert_eq!(c.as_shot_white_xy, None, "a stale AsShotWhiteXY would be read before the neutral");
assert_eq!(**c, ColorData { as_shot_neutral: c.as_shot_neutral, as_shot_white_xy: None, ..tagged.clone() }, "nothing else changes");
// multipliers that aren't usable change nothing
for bad in [[0.0, 1.0, 1.0], [2.0, f32::NAN, 1.6], [2.0, 1.0, f32::INFINITY], [-2.0, 1.0, 1.6]] {
let mut color = FrameColor::Camera(Box::new(ColorData { as_shot_white_xy: Some(D50), ..Default::default() }));
let before = color.clone();
color.set_as_shot_multipliers(bad);
assert_eq!(color, before, "{bad:?}");
}
// linear colour has no white balance to set
let mut linear = Frame::from_linear_rec2020(Rgb32f::new(2, 2), None).color;
let before = linear.clone();
linear.set_as_shot_multipliers([2.0, 1.0, 1.6]);
assert_eq!(linear, before);
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.