GitCorner

Code

ossartcraftlightcraft🌐
5a24289898fd
use super::look_fit::dro_off;
use super::pairs::{Pairs, displayed, luma};
use super::{CameraTone, Rgb32f, ToneMap, luminance_2020};

/// How the per-photo tone curve is fitted ([`fit_pairs_ordered`] tries them in order).
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub(super) enum ToneFit {
    /// [`fit_tone`]: no flat steps, no pixel-exact registration needed.
    Quantile,
    /// [`fit_tone_conditional`]: the earlier fit, kept as the fallback.
    ConditionalMedian,
}

impl ToneFit {
    pub(super) fn fit(self, pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
        match self {
            ToneFit::Quantile => fit_tone(pairs),
            ToneFit::ConditionalMedian => fit_tone_conditional(pairs),
        }
    }
}

/// The bright areas a narrow tone curve is checked against ([`highlights_hold`]): every proxy pixel
/// whose camera JPEG luminance is 0.85 or more, with its sensor value as developed (for a clipped
/// pixel a lower bound of the scene). Taken from the proxies before any training filter (clipped
/// pixels, edges, the sensor value range of [`collect_pairs`]), so no retry of the search can
/// leave them out.
pub(super) fn highlights(sensor: &Rgb32f, reference: &Rgb32f) -> Pairs {
    if sensor.data.len() != reference.data.len() {
        return Vec::new();
    }
    sensor
        .data
        .iter()
        .zip(&reference.data)
        .filter(|(x, y)| x.iter().chain(y.iter()).all(|v| v.is_finite() && *v >= 0.0) && luminance_2020(**y) >= 0.85)
        .map(|(x, y)| (x.map(f64::from), y.map(f64::from)))
        .collect()
}

/// Smallest ratio between the scene luminance of the brightest and of the darkest 1/32 of the
/// pairs a tone curve is fitted to (its last and first knot). Below it the training pairs are
/// nearly uniform (under 0.26 EV from their darker to their brighter parts): the curve would be
/// fitted to a sliver of the tone range and the rest of it extrapolated. Above it the acceptance
/// gates judge the curve. This was 1.5, which rejected low-contrast scenes the curve fits very
/// well: in a hazy ILCE-7RM4 burst the frames at 1.46–1.49 opened with the neutral fallback (no full or partial
/// look, 18 L* darker than the camera JPEG) while the frame at 1.51 fitted at a held-out RMS of
/// 0.013, as the others do once allowed (issue #631). Across 135 public Sony files the lowest is 4.0.
/// A curve under [`NARROW_TONE_RANGE`] must also pass [`highlights_hold`].
const MIN_TONE_RANGE: f32 = 1.2;

/// [`MIN_TONE_RANGE`] before issue #631. A curve fitted to a narrower range than this must also
/// render the photo's bright areas no further from the camera JPEG than the neutral fallback does
/// ([`highlights_hold`]). Neither the fit nor the held-out gates see those areas (camera JPEG
/// luminance 0.85 and up, left out of the training pairs by [`collect_pairs`]), and a curve fitted
/// to a sliver of the tone range extrapolates them: in review, a synthetic frame whose curve passed
/// the gates at a held-out RMS of 0.0004 rendered its bright sky at 0.29 where the camera JPEG has
/// 0.93.
const NARROW_TONE_RANGE: f32 = 1.5;

/// What a narrow tone curve is checked against ([`highlights_hold`]).
pub(super) struct HighlightCheck {
    /// The frame's [`highlights`].
    pub(super) pairs: Pairs,
    /// The photo's curve will be lowered to Sony's without DRO ([`with_dro_off_tone`]) after the
    /// fit, so the check looks at it lowered.
    pub(super) dro: bool,
    /// The camera JPEG is black and white ([`fit_tone_only_with`]): the check compares luminance, as that
    /// fit's gates do, since the JPEG has no colour to match.
    pub(super) grey: bool,
}

/// Whether a look with this `tone` and `colour` (sensor value to the finish stage's input) keeps
/// the frame's [`highlights`] (camera JPEG luminance 0.85 and up, left out of the fit) no further
/// from the camera JPEG than the neutral fallback does, with the tone curve as the photo will get
/// it (lowered for DRO), in luminance alone for a black-and-white JPEG. Only asked of curves narrower than
/// [`NARROW_TONE_RANGE`]: true for wider ones, and for frames without such pixels.
pub(super) fn highlights_hold(tone: &CameraTone, check: &HighlightCheck, colour: impl Fn([f64; 3]) -> [f64; 3]) -> bool {
    let knots = tone.knots();
    if knots[31][0] >= knots[0][0] * NARROW_TONE_RANGE {
        return true;
    }
    // the curve as the photo will get it
    let used = if check.dro { dro_off(tone) } else { *tone };
    let (map, neutral) = (ToneMap::camera(&used, 0.0, 0.0, 0.0), ToneMap::new(0.0, 0.0, 0.0));
    let (mut look, mut fallback) = (0.0, 0.0);
    for (x, target) in &check.pairs {
        let (rendered, base) = (displayed(colour(*x), &map), displayed(*x, &neutral));
        if check.grey {
            look += (luma(rendered) - luma(*target)).powi(2);
            fallback += (luma(base) - luma(*target)).powi(2);
            continue;
        }
        for c in 0..3 {
            look += (rendered[c] - target[c]).powi(2);
            fallback += (base[c] - target[c]).powi(2);
        }
    }
    let hold = look.is_finite() && look <= fallback;
    if !hold && lightcraft_pipeline::profiling() {
        eprintln!("[profile] camera look: narrow tone curve renders the bright areas worse than the fallback ({look:.3} > {fallback:.3})");
    }
    hold
}

/// The camera's tone curve from scene/JPEG luminance pairs, by quantile matching: knot `i` pairs
/// the median of the `i`-th 1/32 of the sorted scene luminances with the median of the `i`-th
/// 1/32 of the sorted JPEG luminances. A tone curve is monotone, so it maps each quantile of the
/// scene to the same quantile of the JPEG; matching the two distributions needs no pixel-exact
/// registration and is increasing by construction.
///
/// Binning the pairs by scene luminance and taking the JPEG's median per bin instead (followed by
/// isotonic pooling) fails on real files: camera JPEGs are lens-corrected and sharpened, so on a
/// 96 px proxy of a busy scene neighbouring pairs disagree by more than the gap between bins.
/// The conditional medians then zig-zag, pooling turns them into flat steps (13 of 31 segments
/// on a Z 6II forest, issue #475) that render as posterised patches, and regression toward the
/// mean flattens both ends of the curve (lifted blacks, dull highlights).
pub(super) fn fit_tone(pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
    if pairs.len() < 128 || !pairs.iter().all(|(x, y)| x.is_finite() && *x > 0.0 && y.is_finite()) {
        return None;
    }
    let mut xs: Vec<f64> = pairs.iter().map(|p| p.0).collect();
    let mut ys: Vec<f64> = pairs.iter().map(|p| p.1).collect();
    xs.sort_by(f64::total_cmp);
    ys.sort_by(f64::total_cmp);
    let n = xs.len();
    let mut knots = [[0.0; 2]; 32];
    for (i, knot) in knots.iter_mut().enumerate() {
        let (a, b) = (i * n / 32, (i + 1) * n / 32);
        let mid = a + (b - a) / 2;
        *knot = [*xs.get(mid)? as f32, *ys.get(mid)? as f32];
    }
    if knots[31][0] < knots[0][0] * MIN_TONE_RANGE {
        return None;
    }
    CameraTone::new(limit_elasticity(knots, &pairs)?)
}

/// Largest log-log slope (elasticity, d ln y / d ln x with [`ELASTICITY_BLACK`] added to y) a fitted tone curve may
/// have between two knots. A much steeper segment is not a camera curve but two pixel populations the camera renders at
/// different brightness for the same scene luminance (by hue): quantile matching puts all of one population's scene
/// range below the other's JPEG range and joins them with a near-vertical step, which amplifies the luminance noise of
/// every pixel on it and renders as contours and blotches. On a dark ISO 1100 D3400 frame of saturated blue and red
/// (issue #1904; reds 1.55x brighter in the JPEG than blues of the same scene luminance) the step had elasticity 12-28,
/// and the render's fine luminance noise in the blue area was 1.46x that of Adobe's DNG of the same frame; with the
/// bound it is 0.99x. Of 514 fitted looks on the user samples the median curve's steepest segment is 2.1 and the 95th
/// percentile 3.6. The smoothed window ends where its end-to-end slope is first under the bound, so a looser bound
/// (5 or 6) only spreads the D3400 step over a few knots and leaves the noise as it was.
pub(super) const MAX_TONE_ELASTICITY: f64 = 4.0;
/// Log-log slope of a curve end left free by [`limit_elasticity`]: about the median steepest segment of fitted looks
/// (2.1), so the knots past the step follow an ordinary camera curve. At the bound itself (4) the D3400 frame's fine
/// luminance noise stays 1.2x Adobe's; at 2 it is 0.99x.
pub(super) const TONE_REPAIR_ELASTICITY: f64 = 2.0;
/// Display level added to both ends of a segment's elasticity, so a toe rising from black is not counted as infinitely
/// steep.
pub(super) const ELASTICITY_BLACK: f64 = 0.002;

/// `knots` with every segment steeper than [`MAX_TONE_ELASTICITY`] smoothed. The knots inside the smallest window
/// around the segment (widened a knot on each side at a time) whose end-to-end elasticity is under the bound move onto
/// the straight log-log line between the window's ends, so the step is spread over the scene range of the populations it
/// joined. A window that reaches an end of the curve is pinned there, which can drag most of the curve onto one line
/// (a step at the top of a Coolpix P340 curve pulled knots 3-29 up, held-out error +4 dE00), so it is weighed against
/// leaving that end free: the knots past the segment on that side move onto [`TONE_REPAIR_ELASTICITY`] from the
/// segment's other knot instead. Of the two the one closer to `pairs` (absolute error) is kept. The curve outside the
/// window and its monotonicity are kept. `None` when nothing brings the curve under the bound. Knots within the bound are returned as
/// they are.
pub(super) fn limit_elasticity(mut knots: [[f32; 2]; 32], pairs: &[(f64, f64)]) -> Option<[[f32; 2]; 32]> {
    let (bound, repair) = (MAX_TONE_ELASTICITY, TONE_REPAIR_ELASTICITY);
    let at = |k: &[[f32; 2]; 32], i: usize| (f64::from(k[i][0]).ln(), (f64::from(k[i][1]) + ELASTICITY_BLACK).ln());
    let slope = |k: &[[f32; 2]; 32], a: usize, b: usize| {
        let ((ua, va), (ub, vb)) = (at(k, a), at(k, b));
        (vb - va) / (ub - ua)
    };
    let level = |v: f64| (v.exp() - ELASTICITY_BLACK).max(0.0) as f32;
    let error = |k: &[[f32; 2]; 32]| {
        let tone = CameraTone::new(*k)?;
        Some(pairs.iter().map(|&(x, y)| (f64::from(tone.apply(x as f32)) - y).abs()).sum::<f64>())
    };
    for _ in 0..64 {
        let Some(i) = (0..31).find(|&i| slope(&knots, i, i + 1) > bound) else { return Some(knots) };
        let (mut lo, mut hi) = (i, i + 1);
        while slope(&knots, lo, hi) > bound && !(lo == 0 && hi == 31) {
            lo = lo.saturating_sub(1);
            hi = (hi + 1).min(31);
        }
        let mut pinned = knots;
        let pinned_ok = slope(&knots, lo, hi) <= bound;
        if pinned_ok {
            let ((ua, va), (ub, vb)) = (at(&knots, lo), at(&knots, hi));
            for k in lo + 1..hi {
                pinned[k][1] = level(va + (vb - va) * (f64::from(knots[k][0]).ln() - ua) / (ub - ua));
            }
        }
        // A window that reaches an end of the curve can leave that end free instead: the knots below the segment rise
        // onto the repair slope down from knot i + 1, or the knots above it drop onto it up from knot i.
        let mut free = None;
        if lo == 0 || hi == 31 {
            let mut k2 = knots;
            if lo == 0 {
                let (ub, vb) = at(&knots, i + 1);
                for k in 0..=i {
                    let (u, v) = at(&knots, k);
                    k2[k][1] = level(v.max(vb - repair * (ub - u)));
                }
            } else {
                let (ua, va) = at(&knots, i);
                for k in i + 1..32 {
                    let (u, v) = at(&knots, k);
                    k2[k][1] = level(v.min(va + repair * (u - ua)));
                }
            }
            free = Some(k2);
        }
        let next = match (pinned_ok, free) {
            (true, Some(free)) => {
                if error(&free).unwrap_or(f64::INFINITY) < error(&pinned).unwrap_or(f64::INFINITY) {
                    free
                } else {
                    pinned
                }
            }
            (true, None) => pinned,
            (false, Some(free)) => free,
            (false, None) => return None,
        };
        if lightcraft_pipeline::profiling() {
            let moved: Vec<usize> = (0..32).filter(|&k| next[k][1] != knots[k][1]).collect();
            eprintln!("[profile] camera tone: elasticity {:.1} at scene {:.4}, knots {moved:?} moved", slope(&knots, i, i + 1), knots[i][0]);
        }
        if next == knots {
            return None;
        }
        knots = next;
    }
    None
}

fn median(values: &mut [f64]) -> Option<f64> {
    values.sort_by(f64::total_cmp);
    values.get(values.len() / 2).copied()
}

/// The earlier tone fit, kept as the fallback of [`ToneFit`]: the JPEG's median per equal-population
/// bin of scene luminance, then pooled-adjacent-violators isotonic regression. It follows the
/// per-pixel pairs closely when the two images are registered, which is why it can clear the
/// acceptance gates where [`fit_tone`] just misses them, but on lens-corrected JPEGs of busy scenes
/// the pooling leaves flat steps (issue #475).
pub(super) fn fit_tone_conditional(mut pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
    if pairs.len() < 128 || !pairs.iter().all(|(x, y)| x.is_finite() && *x > 0.0 && y.is_finite()) {
        return None;
    }
    pairs.sort_by(|a, b| a.0.total_cmp(&b.0));
    let mut knots = [[0.0; 2]; 32];
    for (i, knot) in knots.iter_mut().enumerate() {
        let bin = pairs.get(i * pairs.len() / 32..(i + 1) * pairs.len() / 32)?;
        let mut xs: Vec<_> = bin.iter().map(|p| p.0).collect();
        let mut ys: Vec<_> = bin.iter().map(|p| p.1).collect();
        *knot = [median(&mut xs)? as f32, median(&mut ys)? as f32];
    }
    if knots[31][0] < knots[0][0] * MIN_TONE_RANGE {
        return None;
    }
    // Pool adjacent violating bins (isotonic regression): no reversals or arbitrary polynomial.
    let mut blocks: Vec<(f32, usize)> = Vec::new();
    for knot in knots {
        blocks.push((knot[1], 1));
        while blocks.len() >= 2 {
            let (a, an) = *blocks.get(blocks.len() - 2)?;
            let (b, bn) = *blocks.last()?;
            if a <= b {
                break;
            }
            blocks.truncate(blocks.len() - 2);
            blocks.push(((a * an as f32 + b * bn as f32) / (an + bn) as f32, an + bn));
        }
    }
    let mut i = 0;
    for (y, n) in blocks {
        for knot in knots.get_mut(i..i + n)? {
            knot[1] = y;
        }
        i += n;
    }
    CameraTone::new(limit_elasticity(knots, &pairs)?)
}