GitCorner

Code

ossartcraftlightcraft🌐
5a24289898fd
//! Cutting a face out of a photo the way recognition models expect it.
//!
//! With the five landmarks the detector finds (eyes, nose tip, mouth corners), the face is turned and scaled
//! (a similarity transform, found by least squares) so those points land on the standard 112 × 112
//! ArcFace template. Without landmarks (a box from an XMP sidecar) a padded square around the box is used
//! instead. Both warp with supersampling, so shrinking a large face does not alias.
//!
//! Pure geometry on 8-bit RGB: no image types, no panics (every access is checked).

/// The five template points of a 112 × 112 aligned face, image-left eye first: eye, eye, nose tip, mouth
/// corner, mouth corner. (The standard ArcFace template.)
pub const TEMPLATE_112: [(f32, f32); 5] = [(38.2946, 51.6963), (73.5318, 51.5014), (56.0252, 71.7366), (41.5493, 92.3655), (70.7299, 92.2041)];

/// A borrowed 8-bit RGB picture.
#[derive(Clone, Copy)]
pub struct Rgb<'a> {
    pub data: &'a [u8],
    pub width: usize,
    pub height: usize,
}

/// Face bounds, padding factor, and output dimensions for box-based alignment.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct FaceCrop {
    pub bounds: (f32, f32, f32, f32),
    pub padding: f32,
    pub output_size: (usize, usize),
}

impl Rgb<'_> {
    fn valid(&self) -> bool {
        self.width > 0 && self.height > 0 && self.width.checked_mul(self.height).and_then(|n| n.checked_mul(3)) == Some(self.data.len())
    }

    /// The pixel at integer coordinates, clamped to the picture.
    fn at(&self, x: isize, y: isize) -> [f32; 3] {
        let (x, y) = (x.clamp(0, self.width as isize - 1) as usize, y.clamp(0, self.height as isize - 1) as usize);
        let i = (y * self.width + x) * 3;
        match self.data.get(i..i + 3) {
            Some(p) => [f32::from(p[0]), f32::from(p[1]), f32::from(p[2])],
            None => [0.0; 3],
        }
    }

    /// Bilinear sample at a fractional position (pixel centres at integers).
    fn sample(&self, x: f32, y: f32) -> [f32; 3] {
        let (x0, y0) = (x.floor(), y.floor());
        let (fx, fy) = (x - x0, y - y0);
        let (x0, y0) = (x0 as isize, y0 as isize);
        let (a, b, c, d) = (self.at(x0, y0), self.at(x0 + 1, y0), self.at(x0, y0 + 1), self.at(x0 + 1, y0 + 1));
        let mut out = [0.0; 3];
        for k in 0..3 {
            let top = a[k] * (1.0 - fx) + b[k] * fx;
            let bottom = c[k] * (1.0 - fx) + d[k] * fx;
            out[k] = top * (1.0 - fy) + bottom * fy;
        }
        out
    }
}

/// `q = a · p + t` with complex `a` (scale and rotation) and `t`, mapping source positions to output positions.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Similarity {
    pub a: (f32, f32),
    pub t: (f32, f32),
}

impl Similarity {
    pub fn scale(&self) -> f32 {
        self.a.0.hypot(self.a.1)
    }

    /// Where an output position `q` comes from in the source.
    fn invert(&self, q: (f32, f32)) -> Option<(f32, f32)> {
        let n = self.a.0 * self.a.0 + self.a.1 * self.a.1;
        if !(n.is_finite() && n > 1e-12) {
            return None;
        }
        let (dx, dy) = (q.0 - self.t.0, q.1 - self.t.1);
        Some(((dx * self.a.0 + dy * self.a.1) / n, (dy * self.a.0 - dx * self.a.1) / n))
    }
}

/// The least-squares similarity transform taking `from[i]` to `to[i]` (no reflection). `None` when the points
/// do not determine one (all the same point, or non-finite).
pub fn fit_similarity(from: &[(f32, f32)], to: &[(f32, f32)]) -> Option<Similarity> {
    if from.len() != to.len() || from.len() < 2 {
        return None;
    }
    let n = from.len() as f32;
    let mean = |pts: &[(f32, f32)]| (pts.iter().map(|p| p.0).sum::<f32>() / n, pts.iter().map(|p| p.1).sum::<f32>() / n);
    let (mp, mq) = (mean(from), mean(to));
    // a = Σ conj(p') q' / Σ |p'|²   (complex numbers)
    let (mut re, mut im, mut norm) = (0.0f32, 0.0f32, 0.0f32);
    for (p, q) in from.iter().zip(to) {
        let (px, py, qx, qy) = (p.0 - mp.0, p.1 - mp.1, q.0 - mq.0, q.1 - mq.1);
        re += px * qx + py * qy;
        im += px * qy - py * qx;
        norm += px * px + py * py;
    }
    if !(norm.is_finite() && norm > 1e-9) {
        return None;
    }
    let a = (re / norm, im / norm);
    let t = (mq.0 - (a.0 * mp.0 - a.1 * mp.1), mq.1 - (a.0 * mp.1 + a.1 * mp.0));
    (a.0.is_finite() && a.1.is_finite() && t.0.is_finite() && t.1.is_finite()).then_some(Similarity { a, t })
}

/// Fill an `out_w` × `out_h` picture by taking, for each output pixel, the source under it, averaging
/// enough sub-samples that shrinking does not alias.
fn warp(img: &Rgb, m: &Similarity, out_w: usize, out_h: usize) -> Option<Vec<u8>> {
    if !img.valid() || out_w == 0 || out_h == 0 || out_w > 4096 || out_h > 4096 {
        return None;
    }
    if ![m.a.0, m.a.1, m.t.0, m.t.1].iter().all(|v| v.is_finite()) {
        return None;
    }
    // how many source pixels one output pixel spans, per side
    let span = 1.0 / m.scale();
    let n = if span.is_finite() { (span.ceil() as usize).clamp(1, 4) } else { 1 };
    let mut out = vec![0u8; out_w * out_h * 3];
    for (i, px) in out.as_chunks_mut::<3>().0.iter_mut().enumerate() {
        let (ox, oy) = ((i % out_w) as f32, (i / out_w) as f32);
        let mut acc = [0.0f32; 3];
        for sy in 0..n {
            for sx in 0..n {
                let q = (ox + (sx as f32 + 0.5) / n as f32 - 0.5, oy + (sy as f32 + 0.5) / n as f32 - 0.5);
                let (px_, py_) = m.invert(q)?;
                let s = img.sample(px_, py_);
                acc.iter_mut().zip(s).for_each(|(a, v)| *a += v);
            }
        }
        let k = (n * n) as f32;
        for (o, a) in px.iter_mut().zip(acc) {
            *o = (a / k).round().clamp(0.0, 255.0) as u8;
        }
    }
    Some(out)
}

/// The face aligned to the ArcFace template, `out_w` × `out_h` (the template is for 112 × 112 and is scaled for
/// other sizes). `landmarks` are in pixels of `img`. `None` when the landmarks cannot define a transform.
pub fn align_to_template(img: &Rgb, landmarks: &[(f32, f32); 5], out_w: usize, out_h: usize) -> Option<Vec<u8>> {
    let (sx, sy) = (out_w as f32 / 112.0, out_h as f32 / 112.0);
    let template: Vec<(f32, f32)> = TEMPLATE_112.iter().map(|(x, y)| (x * sx, y * sy)).collect();
    let m = fit_similarity(landmarks, &template)?;
    // a face smaller than a few pixels, or absurdly large, is not a face we can use
    let scale = m.scale();
    if !(scale.is_finite() && (0.01..=100.0).contains(&scale)) {
        return None;
    }
    warp(img, &m, out_w, out_h)
}

/// A square around a face box (pixels of `img`), `pad` times its larger side, as `out_w` × `out_h`.
pub fn crop_box(img: &Rgb, crop: FaceCrop) -> Option<Vec<u8>> {
    let (x0, y0, x1, y1) = crop.bounds;
    let pad = crop.padding;
    let (out_w, out_h) = crop.output_size;
    if ![x0, y0, x1, y1, pad].iter().all(|v| v.is_finite()) {
        return None;
    }
    let side = (x1 - x0).abs().max((y1 - y0).abs()) * pad;
    if !(side.is_finite() && side >= 2.0) {
        return None;
    }
    let (cx, cy) = ((x0 + x1) / 2.0, (y0 + y1) / 2.0);
    let s = out_w.min(out_h) as f32 / side;
    let m = Similarity { a: (s, 0.0), t: (out_w as f32 / 2.0 - s * cx, out_h as f32 / 2.0 - s * cy) };
    warp(img, &m, out_w, out_h)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn close(a: (f32, f32), b: (f32, f32), tol: f32) -> bool {
        (a.0 - b.0).abs() <= tol && (a.1 - b.1).abs() <= tol
    }

    fn apply(m: &Similarity, p: (f32, f32)) -> (f32, f32) {
        (m.a.0 * p.0 - m.a.1 * p.1 + m.t.0, m.a.0 * p.1 + m.a.1 * p.0 + m.t.1)
    }

    #[test]
    fn a_known_similarity_is_recovered() {
        // rotate 20 degrees, scale 0.7, translate
        let (s, th) = (0.7f32, 20f32.to_radians());
        let truth = Similarity { a: (s * th.cos(), s * th.sin()), t: (13.0, -5.0) };
        let from: Vec<(f32, f32)> = TEMPLATE_112.to_vec();
        let to: Vec<(f32, f32)> = from.iter().map(|p| apply(&truth, *p)).collect();
        let m = fit_similarity(&from, &to).unwrap();
        assert!(close(m.a, truth.a, 1e-4) && close(m.t, truth.t, 1e-3), "{m:?} vs {truth:?}");
        // noisy points still give the best fit (the residual is small)
        let noisy: Vec<(f32, f32)> = to.iter().enumerate().map(|(i, p)| (p.0 + (i as f32 - 2.0) * 0.2, p.1 - (i as f32 - 2.0) * 0.1)).collect();
        let m2 = fit_similarity(&from, &noisy).unwrap();
        assert!(from.iter().zip(&noisy).all(|(p, q)| close(apply(&m2, *p), *q, 1.0)));
    }

    #[test]
    fn degenerate_points_give_nothing() {
        let same = vec![(5.0, 5.0); 5];
        assert!(fit_similarity(&same, &TEMPLATE_112).is_none());
        assert!(fit_similarity(&TEMPLATE_112, &TEMPLATE_112[..3]).is_none());
        assert!(fit_similarity(&[(0.0, 0.0)], &[(1.0, 1.0)]).is_none());
        let nan = vec![(f32::NAN, 0.0); 5];
        assert!(fit_similarity(&nan, &TEMPLATE_112).is_none());
        let inf = vec![(f32::INFINITY, 0.0), (1.0, 1.0), (2.0, 2.0), (3.0, 3.0), (4.0, 4.0)];
        assert!(fit_similarity(&inf, &TEMPLATE_112).is_none());
    }

    /// A picture with a bright dot at each of five landmarks, the landmarks being the template moved by a known
    /// similarity: aligning must put the dots back on the template.
    #[test]
    fn alignment_puts_the_landmarks_on_the_template() {
        let (w, h) = (400usize, 300usize);
        let (s, th) = (1.7f32, -15f32.to_radians());
        let place = Similarity { a: (s * th.cos(), s * th.sin()), t: (150.0, 60.0) };
        let landmarks: Vec<(f32, f32)> = TEMPLATE_112.iter().map(|p| apply(&place, *p)).collect();
        let mut data = vec![0u8; w * h * 3];
        for l in &landmarks {
            for dy in -3i32..=3 {
                for dx in -3i32..=3 {
                    let (x, y) = ((l.0 as i32 + dx) as usize, (l.1 as i32 + dy) as usize);
                    if x < w && y < h {
                        data[(y * w + x) * 3] = 255;
                    }
                }
            }
        }
        let img = Rgb { data: &data, width: w, height: h };
        let lm: [(f32, f32); 5] = [landmarks[0], landmarks[1], landmarks[2], landmarks[3], landmarks[4]];
        let out = align_to_template(&img, &lm, 112, 112).unwrap();
        assert_eq!(out.len(), 112 * 112 * 3);
        // the red channel's centre of mass near each template point is on the point
        for t in TEMPLATE_112 {
            let (mut sx, mut sy, mut sw) = (0.0f32, 0.0f32, 0.0f32);
            for y in (t.1 as usize).saturating_sub(8)..(t.1 as usize + 9).min(112) {
                for x in (t.0 as usize).saturating_sub(8)..(t.0 as usize + 9).min(112) {
                    let v = f32::from(out[(y * 112 + x) * 3]);
                    sx += v * x as f32;
                    sy += v * y as f32;
                    sw += v;
                }
            }
            assert!(sw > 100.0, "no dot near {t:?}");
            assert!(close((sx / sw, sy / sw), t, 1.5), "dot at {:?}, template {t:?}", (sx / sw, sy / sw));
        }
    }

    #[test]
    fn box_crops_are_centred_and_bad_inputs_are_refused() {
        let (w, h) = (200usize, 100usize);
        let mut data = vec![0u8; w * h * 3];
        for y in 40..60 {
            for x in 90..110 {
                data[(y * w + x) * 3 + 1] = 255;
            }
        }
        let img = Rgb { data: &data, width: w, height: h };
        let out = crop_box(&img, FaceCrop { bounds: (90.0, 40.0, 110.0, 60.0), padding: 2.0, output_size: (64, 64) }).unwrap();
        // the 20 px block fills the middle half of the 64 px output
        let at = |x: usize, y: usize| out[(y * 64 + x) * 3 + 1];
        assert!(at(32, 32) > 250 && at(20, 20) > 250 && at(44, 44) > 250 && at(5, 5) < 5 && at(60, 32) < 5);
        assert!(crop_box(&img, FaceCrop { bounds: (5.0, 5.0, 5.0, 5.0), padding: 2.0, output_size: (64, 64) }).is_none(), "an empty box");
        assert!(crop_box(&img, FaceCrop { bounds: (f32::NAN, 0.0, 10.0, 10.0), padding: 2.0, output_size: (64, 64) }).is_none());
        assert!(crop_box(&img, FaceCrop { bounds: (0.0, 0.0, 10.0, 10.0), padding: 2.0, output_size: (0, 64) }).is_none());
        assert!(crop_box(&img, FaceCrop { bounds: (0.0, 0.0, 10.0, 10.0), padding: 2.0, output_size: (100_000, 64) }).is_none());
        let bad = Rgb { data: &data[..10], width: w, height: h };
        assert!(crop_box(&bad, FaceCrop { bounds: (0.0, 0.0, 10.0, 10.0), padding: 2.0, output_size: (64, 64) }).is_none(), "wrong data length");
    }

    #[test]
    fn faces_off_the_edge_and_tiny_or_huge_landmarks_do_not_panic() {
        let data = vec![80u8; 50 * 50 * 3];
        let img = Rgb { data: &data, width: 50, height: 50 };
        // a box mostly outside the picture is padded by clamping, not an error
        assert!(crop_box(&img, FaceCrop { bounds: (-30.0, -30.0, 20.0, 20.0), padding: 1.5, output_size: (32, 32) }).is_some());
        let far: [(f32, f32); 5] = [(1e6, 1e6), (1e6 + 5.0, 1e6), (1e6 + 2.0, 1e6 + 3.0), (1e6, 1e6 + 6.0), (1e6 + 5.0, 1e6 + 6.0)];
        let _ = align_to_template(&img, &far, 112, 112);
        let tiny: [(f32, f32); 5] = [(1.0, 1.0), (1.01, 1.0), (1.0, 1.01), (1.01, 1.01), (1.005, 1.005)];
        assert!(align_to_template(&img, &tiny, 112, 112).is_none(), "landmarks a hundredth of a pixel apart");
    }
}