GitCorner

Code

ossartcrafteffectcraft🌐
665bfac0b339
//! MediaPipe Face Landmarker (Google LLC, Apache-2.0) in plain Rust: the official
//! `face_landmarker.task` bundle's BlazeFace short-range detector and Face Mesh V2 (478 points)
//! run on the [`tflite`](crate::tflite) interpreter.
//!
//! The pipeline follows MediaPipe's published face landmarker graph (Apache-2.0): the detector
//! sees a 128×128 crop scaled to −1…1 and decodes 896 SSD anchors (strides 8, 16, 16, 16; two
//! anchors per cell per layer) with weighted non-maximum suppression; the mesh sees a 256×256
//! crop (0…1) of a square 1.5× the face, rotated so the eyes are level (detector keypoints 0 → 1,
//! then mesh points 33 → 263), and reports a presence score. The landmark indices in
//! [`TOPOLOGY`] are points of MediaPipe's canonical face mesh.

use crate::face::{Border, Face, FaceModel, Roi, Topology, crop, sigmoid};
use crate::tflite::Model;
use crate::{FACE_LANDMARKER, ModelInfo, Result};

pub const DETECTOR: &str = "face_detector.tflite";
pub const MESH: &str = "face_landmarks_detector.tflite";

const DET_SIZE: usize = 128;
const MESH_SIZE: usize = 256;
const MESH_POINTS: usize = 478;
/// Detection and presence thresholds (MediaPipe's defaults).
const MIN_SCORE: f32 = 0.5;
const NMS_IOU: f32 = 0.3;
/// Crops are this much larger than the face.
const ROI_SCALE: f32 = 1.5;
/// Mesh points whose direction levels the crop: the outer eye corners.
const EYE_LINE: [usize; 2] = [33, 263];

/// MediaPipe's face mesh: the face oval (`FACEMESH_FACE_OVAL`, in order) and the points the
/// tracker's landmarks are (inner lips for the mouth, iris centres for the pupils).
pub const TOPOLOGY: Topology = Topology {
    outline: &[
        10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377, 152, 148, 176, 149, 150, 136, 172, 58, 132, 93, 234, 127, 162,
        21, 54, 103, 67, 109,
    ],
    landmarks: &[
        ("leftEyebrowInner", 107),
        ("leftEyebrowMiddle", 105),
        ("leftEyebrowOuter", 70),
        ("rightEyebrowInner", 336),
        ("rightEyebrowMiddle", 334),
        ("rightEyebrowOuter", 300),
        ("leftEyeInner", 133),
        ("leftEyeOuter", 33),
        ("leftEyeTop", 159),
        ("leftEyeBottom", 145),
        ("leftPupil", 468),
        ("rightEyeInner", 362),
        ("rightEyeOuter", 263),
        ("rightEyeTop", 386),
        ("rightEyeBottom", 374),
        ("rightPupil", 473),
        ("noseTip", 1),
        ("leftNostril", 79),
        ("rightNostril", 309),
        ("mouthLeft", 61),
        ("mouthRight", 291),
        ("mouthTop", 13),
        ("mouthBottom", 14),
    ],
};

/// A detected face: box `[x0, y0, x1, y1]`, six keypoints (eyes, nose tip, mouth, ears; image
/// left first) and score, in frame pixels.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Detection {
    pub bbox: [f32; 4],
    pub keypoints: [[f32; 2]; 6],
    pub score: f32,
}

pub struct FaceLandmarker {
    detector: Model,
    mesh: Model,
    /// Anchor centres (normalised).
    anchors: Vec<[f32; 2]>,
}

/// SSD anchor centres: per stride group, `fm×fm` cells with two anchors per layer each.
fn anchors() -> Vec<[f32; 2]> {
    let mut out = vec![];
    for (stride, layers) in [(8usize, 1usize), (16, 3)] {
        let fm = DET_SIZE.div_ceil(stride);
        for y in 0..fm {
            for x in 0..fm {
                for _ in 0..2 * layers {
                    out.push([(x as f32 + 0.5) / fm as f32, (y as f32 + 0.5) / fm as f32]);
                }
            }
        }
    }
    out
}

fn iou(a: &[f32; 4], b: &[f32; 4]) -> f32 {
    let iw = (a[2].min(b[2]) - a[0].max(b[0])).max(0.0);
    let ih = (a[3].min(b[3]) - a[1].max(b[1])).max(0.0);
    let inter = iw * ih;
    let union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter;
    if union > 0.0 { inter / union } else { 0.0 }
}

/// Weighted non-maximum suppression: each cluster of overlapping detections becomes their
/// score-weighted average, with the best score.
fn weighted_nms(mut dets: Vec<Detection>) -> Vec<Detection> {
    dets.sort_by(|a, b| b.score.total_cmp(&a.score));
    let mut out = vec![];
    while let Some(top) = dets.first().copied() {
        // `top` always joins its own cluster: its overlap with itself is 0 for a degenerate box
        // (NaN or zero area), so partitioning it with the rest would never shrink `dets`.
        let (mut cluster, rest): (Vec<Detection>, Vec<Detection>) = dets.into_iter().skip(1).partition(|d| iou(&d.bbox, &top.bbox) > NMS_IOU);
        dets = rest;
        cluster.insert(0, top);
        let total: f32 = cluster.iter().map(|d| d.score).sum();
        if total <= 0.0 {
            out.push(top);
            continue;
        }
        let mut avg = Detection { bbox: [0.0; 4], keypoints: [[0.0; 2]; 6], score: top.score };
        for d in &cluster {
            let k = d.score / total;
            for i in 0..4 {
                avg.bbox[i] += d.bbox[i] * k;
            }
            for (a, p) in avg.keypoints.iter_mut().zip(&d.keypoints) {
                a[0] += p[0] * k;
                a[1] += p[1] * k;
            }
        }
        out.push(avg);
    }
    out
}

/// The crop a face's mesh is computed in: `bbox` grown to a square 1.5× its long side, turned so
/// `from → to` is level.
fn roi_of(bbox: [f32; 4], from: [f32; 2], to: [f32; 2]) -> Roi {
    Roi {
        center: [(bbox[0] + bbox[2]) / 2.0, (bbox[1] + bbox[3]) / 2.0],
        size: (bbox[2] - bbox[0]).max(bbox[3] - bbox[1]) * ROI_SCALE,
        angle: (to[1] - from[1]).atan2(to[0] - from[0]),
    }
}

fn bounds(points: &[[f32; 2]]) -> [f32; 4] {
    points
        .iter()
        .fold([f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY], |b, p| [b[0].min(p[0]), b[1].min(p[1]), b[2].max(p[0]), b[3].max(p[1])])
}

impl FaceLandmarker {
    /// From the official `face_landmarker.task` bundle (a zip archive of `.tflite` models).
    pub fn from_task(bytes: &[u8]) -> Result<FaceLandmarker> {
        let detector = Model::read(&crate::pt::zip_file(bytes, DETECTOR)?)?;
        let mesh = Model::read(&crate::pt::zip_file(bytes, MESH)?)?;
        let anchors = anchors();
        if detector.input_shape(0) != Some(&[1, DET_SIZE, DET_SIZE, 3]) || detector.output(0).map(|o| o.0) != Some(&[1, anchors.len(), 16]) {
            return Err("unexpected face detector".into());
        }
        if mesh.input_shape(0) != Some(&[1, MESH_SIZE, MESH_SIZE, 3]) || mesh.output(0).map(|o| o.0.iter().product::<usize>()) != Some(MESH_POINTS * 3) {
            return Err("unexpected face mesh".into());
        }
        Ok(FaceLandmarker { detector, mesh, anchors })
    }

    /// Faces in the square `roi` of a frame (frame pixels), best first.
    pub fn detect(&self, rgb: &[[f32; 3]], w: usize, h: usize, roi: &Roi) -> Result<Vec<Detection>> {
        let x = crop(rgb, w, h, roi, DET_SIZE, [-1.0, 1.0], Border::Zero);
        let out = self.detector.run(&[&x])?;
        let (Some(reg), Some(cls)) = (out.first(), out.get(1)) else { return Err("face detector: missing outputs".into()) };
        let s = DET_SIZE as f32;
        let mut dets = vec![];
        for (i, a) in self.anchors.iter().enumerate() {
            let (Some(r), Some(&c)) = (reg.get(i * 16..i * 16 + 16), cls.get(i)) else { break };
            let score = sigmoid(c.clamp(-100.0, 100.0));
            if score < MIN_SCORE {
                continue;
            }
            let (cx, cy, bw, bh) = (r[0] / s + a[0], r[1] / s + a[1], r[2] / s, r[3] / s);
            let to = |u: f32, v: f32| roi.to_frame(u, v);
            let corners =
                [to(cx - bw / 2.0, cy - bh / 2.0), to(cx + bw / 2.0, cy - bh / 2.0), to(cx - bw / 2.0, cy + bh / 2.0), to(cx + bw / 2.0, cy + bh / 2.0)];
            let mut keypoints = [[0.0f32; 2]; 6];
            for (k, p) in keypoints.iter_mut().enumerate() {
                *p = to(r[4 + 2 * k] / s + a[0], r[5 + 2 * k] / s + a[1]);
            }
            dets.push(Detection { bbox: bounds(&corners), keypoints, score });
        }
        Ok(weighted_nms(dets))
    }

    /// The face mesh in `roi`: 478 points (frame pixels) and the presence score.
    pub fn mesh(&self, rgb: &[[f32; 3]], w: usize, h: usize, roi: &Roi) -> Result<Face> {
        let x = crop(rgb, w, h, roi, MESH_SIZE, [0.0, 1.0], Border::Replicate);
        let out = self.mesh.run(&[&x])?;
        let (Some(pts), Some(flag)) = (out.first(), out.get(1).and_then(|f| f.first())) else { return Err("face mesh: missing outputs".into()) };
        let s = MESH_SIZE as f32;
        let points = pts.as_chunks::<3>().0.iter().take(MESH_POINTS).map(|p| roi.to_frame(p[0] / s, p[1] / s)).collect();
        Ok(Face { points, score: sigmoid(*flag) })
    }

    /// The crop that frames a face found earlier.
    fn roi_of_face(face: &Face) -> Option<Roi> {
        let (a, b) = (face.points.get(EYE_LINE[0])?, face.points.get(EYE_LINE[1])?);
        Some(roi_of(bounds(&face.points), *a, *b))
    }

    /// The mesh from `roi`, refined once from its own points (a detector box is coarse).
    fn mesh_refined(&self, rgb: &[[f32; 3]], w: usize, h: usize, roi: &Roi) -> Result<Option<Face>> {
        let first = self.mesh(rgb, w, h, roi)?;
        if first.score < MIN_SCORE {
            return Ok(None);
        }
        let Some(r) = Self::roi_of_face(&first) else { return Ok(None) };
        let refined = self.mesh(rgb, w, h, &r)?;
        Ok(Some(if refined.score >= MIN_SCORE { refined } else { first }))
    }
}

impl FaceModel for FaceLandmarker {
    fn info(&self) -> &'static ModelInfo {
        &FACE_LANDMARKER
    }

    fn topology(&self) -> &'static Topology {
        &TOPOLOGY
    }

    fn find(&self, rgb: &[[f32; 3]], w: usize, h: usize, region: [f32; 4]) -> Result<Option<Face>> {
        let inside = |d: &Detection| {
            let c = [(d.bbox[0] + d.bbox[2]) / 2.0, (d.bbox[1] + d.bbox[3]) / 2.0];
            c[0] >= region[0] && c[0] <= region[2] && c[1] >= region[1] && c[1] <= region[3]
        };
        // Around the region first (a face filling the mask), then the whole frame (a small face).
        let side = (region[2] - region[0]).max(region[3] - region[1]);
        let around = Roi { center: [(region[0] + region[2]) / 2.0, (region[1] + region[3]) / 2.0], size: side * ROI_SCALE, angle: 0.0 };
        let whole = Roi { center: [w as f32 / 2.0, h as f32 / 2.0], size: w.max(h) as f32, angle: 0.0 };
        for roi in [around, whole] {
            if !roi.size.is_finite() || roi.size <= 4.0 {
                continue;
            }
            if let Some(d) = self.detect(rgb, w, h, &roi)?.into_iter().find(inside) {
                return self.mesh_refined(rgb, w, h, &roi_of(d.bbox, d.keypoints[0], d.keypoints[1]));
            }
        }
        Ok(None)
    }

    fn follow(&self, rgb: &[[f32; 3]], w: usize, h: usize, prev: &Face) -> Result<Option<Face>> {
        let Some(roi) = Self::roi_of_face(prev) else { return Ok(None) };
        let face = self.mesh(rgb, w, h, &roi)?;
        if face.score >= MIN_SCORE {
            return Ok(Some(face));
        }
        // Lost by the mesh (a fast move): look for it again near where it was.
        let b = bounds(&prev.points);
        let (gw, gh) = ((b[2] - b[0]) * 0.5, (b[3] - b[1]) * 0.5);
        self.find(rgb, w, h, [b[0] - gw, b[1] - gh, b[2] + gw, b[3] + gh])
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn anchors_match_the_detector_layout() {
        let a = anchors();
        assert_eq!(a.len(), 896);
        // Stride 8: 16×16 cells, two anchors each; then stride 16: 8×8 cells, six each.
        assert_eq!(a[0], [0.5 / 16.0, 0.5 / 16.0]);
        assert_eq!(a[1], a[0]);
        assert_eq!(a[2], [1.5 / 16.0, 0.5 / 16.0]);
        assert_eq!(a[512], [0.5 / 8.0, 0.5 / 8.0]);
        assert_eq!(a[517], a[512]);
        assert_eq!(a[518], [1.5 / 8.0, 0.5 / 8.0]);
    }

    #[test]
    fn nms_averages_overlapping_detections() {
        let d = |x: f32, s: f32| Detection { bbox: [x, 0.0, x + 10.0, 10.0], keypoints: [[x, 0.0]; 6], score: s };
        let out = weighted_nms(vec![d(0.0, 0.9), d(1.0, 0.9), d(50.0, 0.6)]);
        assert_eq!(out.len(), 2);
        assert!((out[0].bbox[0] - 0.5).abs() < 1e-5 && out[0].score == 0.9);
        assert_eq!(out[1].bbox[0], 50.0);
    }

    /// A degenerate box (NaN or zero area) overlaps nothing, not even itself: each still becomes
    /// its own output instead of looping forever (#367).
    #[test]
    fn nms_terminates_on_degenerate_boxes() {
        let d = Detection { bbox: [f32::NAN; 4], keypoints: [[f32::NAN; 2]; 6], score: 0.7 };
        assert_eq!(weighted_nms(vec![d, d, d]).len(), 3);
        let z = Detection { bbox: [0.0; 4], keypoints: [[0.0; 2]; 6], score: 0.0 };
        assert_eq!(weighted_nms(vec![z, z]).len(), 2);
        let n = Detection { bbox: [0.0; 4], keypoints: [[0.0; 2]; 6], score: f32::NAN };
        assert_eq!(weighted_nms(vec![n, z, d]).len(), 3);
    }

    /// The official bundle (`EFFECTCRAFT_FACE_LANDMARKER` = path to `face_landmarker.task`) on a
    /// fixed input matches TensorFlow Lite's own interpreter (`ai-edge-litert` 2.2.0, run once
    /// to record these sums).
    #[test]
    fn matches_tensorflow_lite_on_the_official_weights() {
        let Some(path) = std::env::var_os("EFFECTCRAFT_FACE_LANDMARKER") else { return };
        let bytes = std::fs::read(path).unwrap();
        let m = FaceLandmarker::from_task(&bytes).unwrap();
        let input = |n: usize| -> Vec<f32> { (0..n).map(|i| ((i as f32) * 0.013).sin() * 0.5).collect() };
        let close = |got: &[f32], want: f64| {
            let s: f64 = got.iter().map(|v| *v as f64).sum();
            assert!((s - want).abs() <= 1e-5 * want.abs().max(1.0), "{s} vs {want}");
        };
        let d = m.detector.run(&[&input(DET_SIZE * DET_SIZE * 3)]).unwrap();
        close(&d[0], 54_202.040_273);
        close(&d[1], -3_061.866_384);
        let y = m.mesh.run(&[&input(MESH_SIZE * MESH_SIZE * 3)]).unwrap();
        close(&y[0], 119_864.749_257);
        close(&y[1], -19.777_233);
    }

    #[test]
    fn topology_indices_are_mesh_points() {
        assert!(TOPOLOGY.outline.iter().chain(TOPOLOGY.landmarks.iter().map(|l| &l.1)).all(|&i| i < MESH_POINTS));
        assert_eq!(TOPOLOGY.outline.len(), 36);
    }
}