665bfac0b339
lib.rs
//! Trained models for Roto Brush and face tracking, each kind behind one swappable interface.
//!
//! - [`MaskModel`]: what Roto Brush asks of a model, the foreground probability of each pixel of a
//! frame given prompts (foreground / background points from the strokes, a box, a prior mask).
//! Anything implementing it can be plugged in; the classical graph-cut segmenter in
//! `effectcraft-track` stays the built-in fallback when no model is chosen.
//! - [`face::FaceModel`]: what face tracking asks of a model: find a face in a region, follow it
//! from frame to frame. The classical face tracker stays the built-in fallback.
//! - [`MODELS`]: the registry. Every entry is open source under a licence compatible with
//! EffectCraft's (MIT OR Apache-2.0), with its authors, source, size and SHA-256. Weights are
//! never bundled: they are fetched on demand (or installed from a file), verified ([`sha256`])
//! and loaded by [`load`].
//! - [`pt`]: a PyTorch checkpoint reader; [`tflite`]: a TensorFlow Lite reader and interpreter;
//! so official weights load as published.
//! - [`mobilesam`]: MobileSAM (Apache-2.0) and [`mediapipe`]: MediaPipe Face Landmarker
//! (Apache-2.0), in plain Rust ([`nn`] kernels on rayon).
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
pub mod face;
pub mod mediapipe;
pub mod mobilesam;
pub mod nn;
pub mod pt;
pub mod tflite;
use std::sync::Arc;
pub type Result<T> = std::result::Result<T, String>;
/// What a model is asked: coordinates in frame pixels.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Prompt {
/// Points and whether each is foreground (`true`) or background.
pub points: Vec<([f32; 2], bool)>,
/// A box around the object: `[x0, y0, x1, y1]`.
pub bbox: Option<[f32; 4]>,
/// A prior: the foreground probability of every pixel (`w×h`), e.g. the previous frame's
/// matte carried forward.
pub mask: Option<Vec<f32>>,
}
/// A trained segmentation model (swappable: Roto Brush only uses this interface).
pub trait MaskModel: Send + Sync {
fn info(&self) -> &'static ModelInfo;
/// Foreground probability (0–1) of each pixel of a frame (straight RGB 0–1, row-major `w×h`).
fn segment(&self, rgb: &[[f32; 3]], w: usize, h: usize, prompt: &Prompt) -> Result<Vec<f32>>;
/// A video tracker built on the same weights, if the model has one.
fn tracker(&self) -> Option<&dyn ObjectTracker> {
None
}
/// The most memory (bytes) the last tracking run held for its memory banks and cached frames (`None`: not tracked).
fn peak_state_bytes(&self) -> Option<u64> {
None
}
/// What runs the model: the GPU's name or "CPU" (`None`: it does not say).
fn device_name(&self) -> Option<String> {
None
}
/// Seconds since the model was last used (`None`: it does not track use, so it is never unloaded
/// for being idle). Large models free their memory after a quiet spell and load again on use.
fn idle_secs(&self) -> Option<f64> {
None
}
}
/// One decoded frame handed to a tracker (straight RGB 0–1, row-major `w×h`).
#[derive(Clone, Debug, Default)]
pub struct ClipFrame {
pub rgb: Vec<[f32; 3]>,
pub w: usize,
pub h: usize,
}
/// What to track: the clip's frames `first..first + count` (the host's frame numbers) and, for
/// each object, its prompts on their frames. Objects are tracked in one pass, so the cost of
/// encoding each frame is shared.
#[derive(Clone, Debug)]
pub struct TrackRequest {
pub first: i64,
pub count: usize,
/// One entry per object: its prompts, each on its frame.
pub objects: Vec<Vec<(i64, Prompt)>>,
}
impl TrackRequest {
/// A request for a single object.
pub fn single(first: i64, count: usize, prompts: Vec<(i64, Prompt)>) -> TrackRequest {
TrackRequest { first, count, objects: vec![prompts] }
}
}
/// A tracker's answer for one frame.
#[derive(Clone, Debug, Default)]
pub struct TrackedFrame {
/// Foreground probability of every pixel (`w×h`); all zero when the object is absent.
pub prob: Vec<f32>,
pub w: usize,
pub h: usize,
/// How sure the tracker is (0–1), for the timeline.
pub confidence: f32,
/// The object is not in this frame (occluded or out of view).
pub occluded: bool,
}
/// A video tracker with memory (SAM 2): given prompts on some frames, it follows the object
/// through the clip in both directions. Optional: a [`MaskModel`] may offer one.
pub trait ObjectTracker: Send + Sync {
/// Track the objects. `frames(i)` supplies frame `i` (`None`: unavailable); `cancel()` is polled
/// between frames; `out(object, i, result)` is called as soon as each object's frame is finished
/// (`object` indexes `req.objects`).
fn track(
&self,
req: &TrackRequest,
frames: &mut dyn FnMut(i64) -> Option<Arc<ClipFrame>>,
cancel: &dyn Fn() -> bool,
out: &mut dyn FnMut(usize, i64, TrackedFrame),
) -> Result<()>;
}
/// What a model is for.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Task {
/// Roto Brush ([`MaskModel`]).
Mask,
/// Face tracking ([`face::FaceModel`]).
Face,
}
/// A model in the registry: what it is, who made it, its licence and where its weights come from.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct ModelInfo {
pub id: &'static str,
pub task: Task,
pub name: &'static str,
pub description: &'static str,
/// Who made it (shown with the model, and in ATTRIBUTION.md).
pub authors: &'static str,
/// SPDX licence of the weights and the architecture.
pub licence: &'static str,
pub licence_url: &'static str,
/// The project's page.
pub homepage: &'static str,
/// App download URL; empty when only Install from File is available.
pub url: &'static str,
/// Checkpoint source for manual installation and notices; never used by the app downloader.
pub source: &'static str,
pub file_name: &'static str,
pub size: u64,
/// SHA-256 of the weights file (lower-case hex).
pub sha256: &'static str,
}
/// Licences a model may have to be listed (open source, compatible with MIT OR Apache-2.0).
pub const ALLOWED_LICENCES: &[&str] = &["MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", "Zlib", "CC0-1.0"];
pub const MOBILE_SAM: ModelInfo = ModelInfo {
id: "mobilesam",
task: Task::Mask,
name: "MobileSAM",
description: "Segment Anything distilled to a 5M-parameter TinyViT encoder (C. Zhang et al., 2023). Turns your strokes into a clean matte and tidies every propagated frame.",
authors: "Chaoning Zhang et al. (MobileSAM), building on Meta AI's Segment Anything and Microsoft's TinyViT",
licence: "Apache-2.0",
licence_url: "https://github.com/ChaoningZhang/MobileSAM/blob/master/LICENSE",
homepage: "https://github.com/ChaoningZhang/MobileSAM",
url: "https://raw.githubusercontent.com/ChaoningZhang/MobileSAM/master/weights/mobile_sam.pt",
source: "https://raw.githubusercontent.com/ChaoningZhang/MobileSAM/master/weights/mobile_sam.pt",
file_name: "mobile_sam.pt",
size: 40_728_226,
sha256: "6dbb90523a35330fedd7f1d3dfc66f995213d81b29a5ca8108dbcdd4e37d6c2f",
};
/// SAM 2.1 Tiny (Meta AI, Apache-2.0): a video object tracker with streaming memory. The weights are
/// the Hugging Face `transformers` checkpoint, identified by SHA-256. Until a storytold mirror
/// is available, install it from a file. `effectcraft-sam2` (native only) loads it, not [`load`].
pub const SAM2_TINY: ModelInfo = ModelInfo {
id: "sam2.1-hiera-tiny",
task: Task::Mask,
name: "SAM 2.1 Tiny",
description: "Meta's Segment Anything Model 2 (Hiera-Tiny, 39M parameters). Follows the object through the clip with a streaming memory: it keeps tracking through occlusion and when the object leaves and re-enters the frame. Slower than MobileSAM on a CPU.",
authors: "Meta AI (Nikhila Ravi et al., SAM 2), Hugging Face transformers checkpoint",
licence: "Apache-2.0",
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
homepage: "https://github.com/facebookresearch/sam2",
url: "",
source: "https://huggingface.co/facebook/sam2.1-hiera-tiny/blob/de431c4043854a71d8101e17995dfe596bf101a5/model.safetensors",
file_name: "sam2.1-hiera-tiny.safetensors",
size: 155_908_064,
sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7",
};
/// SAM 2.1 Small: the Small checkpoint, pinned to a commit like [`SAM2_TINY`].
pub const SAM2_SMALL: ModelInfo = ModelInfo {
id: "sam2.1-hiera-small",
task: Task::Mask,
name: "SAM 2.1 Small",
description: "Meta's Segment Anything Model 2 (Hiera-Small, 46M parameters). The same streaming-memory tracker as SAM 2.1 Tiny with a larger backbone: a little more accurate on thin parts and busy backgrounds, a little slower.",
authors: "Meta AI (Nikhila Ravi et al., SAM 2), Hugging Face transformers checkpoint",
licence: "Apache-2.0",
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
homepage: "https://github.com/facebookresearch/sam2",
url: "",
source: "https://huggingface.co/facebook/sam2.1-hiera-small/blob/ee5bba1d82bb8749febdf90f45e84b687142ba03/model.safetensors",
file_name: "sam2.1-hiera-small.safetensors",
size: 184_305_280,
sha256: "0a4067b11ce1e23d5229203f11c718a823060d15a4b23fa2372a7d4b77cbbc60",
};
/// SAM 2.1 Base+: the Base+ checkpoint, pinned to a commit like [`SAM2_TINY`].
pub const SAM2_BASE_PLUS: ModelInfo = ModelInfo {
id: "sam2.1-hiera-base-plus",
task: Task::Mask,
name: "SAM 2.1 Base+",
description: "Meta's Segment Anything Model 2 (Hiera-Base+, 81M parameters). The same streaming-memory tracker as SAM 2.1 Tiny with a larger backbone: better on difficult footage; wants a GPU for comfortable speed.",
authors: "Meta AI (Nikhila Ravi et al., SAM 2), Hugging Face transformers checkpoint",
licence: "Apache-2.0",
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
homepage: "https://github.com/facebookresearch/sam2",
url: "",
source: "https://huggingface.co/facebook/sam2.1-hiera-base-plus/blob/b7320756a13354e7530a63935656d35b2f91a290/model.safetensors",
file_name: "sam2.1-hiera-base-plus.safetensors",
size: 323_476_296,
sha256: "2012733a0de5d03efd1bba550a2847c4551be9ef2e0d497c83074df66189f780",
};
/// SAM 2.1 Large: the Large checkpoint, pinned to a commit like [`SAM2_TINY`].
pub const SAM2_LARGE: ModelInfo = ModelInfo {
id: "sam2.1-hiera-large",
task: Task::Mask,
name: "SAM 2.1 Large",
description: "Meta's Segment Anything Model 2 (Hiera-Large, 224M parameters). The same streaming-memory tracker as SAM 2.1 Tiny with a larger backbone: the most accurate; about 900 MB and meant for a GPU.",
authors: "Meta AI (Nikhila Ravi et al., SAM 2), Hugging Face transformers checkpoint",
licence: "Apache-2.0",
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
homepage: "https://github.com/facebookresearch/sam2",
url: "",
source: "https://huggingface.co/facebook/sam2.1-hiera-large/blob/665f8e2ad61cf5f53d65644ff27c8ee525124610/model.safetensors",
file_name: "sam2.1-hiera-large.safetensors",
size: 897_897_416,
sha256: "dc407dce21301fd94abb395c5099b4f2c455fdc8a8f261ac3d0ea6d4cd197230",
};
pub const FACE_LANDMARKER: ModelInfo = ModelInfo {
id: "mediapipe-face",
task: Task::Face,
name: "MediaPipe Face Landmarker",
description: "Google's BlazeFace detector and 478-point Face Mesh V2. Follows faces turned to the side, partly covered or in difficult light, where the classic tracker loses them.",
authors: "Google LLC (MediaPipe)",
licence: "Apache-2.0",
licence_url: "https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Face%20Mesh%20V2.pdf",
homepage: "https://developers.google.com/edge/mediapipe/solutions/vision/face_landmarker",
url: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
source: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
file_name: "face_landmarker.task",
size: 3_758_596,
sha256: "64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff",
};
/// The trained models, besides the built-in classical engines.
pub const MODELS: &[ModelInfo] = &[MOBILE_SAM, SAM2_TINY, SAM2_SMALL, SAM2_BASE_PLUS, SAM2_LARGE, FACE_LANDMARKER];
/// The id of the built-in classical engines (graph cut segmenter, shape-model face tracker): no
/// weights, always available.
pub const CLASSICAL: &str = "classical";
pub fn info(id: &str) -> Option<&'static ModelInfo> {
MODELS.iter().find(|m| m.id == id)
}
/// The models for `task`.
pub fn models(task: Task) -> impl Iterator<Item = &'static ModelInfo> {
MODELS.iter().filter(move |m| m.task == task)
}
/// A loaded model.
#[derive(Clone)]
pub enum Loaded {
Mask(Arc<dyn MaskModel>),
Face(Arc<dyn face::FaceModel>),
}
/// Check a weights file against the registry and build the model.
pub fn load(id: &str, bytes: &[u8]) -> Result<Loaded> {
let info = info(id).ok_or_else(|| format!("unknown model `{id}`"))?;
if !ALLOWED_LICENCES.contains(&info.licence) {
return Err(format!("{}: licence {} is not allowed", info.name, info.licence));
}
let got = sha256::hex(bytes);
if got != info.sha256 {
return Err(format!("{}: the file does not match (SHA-256 {got}, expected {})", info.name, info.sha256));
}
match loader(id) {
Some(Loader::MobileSam) => {
let w = pt::read(bytes)?;
Ok(Loaded::Mask(Arc::new(Sam { net: mobilesam::MobileSam::from_weights(&w)? })))
}
Some(Loader::MediaPipe) => Ok(Loaded::Face(Arc::new(mediapipe::FaceLandmarker::from_task(bytes)?))),
Some(Loader::Desktop) => Err(format!("{} is loaded by the desktop application (effectcraft-sam2)", info.name)),
None => Err(format!("no loader for `{id}`")),
}
}
/// What builds a registered model.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Loader {
MobileSam,
MediaPipe,
/// Built outside this crate, by the desktop application (every SAM 2.1 size).
Desktop,
}
/// The loader for model `id`: one answer for every size of a family, so a new size cannot fall
/// through to "no loader" while its sibling gets the helpful error.
fn loader(id: &str) -> Option<Loader> {
match id {
"mobilesam" => Some(Loader::MobileSam),
"mediapipe-face" => Some(Loader::MediaPipe),
id if id.starts_with("sam2.1-hiera-") => Some(Loader::Desktop),
_ => None,
}
}
/// The notice kept next to installed weights: what they are, who made them, their licence.
pub fn notice(m: &ModelInfo) -> String {
format!(
"{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nSHA-256: {}\nProject: {}\n\nInstalled unmodified. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n",
m.name, m.authors, m.licence, m.licence, m.licence_url, m.source, m.sha256, m.homepage
)
}
struct Sam {
net: mobilesam::MobileSam,
}
impl MaskModel for Sam {
fn info(&self) -> &'static ModelInfo {
&MOBILE_SAM
}
fn segment(&self, rgb: &[[f32; 3]], w: usize, h: usize, prompt: &Prompt) -> Result<Vec<f32>> {
let e = self.net.embed(rgb, w, h)?;
Ok(self.net.predict(&e, prompt).0)
}
}
/// SHA-256 (FIPS 180-4), to verify downloaded weights without another dependency.
pub mod sha256 {
const K: [u32; 64] = [
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
];
pub fn digest(data: &[u8]) -> [u8; 32] {
let mut h: [u32; 8] = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19];
let bits = (data.len() as u64).wrapping_mul(8);
let mut tail = data[data.len() - data.len() % 64..].to_vec();
tail.push(0x80);
while tail.len() % 64 != 56 {
tail.push(0);
}
tail.extend_from_slice(&bits.to_be_bytes());
for block in data[..data.len() - data.len() % 64].as_chunks::<64>().0.iter().chain(tail.as_chunks::<64>().0.iter()) {
let mut w = [0u32; 64];
for (i, c) in block.as_chunks::<4>().0.iter().enumerate() {
w[i] = u32::from_be_bytes([c[0], c[1], c[2], c[3]]);
}
for i in 16..64 {
let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3);
let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10);
w[i] = w[i - 16].wrapping_add(s0).wrapping_add(w[i - 7]).wrapping_add(s1);
}
let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut hh] = h;
for i in 0..64 {
let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25);
let ch = (e & f) ^ (!e & g);
let t1 = hh.wrapping_add(s1).wrapping_add(ch).wrapping_add(K[i]).wrapping_add(w[i]);
let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22);
let maj = (a & b) ^ (a & c) ^ (b & c);
let t2 = s0.wrapping_add(maj);
hh = g;
g = f;
f = e;
e = d.wrapping_add(t1);
d = c;
c = b;
b = a;
a = t1.wrapping_add(t2);
}
for (x, y) in h.iter_mut().zip([a, b, c, d, e, f, g, hh]) {
*x = x.wrapping_add(y);
}
}
let mut out = [0u8; 32];
for (i, v) in h.iter().enumerate() {
out[i * 4..i * 4 + 4].copy_from_slice(&v.to_be_bytes());
}
out
}
pub fn hex(data: &[u8]) -> String {
digest(data).iter().map(|b| format!("{b:02x}")).collect()
}
#[cfg(test)]
mod tests {
#[test]
fn known_vectors() {
assert_eq!(super::hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
assert_eq!(super::hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
let long = b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
assert_eq!(super::hex(long), "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
assert_eq!(super::hex(&vec![b'a'; 1_000_000]), "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0");
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn registry_is_open_source_and_verifies_files() {
for m in MODELS {
assert!(ALLOWED_LICENCES.contains(&m.licence), "{}", m.id);
assert_eq!(m.sha256.len(), 64);
assert!(m.url.is_empty() || m.url.starts_with("https://"));
assert!(m.source.starts_with("https://"));
assert!(m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));
assert!(!m.authors.is_empty() && notice(m).contains(m.authors));
}
assert_eq!(
models(Task::Mask).map(|m| m.id).collect::<Vec<_>>(),
["mobilesam", "sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"]
);
for model in [SAM2_TINY, SAM2_SMALL, SAM2_BASE_PLUS, SAM2_LARGE] {
assert!(model.url.is_empty(), "SAM 2 has no app download until a storytold mirror is available");
assert!(model.file_name.ends_with(".safetensors"));
assert!(model.source.contains("/blob/") && model.source.ends_with("/model.safetensors"));
assert!(notice(&model).contains(&format!("Source: {}", model.source)));
assert!(notice(&model).contains(model.sha256));
}
assert_eq!(models(Task::Face).map(|m| m.id).collect::<Vec<_>>(), ["mediapipe-face"]);
assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match")));
// A file that isn't the published one is refused before anything is parsed.
assert!(load("mobilesam", b"not the weights").err().is_some_and(|e| e.contains("does not match")));
assert!(load("nope", b"").is_err());
}
/// Every registered model has a loader, so none falls through to "no loader" after its checksum
/// matches; every SAM 2.1 size is loaded by the desktop application.
#[test]
fn every_registered_model_has_a_loader() {
for m in MODELS {
let l = loader(m.id);
assert!(l.is_some(), "{} has no loader", m.id);
assert_eq!(l == Some(Loader::Desktop), m.id.starts_with("sam2.1-"), "{}", m.id);
}
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.