665bfac0b339
tests.rs
//! Roto Brush on synthetic sequences: a textured disk moving over a textured background.
use super::refine::{MatteParams, decontaminate, matte_alpha};
use super::*;
pub(crate) const W: u32 = 200;
pub(crate) const H: u32 = 150;
pub(crate) const R: f64 = 32.0;
/// Smooth value noise in 0..1.
pub(crate) fn noise(x: f64, y: f64, seed: u32) -> f64 {
fn h(i: i64, j: i64, s: u32) -> f64 {
let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ (s as i64).wrapping_mul(83492791)) as u64;
v ^= v >> 13;
v = v.wrapping_mul(0x5bd1e995);
v ^= v >> 15;
(v % 10_000) as f64 / 10_000.0
}
let mut out = 0.0;
for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] {
let (fx, fy) = (x / cell, y / cell);
let (i, j) = (fx.floor() as i64, fy.floor() as i64);
let (tx, ty) = (fx - i as f64, fy - j as f64);
let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty));
let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx;
let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx;
out += amp * (a + (b - a) * sy);
}
out
}
pub(crate) fn fg_color(x: f64, y: f64) -> [f32; 3] {
let n = noise(x, y, 11);
[(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32]
}
pub(crate) fn bg_color(x: f64, y: f64) -> [f32; 3] {
let n = noise(x, y, 5);
let m = noise(x * 1.7, y * 1.7, 9);
[(0.05 + 0.25 * m) as f32, (0.3 + 0.4 * n) as f32, (0.45 + 0.4 * m) as f32]
}
pub(crate) fn center(f: i64) -> [f64; 2] {
let t = f as f64;
[60.0 + 3.1 * t, 70.0 + 1.3 * t + 4.0 * (t * 0.3).sin()]
}
/// Frame `f`: the disk (texture moving with it) over the static background, with a 1-pixel
/// anti-aliased edge; `c` overrides the centre.
pub(crate) fn frame_at(c: [f64; 2]) -> (Image, Vec<u8>) {
let mut img = Image::new(W, H);
let mut gt = vec![0u8; (W * H) as usize];
for y in 0..H {
for x in 0..W {
let p = [x as f64 + 0.5, y as f64 + 0.5];
let d = (p[0] - c[0]).hypot(p[1] - c[1]);
let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32;
let f = fg_color(p[0] - c[0], p[1] - c[1]);
let b = bg_color(p[0], p[1]);
img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]);
gt[(y * W + x) as usize] = (d <= R) as u8;
}
}
(img, gt)
}
fn line(kind: StrokeKind, frame: i64, a: [f64; 2], b: [f64; 2], radius: f64) -> Stroke {
Stroke {
kind,
frame,
radius,
points: (0..=10).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 10.0, a[1] + (b[1] - a[1]) * k as f64 / 10.0]).collect(),
object: 0,
}
}
/// The strokes an artist would paint on frame `f`: one through the disk, one around it.
pub(crate) fn base_strokes(f: i64) -> Vec<Stroke> {
let c = center(f);
vec![
line(StrokeKind::Fg, f, [c[0] - 18.0, c[1] - 6.0], [c[0] + 18.0, c[1] + 6.0], 5.0),
line(StrokeKind::Bg, f, [c[0] - 50.0, c[1] - 45.0], [c[0] + 50.0, c[1] - 45.0], 5.0),
line(StrokeKind::Bg, f, [c[0] + 50.0, c[1] + 45.0], [c[0] - 45.0, c[1] + 45.0], 5.0),
]
}
#[test]
fn strokes_segment_the_base_frame() {
let (img, gt) = frame_at(center(0));
let st = base_strokes(0);
let refs: Vec<&Stroke> = st.iter().collect();
let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0);
let j = iou(&seg.matte, >);
assert!(j > 0.95, "IoU {j}");
// Deterministic.
let again = segment(&img, &refs, None, &SegOpts::default(), 0.0);
assert_eq!(seg, again);
// The binary form round-trips exactly and is compact (run-length encoded planes).
let bytes = seg.to_bytes();
assert_eq!(FrameSeg::from_bytes(&bytes).as_ref(), Some(&seg));
assert!(bytes.len() < seg.w * seg.h / 4, "{} bytes", bytes.len());
assert!(FrameSeg::from_bytes(&bytes[..bytes.len() - 1]).is_none());
assert!(FrameSeg::from_bytes(b"nope").is_none());
// Untrusted plane dimensions and run lengths are rejected before allocation/resize.
let mut hostile = b"ECSG".to_vec();
hostile.extend_from_slice(&u32::MAX.to_le_bytes());
hostile.extend_from_slice(&u32::MAX.to_le_bytes());
hostile.extend_from_slice(&0u32.to_le_bytes());
assert!(FrameSeg::from_bytes(&hostile).is_none());
let mut overrun = bytes.clone();
let plane_at = 16 + u32::from_le_bytes(bytes[12..16].try_into().unwrap()) as usize;
overrun[plane_at..plane_at + 2].copy_from_slice(&u16::MAX.to_le_bytes());
assert!(FrameSeg::from_bytes(&overrun).is_none());
let mut trailing = bytes.clone();
trailing.push(0);
assert!(FrameSeg::from_bytes(&trailing).is_none());
assert_eq!(FrameSeg::empty(usize::MAX, 2), FrameSeg::default());
}
#[test]
fn coarse_to_fine_segments_large_frames() {
// A 2× upscaled frame goes through the reduced-size path (Standard works at 480 px).
let (small, _) = frame_at(center(0));
let img = effectcraft_raster::resample(&small, W * 3, H * 3);
let c = center(0);
let st: Vec<Stroke> = base_strokes(0)
.into_iter()
.map(|mut s| {
s.points.iter_mut().for_each(|p| *p = [p[0] * 3.0, p[1] * 3.0]);
s.radius *= 3.0;
s
})
.collect();
let refs: Vec<&Stroke> = st.iter().collect();
let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0);
let gt: Vec<u8> =
(0..W * H * 9).map(|i| ((((i % (W * 3)) as f64 + 0.5) / 3.0 - c[0]).hypot((((i / (W * 3)) as f64 + 0.5) / 3.0) - c[1]) <= R) as u8).collect();
let j = iou(&seg.matte, >);
assert!(j > 0.95, "IoU {j}");
}
#[test]
fn propagation_follows_the_disk_for_20_frames() {
let (img0, _) = frame_at(center(0));
let st = base_strokes(0);
let refs: Vec<&Stroke> = st.iter().collect();
let opts = SegOpts::default();
let mut seg = segment(&img0, &refs, None, &opts, 0.0);
let mut prev = img0;
let mut worst: f64 = 1.0;
for f in 1..=20 {
let (img, gt) = frame_at(center(f));
seg = propagate(&prev, &seg, &img, &[], &opts, 0.0);
let j = iou(&seg.matte, >);
worst = worst.min(j);
prev = img;
}
assert!(worst > 0.9, "worst IoU {worst}");
}
#[test]
fn a_correction_stroke_fixes_a_bad_frame() {
let opts = SegOpts::default();
let (img0, _) = frame_at(center(0));
let st = base_strokes(0);
let refs: Vec<&Stroke> = st.iter().collect();
let seg0 = segment(&img0, &refs, None, &opts, 0.0);
// The disk jumps far away on frame 1: propagation cannot follow.
let jump = [150.0, 90.0];
let (img1, gt1) = frame_at(jump);
let bad = propagate(&img0, &seg0, &img1, &[], &opts, 0.0);
assert!(iou(&bad.matte, >1) < 0.5, "the jump should break propagation");
// Paint the disk in and the old place out.
let c0 = center(0);
let fix = [
line(StrokeKind::Fg, 1, [jump[0] - 15.0, jump[1]], [jump[0] + 15.0, jump[1]], 5.0),
line(StrokeKind::Bg, 1, [c0[0] - 10.0, c0[1] - 10.0], [c0[0] + 10.0, c0[1] + 10.0], 6.0),
];
let fr: Vec<&Stroke> = fix.iter().collect();
let good = propagate(&img0, &seg0, &img1, &fr, &opts, 0.0);
let j = iou(&good.matte, >1);
assert!(j > 0.95, "corrected IoU {j}");
// And propagation continues from the corrected frame.
let c2 = [jump[0] + 3.0, jump[1] + 1.0];
let (img2, gt2) = frame_at(c2);
let next = propagate(&img1, &good, &img2, &[], &opts, 0.0);
assert!(iou(&next.matte, >2) > 0.93);
}
/// A stroke corrects the ones painted before it: foreground over an earlier background stroke
/// restores the area, and background over that removes it again.
#[test]
fn later_strokes_override_earlier_ones() {
let c = center(0);
let (img, gt) = frame_at(c);
let spot = [c[0], c[1] + 20.0];
let i = spot[1] as usize * W as usize + spot[0] as usize;
let red = line(StrokeKind::Bg, 0, [spot[0] - 6.0, spot[1]], [spot[0] + 6.0, spot[1]], 6.0);
let green = Stroke { kind: StrokeKind::Fg, ..red.clone() };
let seg = |st: &[Stroke]| segment(&img, &st.iter().collect::<Vec<_>>(), None, &SegOpts::default(), 0.0);
let mut st = base_strokes(0);
st.push(red.clone());
assert_eq!(seg(&st).matte[i], 0, "background over the disk removes it");
st.push(green);
let restored = seg(&st);
assert_eq!(restored.matte[i], 1, "foreground over the background stroke restores it");
assert!(iou(&restored.matte, >) > 0.95, "IoU {}", iou(&restored.matte, >));
st.push(red);
assert_eq!(seg(&st).matte[i], 0, "background again removes it again");
}
/// A soft-edged disk (alpha ramp over `width` pixels) over a background.
fn soft_frame(c: [f64; 2], width: f64, fg: [f32; 3], bg: [f32; 3]) -> (Image, Vec<f32>) {
let mut img = Image::new(W, H);
let mut gt = vec![0.0f32; (W * H) as usize];
for y in 0..H {
for x in 0..W {
let p = [x as f64 + 0.5, y as f64 + 0.5];
let d = (p[0] - c[0]).hypot(p[1] - c[1]);
let a = ((R + width / 2.0 - d) / width).clamp(0.0, 1.0) as f32;
// Faint texture on both layers.
let tf = (noise(p[0], p[1], 3) as f32 - 0.5) * 0.04;
let tb = (noise(p[0], p[1], 4) as f32 - 0.5) * 0.04;
let f = [fg[0] + tf, fg[1] + tf, fg[2] + tf];
let b = [bg[0] + tb, bg[1] + tb, bg[2] + tb];
img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]);
gt[(y * W + x) as usize] = a;
}
}
(img, gt)
}
fn ring(frame: i64, c: [f64; 2], r: f64, radius: f64) -> Stroke {
Stroke {
kind: StrokeKind::Refine,
frame,
radius,
points: (0..=48).map(|k| k as f64 / 48.0 * std::f64::consts::TAU).map(|a| [c[0] + r * a.cos(), c[1] + r * a.sin()]).collect(),
object: 0,
}
}
#[test]
fn refine_edge_recovers_a_soft_edge_and_decontaminates() {
let c = [100.0, 75.0];
let (fgc, bgc) = ([0.9f32, 0.25, 0.1], [0.1f32, 0.3, 0.9]);
let (img, gt) = soft_frame(c, 10.0, fgc, bgc);
let mut st = vec![
line(StrokeKind::Fg, 0, [c[0] - 15.0, c[1]], [c[0] + 15.0, c[1]], 5.0),
line(StrokeKind::Bg, 0, [c[0] - 60.0, c[1] - 55.0], [c[0] + 60.0, c[1] - 55.0], 5.0),
];
st.push(ring(0, c, R, 9.0));
let refs: Vec<&Stroke> = st.iter().collect();
let seg = segment(&img, &refs, None, &SegOpts::default(), 9.0);
assert!(seg.refine.iter().filter(|v| **v != 0).count() > 1000);
let p = MatteParams::default();
let rgb = rgb_of(&img);
let a = matte_alpha(&rgb, &seg, None, None, &p, 9.0, 1.0);
// Inside the ramp the matte is fractional and close to the truth.
let (mut err, mut n, mut frac) = (0.0f64, 0usize, 0usize);
let (mut err_bin, mut n_bin) = (0.0f64, 0usize);
for i in 0..a.len() {
if gt[i] > 0.0 && gt[i] < 1.0 {
err += (a[i] - gt[i]).abs() as f64;
err_bin += (seg.matte[i] as f32 - gt[i]).abs() as f64;
n += 1;
n_bin += 1;
frac += (a[i] > 0.05 && a[i] < 0.95) as usize;
}
}
let (err, err_bin) = (err / n as f64, err_bin / n_bin as f64);
assert!(err < 0.03, "mean alpha error {err} (binary {err_bin})");
assert!(err < err_bin * 0.4);
assert!(frac as f64 > 0.6 * n as f64, "{frac}/{n} fractional");
// Decontamination: edge pixels' colour is the foreground's, not a blue blend.
let mut col = rgb.clone();
let map = decontaminate(&mut col, &a, &seg.refine, (W as usize, H as usize), &p, 9.0, 1.0);
let (mut blue_before, mut blue_after, mut m) = (0.0f64, 0.0f64, 0usize);
for i in 0..a.len() {
if a[i] > 0.2 && a[i] < 0.8 {
blue_before += rgb[i][2] as f64;
blue_after += col[i][2] as f64;
m += 1;
}
}
let (bb, ba) = (blue_before / m as f64, blue_after / m as f64);
assert!(ba < 0.2 && bb > 0.35, "blue before {bb}, after {ba}");
assert!(map.iter().any(|v| *v > 0.2));
// Without the band, the binary matte passes through.
let mut seg2 = seg.clone();
seg2.refine.iter_mut().for_each(|v| *v = 0);
let a2 = matte_alpha(&rgb, &seg2, None, None, &p, 9.0, 1.0);
assert!(a2.iter().zip(&seg.matte).all(|(x, y)| (*x - *y as f32).abs() < 1e-6));
}
/// Refine Edge strokes correct each other the same way: refine over an earlier erase restores
/// the band there.
#[test]
fn later_refine_strokes_override_earlier_erases() {
let c = [100.0, 75.0];
let (img, _) = soft_frame(c, 10.0, [0.9, 0.25, 0.1], [0.1, 0.3, 0.9]);
let spot = [c[0] + R, c[1]];
let i = spot[1] as usize * W as usize + spot[0] as usize;
let erase = Stroke { kind: StrokeKind::RefineErase, frame: 0, radius: 6.0, points: vec![spot], object: 0 };
let refine = Stroke { kind: StrokeKind::Refine, ..erase.clone() };
let band = |st: &[Stroke]| segment(&img, &st.iter().collect::<Vec<_>>(), None, &SegOpts::default(), 9.0).refine[i];
let mut st = vec![
line(StrokeKind::Fg, 0, [c[0] - 15.0, c[1]], [c[0] + 15.0, c[1]], 5.0),
line(StrokeKind::Bg, 0, [c[0] - 60.0, c[1] - 55.0], [c[0] + 60.0, c[1] - 55.0], 5.0),
ring(0, c, R, 9.0),
];
assert_eq!(band(&st), 1);
st.push(erase.clone());
assert_eq!(band(&st), 0, "erase removes the band");
st.push(refine);
assert_eq!(band(&st), 1, "refine over the erase restores it");
st.push(erase);
assert_eq!(band(&st), 0, "erase again removes it again");
}
#[test]
fn reduce_chatter_and_motion_blur_use_the_neighbours() {
let (w, h) = (40usize, 20usize);
let mk = |x0: usize| FrameSeg {
w,
h,
matte: (0..w * h).map(|i| (i % w >= x0 && i % w < x0 + 10) as u8).collect(),
refine: vec![0; w * h],
..Default::default()
};
let (a, b, c) = (mk(10), mk(14), mk(18));
let rgb = vec![[0.5f32; 3]; w * h];
let p = MatteParams { rb_chatter: 100.0, ..Default::default() };
let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0);
// Pixel 12 is only in the previous frame: partially on.
assert!(al[12] > 0.2 && al[12] < 0.5, "{}", al[12]);
let p = MatteParams { motion_blur: true, ..Default::default() };
let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0);
assert!(al[14] > 0.3 && al[14] < 0.9, "{}", al[14]);
assert!(al[19] > 0.99);
}
#[test]
fn chain_keys_restart_at_correction_frames() {
let mut d = RotoData::default();
for s in base_strokes(5) {
d.add(s, 20, [0, 30]);
}
assert_eq!(d.base, Some(5));
assert_eq!(d.span, [0, 25]);
let k1 = d.chain_keys(1);
d.add(line(StrokeKind::Fg, 12, [0.0, 0.0], [1.0, 1.0], 3.0), 20, [0, 30]);
let k2 = d.chain_keys(1);
for f in 0..=11 {
assert_eq!(k1[&f], k2[&f], "frame {f}");
}
for f in 12..=25 {
assert_ne!(k1[&f], k2[&f], "frame {f}");
}
assert_ne!(d.chain_keys(2)[&5], k2[&5]);
let back = RotoData::from_json(&d.to_json());
assert_eq!(back, d);
assert_eq!(d.source_of(12), Some(11));
assert_eq!(d.source_of(3), Some(4));
assert_eq!(d.source_of(5), None);
}
/// `cargo test -p effectcraft-track --release roto_perf -- --ignored --nocapture`
#[test]
#[ignore]
fn roto_perf_1080p() {
let up = |c: [f64; 2]| {
let (img, _) = frame_at(c);
effectcraft_raster::resample(&img, 1920, 1080)
};
let s = 1920.0 / W as f64;
let st: Vec<Stroke> = base_strokes(0)
.into_iter()
.map(|mut k| {
k.points.iter_mut().for_each(|p| *p = [p[0] * s, p[1] * s]);
k.radius *= s;
k
})
.collect();
let refs: Vec<&Stroke> = st.iter().collect();
let opts = SegOpts { search_radius: 15.0 * s / 4.0, ..Default::default() };
let a = up(center(0));
let t = std::time::Instant::now();
let seg = segment(&a, &refs, None, &opts, 0.0);
eprintln!("segment 1080p: {:?}", t.elapsed());
let b = up(center(1));
let t = std::time::Instant::now();
let next = propagate(&a, &seg, &b, &[], &opts, 0.0);
eprintln!("propagate 1080p: {:?} (area {})", t.elapsed(), next.area());
let mut seg2 = next.clone();
let c = center(1);
let ring: Vec<[f64; 2]> = (0..=64).map(|k| k as f64 / 64.0 * std::f64::consts::TAU).map(|t| [(c[0] + R * t.cos()) * s, (c[1] + R * t.sin()) * s]).collect();
let st = [Stroke { kind: StrokeKind::Refine, frame: 1, radius: 20.0, points: ring, object: 0 }];
let r = rasterize(&st.iter().collect::<Vec<_>>(), &[StrokeKind::Refine], 1.0, 1920, 1080);
seg2.refine = r.iter().map(|v| *v as u8).collect();
let rgb = rgb_of(&b);
let t = std::time::Instant::now();
let _ = matte_alpha(&rgb, &seg2, None, None, &MatteParams::default(), 20.0, 1.0);
eprintln!("refine 1080p: {:?}", t.elapsed());
}
/// A stand-in trained model: the foreground is a fixed rectangle (or nothing, or an error).
struct FakeModel(Option<[usize; 4]>, bool);
impl effectcraft_segment::MaskModel for FakeModel {
fn info(&self) -> &'static effectcraft_segment::ModelInfo {
&effectcraft_segment::MOBILE_SAM
}
fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, prompt: &Prompt) -> Result<Vec<f32>, String> {
if self.1 {
return Err("broken".into());
}
assert!(prompt.points.iter().any(|(_, fg)| *fg), "prompted with foreground points");
Ok((0..w * h)
.map(|i| match self.0 {
Some([x0, y0, x1, y1]) => ((i % w) >= x0 && (i % w) < x1 && (i / w) >= y0 && (i / w) < y1) as u8 as f32 * 0.98 + 0.01,
None => 0.01,
})
.collect())
}
}
/// A grey bar on a two-tone background whose halves match the bar's halves: colour alone can't
/// find its edge, a model can. The frame, its truth and the bar's rectangle.
fn bar_frame() -> (Image, Vec<u8>, [usize; 4]) {
let (w, h) = (160usize, 100usize);
let rect = [40, 30, 120, 70];
let mut img = Image::new(w as u32, h as u32);
let mut gt = vec![0u8; w * h];
for y in 0..h {
for x in 0..w {
let inside = x >= rect[0] && x < rect[2] && y >= rect[1] && y < rect[3];
// Left half dark, right half light, inside or out; a faint stripe pattern inside.
let base = if x < w / 2 { 0.2 } else { 0.8 };
let v = if inside { base + 0.02 * ((x / 3) % 2) as f32 } else { base };
img.set(x as u32, y as u32, [v, v, v, 1.0]);
gt[y * w + x] = inside as u8;
}
}
(img, gt, rect)
}
#[test]
fn a_model_prior_shapes_the_cut_and_bad_models_are_ignored() {
let (img, gt, rect) = bar_frame();
let st = [line(StrokeKind::Fg, 0, [50.0, 50.0], [110.0, 50.0], 4.0)];
let refs: Vec<&Stroke> = st.iter().collect();
let opts = SegOpts::default();
let classical = segment(&img, &refs, None, &opts, 0.0);
let good = FakeModel(Some(rect), false);
let with = segment_with(&img, &refs, None, &opts, 0.0, Some(&good));
let (jc, jm) = (iou(&classical.matte, >), iou(&with.matte, >));
assert!(jm > 0.95 && jm > jc + 0.2, "model {jm} vs classical {jc}");
// A failing model: the classical result, unchanged.
let broken = FakeModel(None, true);
assert_eq!(segment_with(&img, &refs, None, &opts, 0.0, Some(&broken)), classical);
// Propagation: a model that loses the object (disagrees with the flow) is ignored.
let next = img.clone();
let lost = FakeModel(None, false);
let a = propagate_with(&img, &with, &next, &[], &opts, 0.0, Some(&lost));
let b = propagate(&img, &with, &next, &[], &opts, 0.0);
assert_eq!(a, b);
// A good model keeps the bar through propagation.
let kept = propagate_with(&img, &with, &next, &[], &opts, 0.0, Some(&good));
assert!(iou(&kept.matte, >) > 0.95, "{}", iou(&kept.matte, >));
}
/// With the real MobileSAM weights (`EFFECTCRAFT_MOBILESAM=path/to/mobile_sam.pt`, else
/// skipped): the base frame and 20 propagated frames of the moving textured disk stay at least
/// as accurate as the classic engine.
#[test]
fn mobilesam_segments_and_propagates_the_disk() {
let Ok(path) = std::env::var("EFFECTCRAFT_MOBILESAM") else { return };
let effectcraft_segment::Loaded::Mask(model) = effectcraft_segment::load("mobilesam", &std::fs::read(path).unwrap()).unwrap() else {
panic!("not a mask model")
};
let m = Some(model.as_ref());
let (img0, gt0) = frame_at(center(0));
let st = base_strokes(0);
let refs: Vec<&Stroke> = st.iter().collect();
let opts = SegOpts::default();
let base = segment_with(&img0, &refs, None, &opts, 0.0, m);
let jb = iou(&base.matte, >0);
let (mut seg, mut cseg) = (base, segment(&img0, &refs, None, &opts, 0.0));
let mut prev = img0;
let (mut worst, mut cworst): (f64, f64) = (1.0, 1.0);
let t = std::time::Instant::now();
for f in 1..=20 {
let (img, gt) = frame_at(center(f));
seg = propagate_with(&prev, &seg, &img, &[], &opts, 0.0, m);
cseg = propagate(&prev, &cseg, &img, &[], &opts, 0.0);
worst = worst.min(iou(&seg.matte, >));
cworst = cworst.min(iou(&cseg.matte, >));
prev = img;
}
eprintln!("MobileSAM: base IoU {jb:.4}, worst propagated {worst:.4} (classic {cworst:.4}), {:.0} ms/frame", t.elapsed().as_secs_f64() * 1000.0 / 20.0);
assert!(jb > 0.95 && worst > 0.93 && worst >= cworst - 0.01, "base {jb}, worst {worst} vs classic {cworst}");
}
/// A box stroke becomes the model's box prompt (scaled, corners in any order); the graph cut
/// ignores it; a prior's box does not replace it.
#[test]
fn box_stroke_is_the_model_box_prompt() {
let fg = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: vec![[50.0, 40.0]], object: 0 };
let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[80.0, 60.0], [20.0, 10.0]], object: 0 };
let p = stroke_prompt(&[&fg, &bx], &[], 0.5, 0, 0);
assert_eq!(p.bbox, Some([10.0, 5.0, 40.0, 30.0]));
assert_eq!(p.points, vec![([25.0, 20.0], true)]);
// Degenerate or non-finite boxes are no prompt.
for pts in [vec![[20.0, 10.0]], vec![[20.0, 10.0], [20.0, 50.0]], vec![[f64::NAN, 10.0], [80.0, 60.0]]] {
let b = Stroke { points: pts, ..bx.clone() };
assert_eq!(stroke_prompt(&[&fg, &b], &[], 1.0, 0, 0).bbox, None);
}
assert!(!StrokeKind::Box.is_segmentation());
assert!(rasterize(&[&bx], &[StrokeKind::Fg, StrokeKind::Bg], 1.0, 100, 80).iter().all(|v| !v));
let mut q = stroke_prompt(&[&fg, &bx], &[], 1.0, 0, 0);
let soft: Vec<f32> = (0..100 * 80).map(|i| ((i % 100) < 10) as u8 as f32).collect();
assert!(add_prior_prompt(&mut q, &soft, &[], 100, 80));
assert_eq!(q.bbox, Some([20.0, 10.0, 80.0, 60.0]));
}
/// A model is never prompted against the latest correction: stroke points that a later stroke
/// of the other kind painted over are dropped, and the propagated matte's points and mask leave
/// out what this frame's background strokes removed.
#[test]
fn model_prompts_follow_the_latest_stroke() {
let green = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 4.0, points: vec![[50.0, 40.0]], object: 0 };
let red = Stroke { kind: StrokeKind::Bg, ..green.clone() };
let (w, h) = (100, 80);
let prompt = |st: &[&Stroke]| stroke_prompt(st, &resolve(st, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h), 1.0, w, h).points;
assert_eq!(prompt(&[&green, &red]), vec![([50.0, 40.0], false)]);
assert_eq!(prompt(&[&green, &red, &green]), vec![([50.0, 40.0], true); 2]);
let soft: Vec<f32> = (0..w * h).map(|i| ((i % w) < 50) as u8 as f32).collect();
let erase = Stroke { kind: StrokeKind::Bg, frame: 0, radius: 15.0, points: vec![[25.0, 40.0]], object: 0 };
let hard = resolve(&[&erase], StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);
let mut q = Prompt::default();
assert!(add_prior_prompt(&mut q, &soft, &hard, w, h));
assert!(!q.points.is_empty());
assert!(q.points.iter().all(|(p, fg)| *fg && owner(&hard, w, h, *p) != Some(HARD_BG)), "{:?}", q.points);
let mask = q.mask.unwrap();
assert!(hard.iter().zip(&mask).all(|(hd, m)| *hd != HARD_BG || *m == 0.0));
}
/// A stand-in model that counts its calls; its foreground is a fixed rectangle.
struct CountingModel([usize; 4], std::sync::atomic::AtomicUsize);
impl effectcraft_segment::MaskModel for CountingModel {
fn info(&self) -> &'static effectcraft_segment::ModelInfo {
&effectcraft_segment::MOBILE_SAM
}
fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, _: &Prompt) -> Result<Vec<f32>, String> {
self.1.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let [x0, y0, x1, y1] = self.0;
Ok((0..w * h).map(|i| ((i % w) >= x0 && (i % w) < x1 && (i / w) >= y0 && (i / w) < y1) as u8 as f32 * 0.98 + 0.01).collect())
}
}
/// A small background correction in the middle of a large foreground dot removes the only point
/// sampled along the dot, but most of the dot is still foreground: the model is still prompted
/// with it, from a point inside what is left. A dot painted over completely prompts nothing.
#[test]
fn a_correction_keeps_the_model_prompted_with_what_is_left() {
let (img, gt, rect) = bar_frame();
let (w, h) = (img.width as usize, img.height as usize);
let dot = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 15.0, points: vec![[80.0, 50.0]], object: 0 };
let tap = Stroke { kind: StrokeKind::Bg, radius: 2.0, ..dot.clone() };
let refs = [&dot, &tap];
let model = CountingModel(rect, Default::default());
let seg = segment_with(&img, &refs, None, &SegOpts::default(), 0.0, Some(&model));
assert_eq!(model.1.load(std::sync::atomic::Ordering::Relaxed), 1, "the model was prompted");
assert!(iou(&seg.matte, >) > 0.95, "IoU {}", iou(&seg.matte, >));
let hard = resolve(&refs, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);
let fg: Vec<[f32; 2]> = stroke_prompt(&refs, &hard, 1.0, w, h).points.into_iter().filter(|p| p.1).map(|p| p.0).collect();
let [q] = fg.as_slice() else { panic!("one foreground point, got {fg:?}") };
let r = (q[0] - 80.0).hypot(q[1] - 50.0);
assert!(r > 5.0 && r < 13.0, "{q:?} sits inside what is left of the dot");
assert_eq!(hard[q[1] as usize * w + q[0] as usize], HARD_FG);
let cover = Stroke { radius: 16.0, ..tap };
let refs = [&dot, &cover];
let hard = resolve(&refs, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);
assert!(stroke_prompt(&refs, &hard, 1.0, w, h).points.iter().all(|p| !p.1));
}
#[test]
fn tracker_prompts_carry_a_handful_of_points() {
let st = base_strokes(0);
let refs: Vec<&Stroke> = st.iter().collect();
let p = tracker_prompt(&refs, 1.0, W as usize, H as usize);
let (fg, bg) = (p.points.iter().filter(|(_, f)| *f).count(), p.points.iter().filter(|(_, f)| !*f).count());
assert!((1..=TRACKER_FG_POINTS).contains(&fg) && (1..=TRACKER_BG_POINTS).contains(&bg), "{fg} fg, {bg} bg");
// a long, finely sampled stroke still gives at most the limit, and an empty one gives none
let long = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: (0..400).map(|i| [i as f64 * 0.4, 50.0]).collect(), object: 0 };
assert!(tracker_prompt(&[&long], 1.0, W as usize, H as usize).points.len() <= TRACKER_FG_POINTS);
assert!(tracker_prompt(&[], 1.0, W as usize, H as usize).points.is_empty());
}
/// Tracker clicks follow the same correction order as the classic matte and model prompts.
#[test]
fn tracker_prompts_follow_latest_corrections_and_keep_remaining_foreground() {
let (w, h) = (100, 80);
let green = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 15.0, points: vec![[50.0, 40.0]], object: 0 };
let red = Stroke { kind: StrokeKind::Bg, radius: 16.0, ..green.clone() };
for strokes in [&[&green, &red][..], &[&red, &green][..]] {
let classic = FrameSeg::empty(w, h);
for prompt in [tracker_prompt(strokes, 1.0, w, h), tracker_prompt_boxed(strokes, 1.0, &classic)] {
let expected = std::ptr::eq(strokes[1], &green);
assert_eq!(prompt.points.iter().any(|(_, fg)| *fg), expected, "{:?}", prompt.points);
let hard = resolve(strokes, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);
for (point, fg) in &prompt.points {
assert_eq!(owner(&hard, w, h, *point), Some(if *fg { HARD_FG } else { HARD_BG }));
}
assert!(!prompt.points.is_empty());
}
}
// A small correction removes the sampled centre, while most of the foreground dot remains.
let tap = Stroke { radius: 2.0, ..red };
let strokes = [&green, &tap];
let hard = resolve(&strokes, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);
for prompt in [tracker_prompt(&strokes, 1.0, w, h), tracker_prompt_boxed(&strokes, 1.0, &FrameSeg::empty(w, h))] {
assert!(prompt.points.iter().any(|(_, fg)| *fg));
assert!(prompt.points.iter().any(|(_, fg)| !*fg));
for (point, fg) in &prompt.points {
assert_eq!(owner(&hard, w, h, *point), Some(if *fg { HARD_FG } else { HARD_BG }));
}
assert!(prompt.points.iter().filter(|(_, fg)| *fg).count() <= TRACKER_FG_POINTS);
assert!(prompt.points.iter().filter(|(_, fg)| !*fg).count() <= TRACKER_BG_POINTS);
}
}
#[test]
fn tracked_segmentation_handles_absent_objects_and_bad_maps() {
let (img, gt) = frame_at(center(0));
let st = base_strokes(0);
let refs: Vec<&Stroke> = st.iter().collect();
let opts = SegOpts::default();
let n = (W * H) as usize;
// a good map (the true disk) reproduces it, with the strokes as constraints
let soft: Vec<f32> = gt.iter().map(|v| f32::from(*v)).collect();
let seg = segment_tracked(&img, &refs, soft, false, &opts, 0.0);
assert!(iou(&seg.matte, >) > 0.95);
// an absent object (no strokes on the frame) is an empty matte, whatever the map says
let none = segment_tracked(&img, &[], vec![1.0; n], true, &opts, 0.0);
assert_eq!(none.area(), 0);
assert_eq!(segment_tracked(&img, &[], vec![0.0; n], false, &opts, 0.0).area(), 0);
// a map of the wrong size is ignored: the strokes alone segment the frame
let wrong = segment_tracked(&img, &refs, vec![1.0; 3], false, &opts, 0.0);
assert!(iou(&wrong.matte, >) > 0.95);
}
#[test]
fn tracker_prompts_carry_the_box_of_the_classic_matte() {
let (img, gt) = frame_at(center(0));
let st = base_strokes(0);
let refs: Vec<&Stroke> = st.iter().collect();
let classic = segment(&img, &refs, None, &SegOpts::default(), 0.0);
let p = tracker_prompt_boxed(&refs, 1.0, &classic);
let b = p.bbox.expect("a box");
let c = center(0);
// the box reaches the disk's edges, with a margin
assert!(b[0] <= (c[0] - R) as f32 && b[0] > (c[0] - R) as f32 - 12.0, "{b:?}");
assert!(b[2] >= (c[0] + R) as f32 && b[2] < (c[0] + R) as f32 + 12.0, "{b:?}");
assert!(b[1] <= (c[1] - R) as f32 && b[3] >= (c[1] + R) as f32, "{b:?}");
assert!(iou(&classic.matte, >) > 0.95);
// no matte, no box; a box the user drew wins
let none = tracker_prompt_boxed(&refs, 1.0, &FrameSeg::empty(W as usize, H as usize));
assert_eq!(none.bbox, None);
let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[10.0, 10.0], [50.0, 60.0]], object: 0 };
let mut with_box: Vec<&Stroke> = refs.clone();
with_box.push(&bx);
assert_eq!(tracker_prompt_boxed(&with_box, 1.0, &classic).bbox, Some([10.0, 10.0, 50.0, 60.0]));
}
#[test]
fn strokes_carry_an_object_and_old_data_is_object_zero() {
// saved before objects existed: no field, and the field is not written for object 0
let old = r#"{"base":0,"span":[0,4],"strokes":[{"kind":"fg","frame":0,"radius":3.0,"points":[[1.0,2.0]]}]}"#;
let d = RotoData::from_json(old);
assert_eq!(d.strokes.first().map(|s| s.object), Some(0));
assert!(!d.to_json().contains("object"), "{}", d.to_json());
// the keys of existing projects do not change
let same = RotoData::from_json(&d.to_json());
assert_eq!(d.chain_keys(1), same.chain_keys(1));
let mut d = d;
d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[9.0, 9.0]], object: 2 }, 4, [0, 10]);
d.add(Stroke { kind: StrokeKind::Bg, frame: 0, radius: 3.0, points: vec![[0.0, 0.0]], object: 2 }, 4, [0, 10]);
d.add(Stroke { kind: StrokeKind::Refine, frame: 0, radius: 3.0, points: vec![[5.0, 5.0]], object: 0 }, 4, [0, 10]);
assert_eq!(d.objects(), vec![0, 2]);
assert!(d.to_json().contains("\"object\":2"));
assert_eq!(RotoData::from_json(&d.to_json()), d);
// an object's own strokes, plus the Refine Edge strokes that belong to the whole matte
assert_eq!(d.strokes_of(0, 2).len(), 3);
assert_eq!(d.strokes_of(0, 0).len(), 2);
assert!(d.strokes_of(0, 1).len() == 1);
// moving a stroke to another object changes the frame keys (the tracked frames are stale)
let before = d.chain_keys(1);
if let Some(s) = d.strokes.first_mut() {
s.object = 1;
}
assert_ne!(before, d.chain_keys(1));
}
#[test]
fn the_union_of_objects_is_their_largest_extent() {
let seg = |cells: &[usize]| {
let mut s = FrameSeg::empty(4, 1);
for c in cells {
s.matte[*c] = 255;
}
s
};
let u = FrameSeg::union(vec![seg(&[0]), seg(&[2, 3]), seg(&[3])]);
assert_eq!(u.matte, vec![255, 0, 255, 255]);
assert_eq!(FrameSeg::union(vec![]).w, 0);
// a part of another size is ignored
let odd = FrameSeg::empty(2, 2);
assert_eq!(FrameSeg::union(vec![seg(&[1]), odd]).matte, vec![0, 255, 0, 0]);
}
#[test]
fn a_stored_span_reaches_a_bounded_number_of_frames() {
// A project can store any span: the chain stops MAX_CHAIN frames either side of the base.
let d = RotoData { base: Some(0), span: [i64::MIN, i64::MAX], strokes: vec![] };
let k = d.chain_keys(1);
assert_eq!(k.len() as i64, 2 * super::MAX_CHAIN + 1);
assert_eq!(k.keys().next(), Some(&-super::MAX_CHAIN));
}
#[test]
fn in_span_matches_the_frames_that_have_keys_and_extreme_bases_do_not_overflow() {
let d = RotoData { base: Some(0), span: [i64::MIN, i64::MAX], strokes: vec![] };
assert!(d.in_span(super::MAX_CHAIN) && !d.in_span(super::MAX_CHAIN + 1));
assert!(d.in_span(-super::MAX_CHAIN) && !d.in_span(-super::MAX_CHAIN - 1));
assert_eq!(d.source_of(super::MAX_CHAIN + 1), None);
assert!(d.chain_keys(1).keys().all(|f| d.in_span(*f)));
// A base at the end of i64: no forward frames, and no `base + 1` overflow.
let d = RotoData { base: Some(i64::MAX), span: [i64::MIN, i64::MAX], strokes: vec![] };
let k = d.chain_keys(1);
assert_eq!(k.len() as i64, super::MAX_CHAIN + 1);
assert!(k.contains_key(&i64::MAX));
}
/// A scene for the band re-cut: `rgb` of a `w`×`h` frame where the foreground is `inside`, with
/// a background that turns red left of `x = 30` (far from most shapes), and a soft prior that is
/// `inside` shifted by (3, 2) px with an edge `soft_edge` px wide.
fn band_scene(w: usize, h: usize, inside: &dyn Fn(f64, f64) -> f64, soft_edge: f64) -> (Vec<[f32; 3]>, Vec<f32>) {
let mut rgb = vec![[0.0f32; 3]; w * h];
let mut soft = vec![0.0f32; w * h];
for y in 0..h {
for x in 0..w {
let (px, py) = (x as f64 + 0.5, y as f64 + 0.5);
let a = inside(px, py).clamp(0.0, 1.0) as f32;
let f = fg_color(px, py);
let b = if px < 30.0 { [0.9, 0.1, 0.1] } else { bg_color(px, py) };
rgb[y * w + x] = [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a)];
soft[y * w + x] = (inside(px - 3.0, py - 2.0) / soft_edge + 0.5).clamp(0.0, 1.0) as f32;
}
}
(rgb, soft)
}
#[test]
fn band_crop_matches_the_full_frame() {
let (w, h) = (200usize, 150usize);
// Signed coverage: > 0.5 inside, falling off by 1 per pixel across the edge.
let disk = |cx: f64, cy: f64, r: f64| move |x: f64, y: f64| r + 0.5 - (x - cx).hypot(y - cy);
let mid = disk(120.0, 80.0, 30.0);
let corner = disk(185.0, 10.0, 28.0);
let donut = move |x: f64, y: f64| (40.0 + 0.5 - (x - 110.0).hypot(y - 75.0)).min((x - 110.0).hypot(y - 75.0) - 14.0 + 0.5);
let nothing = |_: f64, _: f64| -10.0;
let shapes: [(&str, &dyn Fn(f64, f64) -> f64, f32, f64); 8] = [
("disk", &mid, 8.0, 1.0),
("frame corner", &corner, 8.0, 1.0),
("donut", &donut, 6.0, 1.0),
("r = 2, soft edge", &mid, 2.0, 3.0),
// A prior edge wider than the band: the blur sees prior values outside it.
("r = 2, wide soft edge", &mid, 2.0, 10.0),
("r = 3, wide soft edge at the corner", &corner, 3.0, 14.0),
("r = 12, soft edge", &mid, 12.0, 4.0),
("no foreground", ¬hing, 8.0, 1.0),
];
let mut changed = 0;
for (name, inside, r, edge) in shapes {
let (rgb, soft) = band_scene(w, h, inside, edge);
let start: Vec<u8> = soft.iter().map(|v| (*v >= 0.5) as u8).collect();
let fitted = fit_models(&rgb, &start);
// Equal models leave the cut to the prior and the smoothness term, which makes it
// sensitive to the blurred prior.
for models in [None, Some((&fitted.0, &fitted.1)), Some((&fitted.1, &fitted.1))] {
let (mut full, mut crop) = (start.clone(), start.clone());
recut_band(&rgb, (w, h), &mut full, &soft, r, models, false);
recut_band(&rgb, (w, h), &mut crop, &soft, r, models, true);
let diff = full.iter().zip(&crop).filter(|(a, b)| a != b).count();
assert_eq!(diff, 0, "{name}, models {}: {diff} pixels differ", if models.is_some() { "given" } else { "fitted" });
changed += full.iter().zip(&start).filter(|(a, b)| a != b).count();
}
}
// The cut did move edges (the prior is offset from the shapes).
assert!(changed > 100, "{changed}");
}
/// Padded by the blur's reach, the crop blurs the prior exactly as the full frame does at every
/// band pixel (up to the summation rounding of the box sums); a 1 px pad does not when the prior
/// edge is wider than the band.
#[test]
fn band_rect_pad_covers_the_prior_blur() {
let (w, h) = (200usize, 150usize);
let mid = |x: f64, y: f64| 30.5 - (x - 120.0).hypot(y - 80.0);
let corner = |x: f64, y: f64| 28.5 - (x - 185.0).hypot(y - 10.0);
let cases: [(&dyn Fn(f64, f64) -> f64, f32, f64); 4] = [(&mid, 8.0, 1.0), (&mid, 2.0, 10.0), (&corner, 3.0, 14.0), (&mid, 12.0, 30.0)];
for (inside, r, edge) in cases {
let (_, soft) = band_scene(w, h, inside, edge);
let labels: Vec<u8> = soft.iter().map(|v| (*v >= 0.5) as u8).collect();
let sd = matting::signed_distance(&labels, w, h);
let sigma = (r as f64 / 4.0).max(1.0);
let full = matting::gauss(&soft, w, h, sigma);
let worst = |pad: usize| -> f32 {
let rect = band_rect(&sd, (w, h), r, pad).unwrap();
let [x0, y0, x1, y1] = rect;
let blurred = matting::gauss(&crop_of(&soft, w, rect), x1 - x0, y1 - y0, sigma);
let mut worst = 0.0f32;
for y in y0..y1 {
for x in x0..x1 {
if sd[y * w + x].abs() <= r {
worst = worst.max((blurred[(y - y0) * (x1 - x0) + x - x0] - full[y * w + x]).abs());
}
}
}
worst
};
let reach = matting::gauss_reach(sigma);
assert!(worst(reach) < 1e-6, "r {r}, edge {edge}: {}", worst(reach));
if edge > 2.0 * r as f64 {
assert!(worst(1) > 1e-4, "r {r}, edge {edge}: a 1 px pad should differ, {}", worst(1));
}
}
}
#[test]
fn propagate_survives_empty_frames() {
let opts = SegOpts::default();
for (w, h) in [(4u32, 0u32), (0, 4), (0, 0)] {
let img = Image::new(w, h);
let prev = FrameSeg::empty(w as usize, h as usize);
let next = propagate(&img, &prev, &img, &[], &opts, 0.0);
assert_eq!((next.w, next.h), (w as usize, h as usize));
assert!(next.matte.is_empty());
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.