665bfac0b339
track.rs
// Modified work: adapts the ported SAM 2 video tracker to EffectCraft's `ObjectTracker`
// interface (Apache-2.0, see NOTICE).
//! SAM 2 as the Roto Brush's object tracker: prompts on some frames, the whole clip tracked
//! forwards and backwards, each frame reported as it is finished.
use std::cell::RefCell;
use std::collections::HashMap;
use std::rc::Rc;
use std::sync::Arc;
use candle_core::Tensor;
use effectcraft_segment::{ClipFrame, ObjectTracker, TrackRequest, TrackedFrame};
use crate::mask::{Sam2Mask, preprocess, validate_frame_size};
use crate::prompt::{INPUT_SIZE, Prompt};
use crate::resize::mask_to_model;
use crate::session::{Cancel, Direction, FrameMask, FrameSource, ObjectId, VideoTracker};
use crate::{Error, Result};
/// Most frames one tracking request may cover.
const MAX_CLIP: usize = 100_000;
type Frames<'a> = &'a mut dyn FnMut(i64) -> Option<Arc<ClipFrame>>;
/// Feeds the host's frames to the tracker, remembering each frame's size for the output.
struct Source<'a> {
first: i64,
count: usize,
fetch: RefCell<Frames<'a>>,
sizes: Rc<RefCell<HashMap<usize, (usize, usize)>>>,
/// Prompt frames fetched only to learn their size, kept for the encoder (taken on first use).
held: RefCell<Held>,
}
/// Most pixel data (in bytes) kept from the size check of the prompt frames; frames beyond it are
/// fetched again when the tracker needs them.
const HELD_BYTES: usize = 256 << 20;
#[derive(Default)]
struct Held {
frames: HashMap<usize, Arc<ClipFrame>>,
bytes: usize,
}
impl Held {
fn keep(&mut self, index: usize, f: &Arc<ClipFrame>) {
let bytes = f.rgb.len().saturating_mul(std::mem::size_of::<[f32; 3]>());
let fits = self.bytes.checked_add(bytes).filter(|t| *t <= HELD_BYTES);
if let (Some(total), false) = (fits, self.frames.contains_key(&index)) {
self.frames.insert(index, f.clone());
self.bytes = total;
}
}
fn take(&mut self, index: usize) -> Option<Arc<ClipFrame>> {
let f = self.frames.remove(&index)?;
self.bytes = self.bytes.saturating_sub(f.rgb.len().saturating_mul(std::mem::size_of::<[f32; 3]>()));
Some(f)
}
}
impl FrameSource for Source<'_> {
fn num_frames(&self) -> usize {
self.count
}
fn frame(&self, index: usize) -> Result<Tensor> {
let number = i64::try_from(index)
.ok()
.filter(|_| index < self.count)
.and_then(|offset| self.first.checked_add(offset))
.ok_or_else(|| Error::Input(format!("frame index {index} is outside the clip")))?;
let held = self.held.borrow_mut().take(index);
let f = match held {
Some(f) => f,
None => (self.fetch.borrow_mut())(number).ok_or_else(|| Error::Input(format!("frame {number} is not available")))?,
};
self.sizes.borrow_mut().insert(index, (f.w, f.h));
preprocess(&f.rgb, f.w, f.h)
}
}
fn to_frame(m: &FrameMask, size: (usize, usize)) -> Result<TrackedFrame> {
let (w, h) = size;
let logits = Tensor::from_slice(&m.logits, (1, 1, 256, 256), &candle_core::Device::Cpu)?;
let prob = candle_nn::ops::sigmoid(&logits.upsample_bilinear2d(h, w, false)?)?.flatten_all()?.to_vec1::<f32>()?;
let presence = 1.0 / (1.0 + (-m.object_score).exp());
Ok(TrackedFrame { prob, w, h, confidence: (presence * m.iou.clamp(0.0, 1.0)).clamp(0.0, 1.0), occluded: m.occluded })
}
/// Most objects one request may track.
const MAX_OBJECTS: usize = 32;
/// Every prompt as `(object, frame index, prompt)`, in frame order (then object order), after
/// checking each prompt frame and recording its size in `source`. Frame order keeps the tracker's
/// two-frame feature cache useful: all objects prompted on a frame are encoded together, so each
/// prompt frame is encoded once however many objects there are. Objects do not influence each
/// other, so the order of prompts across objects does not change any result.
fn gather_prompts<'r>(req: &'r TrackRequest, source: &Source<'_>) -> Result<Vec<(usize, usize, &'r effectcraft_segment::Prompt)>> {
// Prompt coordinates are in the frame's pixels and the model works at 1024 x 1024, so the
// prompted frames' sizes are needed first.
let mut prompt_frames = vec![];
for (oi, prompts) in req.objects.iter().enumerate() {
for (frame, p) in prompts {
let idx = frame
.checked_sub(req.first)
.and_then(|offset| usize::try_from(offset).ok())
.filter(|i| *i < req.count)
.ok_or_else(|| Error::Input(format!("prompt frame {frame} is outside the clip")))?;
if !source.sizes.borrow().contains_key(&idx) {
let f = (source.fetch.borrow_mut())(*frame).ok_or_else(|| Error::Input(format!("frame {frame} is not available")))?;
validate_frame_size(f.w, f.h, f.rgb.len())?;
source.sizes.borrow_mut().insert(idx, (f.w, f.h));
source.held.borrow_mut().keep(idx, &f);
}
prompt_frames.push((oi, idx, p));
}
}
// stable: prompts of one object on one frame keep their order
prompt_frames.sort_by_key(|(oi, idx, _)| (*idx, *oi));
Ok(prompt_frames)
}
impl Sam2Mask {
fn run_track(&self, req: &TrackRequest, frames: Frames<'_>, cancel: &dyn Fn() -> bool, out: &mut dyn FnMut(usize, i64, TrackedFrame)) -> Result<()> {
if req.count == 0 || req.count > MAX_CLIP {
return Err(Error::Input(format!("a tracking request needs 1 to {MAX_CLIP} frames, got {}", req.count)));
}
req.first.checked_add((req.count - 1) as i64).ok_or_else(|| Error::Input("the clip's last frame overflows its frame number".into()))?;
if req.objects.is_empty() || req.objects.len() > MAX_OBJECTS || req.objects.iter().any(|o| o.is_empty()) {
return Err(Error::Input(format!("tracking needs 1 to {MAX_OBJECTS} objects, each with a prompt on at least one frame")));
}
let sizes = Rc::new(RefCell::new(HashMap::new()));
let source = Source { first: req.first, count: req.count, fetch: RefCell::new(frames), sizes: sizes.clone(), held: RefCell::default() };
let prompt_frames = gather_prompts(req, &source)?;
let mut tracker = VideoTracker::new(self.video_model().clone(), Box::new(source))?;
let object_id = |oi: usize| ObjectId(oi as u32 + 1);
// each object's first prompted frame: its pass starts there, forwards and backwards
let mut first_of: HashMap<usize, usize> = HashMap::new();
for (oi, idx, p) in prompt_frames {
if cancel() {
return Err(Error::Cancelled);
}
let (w, h) = sizes.borrow().get(&idx).copied().ok_or_else(|| Error::Compute("frame size unknown".into()))?;
// A prompt that is only a mask (probability per pixel of the frame) prompts with the mask.
if let (true, true, Some(m)) = (p.points.is_empty(), p.bbox.is_none(), p.mask.as_ref()) {
if m.len() != w * h {
return Err(Error::Input(format!("a mask prompt of {} values does not fit a {w}x{h} frame", m.len())));
}
let binary: Vec<u8> = m.iter().map(|v| u8::from(*v >= 0.5)).collect();
tracker.prompt_mask(idx, object_id(oi), &mask_to_model(&binary, w, h)?)?;
let e = first_of.entry(oi).or_insert(idx);
*e = (*e).min(idx);
continue;
}
let (sx, sy) = (INPUT_SIZE as f32 / w as f32, INPUT_SIZE as f32 / h as f32);
let prompt = Prompt {
points: p.points.iter().map(|(c, fg)| ([c[0] * sx, c[1] * sy], *fg)).collect(),
bbox: p.bbox.map(|b| [b[0] * sx, b[1] * sy, b[2] * sx, b[3] * sy]),
};
tracker.prompt(idx, object_id(oi), &prompt)?;
let e = first_of.entry(oi).or_insert(idx);
*e = (*e).min(idx);
}
let mut starts: Vec<(ObjectId, usize)> = first_of.into_iter().map(|(oi, idx)| (object_id(oi), idx)).collect();
starts.sort_by_key(|(id, _)| id.0);
let flag = Cancel::new();
let mut failure: Option<Error> = None;
let mut emit = |m: FrameMask| {
if cancel() {
flag.cancel();
}
let oi = (m.object.0 as usize).saturating_sub(1);
match to_frame(&m, sizes.borrow().get(&m.frame).copied().unwrap_or((0, 0))) {
Ok(t) => out(oi, req.first + m.frame as i64, t),
Err(e) => {
failure.get_or_insert(e);
flag.cancel();
}
}
};
// One pass each way over all the objects, frame by frame, so the host can finish a frame (and
// let go of its masks) as soon as every object has it, however far apart the prompts are.
let passes = (|| {
tracker.propagate_each(&starts, Direction::Forward, &flag, &mut emit)?;
if starts.iter().any(|(_, start)| *start > 0) {
tracker.propagate_each(&starts, Direction::Backward, &flag, &mut emit)?;
}
Ok(())
})();
self.peak.store(tracker.peak_state_bytes() as u64, std::sync::atomic::Ordering::Relaxed);
// A frame that could not be converted cancelled the passes; that is the error to report, not
// the cancellation it caused.
match failure {
Some(e) => Err(e),
None => passes,
}
}
}
impl ObjectTracker for Sam2Mask {
fn track(
&self,
req: &TrackRequest,
frames: &mut dyn FnMut(i64) -> Option<Arc<ClipFrame>>,
cancel: &dyn Fn() -> bool,
out: &mut dyn FnMut(usize, i64, TrackedFrame),
) -> effectcraft_segment::Result<()> {
self.touch();
let r = self.run_track(req, frames, cancel, out).map_err(|e| e.to_string());
self.touch();
r
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn source_rejects_overflowing_and_out_of_range_indices_before_fetching() {
let mut fetch = |_| panic!("invalid indices must not fetch a frame");
let source = Source { first: i64::MAX, count: 2, fetch: RefCell::new(&mut fetch), sizes: Default::default(), held: RefCell::default() };
assert!(matches!(source.frame(1), Err(Error::Input(_))));
assert!(matches!(source.frame(usize::MAX), Err(Error::Input(_))));
}
fn clip_frame(w: usize, h: usize) -> Arc<ClipFrame> {
Arc::new(ClipFrame { rgb: vec![[0.5, 0.25, 0.75]; w * h], w, h })
}
fn point_prompt() -> effectcraft_segment::Prompt {
effectcraft_segment::Prompt { points: vec![([1.0, 1.0], true)], bbox: None, mask: None }
}
/// Several objects prompted on the same frames: prompts come out frame by frame (so the
/// two-frame feature cache holds each prompt frame for all of its objects), and each prompt
/// frame is fetched from the host once, size check and encoding together.
#[test]
fn prompts_run_frame_major_and_each_prompt_frame_is_fetched_once() {
let frames_prompted = [10i64, 13, 16];
let req = TrackRequest { first: 10, count: 8, objects: (0..4).map(|_| frames_prompted.iter().map(|f| (*f, point_prompt())).collect()).collect() };
let fetched = RefCell::new(vec![]);
let mut fetch = |n: i64| {
fetched.borrow_mut().push(n);
Some(clip_frame(4, 3))
};
let source = Source { first: req.first, count: req.count, fetch: RefCell::new(&mut fetch), sizes: Default::default(), held: RefCell::default() };
let order: Vec<(usize, usize)> = gather_prompts(&req, &source).unwrap().into_iter().map(|(oi, idx, _)| (idx, oi)).collect();
let expected: Vec<(usize, usize)> = [0usize, 3, 6].iter().flat_map(|f| (0..4).map(move |o| (*f, o))).collect();
assert_eq!(order, expected);
assert_eq!(*fetched.borrow(), vec![10, 13, 16], "one size-check fetch per distinct prompt frame");
// the tracker's first look at each prompt frame reuses the frame already fetched
for idx in [0usize, 3, 6] {
source.frame(idx).unwrap();
}
assert_eq!(fetched.borrow().len(), 3, "encoding a prompt frame must not fetch it again");
// a held frame is handed over once; a second look fetches again
source.frame(0).unwrap();
assert_eq!(*fetched.borrow(), vec![10, 13, 16, 10]);
}
#[test]
fn prompts_of_one_object_on_one_frame_keep_their_order() {
let (mut a, mut b) = (point_prompt(), point_prompt());
a.points = vec![([1.0, 1.0], true)];
b.points = vec![([2.0, 2.0], false)];
let req = TrackRequest { first: 0, count: 3, objects: vec![vec![(2, a.clone()), (1, point_prompt()), (2, b.clone())]] };
let mut fetch = |_| Some(clip_frame(2, 2));
let source = Source { first: 0, count: 3, fetch: RefCell::new(&mut fetch), sizes: Default::default(), held: RefCell::default() };
let got = gather_prompts(&req, &source).unwrap();
assert_eq!(got.iter().map(|(_, i, _)| *i).collect::<Vec<_>>(), vec![1, 2, 2]);
assert_eq!(got[1].2.points, a.points);
assert_eq!(got[2].2.points, b.points);
}
#[test]
fn held_frames_are_bounded() {
let mut held = Held::default();
let big = Arc::new(ClipFrame { rgb: vec![[0.0; 3]; HELD_BYTES / 12 + 1], w: 1, h: HELD_BYTES / 12 + 1 });
held.keep(0, &big);
assert!(held.take(0).is_none(), "a frame over the cap is not kept");
let small = clip_frame(4, 4);
held.keep(1, &small);
assert!(held.take(1).is_some());
assert_eq!(held.bytes, 0);
// the cap is cumulative: frames that fit together are kept, the one that breaks it is not
let third = Arc::new(ClipFrame { rgb: vec![[0.0; 3]; HELD_BYTES / 12 / 3 + 1], w: 1, h: HELD_BYTES / 12 / 3 + 1 });
for i in 0..4 {
held.keep(i, &third);
}
assert_eq!(held.frames.len(), 2);
assert!(held.bytes <= HELD_BYTES);
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.