f67d38b35cc5
scene_detect.rs
//! Clip ▸ Scene Edit Detection… (M3.11).
//!
//! `clip.sceneEditDetection {clips?, sensitivity=50, minShotFrames=6, applyCuts=true,
//! createSubclips=false, generateMarkers=false, wait=false}` analyses the selected timeline video
//! clips in a background job (`jobs.list` shows the progress, `jobs.cancel` stops it without
//! changing anything). Every sequence frame of each clip is decoded at reduced size and compared
//! with the previous one ([`filmcraft_render::scene`]). When the job finishes, the results are
//! applied in one undo step ("Scene Edit Detection"):
//!
//! - **Apply a cut at each detected cut point**: the clip (and its linked audio) is cut at the
//! first frame of each new shot;
//! - **Create a bin of subclips from each detected cut point**: one subclip per shot, in a new
//! bin "<clip> Scenes";
//! - **Create a clip marker at each detected cut point**: a marker on the clip's master clip
//! (media time) at each cut.
use std::sync::{Arc, Mutex};
use filmcraft_project::{ClipId, ItemId, ItemKind, Label, Marker, MarkerId, MarkerKind, TrackKind};
use filmcraft_render::scene::{SceneDetector, SceneOptions};
use filmcraft_time::{Tick, TimeRange};
use serde_json::{Value, json};
use crate::commands::{CommandSpec, bad, bool_p, clips_p, f64_p, has_seq, u64_p};
use crate::{EngineError, Result, Session};
/// Frames one clip is analysed for at most (24 h at 60 fps).
pub(crate) const MAX_ANALYSED_FRAMES: i64 = 24 * 3600 * 60;
/// Frames the clips of one job are analysed for in all: repeated or many clips cannot sum past one clip's cap.
pub(crate) const MAX_TOTAL_ANALYSED_FRAMES: i64 = MAX_ANALYSED_FRAMES;
/// Frames to schedule for a clip of `duration`, spent from `remaining` (the job's budget). Checked before any
/// schedule is built, so the memory a job asks for stays bounded.
pub(crate) fn scheduled_frames(duration: Tick, frame_duration: Tick, remaining: &mut i64) -> Result<i64> {
let n = (duration.0 / frame_duration.0.max(1)).clamp(1, MAX_ANALYSED_FRAMES);
if n > *remaining {
return Err(bad("clip.sceneEditDetection", "the selected clips are too long to analyse in one job; select fewer or shorter clips"));
}
*remaining -= n;
Ok(n)
}
/// Longest side of the frames analysed (speed; the measures don't need detail).
const ANALYSIS_WIDTH: f64 = 256.0;
pub(crate) fn commands() -> Vec<CommandSpec> {
vec![CommandSpec {
id: "clip.sceneEditDetection",
label: "Scene Edit Detection…",
menu: &["Clip"],
shortcut: None,
params: r#"{"clips":[id]?,"sensitivity":0..100=50,"minShotFrames":n=6,"applyCuts":bool=true,"createSubclips":bool=false,"generateMarkers":bool=false,"wait":bool=false}"#,
enabled: has_video_selection,
run: detect,
journal: true,
named: crate::commands::NamedKeys::NONE,
}]
}
fn has_video_selection(s: &Session) -> std::result::Result<(), String> {
has_seq(s)?;
let q = s.active_sequence().ok_or("no sequence is open")?;
let any = s.state.selection.iter().any(|c| q.find_item(*c).is_some_and(|(t, _)| q.track(t).is_some_and(|t| t.kind == TrackKind::Video)));
if any { Ok(()) } else { Err("select a video clip in the timeline".into()) }
}
/// What to do with the cuts found.
#[derive(Clone, Copy, Debug, Default)]
pub struct SceneApply {
pub cuts: bool,
pub subclips: bool,
pub markers: bool,
}
/// One analysed clip.
#[derive(Clone, Debug)]
pub struct ClipScenes {
pub clip: ClipId,
pub name: String,
/// The clip's media item (a subclip's parent).
pub media_item: ItemId,
/// Media range the clip shows. When the analysis was capped (`unanalysed` set) it reaches the
/// clip's end, so the last subclip covers the tail too.
pub media_range: TimeRange,
/// The clip is longer than the analysis cap: its tail was not analysed, and no cut was looked for there.
pub unanalysed: bool,
/// (timeline time, media time) of the first frame of each new shot.
pub cuts: Vec<(Tick, Tick)>,
}
/// A running Scene Edit Detection job; applied by [`poll`] when it finishes.
pub struct PendingScene {
pub job: u64,
pub seq: ItemId,
pub apply: SceneApply,
pub results: Arc<Mutex<Option<Vec<ClipScenes>>>>,
/// `Session::document_epoch` when the job started: ids mean something else in another project.
pub epoch: u64,
}
fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
/// The media item a clip shows (a subclip resolves to its parent: subclip clips use the parent's
/// media time).
fn media_root(p: &filmcraft_project::Project, item: ItemId) -> Option<ItemId> {
p.resolve_media(item).map(|(root, _, _)| root)
}
struct Work {
clip: ClipId,
name: String,
media_item: ItemId,
src: filmcraft_media::SharedSource,
scale: f32,
/// (timeline time, media time) of every sequence frame of the clip.
frames: Vec<(Tick, Tick)>,
/// Media time at the clip's end when the schedule was capped short of it.
tail: Option<Tick>,
}
/// The media range of an analysed clip: the scheduled frames from `a` to `b` (`fdm` long each),
/// stretched to `tail`, the clip's own end, when the schedule stopped short of it.
fn analysed_range(a: Tick, b: Tick, fdm: Tick, tail: Option<Tick>) -> TimeRange {
let (mut lo, mut hi) = (a.min(b), a.max(b) + fdm);
if let Some(t) = tail {
lo = lo.min(t);
hi = hi.max(t);
}
TimeRange::from_bounds(lo, hi)
}
/// What to tell the user about clips whose tail was not analysed.
fn unanalysed_warning(results: &[ClipScenes]) -> Option<String> {
let names: Vec<&str> = results.iter().filter(|r| r.unanalysed).map(|r| r.name.as_str()).collect();
(!names.is_empty()).then(|| {
format!("The tail of {} was not analysed (longer than {MAX_ANALYSED_FRAMES} frames); its last scene extends to the clip end", names.join(", "))
})
}
fn detect(s: &mut Session, p: &Value) -> Result<Value> {
let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?;
let q = s.active_sequence().ok_or(EngineError::NoSequence)?;
let fd = q.settings.frame_rate.frame_duration();
let opts = SceneOptions {
sensitivity: f64_p(p, "sensitivity").unwrap_or(50.0).clamp(0.0, 100.0) as f32,
min_shot_frames: u64_p(p, "minShotFrames").unwrap_or(6).clamp(1, 10_000) as usize,
};
let apply = SceneApply {
cuts: bool_p(p, "applyCuts").unwrap_or(true),
subclips: bool_p(p, "createSubclips").unwrap_or(false),
markers: bool_p(p, "generateMarkers").unwrap_or(false),
};
if !(apply.cuts || apply.subclips || apply.markers) {
return Err(bad("clip.sceneEditDetection", "choose at least one of applyCuts, createSubclips, generateMarkers"));
}
if s.scene_jobs.iter().any(|j| j.seq == seq_id) {
return Err(EngineError::Other("Scene Edit Detection is already running on this sequence".into()));
}
let mut work = Vec::new();
let mut remaining = MAX_TOTAL_ANALYSED_FRAMES;
for c in clips_p(s, p) {
let Some((tid, it)) = q.find_item(c) else { continue };
if q.track(tid).is_none_or(|t| t.kind != TrackKind::Video) {
continue;
}
let Some(media_item) = media_root(&s.project, it.item) else { continue };
let Some(src) = s.source(it.item) else { continue };
let Some(v) = src.info().video.clone() else { continue };
// The schedule is built here, before the job can be cancelled; a project can store any
// clip duration, so it is capped (a day at 60 fps) rather than trusted.
let n = scheduled_frames(it.duration, fd, &mut remaining)?;
let capped = it.duration.0 / fd.0.max(1) > n;
// a reversed clip ends at its source in point (source_time_at(end) is one tick before it)
let tail = capped.then(|| if it.reverse { it.source_in } else { it.source_time_at(it.end()) });
let frames: Vec<(Tick, Tick)> = (0..n)
.map(|j| {
let t = it.start + Tick(fd.0 * j);
(t, it.source_time_at(t))
})
.collect();
work.push(Work { clip: c, name: it.name.clone(), media_item, src, scale: (ANALYSIS_WIDTH / v.width.max(1) as f64).min(1.0) as f32, frames, tail });
}
if work.is_empty() {
return Err(EngineError::Other("none of the selected clips is a video clip with media to analyse".into()));
}
let total: usize = work.iter().map(|w| w.frames.len()).sum();
let id = s.jobs.iter().map(|j| j.id).max().unwrap_or(0) + 1;
let label = format!("Scene Edit Detection ({} clip{})", work.len(), if work.len() == 1 { "" } else { "s" });
let job = crate::Job { id, label, progress: Default::default(), result: Default::default() };
job.progress.total.store(total as u64, std::sync::atomic::Ordering::Relaxed);
let results: Arc<Mutex<Option<Vec<ClipScenes>>>> = Arc::default();
let (prog, res, out) = (job.progress.clone(), job.result.clone(), results.clone());
let run = move || {
use std::sync::atomic::Ordering;
let t0 = web_time::Instant::now();
let mut done = 0u64;
let mut found = Vec::new();
let mut err = None;
'clips: for w in &work {
let mut det = SceneDetector::new();
for (k, (_, mt)) in w.frames.iter().enumerate() {
if prog.cancel.load(Ordering::Relaxed) {
err = Some("stopped".to_string());
break 'clips;
}
match w.src.video_frame(filmcraft_media::FrameRequest { time: *mt, scale: w.scale }) {
Ok(f) => {
det.push_rgba(&f.to_rgba8(), f.width as usize, f.height as usize);
}
Err(e) => {
err = Some(format!("{}: can't decode frame {k}: {e}", w.name));
break 'clips;
}
}
done += 1;
prog.done.store(done, Ordering::Relaxed);
*lock(&prog.status) = format!("{}: frame {} of {}", w.name, k + 1, w.frames.len());
}
let cuts: Vec<(Tick, Tick)> = det.cuts(&opts).into_iter().map(|i| w.frames[i]).collect();
let (a, b) = (w.frames.first().map(|f| f.1).unwrap_or_default(), w.frames.last().map(|f| f.1).unwrap_or_default());
let fdm = w.frames.get(1).map(|f| (f.1 - w.frames[0].1).abs()).unwrap_or(Tick(1)).max(Tick(1));
found.push(ClipScenes {
clip: w.clip,
name: w.name.clone(),
media_item: w.media_item,
media_range: analysed_range(a, b, fdm, w.tail),
unanalysed: w.tail.is_some(),
cuts,
});
}
let secs = t0.elapsed().as_secs_f64();
let n_cuts: usize = found.iter().map(|c| c.cuts.len()).sum();
*lock(&prog.status) = match &err {
Some(e) if e == "stopped" => "Stopped: nothing was changed".into(),
Some(e) => e.clone(),
None => format!("Found {n_cuts} cut point(s) in {done} frame(s) ({secs:.1}s)"),
};
let r = match err {
Some(e) => Err(e),
None => {
*lock(&out) = Some(found);
Ok(filmcraft_export::Report {
path: String::new(),
frames: done,
seconds: secs,
bytes: 0,
render_fps: done as f64 / secs.max(1e-6),
..Default::default()
})
}
};
*lock(&res) = Some(r);
prog.finished.store(true, Ordering::Relaxed);
};
let run = crate::export_tools::guard_job(job.progress.clone(), job.result.clone(), run);
s.jobs.push(job);
s.scene_jobs.push(PendingScene { job: id, seq: seq_id, apply, results: results.clone(), epoch: s.document_epoch });
let wait = bool_p(p, "wait").unwrap_or(false);
let mut out = json!({"job": id, "frames": total});
if wait || cfg!(target_arch = "wasm32") {
run();
if let Some(r) = lock(&results).as_ref() {
out["clips"] = json!(
r.iter()
.map(|c| json!({"clip": c.clip.0, "cuts": c.cuts.iter().map(|x| x.0.0).collect::<Vec<_>>(), "mediaCuts": c.cuts.iter().map(|x| x.1.0).collect::<Vec<_>>()}))
.collect::<Vec<_>>()
);
// poll applies the results and drops them, so its own warning list never reaches this reply
if let Some(w) = unanalysed_warning(r) {
out["warnings"] = json!([w]);
}
}
poll(s);
} else {
std::thread::Builder::new().name("filmcraft-scene-detect".into()).spawn(run).map_err(|e| EngineError::Other(e.to_string()))?;
}
if let Some(Err(e)) = s.jobs.iter().find(|j| j.id == id).and_then(|j| lock(&j.result).clone()) {
return Err(EngineError::Other(e));
}
Ok(out)
}
/// Apply finished jobs (one undo step each) and drop finished / cancelled ones. Called once per UI
/// frame from [`Session::poll_persistence`] and after synchronous runs.
pub fn poll(s: &mut Session) {
use std::sync::atomic::Ordering;
let mut i = 0;
while i < s.scene_jobs.len() {
let job = s.jobs.iter().find(|j| j.id == s.scene_jobs[i].job);
let finished = job.is_none_or(|j| j.progress.finished.load(Ordering::Relaxed));
// cancelled before the results were applied: nothing changes
let cancelled = job.is_some_and(|j| j.progress.cancel.load(Ordering::Relaxed));
if !finished {
i += 1;
continue;
}
let pj = s.scene_jobs.remove(i);
let Some(results) = lock(&pj.results).take().filter(|_| !cancelled) else { continue };
if pj.epoch != s.document_epoch {
s.error_toast("clip.sceneEditDetection", "Scene Edit Detection: the project was replaced; nothing was changed");
continue;
}
if let Err(e) = apply_results(s, pj.seq, pj.apply, results) {
s.error_toast("clip.sceneEditDetection", format!("Scene Edit Detection: {e}"));
}
}
}
fn apply_results(s: &mut Session, seq_id: ItemId, apply: SceneApply, results: Vec<ClipScenes>) -> Result<Value> {
let n_cuts: usize = results.iter().map(|r| r.cuts.len()).sum();
let warning = unanalysed_warning(&results);
let warnings: Vec<&String> = warning.iter().collect();
if n_cuts == 0 {
s.toast("Scene Edit Detection found no cut points");
if let Some(w) = &warning {
s.toast(w.clone());
}
return Ok(json!({"cuts": 0, "warnings": warnings}));
}
let media = s.media.clone();
s.edit("Scene Edit Detection", move |pr, _| {
let snapshot = pr.clone();
let durations = move |id: ItemId| crate::media_duration(&snapshot, &media, id);
for r in &results {
if apply.cuts {
let min = pr.sequence(seq_id).map(|q| q.settings.frame_rate.frame_duration()).unwrap_or(Tick(1));
let mut next = pr.next_id;
let q = pr.sequence_mut(seq_id).ok_or(EngineError::NoSequence)?;
let mut ctx = filmcraft_edit::EditCtx { next_id: &mut next, media_duration: &durations, media_start: &|_| Tick::ZERO, min_duration: min };
// latest cut first: the left piece keeps the clip's id for the earlier cuts
for (t, _) in r.cuts.iter().rev() {
let link = q.find_item(r.clip).and_then(|(_, it)| it.link);
let mut items = vec![r.clip];
if let Some(l) = link {
items.extend(q.all_tracks().flat_map(|tr| tr.items.iter()).filter(|it| it.link == Some(l) && it.id != r.clip).map(|it| it.id));
}
filmcraft_edit::razor_items(q, &items, *t, &mut ctx);
}
pr.next_id = next;
pr.sequence(seq_id).ok_or(EngineError::NoSequence)?.check().map_err(EngineError::Other)?;
}
if apply.markers {
let ids: Vec<u64> = r.cuts.iter().map(|_| pr.alloc_id()).collect();
if let Some(m) = pr.item_mut(r.media_item).and_then(|i| i.as_media_mut()) {
for ((_, mt), id) in r.cuts.iter().zip(ids) {
if m.markers.iter().any(|x| x.start == *mt) {
continue;
}
m.markers.push(Marker {
url: String::new(),
author: String::new(),
custom_color: None,
id: MarkerId(id),
start: *mt,
duration: Tick::ZERO,
name: "Scene Edit".into(),
comment: String::new(),
kind: MarkerKind::Comment,
color: Label::Green,
});
}
m.markers.sort_by_key(|x| x.start);
}
}
if apply.subclips {
let base = pr.item(r.media_item).map(|i| i.name.clone()).unwrap_or_else(|| r.name.clone());
let label = pr.item(r.media_item).map(|i| i.label).unwrap_or(Label::Iris);
let bin = pr.add_bin(&format!("{base} Scenes"), None);
let mut bounds: Vec<Tick> = vec![r.media_range.start];
bounds.extend(r.cuts.iter().map(|c| c.1).filter(|t| *t > r.media_range.start && *t < r.media_range.end()));
bounds.push(r.media_range.end());
bounds.sort();
bounds.dedup();
for (k, w) in bounds.windows(2).enumerate() {
let kind = ItemKind::Subclip { parent: r.media_item, range: TimeRange::from_bounds(w[0], w[1]), restrict_trims: false };
pr.add_item(&format!("{base} Scene {:02}", k + 1), label, kind, Some(bin));
}
}
}
Ok(())
})?;
s.toast(format!("Scene Edit Detection: {n_cuts} cut point(s)"));
if let Some(w) = &warning {
s.toast(w.clone());
}
Ok(json!({"cuts": n_cuts, "warnings": warnings}))
}
#[cfg(test)]
mod tests;
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.