665bfac0b339
audio.rs
//! Real-time audio preview.
//!
//! The frontend owns no audio API: the native app supplies an [`AudioDevice`] (cpal) through
//! [`crate::Hooks::audio_device`], the web app one on Web Audio. During preview playback an
//! [`AudioPlayback`] runs a feeder thread (wasm32: [`AudioPlayback::pump`] every UI frame) that
//! mixes the composition ahead of the device with the same mixdown export uses
//! (`effectcraft_render::audio::mix_comp`: Audio switch, solo, Audio Levels, audio effects) and
//! pushes it into an [`AudioFeed`]; the device callback pulls from the feed.
//!
//! **A/V sync:** while audio plays, the audio clock drives playback. The clock is the number of
//! sample frames the device has actually consumed (underruns do not advance it) minus the
//! device's output latency, mapped through the work-area loop ([`clock_sample`]); the viewer
//! shows the comp frame at that time ([`AudioPlayback::clock_time`]), dropping video frames rather
//! than drifting.
//!
//! The feed also tracks per-channel peaks for the Audio panel's VU meters ([`Meter`]).
//!
//! **Scrubbing:** Ctrl/Cmd-dragging the current-time indicator plays a short snippet of the mix
//! at each new frame ([`AudioScrub`], `playback.scrubAudio`), as in After Effects.
use std::collections::{HashSet, VecDeque};
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use effectcraft_engine::project::{GroupKind, ItemId, ItemKind, Layer, LayerSource, Project, PropGroup};
use effectcraft_engine::time::{FrameRate, Tick};
use crate::frames::RenderSource;
mod preview_rate;
use preview_rate::PreviewRate;
/// An audio output (implemented by the native app with cpal).
pub trait AudioDevice: Send {
/// Device sample rate in Hz.
fn sample_rate(&self) -> u32;
/// Start pulling interleaved stereo frames from `feed` (see [`AudioFeed::pull`]).
fn start(&mut self, feed: Arc<AudioFeed>) -> Result<(), String>;
fn stop(&mut self);
/// Output latency in sample frames (device buffering), for A/V sync.
fn latency_frames(&self) -> u64 {
0
}
/// Called every UI frame while playing: devices without an audio thread of their own (Web
/// Audio) move queued samples to the output here.
fn pump(&mut self) {}
}
/// Opens the default output device (`None` when there is none).
pub type AudioDeviceFactory = Box<dyn Fn(&AudioOutput) -> Option<Box<dyn AudioDevice>>>;
/// Which output to open (Settings ▸ Audio): device name (empty = the system default) and the
/// 1-based device channels for left and right.
#[derive(Clone, Debug, PartialEq)]
pub struct AudioOutput {
pub device: String,
pub left: u32,
pub right: u32,
/// Preview Sample Rate: devices that support it are opened at this rate.
pub rate: u32,
}
impl AudioOutput {
pub fn from_prefs(p: &effectcraft_engine::prefs::Prefs) -> AudioOutput {
AudioOutput { device: p.audio.output_device.clone(), left: p.audio.output_left, right: p.audio.output_right, rate: p.audio.preview_sample_rate }
}
}
/// Lock-protected sample queue between the feeder thread and the device callback.
#[derive(Default)]
pub struct AudioFeed {
queue: Mutex<VecDeque<f32>>,
/// Sample frames handed to the device (real audio only: underruns do not count).
consumed: AtomicU64,
/// Peak |sample| per channel since the last [`AudioFeed::take_peaks`] (f32 bits).
peak: [AtomicU32; 2],
/// The feeder reached the end of a non-looping span.
finished: AtomicBool,
}
impl AudioFeed {
/// Fill `out` (interleaved stereo) from the queue; silence where the queue runs dry.
pub fn pull(&self, out: &mut [f32]) {
let mut real = 0usize;
let mut pk = [0.0f32; 2];
if let Ok(mut q) = self.queue.lock() {
let n = q.len().min(out.len()) & !1;
for (i, (o, s)) in out.iter_mut().zip(q.drain(..n)).enumerate() {
*o = s;
pk[i & 1] = pk[i & 1].max(s.abs());
}
real = n;
// Queue removal and the clock advance are one handoff: an edit must not restart
// behind samples already handed to the device.
self.consumed.fetch_add((real / 2) as u64, Ordering::Relaxed);
}
out[real..].iter_mut().for_each(|o| *o = 0.0);
for (c, v) in pk.into_iter().enumerate() {
// Rust 1.99 renamed `fetch_update` to `try_update`; keep the old name while the
// workspace supports Rust 1.95 (`rust-version`).
#[allow(deprecated)]
let _ = self.peak[c].fetch_update(Ordering::Relaxed, Ordering::Relaxed, |old| (v > f32::from_bits(old)).then_some(v.to_bits()));
}
}
/// Drop what is still queued (scrubbing replaces the snippet still playing).
pub fn clear(&self) {
if let Ok(mut q) = self.queue.lock() {
q.clear();
}
}
/// Discard an old mix and snapshot the first unconsumed frame under the same queue lock
/// as [`Self::pull`], so a concurrent device callback cannot leave the cursor behind.
fn discard_pending(&self) -> Option<u64> {
let mut q = self.queue.lock().ok()?;
q.clear();
Some(self.consumed())
}
pub fn push(&self, samples: &[f32]) {
if let Ok(mut q) = self.queue.lock() {
q.extend(samples.iter().copied());
}
}
/// Sample frames waiting in the queue.
pub fn queued_frames(&self) -> usize {
self.queue.lock().map(|q| q.len() / 2).unwrap_or(0)
}
pub fn consumed(&self) -> u64 {
self.consumed.load(Ordering::Relaxed)
}
/// Peak |sample| per channel since the last call (resets them).
pub fn take_peaks(&self) -> [f32; 2] {
[f32::from_bits(self.peak[0].swap(0, Ordering::Relaxed)), f32::from_bits(self.peak[1].swap(0, Ordering::Relaxed))]
}
pub fn finished(&self) -> bool {
self.finished.load(Ordering::Relaxed) && self.queued_frames() == 0
}
}
/// What a preview plays, in output sample coordinates on its playback timeline.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PlaySpan {
pub start: i64,
/// Work area [wa, wb).
pub wa: i64,
pub wb: i64,
pub looping: bool,
}
impl PlaySpan {
/// Span for comp times: start, work area [wa, wb).
pub fn new(start: Tick, wa: Tick, wb: Tick, looping: bool, rate: u32) -> PlaySpan {
let r = rate as i64;
PlaySpan { start: start.to_units_floor(r), wa: wa.to_units_floor(r), wb: wb.to_units_floor(r).max(wa.to_units_floor(r) + 1), looping }
}
}
/// The output sample coordinate audible now: `consumed` frames played by the device minus its
/// `latency`, from the span start, wrapped into the work area when looping (clamped at its end
/// otherwise).
pub fn clock_sample(span: &PlaySpan, consumed: u64, latency: u64) -> i64 {
// Slow previews can span more output samples than fit in a signed difference. Keep the
// elapsed count and loop width wide until the result is back inside the span.
let s = i128::from(span.start) + i128::from(consumed.saturating_sub(latency));
if s < i128::from(span.wb) {
return s as i64;
}
if span.looping {
let width = (i128::from(span.wb) - i128::from(span.wa)).max(1);
let wrapped = i128::from(span.wa) + (s - i128::from(span.wb)).rem_euclid(width);
wrapped.clamp(i64::MIN as i128, i64::MAX as i128) as i64
} else {
span.wb.saturating_sub(1)
}
}
/// Comp time of sample index `s` at `rate` Hz.
pub fn sample_time(s: i64, rate: u32) -> Tick {
Tick::from_units(s, rate.max(1) as i64)
}
/// The comp frame showing sample `s` (the frame whose interval contains it).
pub fn frame_of_sample(s: i64, rate: u32, fps: FrameRate) -> i64 {
fps.frame_at(sample_time(s, rate))
}
/// Block size the feeder mixes at a time.
const BLOCK: usize = 1024;
/// A running audio preview: device + feeder thread.
pub struct AudioPlayback {
pub feed: Arc<AudioFeed>,
device: Box<dyn AudioDevice>,
stop: Arc<AtomicBool>,
thread: Option<JoinHandle<()>>,
/// What the mixer reads and advances: the feeder thread, or [`AudioPlayback::pump`] on wasm32
/// (no threads).
state: Arc<Mutex<Feeder>>,
comp: ItemId,
mix: u32,
pub rate: u32,
pub span: PlaySpan,
timeline: PreviewRate,
}
/// The mixer's position and its view of the project.
struct Feeder {
src: RenderSource,
/// The next output sample index to mix (mapped to composition time by the preview rate).
cursor: i64,
/// Bumped when the project changes: a block mixed from an older project is dropped.
generation: u64,
}
/// Audio kept queued ahead of the device: ~200 ms (a feeder thread tops it up continuously);
/// wasm32 tops it up once per UI frame, so it keeps ~350 ms to ride out slow frames.
fn queue_target(rate: u32) -> usize {
if cfg!(target_arch = "wasm32") { rate as usize * 7 / 20 } else { rate as usize / 5 }
}
impl AudioPlayback {
/// Start mixing `comp` over `span` into `device`.
/// `mix_rate`: Settings ▸ Audio ▸ Preview Sample Rate, the rate the comp's audio is mixed
/// at (resampled to the device's rate when they differ; 0 = the device's rate).
#[allow(clippy::too_many_arguments)]
pub fn start(
device: Box<dyn AudioDevice>,
src: RenderSource,
comp: ItemId,
start: Tick,
wa: Tick,
wb: Tick,
looping: bool,
mix_rate: u32,
) -> Result<AudioPlayback, String> {
Self::start_at_speed(device, src, comp, start, wa, wb, looping, mix_rate, 1.0)
}
/// Start at a preview speed relative to composition time. Both the sound (including pitch)
/// and its composition clock follow this speed; the physical device rate stays unchanged.
#[allow(clippy::too_many_arguments)]
pub fn start_at_speed(
mut device: Box<dyn AudioDevice>,
src: RenderSource,
comp: ItemId,
start: Tick,
wa: Tick,
wb: Tick,
looping: bool,
mix_rate: u32,
speed: f64,
) -> Result<AudioPlayback, String> {
let rate = device.sample_rate().max(8000);
let mix = if mix_rate == 0 { rate } else { mix_rate.clamp(8000, 192_000) };
let timeline = PreviewRate::new(rate, speed, start)?;
let span = timeline.span(start, wa, wb, looping);
let feed = Arc::new(AudioFeed::default());
let stop = Arc::new(AtomicBool::new(false));
let state = Arc::new(Mutex::new(Feeder { src, cursor: span.start, generation: 0 }));
if cfg!(target_arch = "wasm32") {
// The device takes samples only in `pump`, which fills the queue first.
let mut p = AudioPlayback { feed: feed.clone(), device, stop, thread: None, state, comp, mix, rate, span, timeline };
p.device.start(feed)?;
p.pump();
return Ok(p);
}
// All mixing happens on the feeder thread, the first block too: reading footage for the
// first time opens it (a FLAC or MP3 file decodes whole), which froze the UI for as long
// as that took (#408). The device plays silence until the first block arrives, and the
// clock doesn't count it.
let thread = {
let (feed, stop, state) = (feed.clone(), stop.clone(), state.clone());
std::thread::Builder::new()
.name("ec-audio-feed".into())
.spawn(move || {
let target = queue_target(rate);
while !stop.load(Ordering::Relaxed) {
if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) {
feed_block(&state, comp, &feed, &span, timeline, mix);
} else {
std::thread::sleep(std::time::Duration::from_millis(4));
}
}
})
.map_err(|e| e.to_string())?
};
device.start(feed.clone())?;
Ok(AudioPlayback { feed, device, stop, thread: Some(thread), state, comp, mix, rate, span, timeline })
}
/// Every UI frame while playing: on wasm32 mix ahead into the feed (what the feeder thread
/// does elsewhere), then let the device take what it needs.
pub fn pump(&mut self) {
if cfg!(target_arch = "wasm32") {
let target = queue_target(self.rate);
while self.feed.queued_frames() < target && !self.feed.finished.load(Ordering::Relaxed) {
feed_block(&self.state, self.comp, &self.feed, &self.span, self.timeline, self.mix);
}
}
self.device.pump();
}
/// The project the sound is mixed from.
pub fn project(&self) -> Option<Arc<Project>> {
self.state.lock().ok().map(|s| s.src.project.clone())
}
/// Mix from `project` from now on (the project was edited while playing, e.g. a layer's Audio
/// switch, #2104): what is queued came from the old project, so it is dropped and mixing
/// resumes at the first sample the device has not taken yet (also when the feeder had
/// already mixed up to the end of a non-looping span).
pub fn set_project(&mut self, project: Arc<Project>) {
let Ok(mut s) = self.state.lock() else { return };
let Some(consumed) = self.feed.discard_pending() else { return };
s.src.project = project;
s.generation = s.generation.wrapping_add(1);
self.feed.finished.store(false, Ordering::Relaxed);
// The display clock clamps to the last *audible* sample. A finished non-looping
// feeder must instead stay at the exclusive end, never replay that last sample.
let next = i128::from(self.span.start) + i128::from(consumed);
s.cursor = if !self.span.looping && next >= i128::from(self.span.wb) { self.span.wb } else { clock_sample(&self.span, consumed, 0) };
}
/// Follow an edited `project`: mixing continues from it, and the queued sound is dropped only
/// when the edit changed what is heard ([`audio_changed`]); a transform or colour edit leaves
/// the queue alone, so playback does not drop out.
pub fn follow_project(&mut self, project: Arc<Project>) {
let Some(old) = self.project() else { return };
if audio_changed(&old, &project, self.comp) {
self.set_project(project);
} else if let Ok(mut s) = self.state.lock() {
s.src.project = project;
}
}
/// The comp sample index audible now, expressed at the device's rate.
pub fn clock(&self) -> i64 {
let sample = clock_sample(&self.span, self.feed.consumed(), self.device.latency_frames());
if self.timeline.normal() { sample } else { self.timeline.time(sample).to_units_floor(i64::from(self.rate)) }
}
/// Composition time under the audible output sample, including preview speed and latency.
pub fn clock_time(&self) -> Tick {
self.timeline.time(clock_sample(&self.span, self.feed.consumed(), self.device.latency_frames()))
}
pub fn finished(&self) -> bool {
self.feed.finished()
}
pub fn stop(&mut self) {
self.device.stop();
self.stop.store(true, Ordering::Relaxed);
if let Some(t) = self.thread.take() {
let _ = t.join();
}
}
}
impl Drop for AudioPlayback {
fn drop(&mut self) {
self.stop();
}
}
/// Whether `new` sounds different from `old` for `comp`: a layer's timing, source, Audio / Solo /
/// Effects switch, Audio Levels or effects, or the same for a nested comp. Other edits (video
/// properties, names, labels) do not change the mix unless an audio expression can read them.
fn audio_changed(old: &Project, new: &Project, comp: ItemId) -> bool {
audio_changed_inner(old, new, comp, &mut HashSet::new())
}
/// Each shared precomp is compared once per depth. A depth limit alone still visits an
/// exponentially large number of paths in a small graph of reused precomps, blocking the UI.
fn audio_changed_inner(old: &Project, new: &Project, comp: ItemId, seen: &mut HashSet<(ItemId, usize)>) -> bool {
/// Compare one comp and its reachable audio sources, without revisiting a shared path depth.
fn comps_differ(old: &Project, new: &Project, comp: ItemId, depth: usize, seen: &mut HashSet<(ItemId, usize)>) -> bool {
if depth >= 16 || !seen.insert((comp, depth)) {
return false;
}
let (Some(a), Some(b)) = (old.comp(comp), new.comp(comp)) else { return old.comp(comp).is_some() != new.comp(comp).is_some() };
a.layers.len() != b.layers.len()
|| a.layers.iter().zip(&b.layers).any(|(x, y)| {
x.id != y.id
|| x.source != y.source
|| x.start_time != y.start_time
|| x.in_point != y.in_point
|| x.out_point != y.out_point
|| x.stretch != y.stretch
|| x.switches.audio != y.switches.audio
|| x.switches.solo != y.switches.solo
|| x.switches.effects != y.switches.effects
|| x.props.sub("audio") != y.props.sub("audio")
|| audio_effects(x) != audio_effects(y)
|| audio_has_expression(x)
|| audio_has_expression(y)
|| match &x.source {
LayerSource::Footage { item } => footage_differs(old, new, *item),
LayerSource::Comp { item } => comps_differ(old, new, *item, depth + 1, seen),
_ => false,
}
})
}
/// The layer's audio effects (Effect > Audio), the only effects the mixer applies; a video effect
/// edit leaves them as they were.
fn audio_effects(l: &Layer) -> Vec<&PropGroup> {
let is_audio = |g: &&PropGroup| matches!(&g.kind, GroupKind::Effect { effect } if effectcraft_engine::effects::audio_fx::is_audio_effect(effect));
l.effects().into_iter().flat_map(|fx| fx.groups()).filter(is_audio).collect()
}
/// Audio expressions can read any property in any comp (including video-only effects and
/// names). Without a complete dependency graph, keeping the old mix would be stale.
fn audio_has_expression(l: &Layer) -> bool {
l.switches.audio
&& (l.props.sub("audio").and_then(|g| g.get("levels")).is_some_and(|p| p.has_expression())
|| (l.switches.effects && audio_effects(l).iter().any(|g| g.enabled && g.props().any(|p| p.has_expression()))))
}
/// The footage item's audio availability changed (relinked, missing, no audio).
fn footage_differs(old: &Project, new: &Project, item: ItemId) -> bool {
let f = |p: &Project| match p.item(item).map(|i| &i.kind) {
Some(ItemKind::Footage(f)) => Some((f.has_audio, f.missing, f.path.clone())),
_ => None,
};
f(old) != f(new)
}
comps_differ(old, new, comp, 0, seen)
}
/// Mix one block at the feeder's cursor into the feed and move the cursor on (wrapping or
/// finishing at the work-area end). The block is mixed without holding the lock, so a project
/// change never waits for a slow mix; a block mixed from the old project is dropped.
fn feed_block(state: &Mutex<Feeder>, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, timeline: PreviewRate, mix: u32) {
let Ok((src, mut cursor, generation)) = state.lock().map(|s| (s.src.clone(), s.cursor, s.generation)) else {
// Poisoned: nothing more can be mixed.
feed.finished.store(true, Ordering::Relaxed);
return;
};
let mut block = None;
if cursor >= span.wb && span.looping {
cursor = span.wa;
}
if cursor < span.wb {
let n = span.wb.saturating_sub(cursor).clamp(0, BLOCK as i64) as usize;
block = Some(if timeline.normal() {
mix_block(&src, comp, timeline.time(cursor), n, timeline.rate(), mix)
} else {
timeline.mix(cursor, n, mix, |t, frames| {
effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, frames, mix)
})
});
cursor += n as i64;
}
let Ok(mut s) = state.lock() else { return };
if s.generation != generation {
return;
}
match block {
Some(b) => feed.push(&b),
None => feed.finished.store(true, Ordering::Relaxed),
}
s.cursor = cursor;
}
/// `n` stereo frames of `comp`'s mix from comp time `t` at the device `rate`, mixed at `mix` Hz
/// (resampled when they differ).
fn mix_block(src: &RenderSource, comp: ItemId, t: Tick, n: usize, rate: u32, mix: u32) -> Vec<f32> {
let render =
|frames: usize, at: u32| effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, frames, at);
if mix == rate {
return render(n, rate);
}
let m = (n as u64 * mix as u64).div_ceil(rate as u64) as usize + 1;
resample_stereo(&render(m, mix), mix, rate, n)
}
/// Audio scrubbing: an open output that plays a short, faded snippet of the comp's mix each time
/// the current time moves to another frame (each snippet replaces what is still queued, so the
/// sound follows the pointer without lagging behind).
pub struct AudioScrub {
pub feed: Arc<AudioFeed>,
device: Box<dyn AudioDevice>,
pub rate: u32,
/// The comp and frame last played (holding still repeats nothing).
last: Option<(ItemId, i64)>,
/// When it last played (UI seconds): an idle scrub closes the device.
pub last_used: f64,
}
impl AudioScrub {
pub fn open(mut device: Box<dyn AudioDevice>, now: f64) -> Result<AudioScrub, String> {
let rate = device.sample_rate().max(8000);
let feed = Arc::new(AudioFeed::default());
device.start(feed.clone())?;
Ok(AudioScrub { feed, device, rate, last: None, last_used: now })
}
/// Play the frame of `comp` at `t` (one frame long, 30–100 ms) unless it was the last one
/// played. Returns whether a snippet was queued.
pub fn play(&mut self, src: &RenderSource, comp: ItemId, t: Tick, mix_rate: u32, now: f64) -> bool {
self.last_used = now;
let Some(c) = src.project.comp(comp) else { return false };
let frame = c.frame_rate.frame_at(t);
if self.last == Some((comp, frame)) {
return false;
}
self.last = Some((comp, frame));
let rate = self.rate;
let mix = if mix_rate == 0 { rate } else { mix_rate.clamp(8000, 192_000) };
let n = (c.frame_duration().seconds().clamp(0.03, 0.1) * rate as f64) as usize;
let mut buf = mix_block(src, comp, c.frame_rate.tick_of(frame), n, rate, mix);
// 3 ms fades: no clicks where snippets meet.
let fade = (rate as usize * 3 / 1000).clamp(1, n / 2 + 1);
let frames = buf.len() / 2;
for i in 0..frames {
let g = (i.min(frames - 1 - i) as f32 / fade as f32).min(1.0);
buf[2 * i] *= g;
buf[2 * i + 1] *= g;
}
self.feed.clear();
self.feed.push(&buf);
true
}
/// Every UI frame while open (Web Audio moves queued samples to the output here).
pub fn pump(&mut self) {
self.device.pump();
}
}
impl Drop for AudioScrub {
fn drop(&mut self) {
self.device.stop();
}
}
/// Linear resampling of interleaved stereo from `from` Hz to `n` frames at `to` Hz.
pub fn resample_stereo(src: &[f32], from: u32, to: u32, n: usize) -> Vec<f32> {
let frames = src.len() / 2;
let mut out = Vec::with_capacity(n * 2);
if frames == 0 {
out.resize(n * 2, 0.0);
return out;
}
let step = from as f64 / to as f64;
for i in 0..n {
let x = i as f64 * step;
let a = (x.floor() as usize).min(frames - 1);
let b = (a + 1).min(frames - 1);
let f = (x - a as f64) as f32;
for c in 0..2 {
out.push(src[a * 2 + c] * (1.0 - f) + src[b * 2 + c] * f);
}
}
out
}
/// VU meter state for the Audio panel: smoothed level with a falling decay and peak hold, in
/// dBFS per channel.
#[derive(Clone, Copy, Debug)]
pub struct Meter {
pub level_db: [f32; 2],
pub peak_db: [f32; 2],
peak_at: [f64; 2],
/// Clip indicators (latched until clicked / playback restarts).
pub clipped: [bool; 2],
last: f64,
}
impl Default for Meter {
fn default() -> Self {
Meter { level_db: [METER_FLOOR; 2], peak_db: [METER_FLOOR; 2], peak_at: [0.0; 2], clipped: [false; 2], last: 0.0 }
}
}
/// Bottom of the meter scale (dB).
pub const METER_FLOOR: f32 = -48.0;
/// How long the peak marker holds (s) and how fast levels fall (dB/s).
const PEAK_HOLD: f64 = 1.5;
const FALL_DB_PER_S: f32 = 30.0;
pub fn to_db(v: f32) -> f32 {
if v <= 1e-6 { METER_FLOOR } else { (20.0 * v.log10()).max(METER_FLOOR) }
}
impl Meter {
/// Feed the peaks measured since the last update at wall time `now` (seconds).
pub fn update(&mut self, peaks: [f32; 2], now: f64) {
let dt = if self.last > 0.0 { (now - self.last).clamp(0.0, 0.5) as f32 } else { 0.0 };
self.last = now;
for c in 0..2 {
let db = to_db(peaks[c]);
self.level_db[c] = db.max(self.level_db[c] - FALL_DB_PER_S * dt);
if db >= self.peak_db[c] || now - self.peak_at[c] > PEAK_HOLD {
self.peak_db[c] = db.max(self.level_db[c]);
self.peak_at[c] = now;
}
if peaks[c] >= 1.0 {
self.clipped[c] = true;
}
}
}
/// Anything to animate (levels still falling)?
pub fn active(&self) -> bool {
self.level_db.iter().chain(self.peak_db.iter()).any(|d| *d > METER_FLOOR)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn clock_follows_consumed_frames_and_loops() {
let span = PlaySpan { start: 4800, wa: 0, wb: 48000, looping: true };
assert_eq!(clock_sample(&span, 0, 0), 4800);
// Latency holds the clock back until the device has played through its buffer.
assert_eq!(clock_sample(&span, 512, 1024), 4800);
assert_eq!(clock_sample(&span, 2048, 1024), 4800 + 1024);
// Wrap at the work-area end.
assert_eq!(clock_sample(&span, 48000 - 4800 + 100, 0), 100);
let once = PlaySpan { looping: false, ..span };
assert_eq!(clock_sample(&once, 100_000, 0), 47999);
}
#[test]
fn slow_preview_clocks_do_not_overflow_a_wide_output_span() {
let span = PlaySpan { start: 0, wa: i64::MIN, wb: i64::MAX, looping: true };
assert_eq!(clock_sample(&span, u64::MAX, 0), 0);
assert_eq!(clock_sample(&PlaySpan { looping: false, ..span }, u64::MAX, 0), i64::MAX - 1);
}
#[test]
fn frame_mapping_matches_comp_frames() {
let fps = FrameRate::new(30, 1);
// 48 kHz at 30 fps: 1600 samples per frame.
assert_eq!(frame_of_sample(0, 48000, fps), 0);
assert_eq!(frame_of_sample(1599, 48000, fps), 0);
assert_eq!(frame_of_sample(1600, 48000, fps), 1);
assert_eq!(frame_of_sample(48000 * 2, 48000, fps), 60);
let span = PlaySpan::new(Tick::from_seconds_f64(1.0), Tick::ZERO, Tick::from_seconds_f64(2.0), true, 48000);
assert_eq!((span.start, span.wa, span.wb), (48000, 0, 96000));
// 29.97: frame boundaries are fractional in samples.
let ntsc = FrameRate::new(30000, 1001);
assert_eq!(frame_of_sample(1601, 48000, ntsc), 0);
assert_eq!(frame_of_sample(1602, 48000, ntsc), 1);
}
#[test]
fn feed_counts_real_frames_and_peaks() {
let f = AudioFeed::default();
f.push(&[0.5, -0.25, 0.1, 0.9]);
let mut out = [1.0f32; 8];
f.pull(&mut out);
assert_eq!(out, [0.5, -0.25, 0.1, 0.9, 0.0, 0.0, 0.0, 0.0]);
assert_eq!(f.consumed(), 2);
assert_eq!(f.take_peaks(), [0.5, 0.9]);
assert_eq!(f.take_peaks(), [0.0, 0.0]);
}
/// #408: starting a preview with sound returns at once even when reading the footage takes
/// long (the first read of a long FLAC decodes all of it): the first block was mixed on the
/// calling (UI) thread. The sound arrives from the feeder thread, and the clock waits for it.
#[test]
fn starting_audio_does_not_wait_for_slow_footage() {
use effectcraft_engine::color::Label;
use effectcraft_engine::project::{Comp, Footage, FootageKind, ItemKind, LayerSource, Project, build};
use effectcraft_engine::render::FootageSource;
use effectcraft_engine::render::cache::LayerCache;
use std::time::{Duration, Instant};
/// Footage audio that takes `delay` to read the first time (then a constant 0.5).
struct Slow {
delay: Duration,
opened: Mutex<bool>,
}
impl FootageSource for Slow {
fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option<Arc<effectcraft_engine::raster::Image>> {
None
}
fn audio(&self, _: ItemId, _: &Footage, _: Tick, frames: usize, _: u32) -> Option<Vec<f32>> {
let mut opened = self.opened.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if !*opened {
std::thread::sleep(self.delay);
*opened = true;
}
Some(vec![0.5; frames * 2])
}
}
/// An output that only takes the feed.
struct Device;
impl AudioDevice for Device {
fn sample_rate(&self) -> u32 {
48000
}
fn start(&mut self, _: Arc<AudioFeed>) -> Result<(), String> {
Ok(())
}
fn stop(&mut self) {}
}
let mut p = Project::default();
let footage =
Footage { path: "long.flac".into(), kind: FootageKind::Audio, duration: Tick::from_seconds_f64(600.0), has_audio: true, ..Default::default() };
let item = p.add_item("long.flac", Label::SeaFoam, None, ItemKind::Footage(footage));
let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(10.0));
let layer = build::layer(&mut p, &comp, "long.flac", LayerSource::Footage { item }, (0, 0), None);
comp.layers = vec![layer].into();
let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into()));
let src = RenderSource {
project: Arc::new(p),
footage: Arc::new(Slow { delay: Duration::from_secs(2), opened: Mutex::new(false) }),
expr: None,
layer_cache: Arc::new(LayerCache::default()),
gpu: None,
gpu_display: false,
disk: None,
};
let t = Instant::now();
let mut playback =
AudioPlayback::start(Box::new(Device), src, cid, Tick::ZERO, Tick::ZERO, Tick::from_seconds_f64(10.0), true, 0).expect("the preview starts");
assert!(t.elapsed() < Duration::from_secs(1), "start waited {:?} for the footage", t.elapsed());
assert_eq!(playback.clock(), 0, "the clock waits for the sound");
// The feeder thread delivers it once the footage is open.
while playback.feed.queued_frames() == 0 {
assert!(t.elapsed() < Duration::from_secs(30), "the feeder never delivered");
std::thread::sleep(Duration::from_millis(10));
}
let mut out = [0.0f32; 8];
playback.feed.pull(&mut out);
assert_eq!(out, [0.5; 8]);
playback.stop();
}
/// #2104: muting a layer while the preview plays silences it: the queued sound is dropped
/// and everything mixed afterwards comes from the edited project.
#[test]
fn muting_a_layer_while_playing_silences_the_preview() {
use effectcraft_engine::color::Label;
use effectcraft_engine::project::{Comp, Footage, FootageKind, ItemKind, LayerSource, build};
use effectcraft_engine::render::FootageSource;
use effectcraft_engine::render::cache::LayerCache;
use std::time::{Duration, Instant};
/// Footage audio that is a constant 0.5.
struct Steady;
impl FootageSource for Steady {
fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option<Arc<effectcraft_engine::raster::Image>> {
None
}
fn audio(&self, _: ItemId, _: &Footage, _: Tick, frames: usize, _: u32) -> Option<Vec<f32>> {
Some(vec![0.5; frames * 2])
}
}
struct Device;
impl AudioDevice for Device {
fn sample_rate(&self) -> u32 {
48000
}
fn start(&mut self, _: Arc<AudioFeed>) -> Result<(), String> {
Ok(())
}
fn stop(&mut self) {}
}
let mut p = Project::default();
let footage =
Footage { path: "tone.flac".into(), kind: FootageKind::Audio, duration: Tick::from_seconds_f64(60.0), has_audio: true, ..Default::default() };
let item = p.add_item("tone.flac", Label::SeaFoam, None, ItemKind::Footage(footage));
let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(10.0));
let layer = build::layer(&mut p, &comp, "tone.flac", LayerSource::Footage { item }, (0, 0), None);
comp.layers = vec![layer].into();
let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into()));
let mut project = Arc::new(p);
let src = RenderSource {
project: project.clone(),
footage: Arc::new(Steady),
expr: None,
layer_cache: Arc::new(LayerCache::default()),
gpu: None,
gpu_display: false,
disk: None,
};
let mut playback =
AudioPlayback::start(Box::new(Device), src, cid, Tick::ZERO, Tick::ZERO, Tick::from_seconds_f64(10.0), true, 0).expect("the preview starts");
let t = Instant::now();
while playback.feed.queued_frames() == 0 {
assert!(t.elapsed() < Duration::from_secs(30), "the feeder never delivered");
std::thread::sleep(Duration::from_millis(10));
}
// Mute the layer while the sound plays.
if let Some(c) = Arc::make_mut(&mut project).comp_mut(cid) {
c.layers[0].switches.audio = false;
}
playback.set_project(project.clone());
assert!(playback.project().is_some_and(|p| Arc::ptr_eq(&p, &project)));
// The feeder tops the queue up again, now from the muted project.
while playback.feed.queued_frames() == 0 {
assert!(t.elapsed() < Duration::from_secs(30), "the feeder never delivered");
std::thread::sleep(Duration::from_millis(10));
}
let mut out = vec![1.0f32; playback.feed.queued_frames() * 2];
playback.feed.pull(&mut out);
assert!(out.iter().all(|v| *v == 0.0), "the muted layer still plays");
playback.stop();
}
/// #2104: only edits that change the mix drop the queued sound.
#[test]
fn only_audible_edits_change_the_audio() {
use effectcraft_engine::color::Label;
use effectcraft_engine::project::{Comp, Footage, FootageKind, build};
let mut p = Project::default();
let footage =
Footage { path: "tone.flac".into(), kind: FootageKind::Audio, duration: Tick::from_seconds_f64(60.0), has_audio: true, ..Default::default() };
let item = p.add_item("tone.flac", Label::SeaFoam, None, ItemKind::Footage(footage));
let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(10.0));
let layer = build::layer(&mut p, &comp, "tone.flac", LayerSource::Footage { item }, (0, 0), None);
comp.layers = vec![layer].into();
let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into()));
let old = p;
// A video-only edit: the layer is hidden, its name and label change.
let mut video = old.clone();
if let Some(l) = video.comp_mut(cid).and_then(|c| c.layers.first_mut()) {
l.switches.video = false;
l.name = "renamed".into();
}
assert!(!audio_changed(&old, &video, cid), "a video edit must not drop the sound");
// Muting, solo and a moved in point do change it.
for edit in 0..3 {
let mut edited = old.clone();
if let Some(l) = edited.comp_mut(cid).and_then(|c| c.layers.first_mut()) {
match edit {
0 => l.switches.audio = false,
1 => l.switches.solo = true,
_ => l.in_point = Tick::from_seconds_f64(1.0),
}
}
assert!(audio_changed(&old, &edited, cid), "audible edit {edit} must be followed");
}
}
/// #2104: a video effect edit leaves the mix alone, an audio effect changes it.
#[test]
fn only_audio_effects_change_the_audio() {
use effectcraft_engine::color::Label;
use effectcraft_engine::project::build::{self, Ids};
use effectcraft_engine::project::{Comp, Footage, FootageKind};
let mut p = Project::default();
let footage =
Footage { path: "tone.flac".into(), kind: FootageKind::Audio, duration: Tick::from_seconds_f64(60.0), has_audio: true, ..Default::default() };
let item = p.add_item("tone.flac", Label::SeaFoam, None, ItemKind::Footage(footage));
let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(10.0));
let layer = build::layer(&mut p, &comp, "tone.flac", LayerSource::Footage { item }, (0, 0), None);
comp.layers = vec![layer].into();
let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into()));
let with_effect = |id: &str| {
let mut edited = p.clone();
let spec = effectcraft_engine::effects::find(id).expect("the effect exists");
let mut next = edited.next_id;
let g = effectcraft_engine::effects::instantiate(spec, &mut Ids(&mut next), spec.name, [64.0, 36.0]);
edited.next_id = next;
if let Some(fx) = edited.comp_mut(cid).and_then(|c| c.layers.first_mut()).and_then(|l| l.props.sub_mut("effects")) {
fx.children.push(g.into());
}
edited
};
assert!(!audio_changed(&p, &with_effect("ec.blur.gaussian"), cid), "a video effect must not drop the sound");
assert!(audio_changed(&p, &with_effect("ec.audio.backwards"), cid), "an audio effect must be followed");
}
#[test]
fn meter_decays_and_holds_peaks() {
let mut m = Meter::default();
m.update([1.0, 0.5], 1.0);
assert!((m.level_db[0] - 0.0).abs() < 1e-4 && (m.level_db[1] + 6.0206).abs() < 1e-3);
assert!(m.clipped[0] && !m.clipped[1]);
m.update([0.0, 0.0], 1.1);
assert!((m.level_db[0] + 3.0).abs() < 1e-3, "{}", m.level_db[0]);
assert_eq!(m.peak_db[0], 0.0, "peak holds");
// After the hold time the peak marker follows the falling level.
m.update([0.0, 0.0], 3.0);
assert!(m.peak_db[0] < -10.0, "{}", m.peak_db[0]);
}
}
#[cfg(test)]
#[path = "audio/live_tests.rs"]
mod live_tests;
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.