f67d38b35cc5
lib.rs
//! Media sources for FilmCraft.
//!
//! A [`MediaSource`] is anything that yields video frames and/or audio for a media time: decoded
//! files (containers + codecs), stills, image sequences and synthetic generators (bars & tone,
//! colour mattes, counting leader, procedural demo scenes). Sources are `Send + Sync`; decoder state
//! lives behind interior mutability so monitors, thumbnails, playback and export can share them.
//!
//! Time passed to a source is **media time** (0 = first frame of the media), in ticks.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
pub mod animated;
pub mod cache;
pub mod cancel;
pub mod digits;
pub mod dpx;
pub mod generators;
pub mod pending;
pub mod reader;
pub mod sequence;
pub mod still;
pub mod tga;
pub mod wav;
use std::path::Path;
use std::sync::Arc;
use filmcraft_color::ColorInfo;
use filmcraft_frame::{AudioBuffer, VideoFrame};
use filmcraft_time::{FrameRate, Tick};
use serde::{Deserialize, Serialize};
pub use cache::FrameCache;
pub use generators::{DemoScene, Generator};
pub use reader::{ByteReader, ReaderOpener, SharedReader};
#[derive(Debug, thiserror::Error)]
pub enum MediaError {
#[error("unsupported media: {0}")]
Unsupported(String),
#[error("I/O error: {0}")]
Io(String),
#[error("decode error: {0}")]
Decode(String),
#[error("no {0} stream")]
NoStream(&'static str),
#[error("media offline: {0}")]
Offline(String),
/// The request was cancelled (see [`cancel`]).
#[error("cancelled")]
Cancelled,
}
pub type Result<T> = std::result::Result<T, MediaError>;
/// Up to 384 kHz audio (surpassing professional 192 kHz formats).
pub const MAX_AUDIO_SAMPLE_RATE: u32 = 384_000;
/// Up to 64 discrete channels (exceeding standard 32-channel immersive layouts).
pub const MAX_AUDIO_CHANNELS: u32 = 64;
/// Bounded planar audio buffer: 64 channels * 192 kHz * 10 seconds of f32 is ~492 MiB.
pub const MAX_AUDIO_BUFFER_BYTES: usize = 512 * 1024 * 1024;
/// Bound container-advertised audio stream dimensions before they are trusted by the app or
/// allocator.
///
/// Out-of-range values are rejected rather than clamped: publishing a lower sample rate without
/// resampling the samples would make the stream play at the wrong speed, and a clamped channel
/// count would misdescribe the interleaving.
pub fn bounded_audio_dimensions(sample_rate: u32, channels: u32) -> Result<(u32, u32)> {
if sample_rate == 0 || sample_rate > MAX_AUDIO_SAMPLE_RATE {
return Err(MediaError::Decode(format!("unsupported audio sample rate {sample_rate}")));
}
if channels == 0 || channels > MAX_AUDIO_CHANNELS {
return Err(MediaError::Decode(format!("unsupported audio channel count {channels}")));
}
Ok((sample_rate, channels))
}
/// Check an adapter-owned planar `f32` audio allocation before constructing it.
pub fn validate_audio_buffer_shape(channels: usize, frames: usize) -> Result<()> {
if channels == 0 || channels > MAX_AUDIO_CHANNELS as usize {
return Err(MediaError::Decode(format!("unsupported audio channel count {channels}")));
}
let bytes = channels
.checked_mul(frames)
.and_then(|samples| samples.checked_mul(std::mem::size_of::<f32>()))
.ok_or_else(|| MediaError::Decode("audio buffer dimensions overflow".into()))?;
if bytes > MAX_AUDIO_BUFFER_BYTES {
return Err(MediaError::Decode(format!("audio buffer of {bytes} bytes exceeds the {} byte limit", MAX_AUDIO_BUFFER_BYTES)));
}
Ok(())
}
/// Number of source frames needed to resample `frames` output frames. Arithmetic is checked so
/// corrupted rates cannot wrap into a small allocation.
pub fn resample_source_frames(frames: usize, source_rate: u32, output_rate: u32) -> Result<usize> {
if output_rate == 0 {
return Err(MediaError::Decode("zero requested audio sample rate".into()));
}
let source = usize::try_from(source_rate).map_err(|_| MediaError::Decode("invalid source sample rate".into()))?;
let output = usize::try_from(output_rate).map_err(|_| MediaError::Decode("invalid output sample rate".into()))?;
frames
.checked_mul(source)
.and_then(|n| n.checked_add(output.saturating_sub(1)))
.map(|n| n / output)
.ok_or_else(|| MediaError::Decode("resampled audio window overflows".into()))
}
/// Linearly resamples a decoded source window into `out`.
///
/// `window[c][j]` holds source sample `window_start + j` at `source_rate`; `out[c][k]` receives
/// output sample `start + k` at `output_rate`. Each output position is computed from its absolute
/// index, `(start + k) * source_rate / output_rate`, so a sample gets the same value however the
/// caller cuts the timeline into requests (the mixer relies on that). Source samples outside the
/// window read as silence (each neighbour is looked up on its own, so a left sample before the
/// window still blends with an in-window right one), and a missing right neighbour repeats the left
/// one. A zero output rate leaves `out` untouched.
pub fn resample_window_into(window: &[Vec<f32>], window_start: i64, source_rate: u32, start: i64, output_rate: u32, out: &mut [Vec<f32>]) {
if output_rate == 0 {
return;
}
let ratio = f64::from(source_rate) / f64::from(output_rate);
for (o, s) in out.iter_mut().zip(window) {
for (k, d) in o.iter_mut().enumerate() {
let pos = start.saturating_add(i64::try_from(k).unwrap_or(i64::MAX)) as f64 * ratio;
let base = pos.floor();
let f = (pos - base) as f32;
let left = (base as i64).saturating_sub(window_start);
let at = |i: i64| usize::try_from(i).ok().and_then(|i| s.get(i)).copied();
let a = at(left).unwrap_or(0.0);
let b = at(left.saturating_add(1)).unwrap_or(a);
*d = a + (b - a) * f;
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum MediaKind {
/// Video, possibly with audio.
Movie,
AudioOnly,
Still,
ImageSequence,
Synthetic,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct VideoStreamInfo {
pub width: u32,
pub height: u32,
pub frame_rate: FrameRate,
pub par: (u32, u32),
pub codec: String,
pub pixel_format: String,
pub color: ColorInfo,
pub has_alpha: bool,
/// Bitrate in bits/s when known.
pub bitrate: Option<u64>,
/// HDR static metadata (mastering display, content light level) when the file carries it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub hdr: Option<filmcraft_color::HdrMetadata>,
/// Which values of `color` are FilmCraft's defaults rather than read from the file. `None`
/// where the source doesn't say: stills, generated media, MPEG program and transport streams,
/// and media info saved before this was recorded.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub color_assumed: Option<filmcraft_color::ColorAssumed>,
}
/// Most audio streams a source reports (the count comes from the file: never trusted).
pub const MAX_AUDIO_STREAMS: usize = 64;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AudioStreamInfo {
pub sample_rate: u32,
pub channels: u32,
pub codec: String,
pub bits_per_sample: Option<u32>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct MediaInfo {
pub name: String,
pub kind: MediaKind,
/// Media duration. Stills report a default duration (the still-image default preference).
pub duration: Tick,
pub video: Option<VideoStreamInfo>,
/// Every audio stream the container carries, in file order. Stream 0 is the one
/// [`MediaSource::audio`] reads; the others are read with [`MediaSource::audio_stream`].
/// Project files written before multi-stream support hold a single `audio` object (or
/// `null`): both still load.
#[serde(default, alias = "audio", deserialize_with = "deserialize_audio_streams")]
pub audio_streams: Vec<AudioStreamInfo>,
pub container: String,
/// Timecode of the first frame (in frames at `video.frame_rate`), if the file carries one.
pub start_timecode: Option<i64>,
pub file_size: Option<u64>,
}
/// Reads `audio_streams` from a list, or from the single optional stream older project files wrote.
fn deserialize_audio_streams<'de, D: serde::Deserializer<'de>>(d: D) -> std::result::Result<Vec<AudioStreamInfo>, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum Streams {
Many(Vec<AudioStreamInfo>),
One(Option<AudioStreamInfo>),
}
Ok(match Streams::deserialize(d)? {
Streams::Many(v) => v,
Streams::One(o) => o.into_iter().collect(),
})
}
impl MediaInfo {
pub fn frame_rate(&self) -> FrameRate {
self.video.as_ref().map(|v| v.frame_rate).unwrap_or_default()
}
pub fn has_video(&self) -> bool {
self.video.is_some()
}
pub fn has_audio(&self) -> bool {
!self.audio_streams.is_empty()
}
/// The primary audio stream (stream 0), the one [`MediaSource::audio`] reads.
pub fn audio(&self) -> Option<&AudioStreamInfo> {
self.audio_streams.first()
}
}
/// A request for a video frame.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct FrameRequest {
/// Media time.
pub time: Tick,
/// Desired scale relative to full resolution (1.0 = full). Sources may return a larger frame.
pub scale: f32,
}
impl FrameRequest {
pub fn full(time: Tick) -> Self {
Self { time, scale: 1.0 }
}
}
pub trait MediaSource: Send + Sync {
fn info(&self) -> &MediaInfo;
/// The frame displayed at media time `req.time`.
fn video_frame(&self, req: FrameRequest) -> Result<Arc<VideoFrame>>;
/// Like [`video_frame`](Self::video_frame), but a source may leave the container's display
/// rotation to the caller: the frame comes with the clockwise quarter turns still to apply
/// to it (0: upright, as `video_frame` returns it).
fn video_frame_unturned(&self, req: FrameRequest) -> Result<(Arc<VideoFrame>, u8)> {
self.video_frame(req).map(|f| (f, 0))
}
/// `frames` audio frames starting at sample index `start` (at `sample_rate`, media time).
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer>;
/// Like [`audio`](Self::audio) for audio stream `stream` (an index into
/// [`MediaInfo::audio_streams`]). Sources with a single audio stream need not override this:
/// stream 0 is `audio`, any other index is [`MediaError::NoStream`].
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer> {
if stream == 0 { self.audio(start, frames, sample_rate) } else { Err(MediaError::NoStream("audio")) }
}
}
pub type SharedSource = Arc<dyn MediaSource>;
/// File extensions we recognise at import (the media browser filters on these).
pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "tga", "dpx", "exr", "hdr", "dng", "psd"];
pub const AUDIO_EXTENSIONS: &[&str] = &["wav", "wave", "bwf", "aif", "aiff", "mp3", "mp2", "m4a", "aac", "flac", "ogg", "opus"];
pub const VIDEO_EXTENSIONS: &[&str] = &[
"mp4", "m4v", "mov", "mkv", "webm", "avi", "mxf", "mts", "m2ts", "m2t", "ts", "mpg", "mpeg", "vob", "mod", "tod", "m2v", "m1v", "mpv", "3gp", "y4m", "apv",
];
pub fn is_importable(path: &Path) -> bool {
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase();
STILL_EXTENSIONS.contains(&ext.as_str()) || AUDIO_EXTENSIONS.contains(&ext.as_str()) || VIDEO_EXTENSIONS.contains(&ext.as_str())
}
/// A pluggable opener (containers/codecs register themselves here from higher-level crates).
pub type Opener = fn(name: &str, bytes: Arc<[u8]>) -> Option<Result<SharedSource>>;
/// Open media from bytes, trying `extra` openers first, then the built-in ones (WAV, animated
/// images, stills).
pub fn open_bytes(name: &str, bytes: Arc<[u8]>, extra: &[Opener]) -> Result<SharedSource> {
for o in extra {
if let Some(r) = o(name, bytes.clone()) {
return r;
}
}
if still::exr::sniff(&bytes) {
return Ok(Arc::new(still::exr::still(name, &bytes)?));
}
if still::hdr::sniff(&bytes) {
return Ok(Arc::new(still::hdr::still(name, &bytes)?));
}
if wav::sniff(&bytes) {
return Ok(Arc::new(wav::WavSource::parse(name, bytes)?));
}
// APNG, animated GIF, and animated WebP play as movies (#402); single-frame files fall through to the stills
if let Some(r) = animated::from_bytes(name, bytes.clone()) {
return Ok(Arc::new(r?));
}
if still::sniff(name, &bytes) {
return Ok(Arc::new(still::StillSource::decode(name, &bytes)?));
}
Err(MediaError::Unsupported(format!("{name}: unrecognised format")))
}
/// Open a file from disk (native only).
#[cfg(not(target_arch = "wasm32"))]
pub fn open_path(path: &Path, extra: &[Opener]) -> Result<SharedSource> {
let bytes = std::fs::read(path).map_err(|e| MediaError::Io(format!("{}: {e}", path.display())))?;
let name = path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
open_bytes(&name, bytes.into(), extra)
}
/// A source that renders nothing (used for offline media): the "Media Offline" slate.
pub struct OfflineSource {
pub info: MediaInfo,
}
impl MediaSource for OfflineSource {
fn info(&self) -> &MediaInfo {
&self.info
}
fn video_frame(&self, req: FrameRequest) -> Result<Arc<VideoFrame>> {
let v = self.info.video.as_ref().ok_or(MediaError::NoStream("video"))?;
let s = req.scale.clamp(0.02, 1.0);
let (w, h) = (((v.width as f32 * s) as u32).max(2), ((v.height as f32 * s) as u32).max(2));
let mut px = vec![0u8; (w * h * 4) as usize];
for p in px.as_chunks_mut::<4>().0 {
p.copy_from_slice(&[140, 16, 16, 255]);
}
Ok(Arc::new(VideoFrame::rgba8(w, h, px).with_pts(req.time)))
}
fn audio(&self, _start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer> {
// the shape comes from the project's stored metadata (the media is not there to check)
let channels = self.info.audio().map_or(2, |a| a.channels as usize);
validate_audio_buffer_shape(channels, frames)?;
Ok(AudioBuffer::silence(sample_rate, channels, frames))
}
}
#[cfg(test)]
mod importable_tests {
use super::*;
use std::path::Path;
#[test]
fn camera_extensions_match_regardless_of_case() {
for name in ["clip.mov", "clip.MOV", "clip.Mov", "shot.MP4", "IMG_0001.JPG", "sound.WAV", "take.M2TS"] {
assert!(is_importable(Path::new(name)), "{name}");
}
assert!(!is_importable(Path::new("notes.txt")));
}
#[test]
fn offline_media_does_not_allocate_from_stored_metadata() {
let info = MediaInfo {
name: "gone.wav".into(),
kind: MediaKind::AudioOnly,
duration: Tick(0),
video: None,
audio_streams: vec![AudioStreamInfo { sample_rate: 48_000, channels: 60_000, codec: "pcm".into(), bits_per_sample: None }],
container: String::new(),
start_timecode: None,
file_size: None,
};
let src = OfflineSource { info };
// a 100-day transcription request at the stored channel count
// computed as u64: the product overflows a 32-bit usize (x86 Windows), where it must still be rejected
let frames = usize::try_from(48_000u64 * 86_400 * 100).unwrap_or(usize::MAX);
assert!(src.audio(0, frames, 48_000).is_err());
}
#[test]
fn hostile_audio_dimensions_are_bounded_before_adapters_allocate() {
assert!(bounded_audio_dimensions(0, 0).is_err());
assert!(bounded_audio_dimensions(u32::MAX, u32::MAX).is_err());
assert!(bounded_audio_dimensions(MAX_AUDIO_SAMPLE_RATE + 1, 2).is_err());
assert_eq!(bounded_audio_dimensions(48_000, 2).ok(), Some((48_000, 2)));
assert!(validate_audio_buffer_shape(MAX_AUDIO_CHANNELS as usize, MAX_AUDIO_BUFFER_BYTES / 4 / MAX_AUDIO_CHANNELS as usize).is_ok());
assert!(validate_audio_buffer_shape(MAX_AUDIO_CHANNELS as usize, MAX_AUDIO_BUFFER_BYTES / 4 / MAX_AUDIO_CHANNELS as usize + 1).is_err());
assert!(resample_source_frames(usize::MAX, MAX_AUDIO_SAMPLE_RATE, 1).is_err());
}
/// A window decoded from any start gives the samples of the whole source, and cuts don't matter.
#[test]
fn resampled_window_reads_absolute_positions() {
let source = filmcraft_testkit::cuts::noise(40_000, 3);
let read = |start: i64, frames: usize| {
let ratio = 48_000f64 / 47_000f64;
let s0 = (start as f64 * ratio).floor() as i64;
let need = resample_source_frames(frames, 48_000, 47_000).unwrap() + 2;
let window = vec![source[s0 as usize..(s0 as usize + need).min(source.len())].to_vec()];
let mut out = vec![vec![0.0; frames]];
resample_window_into(&window, s0, 48_000, start, 47_000, &mut out);
out
};
filmcraft_testkit::cuts::assert_cut_independent("window", 1_000, 20_000, read);
let mut whole = vec![vec![0.0; 20_000]];
resample_window_into(std::slice::from_ref(&source), 0, 48_000, 1_000, 47_000, &mut whole);
assert_eq!(whole, read(1_000, 20_000));
// outside the window: silence, never a panic; a zero output rate leaves the buffer alone
let mut out = vec![vec![7.0; 4]];
resample_window_into(&[vec![1.0; 4]], 100, 48_000, i64::MIN, 44_100, &mut out);
assert_eq!(out, vec![vec![0.0; 4]]);
let mut out = vec![vec![7.0; 4]];
resample_window_into(&[vec![1.0; 4]], 0, 48_000, 0, 0, &mut out);
assert_eq!(out, vec![vec![7.0; 4]]);
// a left neighbour before the window still blends with the in-window right one
let mut edge = vec![vec![0.0; 1]];
resample_window_into(&[vec![1.0]], 1, 1, 1, 2, &mut edge);
assert_eq!(edge, vec![vec![0.5]]);
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.