f67d38b35cc5
audio.rs
//! Audio decoding: PCM directly; compressed codecs through symphonia (bootstrap, MPL-2.0 unmodified).
use std::collections::HashMap;
use std::sync::Arc;
use filmcraft_frame::{AudioBuffer, VideoFrame};
use filmcraft_media::{AudioStreamInfo, FrameRequest, MediaError, MediaInfo, MediaKind, MediaSource, SharedSource};
use filmcraft_time::Tick;
use symphonia::core::audio::SampleBuffer;
use symphonia::core::codecs::{CODEC_TYPE_AAC, CODEC_TYPE_ALAC, CODEC_TYPE_FLAC, CODEC_TYPE_MP3, CodecParameters, CodecType, Decoder, DecoderOptions};
use symphonia::core::formats::{FormatOptions, Packet};
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
use symphonia::core::probe::Hint;
use crate::{CodecError, Result};
#[cfg(test)]
#[path = "audio_panic_tests.rs"]
mod panic_tests;
/// A decoder that unwound may have partially changed its state. Keep it quarantined, including
/// across seeks, until its owner creates a fresh decoder. Ordinary decode errors remain retryable.
struct SymphoniaDecoder {
decoder: Box<dyn Decoder>,
failed: bool,
}
fn audio_panic(operation: &str) -> CodecError {
CodecError::Decode(format!("audio decoder panicked while {operation}; reopen the media to retry"))
}
/// Converts decoder panics to `CodecError::Decode` only when built with `panic = "unwind"`.
/// The default `wasm32-unknown-unknown` target uses `panic = "abort"`, so decoder panics cannot
/// be caught or converted. Process aborts and allocation failures that abort remain unrecoverable.
fn guard_audio<T>(operation: &str, f: impl FnOnce() -> Result<T>) -> Result<T> {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(|_| audio_panic(operation))?
}
impl SymphoniaDecoder {
fn new(params: &CodecParameters) -> Result<Self> {
guard_audio("initializing Symphonia", || {
let decoder = symphonia::default::get_codecs().make(params, &DecoderOptions::default()).map_err(|e| CodecError::Unsupported(e.to_string()))?;
Ok(Self { decoder, failed: false })
})
}
fn call<T>(&mut self, operation: &str, f: impl FnOnce(&mut dyn Decoder) -> T) -> Result<T> {
if self.failed {
return Err(CodecError::Decode("audio decoder is unavailable after a panic; reopen the media to retry".into()));
}
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(self.decoder.as_mut()))) {
Ok(result) => Ok(result),
Err(_) => {
self.failed = true;
Err(audio_panic(operation))
}
}
}
// Keep the two errors distinct: standalone imports skip ordinary corrupt packets, but must
// stop on a panic rather than report partially decoded audio as a successful import.
fn decode(&mut self, packet: &Packet) -> Result<symphonia::core::errors::Result<Vec<Vec<f32>>>> {
self.call("decoding a Symphonia packet", |decoder| {
let buf = decoder.decode(packet)?;
let spec = *buf.spec();
let ch = spec.channels.count().max(1);
let mut sb = SampleBuffer::<f32>::new(buf.capacity() as u64, spec);
sb.copy_interleaved_ref(buf);
let mut out = vec![Vec::with_capacity(sb.samples().len() / ch); ch];
for frame in sb.samples().chunks_exact(ch) {
for (channel, sample) in out.iter_mut().zip(frame) {
channel.push(*sample);
}
}
Ok(out)
})
}
}
enum Inner {
/// Our own AAC decoder (craft-audio): AAC-LC, and HE-AAC with its SBR high band when the
/// stream plays at twice the core rate (see [`craft_audio::aac::Decoder::set_sbr`]).
/// Surround output is reordered to WAV/SMPTE order (see [`aac_to_wav_order`]).
Aac { dec: Box<craft_audio::aac::Decoder>, asc: Vec<u8> },
/// AAC whose configuration arrives with the first frame (ADTS / LATM).
AacPending,
/// MPEG audio at this rate whose layer arrives with the first frame (MP4 / QuickTime name the
/// codec, not the layer; layers I / II go to our decoder, layer III to symphonia's).
MpegPending(u32),
/// Our own Opus decoder (always 48 kHz output; pre-skip is left to container timestamps).
/// `order` maps output channel → decoded channel (Vorbis → WAV/SMPTE order for surround).
Opus { dec: Box<craft_audio::opus::Decoder>, order: Option<&'static [usize]> },
/// Bootstrap decoders (MP3, ALAC, FLAC, HE-AAC…) via symphonia.
Symphonia(SymphoniaDecoder),
/// Our AC-3 and E-AC-3 decoder (ATSC A/52, Annex E).
Ac3(Box<craft_audio::ac3::Decoder>),
/// Our MPEG-1/2 audio Layer I and II decoder (ISO/IEC 11172-3, 13818-3).
Mpa(Box<craft_audio::mpa::Decoder>),
}
/// Opus always decodes at 48 kHz (RFC 7845 §5.1: the input rate in the header is informational).
pub const OPUS_RATE: u32 = 48_000;
/// Opus pre-roll decoded before a random-access target, in 48 kHz frames: 320 ms.
///
/// The containers' 80 ms (RFC 7845 §4.6, Matroska `SeekPreRoll`) is enough to sound right, but the
/// CELT energy predictor converges only ~24 dB per 80 ms: measured against a continuous decode, a
/// cold start 80 ms early is ~26 dB off, 320 ms early ~89 dB (float-exact). Seeking must give the
/// same samples as playback (render caches, scrubbing), so we pay the extra few packets.
pub const OPUS_PRE_ROLL: u32 = 4 * 3840;
/// Channel order for Vorbis-ordered surround (Opus mapping family 1, RFC 7845 §5.1.1.2): output
/// channel → decoded channel, giving the WAV/SMPTE order (L R C LFE back… side…) used elsewhere.
fn vorbis_to_wav_order(channels: usize) -> Option<&'static [usize]> {
Some(match channels {
3 => &[0, 2, 1],
5 => &[0, 2, 1, 3, 4],
6 => &[0, 2, 1, 5, 3, 4],
7 => &[0, 2, 1, 6, 5, 3, 4],
8 => &[0, 2, 1, 7, 5, 6, 3, 4],
_ => return None,
})
}
/// Channel order for AAC surround (ISO/IEC 14496-3 Table 1.19, our decoder's element order: C, L R,
/// surrounds, LFE): output channel → decoded channel, giving the WAV/SMPTE order (L R C LFE Ls Rs)
/// used elsewhere, as the AC-3 decoder does and the AAC encoder's caller undoes on export. Only the
/// unambiguous channel configurations 3–6, 11 (6.1: … LFE, back centre, sides) and 12 (7.1: … LFE,
/// backs, sides); 1, 2, 7 and PCE layouts keep the decoded order.
fn aac_to_wav_order(channel_config: u8) -> Option<&'static [usize]> {
Some(match channel_config {
3 => &[1, 2, 0],
4 => &[1, 2, 0, 3],
5 => &[1, 2, 0, 3, 4],
6 => &[1, 2, 0, 5, 3, 4],
11 => &[1, 2, 0, 6, 5, 3, 4],
12 => &[1, 2, 0, 7, 5, 6, 3, 4],
_ => return None,
})
}
/// [`aac_to_wav_order`] for a stream's configuration. Configuration 0 takes the speakers its
/// program config element spells out (`craft_audio::aac::speaker_config`), so a PCE carried in an
/// MP4 `esds` or in LATM's `StreamMuxConfig` in the shape of 5.1 is reordered like configuration 6;
/// a layout that cannot be told keeps the decoded order.
fn aac_wav_order(asc: &craft_audio::aac::AudioSpecificConfig) -> Option<&'static [usize]> {
aac_to_wav_order(craft_audio::aac::speaker_config(asc))
}
/// `channels` reordered by `order` (output channel → decoded channel). A decode whose channel count
/// does not match the order is returned unchanged.
fn reorder(mut channels: Vec<Vec<f32>>, order: Option<&[usize]>) -> Vec<Vec<f32>> {
match order {
Some(order) if order.len() == channels.len() => order.iter().map(|&c| channels.get_mut(c).map(std::mem::take).unwrap_or_default()).collect(),
_ => channels,
}
}
/// Duration of an Opus packet in 48 kHz samples from its TOC and frame count (RFC 6716 §3.1),
/// without parsing frame lengths. For a multistream packet this is the first stream's TOC, which
/// every stream shares. `None` for an empty or invalid (> 120 ms) packet.
pub fn opus_packet_samples(data: &[u8]) -> Option<usize> {
let toc = craft_audio::opus::Toc::parse(*data.first()?);
let frames = match data[0] & 3 {
0 => 1,
1 | 2 => 2,
_ => (*data.get(1)? & 0x3F) as usize,
};
let n = toc.frame_samples_48k() * frames;
(n > 0 && n <= 5760).then_some(n)
}
/// A packet decoder producing planar f32.
pub struct PacketDecoder {
inner: Inner,
pub channels: usize,
}
impl PacketDecoder {
pub fn new(codec: CodecType, sample_rate: u32, extra: Option<Vec<u8>>) -> Result<Self> {
let mut p = CodecParameters::new();
p.for_codec(codec).with_sample_rate(sample_rate);
if let Some(x) = extra {
p.with_extra_data(x.into_boxed_slice());
}
let dec = SymphoniaDecoder::new(&p)?;
Ok(Self { inner: Inner::Symphonia(dec), channels: 0 })
}
/// AAC: our decoder for AAC-LC and HE-AAC (v1; the mono core of v2); symphonia for other
/// object types. `sample_rate` is the stream's output rate: when it is twice the core rate
/// (HE-AAC, see [`aac_output_rate`]) the SBR high band is decoded at that rate.
pub fn aac(asc: &[u8], sample_rate: u32) -> Result<Self> {
if let Ok(mut dec) = craft_audio::aac::Decoder::new(asc) {
let sbr = decodes_sbr(dec.sample_rate(), sample_rate);
dec.set_sbr(sbr);
return Ok(Self { inner: Inner::Aac { dec: Box::new(dec), asc: asc.to_vec() }, channels: 0 });
}
Self::new(CODEC_TYPE_AAC, sample_rate, Some(asc.to_vec()))
}
/// Opus from an `OpusHead` (Matroska `CodecPrivate`, Ogg) or a `dOps` box payload. Output is
/// 48 kHz; pre-skip is *not* trimmed here (Matroska `CodecDelay` / MP4 edit lists do that).
pub fn opus(head: craft_audio::opus::OpusHead) -> Result<Self> {
let dec = craft_audio::opus::Decoder::from_head(head, OPUS_RATE).map_err(|e| CodecError::Unsupported(format!("Opus: {e}")))?;
let channels = dec.channels();
let order = if dec.head().mapping_family == 1 { vorbis_to_wav_order(channels) } else { None };
Ok(Self { inner: Inner::Opus { dec: Box::new(dec), order }, channels })
}
/// MPEG-1/2 audio layer I, II (our decoder: symphonia rejects the bit rate / mode
/// combinations ISO/IEC 11172-3 does not allow, such as ffmpeg's default 384 kbit/s mono MP2)
/// or III (symphonia).
pub fn mpeg_audio(layer: u8, sample_rate: u32) -> Result<Self> {
let codec = match layer {
1 | 2 => return Ok(Self { inner: Inner::Mpa(Box::default()), channels: 0 }),
_ => CODEC_TYPE_MP3,
};
Self::new(codec, sample_rate, None)
}
/// MPEG-1/2 audio of a layer the frame headers name (see [`mpeg_audio_layer`]): MP4's
/// `esds` object types 0x69 / 0x6B and QuickTime's `.mp3` / `.mp2` cover layers I–III alike
/// (ffmpeg stores MP2 as `mp4a` + 0x6B).
pub fn lazy_mpeg_audio(sample_rate: u32) -> Self {
Self { inner: Inner::MpegPending(sample_rate), channels: 0 }
}
/// AAC configured by [`Self::ensure_aac`] from in-band headers (ADTS, LATM).
pub fn lazy_aac() -> Self {
Self { inner: Inner::AacPending, channels: 0 }
}
/// (Re)configure for the AudioSpecificConfig `asc` unless it is the current one.
pub fn ensure_aac(&mut self, asc: &[u8]) -> Result<()> {
if let Inner::Aac { asc: cur, .. } = &self.inner
&& cur == asc
{
return Ok(());
}
let dec = craft_audio::aac::Decoder::new(asc).map_err(|e| CodecError::Unsupported(format!("AAC: {e}")))?;
self.inner = Inner::Aac { dec: Box::new(dec), asc: asc.to_vec() };
Ok(())
}
/// AC-3 and E-AC-3 (ATSC A/52, Annex E).
pub fn ac3() -> Result<Self> {
Ok(Self { inner: Inner::Ac3(Box::new(craft_audio::ac3::Decoder::new())), channels: 0 })
}
/// Whether this decodes Opus (which needs [`OPUS_PRE_ROLL`] of pre-roll after a seek).
pub fn is_opus(&self) -> bool {
matches!(self.inner, Inner::Opus { .. })
}
pub fn for_isobmff(c: &filmcraft_isobmff::CodecConfig, rate: u32) -> Result<Self> {
use filmcraft_isobmff::CodecConfig as C;
match c {
C::Aac(a) => Self::aac(&a.asc, if rate > 0 { rate } else { a.sample_rate }),
C::Mp3 => Ok(Self::lazy_mpeg_audio(rate)),
C::Ac3 { .. } | C::Eac3 { .. } => Self::ac3(),
C::Alac { cookie } => Self::new(CODEC_TYPE_ALAC, rate, Some(cookie.clone())),
// the decoder needs the STREAMINFO body: `dfLa` holds it after the 4-byte block header
C::Flac(f) => Self::new(CODEC_TYPE_FLAC, rate, f.metadata_blocks.get(4..38).map(<[u8]>::to_vec)),
C::Opus(o) => Self::opus(craft_audio::opus::OpusHead::from_dops(&o.to_bytes()).map_err(|e| CodecError::Unsupported(format!("Opus: {e}")))?),
other => Err(CodecError::Unsupported(format!("{} audio", other.name()))),
}
}
/// Decode one packet into planar channels.
pub fn decode(&mut self, data: &[u8], ts: u64) -> Result<Vec<Vec<f32>>> {
if let Inner::MpegPending(rate) = self.inner {
let layer = mpeg_audio_layer(data).ok_or_else(|| CodecError::Decode("MPEG audio: no frame header".into()))?;
self.inner = Self::mpeg_audio(layer, rate)?.inner;
}
let dec = match &mut self.inner {
Inner::Aac { dec, .. } => {
let out = dec.decode(data).map_err(|e| CodecError::Decode(e.to_string()))?;
let out = reorder(out, aac_wav_order(dec.config()));
self.channels = out.len();
return Ok(out);
}
Inner::Opus { dec, order } => {
// A corrupt packet is concealed like a lost one (keeps timing and decoder state).
let out = match dec.decode(Some(data)) {
Ok(o) => o,
Err(_) => dec.decode(None).map_err(|e| CodecError::Decode(e.to_string()))?,
};
let out = reorder(out, *order);
self.channels = out.len();
return Ok(out);
}
Inner::Symphonia(d) => d,
Inner::AacPending => return Err(CodecError::Decode("AAC: no configuration yet".into())),
Inner::MpegPending(_) => return Err(CodecError::Decode("MPEG audio: no frame header".into())),
Inner::Ac3(d) => {
// a packet may hold several syncframes (MP4 / Matroska); E-AC-3 dependent
// substreams and further programs decode to no channels and are skipped
let mut out: Vec<Vec<f32>> = Vec::new();
let mut p = 0;
while p < data.len() {
let f = d.decode(&data[p..]).map_err(|e| CodecError::Decode(e.to_string()))?;
p += f.header.frame_bytes;
if f.channels.is_empty() {
continue;
}
if out.is_empty() {
out = f.channels;
} else {
for (o, c) in out.iter_mut().zip(f.channels) {
o.extend(c);
}
}
}
self.channels = out.len();
return Ok(out);
}
Inner::Mpa(d) => {
// a packet holds one frame or several; a damaged first frame is an error, damage
// after a good frame ends the packet
let mut out: Vec<Vec<f32>> = Vec::new();
let mut p = 0;
while let Some(rest) = data.get(p..).filter(|r| !r.is_empty()) {
let f = match d.decode(rest) {
Ok(f) => f,
Err(e) if out.is_empty() => return Err(CodecError::Decode(e.to_string())),
Err(_) => break,
};
p += f.header.frame_bytes;
if out.is_empty() {
out = f.channels;
} else if out.len() == f.channels.len() {
for (o, c) in out.iter_mut().zip(f.channels) {
o.extend(c);
}
} else {
break;
}
}
self.channels = out.len();
return Ok(out);
}
};
let pkt = Packet::new_from_slice(0, ts, 0, data);
let out = dec.decode(&pkt)?.map_err(|e| CodecError::Decode(e.to_string()))?;
self.channels = out.len();
Ok(out)
}
pub fn reset(&mut self) {
match &mut self.inner {
// craft-audio's reset clears the overlap, noise and SBR signal state but keeps the
// SBR header the stream last sent, so packets after a seek that carry no header still
// get their high band. Rebuilding the decoder would drop it until the next header.
Inner::Aac { dec, .. } => dec.reset(),
Inner::Mpa(d) => d.reset(),
Inner::Opus { dec, .. } => dec.reset(),
Inner::Symphonia(d) => {
// The legacy reset API cannot return an error. Log it now and keep the failure
// latched so the next decode reports it to the caller as well.
if !d.failed
&& let Err(error) = d.call("resetting Symphonia", |decoder| decoder.reset())
{
log::error!("{error}");
}
}
Inner::Ac3(d) => d.reset(),
Inner::AacPending | Inner::MpegPending(_) => {}
}
}
}
/// The layer (1, 2 or 3) named by the MPEG-1/2 audio frame header that starts `frame`: the 12-bit
/// sync word (11 bits for MPEG-2.5), then `ID` and the 2-bit `layer` field, 3 for layer I down to 1
/// for layer III (ISO/IEC 11172-3 §2.4.2.3). `None` without a sync word or for the reserved layer 0.
pub fn mpeg_audio_layer(frame: &[u8]) -> Option<u8> {
match *frame {
[0xFF, b1, ..] if b1 & 0xE0 == 0xE0 && (b1 >> 1) & 3 != 0 => Some(4 - ((b1 >> 1) & 3)),
_ => None,
}
}
/// The output rate of an AAC stream: twice the core rate for HE-AAC (ISO/IEC 14496-3 §1.6.5),
/// from the `AudioSpecificConfig` or, with implicit signalling, the first access units
/// ([`craft_audio::aac::output_rate`]). `None` if `asc` is not one our decoder reads.
pub fn aac_output_rate<'a>(asc: &[u8], first_units: impl IntoIterator<Item = &'a [u8]>) -> Option<u32> {
craft_audio::aac::output_rate(asc, first_units)
}
/// Whether [`PacketDecoder::aac`] decodes the SBR high band of an AAC core at `core` Hz to play
/// it at `rate` (HE-AAC).
fn decodes_sbr(core: u32, rate: u32) -> bool {
core.checked_mul(2) == Some(rate)
}
/// A codec whose packets all decode to the same number of samples (see [`fixed_packet_samples`]).
#[derive(Clone, Copy, Debug)]
pub enum FixedFrames<'a> {
/// AAC with this AudioSpecificConfig.
Aac(&'a [u8]),
/// MPEG-1/2 audio, layer I, II or III.
MpegAudio,
/// AC-3 (and E-AC-3 in a stream of single syncframes, MPEG-TS).
Ac3,
/// E-AC-3 packets of whole syncframes (MP4, Matroska): an access unit may hold several
/// syncframes and dependent substreams.
Eac3,
}
/// Samples each packet decodes to at the stream's output `rate`, for codecs where that is the same
/// for every packet: AAC 1024 (2048 when [`PacketDecoder::aac`] decodes HE-AAC with SBR); MPEG
/// audio and AC-3 from the frame header of `first`, the stream's first packet; E-AC-3 from the
/// syncframes of the program we decode in `first`. `None` when it is not known: an AAC
/// configuration our decoder does not read, a header that does not parse, or a header whose rate
/// is not `rate`.
pub fn fixed_packet_samples(codec: FixedFrames, first: &[u8], rate: u32) -> Option<i64> {
let mpegts = match codec {
FixedFrames::Aac(asc) => {
let core = craft_audio::aac::Decoder::new(asc).ok()?.sample_rate();
return Some(if decodes_sbr(core, rate) { 2048 } else { 1024 });
}
FixedFrames::Eac3 => {
let mut samples = 0usize;
let mut p = 0;
while let Some(h) = first.get(p..).and_then(|b| craft_audio::ac3::parse_header(b).ok()) {
if h.is_primary() {
if h.sample_rate != rate {
return None;
}
samples += h.samples();
}
p += h.frame_bytes;
}
return i64::try_from(samples).ok().filter(|&n| n > 0);
}
FixedFrames::MpegAudio => craft_media::mpegts::Codec::MpegAudio,
FixedFrames::Ac3 => craft_media::mpegts::Codec::Ac3,
};
let f = craft_media::mpegts::frame_info(&mpegts, first)?;
(f.sample_rate == rate && f.samples > 0).then_some(f.samples as i64)
}
/// Where the container puts one audio packet, for [`contiguous_starts`].
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PacketTime {
/// Its timestamp, in output samples.
pub stamp: i64,
/// Whether `stamp` is the packet's own. A Matroska frame after the first in a laced block
/// without a duration repeats the block's timestamp.
pub stamped: bool,
/// Samples it decodes to, when known.
pub samples: Option<i64>,
}
/// Packet start positions in output samples: each packet starts where the previous one ends,
/// unless its own timestamp is more than `tolerance` samples from there (a real gap or overlap in
/// the recording) or the previous packet's length is unknown; then it starts at its timestamp.
///
/// Container timestamps are coarser than a sample. Matroska stores milliseconds, and a remux to MP4
/// (OBS's) turns that rounding into uneven sample durations: a 1024-sample AAC frame at 48 kHz lasts
/// 21.33 ms, so frames are stamped at 0, 1008, 2064, 3072… Placing each decoded packet at its
/// stamp overwrote the end of one packet and left a hole of silence after another, a click every
/// few packets: crackling throughout playback and export (#236).
pub fn contiguous_starts(packets: impl IntoIterator<Item = PacketTime>, tolerance: i64) -> Vec<i64> {
let tolerance = tolerance.max(0).unsigned_abs();
let mut next: Option<i64> = None;
packets
.into_iter()
.map(|p| {
let start = match next {
Some(n) if !p.stamped || n.abs_diff(p.stamp) <= tolerance => n,
_ => p.stamp,
};
next = p.samples.and_then(|k| start.checked_add(k));
start
})
.collect()
}
/// Whether AC-3 and E-AC-3 audio decode (our ATSC A/52 decoder).
pub const AC3_DECODER: bool = true;
/// An AudioSpecificConfig for AAC with these parameters (ISO/IEC 14496-3 §1.6.2.1, GA specific
/// config with no extension flags).
fn asc_bytes(object_type: u8, sf_index: u8, channel_config: u8) -> Vec<u8> {
let v = ((object_type as u16) << 11) | ((sf_index as u16 & 15) << 7) | ((channel_config as u16 & 15) << 3);
v.to_be_bytes().to_vec()
}
/// An ADTS frame (ISO/IEC 13818-7 §6.2, parsed by [`craft_audio::aac::AdtsHeader`]): the
/// equivalent AudioSpecificConfig and the raw data block. Frames with several raw data blocks are
/// not split (`None`).
pub fn adts_split(frame: &[u8]) -> Option<(Vec<u8>, &[u8])> {
let h = craft_audio::aac::AdtsHeader::parse(frame).ok().filter(|h| h.raw_data_blocks == 1)?;
let end = h.frame_length.min(frame.len());
(end > h.header_length).then(|| (asc_bytes(h.object_type, h.sf_index, h.channel_config), &frame[h.header_length..end]))
}
/// LOAS / LATM (AAC in MPEG transport streams): craft-audio reads the StreamMuxConfig and the
/// payload of the single sub-frame.
pub use craft_audio::aac::{LatmConfig, latm_config, latm_split};
/// Blu-ray and DVD LPCM packet decoding from the shared audio library.
pub use craft_audio::lpcm::{BlurayLpcm, bluray_lpcm_header, decode_bluray_lpcm, decode_dvd_lpcm, dvd_lpcm_group_bytes};
/// Decode interleaved PCM bytes into planar f32.
pub fn decode_pcm(data: &[u8], cfg: &filmcraft_isobmff::PcmConfig) -> Vec<Vec<f32>> {
let mut out = vec![Vec::new(); cfg.channels.max(1) as usize];
let format = craft_audio::pcm_interleaved::Format { bits: cfg.bits, float: cfg.float, big_endian: cfg.big_endian, signed: cfg.signed };
craft_audio::pcm_interleaved::decode_into(data, format, &mut out);
out
}
/// Resample-read `frames` from planar source audio at `src_rate` into a buffer at `rate`,
/// starting at output sample `start` (linear interpolation; the audio crate adds sinc later).
pub fn read_resampled(src: &[Vec<f32>], src_rate: u32, start: i64, frames: usize, rate: u32) -> filmcraft_media::Result<AudioBuffer> {
if rate == 0 {
return Err(MediaError::Decode("audio sample rate must be nonzero".into()));
}
let ch = src.len().max(1);
filmcraft_media::validate_audio_buffer_shape(ch, frames)?;
let mut out = AudioBuffer::silence(rate, ch, frames);
let total = src.first().map_or(0, Vec::len);
let ratio = src_rate as f64 / rate as f64;
for i in 0..frames {
let pos = (start + i as i64) as f64 * ratio;
if pos < 0.0 {
continue;
}
let i0 = pos.floor() as usize;
if i0 >= total {
break;
}
let f = (pos - i0 as f64) as f32;
for (c, s) in src.iter().enumerate() {
let a = s[i0];
let b = s.get(i0 + 1).copied().unwrap_or(a);
out.channels[c][i] = a + (b - a) * f;
}
}
Ok(out)
}
/// A standalone audio file decoded fully into memory (audio files are small next to video).
pub struct AudioFileSource {
info: MediaInfo,
rate: u32,
samples: Vec<Vec<f32>>,
}
impl AudioFileSource {
pub fn decode(name: &str, bytes: Arc<[u8]>) -> Result<Self> {
Self::decode_with(name, bytes, SymphoniaDecoder::new)
}
fn decode_with(name: &str, bytes: Arc<[u8]>, make_decoder: impl FnOnce(&CodecParameters) -> Result<SymphoniaDecoder>) -> Result<Self> {
// Probing, demuxing, decoder construction and sample conversion can all see hostile input.
// Everything owned by this attempt is discarded if it unwinds; no partial source escapes.
guard_audio(&format!("opening {name}"), || Self::decode_inner(name, bytes, make_decoder))
}
fn decode_inner(name: &str, bytes: Arc<[u8]>, make_decoder: impl FnOnce(&CodecParameters) -> Result<SymphoniaDecoder>) -> Result<Self> {
let ext = name.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
if ext == "mp3" {
let (audio, decoded) = craft_audio::decode(&bytes, Some("mp3")).map_err(|e| match e {
craft_audio::AudioError::Unsupported(message) | craft_audio::AudioError::TooLarge(message) => {
CodecError::Unsupported(format!("{name}: {message}"))
}
error => CodecError::Decode(format!("{name}: {error}")),
})?;
let frames = decoded.frames();
let rate = decoded.sample_rate;
let samples = decoded.channels;
if samples.is_empty() || frames == 0 {
return Err(CodecError::Decode(format!("{name}: no audio decoded")));
}
let info = MediaInfo {
name: name.to_string(),
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames as i64, rate as i64),
video: None,
audio_streams: vec![AudioStreamInfo {
sample_rate: rate,
channels: samples.len() as u32,
codec: audio.codec.name().to_uppercase(),
bits_per_sample: None,
}],
container: ext.to_uppercase(),
start_timecode: None,
file_size: Some(bytes.len() as u64),
};
return Ok(Self { info, rate, samples });
}
if ext == "mp2"
&& let Some((layer, rate, samples)) = decode_mpeg_layer12(&bytes)
{
let frames = samples.first().map_or(0, Vec::len);
let info = MediaInfo {
name: name.to_string(),
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames as i64, rate as i64),
video: None,
audio_streams: vec![AudioStreamInfo { sample_rate: rate, channels: samples.len() as u32, codec: format!("MP{layer}"), bits_per_sample: None }],
container: ext.to_uppercase(),
start_timecode: None,
file_size: Some(bytes.len() as u64),
};
return Ok(Self { info, rate, samples });
}
if ext == "aac" {
let (rate, samples) = decode_adts(&bytes)?.ok_or_else(|| CodecError::Decode(format!("{name}: no ADTS AAC frames")))?;
let frames = samples.first().map_or(0, Vec::len);
let info = MediaInfo {
name: name.to_string(),
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames as i64, rate as i64),
video: None,
audio_streams: vec![AudioStreamInfo { sample_rate: rate, channels: samples.len() as u32, codec: "AAC".into(), bits_per_sample: None }],
container: "ADTS".into(),
start_timecode: None,
file_size: Some(bytes.len() as u64),
};
return Ok(Self { info, rate, samples });
}
let mss = MediaSourceStream::new(Box::new(std::io::Cursor::new(bytes.clone())), Default::default());
let mut hint = Hint::new();
hint.with_extension(&ext);
// gapless: trim the encoder delay/padding that MP3 (LAME/Xing tag) files declare, so audio lands on the timeline where other editors put it.
// MP3 only: the other formats' timing (AAC priming in MP4 edit lists, …) is handled elsewhere and checked against ffmpeg.
let fmt_opts = FormatOptions { enable_gapless: ext == "mp3", ..FormatOptions::default() };
let probed =
symphonia::default::get_probe().format(&hint, mss, &fmt_opts, &MetadataOptions::default()).map_err(|e| CodecError::Unsupported(e.to_string()))?;
let mut format = probed.format;
let track = format.default_track().ok_or_else(|| CodecError::Unsupported("no audio track".into()))?.clone();
let rate = track.codec_params.sample_rate.unwrap_or(48_000);
let mut dec = make_decoder(&track.codec_params)?;
let mut samples: Vec<Vec<f32>> = Vec::new();
while let Ok(pkt) = format.next_packet() {
if pkt.track_id() != track.id {
continue;
}
let Ok(channels) = dec.decode(&pkt)? else { continue };
if samples.is_empty() {
samples = vec![Vec::new(); channels.len()];
}
let add: usize = channels.iter().map(Vec::len).sum();
check_decoded_budget(samples.iter().map(Vec::len).sum(), add)?;
for (out, channel) in samples.iter_mut().zip(channels) {
out.extend(channel);
}
}
if samples.is_empty() {
return Err(CodecError::Decode("no audio decoded".into()));
}
let frames = samples[0].len();
let codec_name =
symphonia::default::get_codecs().get_codec(track.codec_params.codec).map(|d| d.short_name.to_uppercase()).unwrap_or_else(|| ext.to_uppercase());
let info = MediaInfo {
name: name.to_string(),
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames as i64, rate as i64),
video: None,
audio_streams: vec![AudioStreamInfo {
sample_rate: rate,
channels: samples.len() as u32,
codec: codec_name,
bits_per_sample: track.codec_params.bits_per_sample,
}],
container: ext.to_uppercase(),
start_timecode: None,
file_size: Some(bytes.len() as u64),
};
Ok(Self { info, rate, samples })
}
}
/// The length of an ID3v2 tag at the start of `bytes` (0 without one): a 10-byte header, a
/// syncsafe size, and a 10-byte footer when flagged (ID3v2.4 §3).
fn id3v2_len(bytes: &[u8]) -> usize {
match bytes {
[b'I', b'D', b'3', _, _, flags, s0, s1, s2, s3, ..] => {
let size = [*s0, *s1, *s2, *s3].iter().fold(0usize, |a, &b| (a << 7) | (b & 0x7F) as usize);
size.saturating_add(if flags & 0x10 != 0 { 20 } else { 10 })
}
_ => 0,
}
}
/// Decoded-packet caches (AVI, MP4, Matroska) keep at most this many packets and this many bytes:
/// one packet can decode to far more than it holds (a PCM chunk, or overlapping references that
/// each decode in full), so the count alone does not bound memory.
pub(crate) const MAX_PACKET_CACHE_ENTRIES: usize = 4096;
pub(crate) const MAX_PACKET_CACHE_BYTES: usize = 256 << 20;
/// A cached decoded packet: how many bytes it holds.
pub(crate) trait PacketBytes {
fn packet_bytes(&self) -> usize;
}
impl PacketBytes for Vec<Vec<f32>> {
fn packet_bytes(&self) -> usize {
self.iter().map(|c| c.len().saturating_mul(4)).fold(0, usize::saturating_add)
}
}
/// Add packet `i` to a cache (`packets`, oldest-first `order`, running `bytes`), then drop the
/// oldest packets, keeping the newest, until the cache is within both limits.
pub(crate) fn cache_packet<P: PacketBytes>(packets: &mut HashMap<usize, Arc<P>>, order: &mut Vec<usize>, bytes: &mut usize, i: usize, p: &Arc<P>) {
packets.insert(i, p.clone());
order.push(i);
*bytes = bytes.saturating_add(p.packet_bytes());
while order.len() > 1 && (order.len() > MAX_PACKET_CACHE_ENTRIES || *bytes > MAX_PACKET_CACHE_BYTES) {
let old = order.remove(0);
if let Some(q) = packets.remove(&old) {
*bytes = bytes.saturating_sub(q.packet_bytes());
}
}
}
/// Most samples (all channels together) a standalone ADTS file decodes into memory: 4 GiB of f32,
/// about three hours of 48 kHz stereo. Longer files are refused rather than allocated.
const MAX_ADTS_SAMPLES: usize = 1 << 30;
/// Most samples (all channels together) a standalone file or an Ogg Vorbis stream keeps decoded in
/// memory: the same 4 GiB of f32 as [`MAX_ADTS_SAMPLES`]. A file that decodes to more is refused.
pub(crate) const MAX_DECODED_SAMPLES: usize = MAX_ADTS_SAMPLES;
/// Whether `retained` samples plus `add` more stay within [`MAX_DECODED_SAMPLES`] (compact packets
/// can decode to blocks far larger than themselves, so a small file could otherwise exhaust memory).
pub(crate) fn check_decoded_budget(retained: usize, add: usize) -> Result<()> {
if retained.saturating_add(add) > MAX_DECODED_SAMPLES {
return Err(CodecError::Unsupported("audio too long to decode into memory (over 4 GiB decoded): shorten or transcode it".into()));
}
Ok(())
}
/// The length of the ADTS frame starting `b` ([`craft_audio::aac::AdtsHeader`]: sync word, layer
/// `00`, 13-bit `frame_length` including the header), when it starts with a header and fits in `b`.
fn adts_frame_len(b: &[u8]) -> Option<usize> {
let h = craft_audio::aac::AdtsHeader::parse(b).ok()?;
(h.frame_length > h.header_length && h.frame_length <= b.len()).then_some(h.frame_length)
}
/// A raw ADTS stream (`.aac`) decoded with our AAC decoder: (output sample rate, planar samples).
/// Frames follow each other from the first header (after an ID3v2 tag); damaged data is skipped
/// up to the next header, frames with another configuration than the first are skipped, and a
/// frame that does not decode keeps its duration as silence. HE-AAC decodes with its SBR high band
/// at its output rate. `Ok(None)` when no frame decodes.
fn decode_adts(bytes: &[u8]) -> Result<Option<(u32, Vec<Vec<f32>>)>> {
let mut frames: Vec<(Vec<u8>, &[u8])> = Vec::new();
let mut p = id3v2_len(bytes);
while let Some(rest) = bytes.get(p..).filter(|r| r.len() >= 7) {
match adts_frame_len(rest).and_then(|len| Some((len, adts_split(rest.get(..len)?)?))) {
Some((len, f)) => {
frames.push(f);
p += len;
}
None => p += 1,
}
}
let Some(asc) = frames.first().map(|f| f.0.clone()) else { return Ok(None) };
let Some(rate) = aac_output_rate(&asc, frames.iter().take(8).map(|f| f.1)) else { return Ok(None) };
let mut dec = PacketDecoder::aac(&asc, rate)?;
let mut out: Vec<Vec<f32>> = Vec::new();
let mut total = 0usize;
let mut last = 0usize;
for (cfg, payload) in frames {
if cfg != asc {
continue;
}
let decoded = dec.decode(payload, 0).ok().filter(|c| out.is_empty() || c.len() == out.len());
match decoded {
Some(c) if out.is_empty() => {
last = c.first().map_or(0, Vec::len);
out = c;
}
Some(c) => {
last = c.first().map_or(0, Vec::len);
for (o, s) in out.iter_mut().zip(c) {
o.extend(s);
}
}
None => {
for o in out.iter_mut() {
o.resize(o.len() + last, 0.0);
}
}
}
total = total.saturating_add(last.saturating_mul(out.len()));
if total > MAX_ADTS_SAMPLES {
return Err(CodecError::Unsupported("AAC file too long to import as raw ADTS (over 4 GiB decoded): remux it to .m4a".into()));
}
}
Ok(out.first().is_some_and(|c| !c.is_empty()).then_some((rate, out)))
}
/// A raw MPEG audio Layer I / II stream (`.mp2`) decoded with our decoder: (layer, sample rate,
/// planar samples). Frames follow each other from the first frame header (after an ID3v2 tag);
/// damaged data is skipped up to the next frame header of the same layer and rate, and a frame
/// with another channel count keeps its duration as silence. `None` when the stream does not start
/// with a Layer I / II frame followed by another frame (or the end), e.g. MP3.
fn decode_mpeg_layer12(bytes: &[u8]) -> Option<(u8, u32, Vec<Vec<f32>>)> {
let mut p = id3v2_len(bytes);
let frame_at = |p: usize| bytes.get(p..).and_then(|b| craft_audio::mpa::parse_header(b).ok()).filter(|h| h.frame_bytes > 0);
let first = frame_at(p)?;
let next = p.saturating_add(first.frame_bytes);
if first.layer == 3 || (next < bytes.len() && frame_at(next).is_none_or(|h| h.layer != first.layer || h.sample_rate != first.sample_rate)) {
return None;
}
let mut dec = craft_audio::mpa::Decoder::new();
let mut out: Vec<Vec<f32>> = vec![Vec::new(); first.channels()];
while let Some(rest) = bytes.get(p..).filter(|r| r.len() >= 4) {
match dec.decode(rest) {
Ok(f) if f.header.layer == first.layer && f.header.sample_rate == first.sample_rate => {
let n = f.header.samples();
if f.channels.len() == out.len() {
for (o, c) in out.iter_mut().zip(f.channels) {
o.extend(c);
}
} else {
for o in out.iter_mut() {
o.resize(o.len() + n, 0.0);
}
}
p += f.header.frame_bytes;
}
_ => p += 1,
}
}
out.first().is_some_and(|c| !c.is_empty()).then_some((first.layer, first.sample_rate, out))
}
impl MediaSource for AudioFileSource {
fn info(&self) -> &MediaInfo {
&self.info
}
fn video_frame(&self, _req: FrameRequest) -> std::result::Result<Arc<VideoFrame>, MediaError> {
Err(MediaError::NoStream("video"))
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> std::result::Result<AudioBuffer, MediaError> {
read_resampled(&self.samples, self.rate, start, frames, sample_rate)
}
}
pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option<std::result::Result<SharedSource, MediaError>> {
let ext = name.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
if !matches!(ext.as_str(), "mp3" | "mp2" | "aac" | "flac" | "ogg" | "oga" | "aif" | "aiff" | "aifc") {
return None;
}
Some(AudioFileSource::decode(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}
#[cfg(test)]
mod resample_bounds_tests {
use super::*;
#[test]
fn read_resampled_rejects_an_oversized_window() {
let src = vec![vec![0.0f32; 48]; 2];
assert!(read_resampled(&src, 48_000, 0, usize::MAX / 8, 48_000).is_err());
assert!(read_resampled(&src, 48_000, 0, filmcraft_media::MAX_AUDIO_BUFFER_BYTES, 48_000).is_err());
assert_eq!(read_resampled(&src, 48_000, 0, 24, 48_000).unwrap().channels[0].len(), 24);
assert!(read_resampled(&src, 48_000, 0, 24, 0).is_err(), "a zero output rate must not divide");
}
}
#[cfg(test)]
mod mpeg_layer12_tests {
use super::*;
/// A silent 48 kHz MPEG-1 Layer II frame (all-zero allocations) of `bitrate_index` and `mode`.
fn silent_frame(bitrate_index: u8, mode: u8) -> Vec<u8> {
let h = [0xFF, 0xFD, bitrate_index << 4 | 0x04, mode << 6];
let mut f = h.to_vec();
f.resize(craft_audio::mpa::parse_header(&h).unwrap().frame_bytes, 0);
f
}
/// #809: mono MP2 at 384 kbit/s (ffmpeg's default for mono) is outside ISO/IEC 11172-3's
/// allowed combinations, and symphonia rejected every frame, so the track played as silence.
#[test]
fn mono_mp2_at_384k_decodes() {
let mut d = PacketDecoder::mpeg_audio(2, 48_000).unwrap();
let out = d.decode(&silent_frame(14, 3), 0).unwrap();
assert_eq!((out.len(), out[0].len(), d.channels), (1, 1152, 1));
// a packet of two stereo frames decodes to both
let two = [silent_frame(10, 0), silent_frame(10, 0)].concat();
let out = d.decode(&two, 1152).unwrap();
assert_eq!((out.len(), out[0].len()), (2, 2304));
// damage: an error for the first frame, the good frames before it otherwise
assert!(d.decode(&[0u8; 8], 0).is_err());
assert!(d.decode(&silent_frame(10, 0)[..100], 0).is_err());
let partial = [silent_frame(10, 0), vec![0xFF, 0xFD, 0xA4]].concat();
assert_eq!(d.decode(&partial, 0).unwrap()[0].len(), 1152);
d.reset();
}
/// Raw `.mp2` files: after an ID3v2 tag, frame after frame, skipping damage.
#[test]
fn raw_streams_resync_after_damage() {
let mono = silent_frame(14, 3);
let file = [&b"ID3\x04\0\0\0\0\0\x05abcde"[..], &mono, &mono, &[0xFF, 0xFD, 0x13, 0x37, 0x42], &mono].concat();
let (layer, rate, samples) = decode_mpeg_layer12(&file).unwrap();
assert_eq!((layer, rate, samples.len(), samples[0].len()), (2, 48_000, 1, 3 * 1152));
let src = AudioFileSource::decode("x.mp2", file.clone().into()).unwrap();
assert_eq!(src.info().audio().map(|a| (a.channels, a.codec.as_str())), Some((1, "MP2")));
// MP3, junk and truncated data are not ours (symphonia gets them)
assert!(decode_mpeg_layer12(&[0xFF, 0xFB, 0x90, 0x44, 0, 0]).is_none());
for n in 0..40 {
assert!(decode_mpeg_layer12(&file[..n]).is_none(), "{n}");
}
assert!(decode_mpeg_layer12(&[&mono[..], &[0x12; 50]].concat()).is_none());
// an ID3v2 tag claiming the largest size (with a footer) points past the end
assert!(decode_mpeg_layer12(&[&b"ID3\x04\0\x10\x7F\x7F\x7F\x7F"[..], &mono].concat()).is_none());
}
}
#[cfg(test)]
mod he_aac_tests {
use super::*;
/// AAC-LC config, 22.05 kHz stereo.
const LC_22K: [u8; 2] = [0x13, 0x90];
/// An access unit holding only a fill element with SBR data (ID_FIL, EXT_SBR_DATA, ID_END).
const SBR_FILL: [u8; 3] = [0b1100_0011, 0b1010_0001, 0b1100_0000];
#[test]
fn output_rate_doubles_for_he_aac() {
// explicit: the extension rate from the config
assert_eq!(aac_output_rate(&[0x2B, 0x92, 0x08, 0x00], []), Some(44_100));
// implicit: SBR data in the first access units of a ≤ 24 kHz AAC-LC stream
assert_eq!(aac_output_rate(&LC_22K, [&SBR_FILL[..]]), Some(44_100));
assert_eq!(aac_output_rate(&LC_22K, []), Some(22_050));
// AAC-LC at 44.1 kHz stays as it is, even with SBR-looking fill data
assert_eq!(aac_output_rate(&[0x12, 0x10], [&SBR_FILL[..]]), Some(44_100));
}
/// HE-AAC played at its output rate decodes through craft-audio's SBR tool (2048 samples an
/// access unit, the core's band carried through the filter banks), not an interpolated core;
/// the same stream at its core rate stays AAC-LC. A reset keeps the SBR choice.
#[test]
fn he_aac_decodes_its_sbr_band_at_the_output_rate() {
let mut enc = craft_audio::aac::Encoder::new(craft_audio::aac::EncoderConfig::cbr(22_050, 2, 64_000)).unwrap();
let tone: Vec<f32> = (0..22_050).map(|i| (i as f32 * std::f32::consts::TAU * 440.0 / 22_050.0).sin() * 0.5).collect();
let mut aus = enc.encode(&[&tone, &tone]);
aus.extend(enc.flush());
// explicit signalling: AOT 5, 22.05 kHz core, stereo, 44.1 kHz output
let he = [0x2B, 0x92, 0x08, 0x00];
let mut dec = PacketDecoder::aac(&he, 44_100).unwrap();
let mut core = PacketDecoder::aac(&he, 22_050).unwrap();
let mut energy = 0.0f32;
for (i, au) in aus.iter().enumerate() {
let out = dec.decode(au, 0).unwrap();
assert_eq!((out.len(), out[0].len()), (2, 2048), "access unit {i}");
energy += out[0].iter().map(|v| v * v).sum::<f32>();
assert_eq!(core.decode(au, 0).unwrap()[0].len(), 1024);
}
assert!(energy > 1.0, "the core's tone comes through the SBR filter banks");
dec.reset();
assert_eq!(dec.decode(&aus[1], 0).unwrap()[0].len(), 2048);
}
}
#[cfg(test)]
mod surround_order_tests {
use super::*;
/// Power of `x` at `hz` (Goertzel), normalised by length.
fn power(x: &[f32], hz: f32, rate: f32) -> f32 {
let k = 2.0 * (std::f32::consts::TAU * hz / rate).cos();
let (mut s1, mut s2) = (0.0f32, 0.0f32);
for &v in x {
let s0 = v + k * s1 - s2;
s2 = s1;
s1 = s0;
}
(s1 * s1 + s2 * s2 - k * s1 * s2) / x.len() as f32
}
/// #721: 5.1 AAC (configuration 6, coded C L R Ls Rs LFE) decodes to WAV/SMPTE order
/// L R C LFE Ls Rs, as the rest of FilmCraft and the AAC export path expect.
#[test]
fn aac_51_decodes_in_wav_order() {
let rate = 48_000u32;
// tones per speaker, in WAV order: L, R, C, LFE, Ls, Rs
let hz = [300.0f32, 400.0, 500.0, 60.0, 700.0, 800.0];
let tone = |f: f32| (0..rate as usize).map(|i| 0.25 * (std::f32::consts::TAU * f * i as f32 / rate as f32).sin()).collect::<Vec<f32>>();
let wav: Vec<Vec<f32>> = hz.iter().map(|&f| tone(f)).collect();
// the encoder takes AAC element order (as the exporter feeds it): C L R Ls Rs LFE
let coded: Vec<&[f32]> = [2usize, 0, 1, 4, 5, 3].iter().map(|&c| wav[c].as_slice()).collect();
let mut enc = craft_audio::aac::Encoder::new(craft_audio::aac::EncoderConfig::cbr(rate, 6, 384_000)).unwrap();
let mut units = enc.encode(&coded);
units.extend(enc.flush());
let mut dec = PacketDecoder::aac(&enc.audio_specific_config(), rate).unwrap();
let mut out = vec![Vec::new(); 6];
for u in &units {
for (o, c) in out.iter_mut().zip(dec.decode(u, 0).unwrap()) {
o.extend(c);
}
}
assert_eq!(dec.channels, 6);
for (ch, samples) in out.iter().enumerate() {
let window = &samples[4800..38_400];
let own = power(window, hz[ch], rate as f32);
for (other, &f) in hz.iter().enumerate().filter(|&(o, _)| o != ch) {
let p = power(window, f, rate as f32);
assert!(own > 100.0 * p, "channel {ch}: {} Hz {own} vs {f} Hz (channel {other}) {p}", hz[ch]);
}
}
}
#[test]
fn reorder_leaves_mismatched_channel_counts_alone() {
let chans = vec![vec![1.0], vec![2.0]];
assert_eq!(reorder(chans.clone(), aac_to_wav_order(6)), chans);
assert_eq!(reorder(vec![vec![0.0], vec![1.0], vec![2.0]], aac_to_wav_order(3)), vec![vec![1.0], vec![2.0], vec![0.0]]);
}
/// Configuration 0 with a program config element listing centre, front pair, back pair and LFE
/// (5.1 as configuration 6 lays it out) is reordered to WAV order like configuration 6; a
/// pair-first PCE (ffmpeg's 5.1(side)) already decodes in WAV order and is left alone.
#[test]
fn a_program_config_element_in_the_shape_of_5_1_is_reordered_like_configuration_6() {
use craft_audio::aac::{AudioSpecificConfig, ProgramConfig};
let with = |front: Vec<(bool, u8)>, side: Vec<(bool, u8)>, back: Vec<(bool, u8)>, lfe: Vec<u8>| {
let mut pce = ProgramConfig::default();
(pce.front, pce.side, pce.back, pce.lfe) = (front, side, back, lfe);
let mut asc = AudioSpecificConfig::lc(3, 0);
asc.pce = Some(pce);
asc
};
assert!(aac_to_wav_order(6).is_some());
let centre_first = with(vec![(false, 0), (true, 0)], vec![], vec![(true, 1)], vec![0]);
assert_eq!(aac_wav_order(¢re_first), aac_to_wav_order(6));
let pair_first = with(vec![(true, 0), (false, 0)], vec![(false, 1)], vec![(true, 1)], vec![]);
assert_eq!(aac_wav_order(&pair_first), None);
assert_eq!(aac_wav_order(&AudioSpecificConfig::lc(3, 6)), aac_to_wav_order(6));
}
}
#[cfg(test)]
mod packet_time_tests {
use super::*;
/// 1024-sample frames at 48 kHz stamped to the nearest millisecond, then back in samples.
fn ms_stamps(count: i64) -> Vec<i64> {
(0..count).map(|k| (k * 1024 * 1000 + 24_000) / 48_000 * 48).collect()
}
fn timed(stamps: &[i64], samples: Option<i64>) -> Vec<PacketTime> {
stamps.iter().map(|&stamp| PacketTime { stamp, stamped: true, samples }).collect()
}
#[test]
fn rounded_stamps_give_back_to_back_packets() {
let stamps = ms_stamps(200);
assert_eq!(&stamps[..4], [0, 1008, 2064, 3072]);
let starts = contiguous_starts(timed(&stamps, Some(1024)), 512);
assert_eq!(starts, (0..200).map(|k| k * 1024).collect::<Vec<_>>());
}
#[test]
fn real_gaps_and_overlaps_resynchronise() {
// a 100 ms gap after packet 3, then a packet stamped 600 samples early
let mut stamps: Vec<i64> = (0..8).map(|k| k * 1024).collect();
for s in &mut stamps[4..] {
*s += 4800;
}
stamps[7] -= 600;
let starts = contiguous_starts(timed(&stamps, Some(1024)), 512);
assert_eq!(starts, [0, 1024, 2048, 3072, 8896, 9920, 10944, 11368]);
// within the tolerance: runs on
stamps[7] += 200;
assert_eq!(contiguous_starts(timed(&stamps, Some(1024)), 512)[7], 11968);
}
#[test]
fn unstamped_packets_follow_on_and_unknown_lengths_resynchronise() {
// a laced block of three frames that repeat its timestamp
let packets = [0, 1024, 1024, 1024, 4096].map(|stamp| PacketTime { stamp, stamped: true, samples: Some(1024) });
let mut laced = packets;
laced[2].stamped = false;
laced[3].stamped = false;
assert_eq!(contiguous_starts(laced, 512), [0, 1024, 2048, 3072, 4096]);
// a packet whose length is unknown: the next one starts at its own stamp
let mut unknown = timed(&[0, 1024, 2100, 3124], Some(1024));
unknown[1].samples = None;
assert_eq!(contiguous_starts(unknown, 512), [0, 1024, 2100, 3124]);
}
#[test]
fn hostile_values_stay_finite() {
let packets = [
PacketTime { stamp: i64::MAX, stamped: true, samples: Some(i64::MAX) },
PacketTime { stamp: i64::MIN, stamped: false, samples: Some(-5) },
PacketTime { stamp: 0, stamped: true, samples: Some(i64::MIN) },
PacketTime { stamp: 7, stamped: true, samples: None },
];
assert_eq!(contiguous_starts(packets, i64::MIN).len(), 4);
assert_eq!(contiguous_starts(packets, i64::MAX).len(), 4);
assert!(contiguous_starts([], 512).is_empty());
}
#[test]
fn fixed_frame_lengths() {
// AAC-LC 48 kHz stereo; HE-AAC (24 kHz core) played at 48 kHz decodes its SBR band
assert_eq!(fixed_packet_samples(FixedFrames::Aac(&[0x11, 0x90]), &[], 48_000), Some(1024));
assert_eq!(fixed_packet_samples(FixedFrames::Aac(&[0x13, 0x10]), &[], 48_000), Some(2048));
assert_eq!(fixed_packet_samples(FixedFrames::Aac(&[]), &[], 48_000), None);
// MPEG-1 layer III 44.1 kHz, 128 kbps; MPEG-2 layer III 22.05 kHz
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xFB, 0x90, 0x44], 44_100), Some(1152));
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xF3, 0x90, 0x44], 22_050), Some(576));
// the header's rate must be the stream's
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF, 0xFB, 0x90, 0x44], 48_000), None);
assert_eq!(fixed_packet_samples(FixedFrames::MpegAudio, &[0xFF], 44_100), None);
assert_eq!(fixed_packet_samples(FixedFrames::Ac3, &[0x0B, 0x77], 48_000), None);
// E-AC-3 access units: 8-byte syncframes (frmsiz 3) at 48 kHz, stereo. A six-block
// independent frame with a dependent substream (7.1) is 1536 samples; two one-block
// frames are 512
let frame = |strmtyp: u8, numblkscod: u8| [0x0B, 0x77, strmtyp << 6, 3, (numblkscod << 4) | (2 << 1), 16 << 3, 0, 0];
let au = |frames: &[[u8; 8]]| frames.concat();
assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &au(&[frame(0, 3), frame(1, 3)]), 48_000), Some(1536));
assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &au(&[frame(0, 0), frame(0, 0)]), 48_000), Some(512));
assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &au(&[frame(0, 3)]), 44_100), None);
assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &[0x0B, 0x77], 48_000), None);
}
}
#[cfg(test)]
mod mpeg_audio_tests {
use super::*;
/// A silent mono 48 kHz MPEG-1 frame of `layer` (2: 64 kbps, 192 bytes; 3: 56 kbps, 168
/// bytes): header, then all-zero bit allocation / side information.
fn silent_frame(layer: u8) -> Vec<u8> {
let (b1, len) = if layer == 2 { (0xFD, 192) } else { (0xFB, 168) };
let mut f = vec![0xFF, b1, 0x44, 0xC0];
f.resize(len, 0);
f
}
#[test]
fn layer_comes_from_the_frame_header() {
assert_eq!(mpeg_audio_layer(&[0xFF, 0xFF, 0x10, 0xC0]), Some(1));
assert_eq!(mpeg_audio_layer(&silent_frame(2)), Some(2));
assert_eq!(mpeg_audio_layer(&silent_frame(3)), Some(3));
// MPEG-2 layer III, MPEG-2.5 layer II
assert_eq!(mpeg_audio_layer(&[0xFF, 0xF3, 0x90, 0x44]), Some(3));
assert_eq!(mpeg_audio_layer(&[0xFF, 0xE5, 0x90, 0x44]), Some(2));
// reserved layer, no sync word, too short
for bad in [&[0xFF, 0xF9, 0x44, 0xC0][..], &[0xFF, 0x1D], &[0x00, 0xFD], &[0xFF], &[]] {
assert_eq!(mpeg_audio_layer(bad), None, "{bad:?}");
}
}
/// #800: ffmpeg stores MP2 in MP4 as `mp4a` + object type 0x6B, which names no layer. Handing
/// it to the layer III decoder rejected every frame and the track played as silence.
#[test]
fn mp4_mpeg_audio_decodes_the_layer_its_frames_name() {
for layer in [2u8, 3] {
let mut d = PacketDecoder::for_isobmff(&filmcraft_isobmff::CodecConfig::Mp3, 48_000).unwrap();
// a packet without a frame header is an error, not a crash, and configures nothing
assert!(d.decode(&[0u8; 16], 0).is_err());
d.reset();
for k in 0..3u64 {
let out = d.decode(&silent_frame(layer), k * 1152).unwrap_or_else(|e| panic!("layer {layer}: {e}"));
assert_eq!(out.len(), 1, "layer {layer}");
assert_eq!(out[0].len(), 1152, "layer {layer}");
assert!(out[0].iter().all(|&s| s == 0.0), "layer {layer}");
}
}
}
}
#[cfg(test)]
mod gapless_tests {
use super::*;
const FRAME_LEN: usize = 417;
const AUDIO_FRAMES: usize = 10;
/// CRC-16 as used by the LAME tag (polynomial 0x8005, reflected, start 0).
fn crc16(data: &[u8]) -> u16 {
let mut crc = 0u16;
for &b in data {
crc ^= b as u16;
for _ in 0..8 {
crc = if crc & 1 != 0 { (crc >> 1) ^ 0xA001 } else { crc >> 1 };
}
}
crc
}
/// Silent mono 44.1 kHz 128 kbps MPEG-1 layer III file: an Info frame carrying a LAME tag with
/// an encoder delay of 576 samples and no padding, then ten frames of silence.
fn tagged_mp3() -> Vec<u8> {
let header = [0xFF, 0xFB, 0x90, 0xC0];
let mut info = vec![0u8; FRAME_LEN];
info[..4].copy_from_slice(&header);
info[21..25].copy_from_slice(b"Info");
info[25..29].copy_from_slice(&0x0Fu32.to_be_bytes());
info[29..33].copy_from_slice(&(AUDIO_FRAMES as u32).to_be_bytes());
info[33..37].copy_from_slice(&((FRAME_LEN * (AUDIO_FRAMES + 1)) as u32).to_be_bytes());
info[141..150].copy_from_slice(b"LAME3.100");
// encoder delay 576 (12 bits), padding 0 (12 bits)
info[162..165].copy_from_slice(&[0x24, 0x00, 0x00]);
let crc = crc16(&info[..175]);
info[175..177].copy_from_slice(&crc.to_be_bytes());
let mut file = info;
for _ in 0..AUDIO_FRAMES {
let mut f = vec![0u8; FRAME_LEN];
f[..4].copy_from_slice(&header);
file.extend_from_slice(&f);
}
file
}
/// #1137: the LAME tag's encoder delay (576) plus the decoder delay (529) is cut from the start.
#[test]
fn decoded_budget_is_enforced_without_allocating() {
assert!(check_decoded_budget(0, MAX_DECODED_SAMPLES).is_ok());
assert!(check_decoded_budget(MAX_DECODED_SAMPLES, 1).is_err());
assert!(check_decoded_budget(usize::MAX, usize::MAX).is_err());
}
#[test]
fn mp3_encoder_delay_is_trimmed() {
let src = AudioFileSource::decode("tagged.mp3", Arc::from(tagged_mp3())).unwrap();
assert_eq!(src.rate, 44_100);
assert_eq!(src.info.audio_streams.len(), 1);
assert_eq!(src.info.audio_streams[0].sample_rate, 44_100);
assert_eq!(src.info.audio_streams[0].channels, 1);
assert_eq!(src.info.audio_streams[0].codec, "MP3");
assert_eq!(src.samples[0].len(), AUDIO_FRAMES * 1152 - (576 + 529));
assert!(src.samples[0].iter().all(|sample| sample.abs() < 1e-6));
}
}
#[cfg(test)]
mod decode_pcm_tests {
use super::*;
#[test]
fn pcm_adapter_decodes_interleaved_big_endian_samples() {
let cfg = filmcraft_isobmff::PcmConfig { bits: 16, float: false, big_endian: true, signed: true, channels: 2, sample_rate: 48_000.0 };
assert_eq!(decode_pcm(&[0x40, 0, 0xc0, 0, 0xff], &cfg), [vec![0.5], vec![-0.5]]);
}
#[test]
fn zero_bits_decode_to_empty_channels_without_panicking() {
// A hostile `ipcm`/`fpcm` sample entry with `sample_size == 0` used to divide by zero
// (`bps * ch == 0`) and panic the audio worker on import/playback.
let cfg = filmcraft_isobmff::PcmConfig { bits: 0, float: false, big_endian: true, signed: true, channels: 2, sample_rate: 48_000.0 };
let out = decode_pcm(&[0u8; 64], &cfg);
assert_eq!(out.len(), 2);
assert!(out.iter().all(Vec::is_empty));
}
}
#[cfg(test)]
mod packet_cache_tests {
use super::*;
/// A packet that claims a size (nothing is allocated).
struct Claim(usize);
impl PacketBytes for Claim {
fn packet_bytes(&self) -> usize {
self.0
}
}
#[test]
fn the_cache_is_held_to_its_byte_budget_and_keeps_the_newest() {
let (mut packets, mut order, mut bytes) = (HashMap::new(), Vec::new(), 0usize);
let third = MAX_PACKET_CACHE_BYTES / 3 + 1;
for i in 0..5 {
cache_packet(&mut packets, &mut order, &mut bytes, i, &Arc::new(Claim(third)));
}
assert_eq!(order, [3, 4], "only what fits in the budget stays, newest last");
assert_eq!(bytes, 2 * third);
// one packet over the whole budget is still kept (it is the newest)
cache_packet(&mut packets, &mut order, &mut bytes, 9, &Arc::new(Claim(MAX_PACKET_CACHE_BYTES * 2)));
assert_eq!(order, [9]);
assert!(packets.contains_key(&9) && packets.len() == 1);
}
#[test]
fn the_entry_limit_still_applies() {
let (mut packets, mut order, mut bytes) = (HashMap::new(), Vec::new(), 0usize);
for i in 0..MAX_PACKET_CACHE_ENTRIES + 10 {
cache_packet(&mut packets, &mut order, &mut bytes, i, &Arc::new(Claim(1)));
}
assert_eq!(order.len(), MAX_PACKET_CACHE_ENTRIES);
assert_eq!(packets.len(), MAX_PACKET_CACHE_ENTRIES);
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.