GitCorner

Code

ossartcraftfilmcraft🌐
f67d38b35cc5
//! MPEG-2 transport streams (`.ts`, `.m2ts`, `.mts`), program streams (`.mpg`, `.vob`, `.mod`)
//! and MPEG-1/2 video elementary streams (`.m2v`, `.m1v`).
//!
//! - Demux and the access-unit index come from `craft_media::mpegts` (built when the file opens).
//! - Video: MPEG-1/2 (our decoder), H.264 and HEVC (Annex B byte streams, our decoders) with
//!   GOP-aware random access ([`crate::gop`]). Frames are presented in display order, from the
//!   first random-access picture; display order comes from the PTS (or, without timestamps, the
//!   GOP and temporal reference). Leading pictures of an open GOP decode from the previous GOP.
//! - Audio: MPEG audio layers I-III (symphonia), AAC in ADTS or LATM (our decoder), AC-3 (our
//!   decoder), Blu-ray / AVCHD and DVD LPCM. Every audio PID is a stream (capped). Audio frames
//!   are placed on the timeline from their PTS relative to the first video frame,
//!   sample-contiguously (re-anchored only at gaps).

use std::collections::{HashMap, VecDeque};
use std::sync::{Arc, Mutex};

use craft_media::mpegts::{Codec, File, Kind, Unit};
use filmcraft_frame::{AudioBuffer, VideoFrame};
use filmcraft_media::{AudioStreamInfo, FrameRequest, MediaError, MediaInfo, MediaKind, MediaSource, SharedSource, VideoStreamInfo};
use filmcraft_time::{FrameRate, Tick};

use crate::CodecError;
use crate::gop::{GopCache, VideoSamples};
use crate::video::{H264Decoder, HevcDecoder, Mpeg2Decoder, VideoDecoder};

/// An MPEG-1/2 video elementary stream starts with a sequence header.
fn is_video_es(head: &[u8]) -> bool {
    head.starts_with(&[0, 0, 1, 0xB3])
}

pub fn sniff(head: &[u8]) -> bool {
    craft_media::mpegts::sniff(head).is_some() || is_video_es(head)
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum VKind {
    Mpeg2,
    H264,
    Hevc,
}

/// The presented video: samples in decode order from the first random-access unit.
struct VideoTrack {
    kind: VKind,
    /// Stream index in the demuxed file (unused for elementary streams).
    stream: usize,
    /// Unit indices (into the stream's units, or the elementary stream's access units).
    units: Vec<usize>,
    keys: Vec<bool>,
    /// Display position of each sample (negative: not presented).
    pts: Vec<i64>,
    /// Display position → sample.
    order: Vec<usize>,
    /// MPEG-2 sequence header for priming decoders.
    header: Vec<u8>,
}

#[derive(Clone, Copy, Debug)]
enum ACodec {
    Mpa(u8),
    Adts,
    Latm,
    Ac3,
    LpcmBluray,
    LpcmDvd(craft_media::mpegts::LpcmFormat),
}

struct AudioState {
    dec: Option<AudioDecoder>,
    cache: HashMap<usize, Arc<Vec<Vec<f32>>>>,
    order: VecDeque<usize>,
    last: Option<usize>,
}

struct AudioTrack {
    stream: usize,
    codec: ACodec,
    rate: u32,
    channels: usize,
    /// Presentation sample (at `rate`, relative to the first video frame) of each unit.
    starts: Vec<i64>,
    /// Samples in each unit.
    lens: Vec<u32>,
    /// LATM: the stream's configuration (frames may only reference it).
    latm: Option<crate::audio::LatmConfig>,
    /// DVD LPCM: byte offset of each unit in the elementary stream (and the end).
    byte_starts: Vec<u64>,
    state: Mutex<AudioState>,
}

pub struct MpegSource {
    info: MediaInfo,
    bytes: crate::Src,
    file: Option<File>,
    /// Elementary stream: the whole file and its access units.
    es: Option<(Arc<[u8]>, Vec<std::ops::Range<usize>>)>,
    video: Option<VideoTrack>,
    gop: GopCache,
    /// Playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`.
    audios: Vec<AudioTrack>,
    /// Why the video or audio stream cannot be decoded.
    unsupported_video: Option<String>,
    unsupported_audio: Option<String>,
}

fn vkind(c: &Codec) -> Option<VKind> {
    match c {
        Codec::Mpeg1Video | Codec::Mpeg2Video => Some(VKind::Mpeg2),
        Codec::H264 => Some(VKind::H264),
        Codec::Hevc => Some(VKind::Hevc),
        _ => None,
    }
}

fn audio_supported(c: &Codec) -> bool {
    matches!(c, Codec::MpegAudio | Codec::AacAdts | Codec::AacLatm | Codec::LpcmBluray | Codec::LpcmDvd | Codec::Ac3 | Codec::Eac3)
}

/// The coded rate unless the timestamps say otherwise (3:2 pulldown, field-rate coding). Never
/// zero: damaged timestamps spread over a huge span give a rate that rounds to 0 fps, which the
/// duration computation divided by.
fn stream_rate(coded: Option<FrameRate>, from_pts: Option<f64>) -> FrameRate {
    let rate = match (coded, from_pts) {
        (Some(r), Some(p)) if (r.as_f64() - p).abs() / p > 0.01 => FrameRate::from_f64(p),
        (Some(r), _) => r,
        (None, Some(p)) => FrameRate::from_f64(p),
        (None, None) => FrameRate::FPS_25,
    };
    if rate.num > 0 && rate.den > 0 { rate } else { FrameRate::FPS_25 }
}

/// Display-order keys of samples (decode order): the PTS when every unit has one, else (MPEG
/// video) GOP start + temporal_reference, else the decode order.
fn display_keys(units: &[&Unit], mpeg: bool) -> Vec<i64> {
    if units.iter().all(|u| u.pts.is_some()) {
        return units.iter().map(|u| u.pts.unwrap_or(0)).collect();
    }
    if mpeg {
        let mut out = Vec::with_capacity(units.len());
        let mut base = 0i64;
        for (i, u) in units.iter().enumerate() {
            if let Some(p) = u.picture
                && (p.gop || i == 0)
            {
                base = i as i64;
            }
            out.push(base + u.picture.map_or(0, |p| p.temporal_reference as i64));
        }
        // scale so that a later GOP never sorts before an earlier one
        return out.iter().map(|&k| k * 4).collect();
    }
    let mut last = i64::MIN / 4;
    units
        .iter()
        .map(|u| {
            last = u.pts.unwrap_or(last + 1);
            last
        })
        .collect()
}

/// Presentation order of `keys` (decode order) from sample 0 (a random-access unit): samples
/// shown before it (leading pictures without their references) are not presented.
fn presentation(keys: &[i64]) -> (Vec<i64>, Vec<usize>) {
    let mut idx: Vec<usize> = (0..keys.len()).filter(|&i| keys[i] >= keys[0]).collect();
    idx.sort_by_key(|&i| (keys[i], i));
    let mut pts = vec![0i64; keys.len()];
    for (i, p) in pts.iter_mut().enumerate() {
        *p = -(i as i64) - 1;
    }
    for (d, &i) in idx.iter().enumerate() {
        pts[i] = d as i64;
    }
    (pts, idx)
}

impl MpegSource {
    pub fn open(name: &str, bytes: Arc<[u8]>) -> crate::Result<Self> {
        Self::open_reader(name, Arc::new(filmcraft_media::reader::MemReader(bytes)))
    }

    /// Open from a random-access reader: the whole file is scanned once to index its access units.
    pub fn open_reader(name: &str, reader: filmcraft_media::SharedReader) -> crate::Result<Self> {
        let bytes = crate::Src(reader);
        let head = filmcraft_media::reader::read_range(&*bytes.0, 0, 4096).map_err(|e| CodecError::Container(e.to_string()))?;
        if is_video_es(&head) {
            return Self::open_es(name, bytes);
        }
        let file = craft_media::mpegts::open(&bytes).map_err(|e| match e {
            craft_media::mpegts::Error::NotMpeg => CodecError::Unsupported("not an MPEG stream".into()),
            other => CodecError::Container(other.to_string()),
        })?;
        for w in &file.warnings {
            log::warn!("{name}: {w}");
        }
        let vstream = file.find(Kind::Video, |c| vkind(c).is_some()).or_else(|| file.find(Kind::Video, |_| true));
        let has_audio = file.streams.iter().any(|s| s.kind() == Kind::Audio && !s.units.is_empty());
        if vstream.is_none() && !has_audio {
            return Err(CodecError::Unsupported(format!("{}: no video or audio streams", file.format.name())));
        }
        let mut src = MpegSource {
            info: MediaInfo {
                name: name.to_string(),
                kind: MediaKind::Movie,
                duration: Tick::ZERO,
                video: None,
                audio_streams: Vec::new(),
                container: file.format.name().to_string(),
                start_timecode: None,
                file_size: Some(bytes.0.len()),
            },
            bytes,
            file: None,
            es: None,
            video: None,
            gop: GopCache::new(None),
            audios: Vec::new(),
            unsupported_video: None,
            unsupported_audio: None,
        };
        // video
        let origin;
        if let Some(vi) = vstream {
            let st = &file.streams[vi];
            match vkind(&st.codec) {
                Some(kind) => {
                    let first_key =
                        st.units.iter().position(|u| u.key).ok_or_else(|| CodecError::Decode("no random-access picture in the video stream".into()))?;
                    let sel: Vec<&Unit> = st.units[first_key..].iter().collect();
                    let keys = display_keys(&sel, kind == VKind::Mpeg2);
                    let (pts, order) = presentation(&keys);
                    // timeline origin: the PTS of the first presented picture
                    origin = order.first().and_then(|&i| sel[i].pts).or_else(|| sel.iter().filter_map(|u| u.pts).min());
                    let rate_from_pts = {
                        let shown: Vec<i64> = order.iter().filter_map(|&i| sel[i].pts).collect();
                        (shown.len() >= 2 && shown.len() == order.len())
                            .then(|| (shown.len() - 1) as f64 * 90_000.0 / (shown[shown.len() - 1] - shown[0]).max(1) as f64)
                    };
                    let track = VideoTrack {
                        kind,
                        stream: vi,
                        units: (first_key..st.units.len()).collect(),
                        keys: sel.iter().map(|u| u.key).collect(),
                        pts,
                        order,
                        header: Vec::new(),
                    };
                    src.file = Some(file);
                    src.video = Some(track);
                    src.describe_video(rate_from_pts)?;
                    if let Some(file) = src.file.take() {
                        src.setup_audio(file, origin);
                    }
                }
                None => {
                    src.unsupported_video = Some(format!("{} video in {} (FilmCraft has no decoder for it)", st.codec.name(), file.format.name()));
                    src.info.video = Some(VideoStreamInfo {
                        width: 0,
                        height: 0,
                        frame_rate: FrameRate::FPS_25,
                        par: (1, 1),
                        codec: format!("{} (unsupported)", st.codec.name()),
                        pixel_format: String::new(),
                        color: filmcraft_color::ColorInfo::REC709,
                        has_alpha: false,
                        bitrate: None,
                        hdr: None,
                        color_assumed: None,
                    });
                    src.setup_audio(file, None);
                }
            }
        } else {
            src.info.kind = MediaKind::AudioOnly;
            src.setup_audio(file, None);
        }
        Ok(src)
    }

    /// A raw MPEG-1/2 video elementary stream (no timestamps: GOP / temporal reference order).
    fn open_es(name: &str, bytes: crate::Src) -> crate::Result<Self> {
        let all: Arc<[u8]> = filmcraft_media::reader::read_range(&*bytes.0, 0, usize::try_from(bytes.0.len()).unwrap_or(usize::MAX))
            .map_err(|e| CodecError::Container(e.to_string()))?
            .into();
        let aus = craft_video::mpeg2::access_units(&all);
        let infos: Vec<craft_video::mpeg2::AccessUnitInfo> = aus.iter().map(|r| craft_video::mpeg2::scan_access_unit(&all[r.clone()])).collect();
        let first_key = infos.iter().position(|a| a.is_intra()).ok_or_else(|| CodecError::Decode("no I picture in the stream".into()))?;
        // fake units carrying the picture info for display ordering
        let mut keys = Vec::new();
        let mut base = 0i64;
        for (k, a) in infos[first_key..].iter().enumerate() {
            if a.gop.is_some() || k == 0 {
                base = k as i64;
            }
            keys.push((base + a.pictures.first().map_or(0, |p| p.0.temporal_reference as i64)) * 4);
        }
        let (pts, order) = presentation(&keys);
        let track = VideoTrack {
            kind: VKind::Mpeg2,
            stream: 0,
            units: (first_key..aus.len()).collect(),
            keys: infos[first_key..].iter().map(|a| a.is_intra()).collect(),
            pts,
            order,
            header: Vec::new(),
        };
        let mut src = MpegSource {
            info: MediaInfo {
                name: name.to_string(),
                kind: MediaKind::Movie,
                duration: Tick::ZERO,
                video: None,
                audio_streams: Vec::new(),
                container: "MPEG video elementary stream".into(),
                start_timecode: None,
                file_size: Some(bytes.0.len()),
            },
            bytes,
            file: None,
            es: Some((all, aus)),
            video: Some(track),
            gop: GopCache::new(None),
            audios: Vec::new(),
            unsupported_video: None,
            unsupported_audio: None,
        };
        src.describe_video(None)?;
        // the first GOP's time code (non-drop-frame count at the nominal rate)
        if let Some(tc) = infos[first_key].gop {
            let fps = src.info.frame_rate().as_f64().round().max(1.0) as i64;
            src.info.start_timecode = Some(((tc.hours as i64 * 60 + tc.minutes as i64) * 60 + tc.seconds as i64) * fps + tc.pictures as i64);
        }
        Ok(src)
    }

    /// The demuxed file (streams and their access-unit tables); `None` for elementary streams.
    pub fn file(&self) -> Option<&File> {
        self.file.as_ref()
    }

    /// Timeline position (samples at the audio stream's rate) of the first audio frame, and the
    /// number of samples in the audio stream.
    pub fn audio_extent(&self) -> Option<(i64, i64)> {
        let a = self.audios.first()?;
        let first = *a.starts.first()?;
        let end = a.starts.last()? + *a.lens.last()? as i64;
        Some((first, end - first))
    }

    /// The demuxed stream indices of the presented video and first audio.
    pub fn stream_indices(&self) -> (Option<usize>, Option<usize>) {
        (self.video.as_ref().filter(|_| self.es.is_none()).map(|v| v.stream), self.audios.first().map(|a| a.stream))
    }

    /// Presentation timestamps (90 kHz) of the presented video frames in display order. Frames
    /// whose PES packet carried none are interpolated at the frame duration from the nearest
    /// frame with one (`None` only when no frame has a timestamp, and for elementary streams).
    pub fn video_pts(&self) -> Vec<Option<i64>> {
        let (Some(v), Some(f)) = (&self.video, &self.file) else { return Vec::new() };
        let units = &f.streams[v.stream].units;
        let raw: Vec<Option<i64>> = v.order.iter().map(|&i| units[v.units[i]].pts).collect();
        let rate = self.info.frame_rate();
        let dur = |k: i64| (k as i128 * 90_000 * rate.den as i128 / rate.num.max(1) as i128) as i64;
        interpolate_pts(&raw, dur)
    }

    fn read_unit(&self, v: &VideoTrack, i: usize) -> crate::Result<Vec<u8>> {
        let u = v.units[i];
        if let Some((data, aus)) = &self.es {
            return Ok(data[aus[u].clone()].to_vec());
        }
        let file = self.file.as_ref().ok_or_else(|| CodecError::Container("no stream file".into()))?;
        file.read_unit(&self.bytes, v.stream, u).map_err(|e| CodecError::Container(e.to_string()))
    }

    /// Fill in the video stream info from the first picture.
    fn describe_video(&mut self, rate_from_pts: Option<f64>) -> crate::Result<()> {
        let mut seq_header = None;
        let v = self.video.as_ref().ok_or_else(|| CodecError::Unsupported("no video stream".into()))?;
        let first = self.read_unit(v, 0)?;
        let n = v.order.len();
        let (width, height, par, codec, pixel_format, mut color);
        let rate;
        color = filmcraft_color::ColorInfo::REC709;
        match v.kind {
            VKind::Mpeg2 => {
                let info =
                    craft_video::mpeg2::probe(&first).ok_or_else(|| CodecError::Decode("no MPEG video sequence header at the first I picture".into()))?;
                let header = crate::video::mpeg2_sequence_header(&first).unwrap_or_default();
                // field order from the first frames' picture coding extensions
                let mut d = craft_video::mpeg2::Decoder::new();
                let mut fo = None;
                for i in 0..v.units.len().min(4) {
                    if let Ok(pics) = d.decode(&self.read_unit(v, i)?, i as i64)
                        && let Some(p) = pics.first()
                    {
                        fo = Some(p.field_order());
                        color = crate::video::mpeg2_to_video_frame(p.clone()).color;
                        break;
                    }
                }
                let fo = fo.unwrap_or_else(|| d.flush().first().and_then(|p| p.field_order()));
                width = info.width;
                height = info.height;
                par = info.sar;
                codec = info.codec_name();
                pixel_format = crate::video::mpeg2_pixel_format(&info, fo);
                rate = info.frame_rate.map(|(n, d)| FrameRate::new(n as i64, d as i64));
                seq_header = Some(header);
            }
            VKind::H264 | VKind::Hevc => {
                // decode up to the first frame for size, aspect and colour
                let mut dec: Box<dyn VideoDecoder> = if v.kind == VKind::H264 { Box::new(H264Decoder::annexb()) } else { Box::new(HevcDecoder::annexb()) };
                let mut frame = None;
                for i in 0..v.units.len().min(32) {
                    if let Some(f) = dec.decode(&self.read_unit(v, i)?, i as i64)?.into_iter().next() {
                        frame = Some(f.frame);
                        break;
                    }
                }
                let frame = frame.or_else(|| dec.flush().into_iter().next().map(|f| f.frame)).ok_or_else(|| CodecError::Decode("no picture decoded".into()))?;
                width = frame.width;
                height = frame.height;
                par = frame.par;
                color = frame.color;
                codec = if v.kind == VKind::H264 { "H.264".into() } else { "HEVC".into() };
                rate = None;
                pixel_format = match &frame.data {
                    filmcraft_frame::PixelData::Yuv8 { chroma, .. } => format!("YUV {} 8-bit", chroma_name(*chroma)),
                    filmcraft_frame::PixelData::Yuv16 { chroma, bits, .. } => format!("YUV {} {bits}-bit", chroma_name(*chroma)),
                    _ => String::new(),
                };
            }
        }
        // the coded rate unless the timestamps say otherwise (3:2 pulldown, field-rate coding)
        let rate = stream_rate(rate, rate_from_pts);
        let duration = rate.tick_of(n as i64);
        let total_bytes: u64 = match (&self.es, &self.file) {
            (Some((_, aus)), _) => v.units.iter().map(|&u| aus[u].len() as u64).sum(),
            (None, Some(f)) => v.units.iter().map(|&u| f.streams[v.stream].units[u].size as u64).sum(),
            _ => 0,
        };
        let secs = duration.seconds();
        if let Some(h) = seq_header
            && let Some(v) = self.video.as_mut()
        {
            v.header = h;
        }
        self.info.duration = duration;
        self.info.video = Some(VideoStreamInfo {
            width,
            height,
            frame_rate: rate,
            par,
            codec,
            pixel_format,
            color,
            has_alpha: false,
            bitrate: (secs > 0.0).then(|| (total_bytes as f64 * 8.0 / secs) as u64),
            hdr: None,
            color_assumed: None,
        });
        Ok(())
    }

    /// Index every audio stream the container carries (capped); `origin`: PTS of the first video frame.
    fn setup_audio(&mut self, file: File, origin: Option<i64>) {
        let indices: Vec<usize> = file
            .streams
            .iter()
            .enumerate()
            .filter(|(_, s)| s.kind() == Kind::Audio && !s.units.is_empty())
            .map(|(i, _)| i)
            .take(filmcraft_media::MAX_AUDIO_STREAMS)
            .collect();
        let mut tracks = Vec::new();
        for ai in indices {
            if let Some(t) = self.audio_track(&file, ai, origin) {
                tracks.push(t);
            }
        }
        self.file = Some(file);
        self.audios = tracks;
    }

    /// Mark the audio stream as not decodable (it still shows in the media info when nothing plays).
    fn audio_unsupported(&mut self, codec_name: &str, why: String) -> Option<AudioTrack> {
        if !self.info.audio_streams.is_empty() {
            return None;
        }
        self.unsupported_audio = Some(why);
        self.info.audio_streams =
            vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: format!("{codec_name} (unsupported)"), bits_per_sample: None }];
        None
    }

    fn audio_track(&mut self, file: &File, ai: usize, origin: Option<i64>) -> Option<AudioTrack> {
        let st = &file.streams[ai];
        let codec_name = st.codec.name();
        if !audio_supported(&st.codec) || st.units.is_empty() {
            let why = format!("{codec_name} audio in {} (FilmCraft has no decoder for it)", file.format.name());
            return self.audio_unsupported(&codec_name, why);
        }
        let first = match file.read_unit(&self.bytes, ai, 0) {
            Ok(b) => b,
            Err(e) => {
                return self.audio_unsupported(&codec_name, format!("{codec_name}: {e}"));
            }
        };
        let mut latm = None;
        let (codec, rate, channels, per_frame, bits) = match &st.codec {
            Codec::MpegAudio | Codec::AacAdts | Codec::Ac3 | Codec::Eac3 => {
                let Some(fi) = craft_media::mpegts::frame_info(&st.codec, &first) else {
                    return self.audio_unsupported(&codec_name, format!("{codec_name}: bad frame header"));
                };
                let c = match st.codec {
                    Codec::MpegAudio => ACodec::Mpa(fi.variant),
                    Codec::AacAdts => ACodec::Adts,
                    _ => ACodec::Ac3,
                };
                (c, fi.sample_rate, fi.channels as usize, Some(fi.samples), None)
            }
            Codec::AacLatm => match crate::audio::latm_config(&first) {
                Some(cfg) => {
                    latm = Some(cfg.clone());
                    (ACodec::Latm, cfg.sample_rate, cfg.channels as usize, Some(1024), None)
                }
                None => {
                    return self.audio_unsupported(&codec_name, "AAC (LATM): unsupported StreamMuxConfig (only AAC-LC is decoded)".into());
                }
            },
            Codec::LpcmBluray => match crate::audio::bluray_lpcm_header(&first) {
                Some(h) => (ACodec::LpcmBluray, h.sample_rate, h.channels, None, Some(h.bits)),
                None => {
                    return self.audio_unsupported(&codec_name, "LPCM: bad header".into());
                }
            },
            Codec::LpcmDvd => match st.lpcm {
                Some(f) => (ACodec::LpcmDvd(f), f.sample_rate, f.channels as usize, None, Some(f.bits)),
                None => {
                    return self.audio_unsupported(&codec_name, "DVD LPCM: no format header".into());
                }
            },
            _ => return self.audio_unsupported(&codec_name, format!("{codec_name} audio")),
        };
        if codec_is_ac3_unsupported(codec) {
            return self.audio_unsupported(&codec_name, format!("{codec_name} audio (FilmCraft has no AC-3 decoder yet)"));
        }
        // DVD LPCM packets split sample groups: units own the groups that start in them
        let mut byte_starts = Vec::new();
        if let ACodec::LpcmDvd(_) = codec {
            let mut acc = 0u64;
            for u in &st.units {
                byte_starts.push(acc);
                acc += u.size as u64;
            }
            byte_starts.push(acc);
        }
        let group = |f: craft_media::mpegts::LpcmFormat| crate::audio::dvd_lpcm_group_bytes(f.channels as usize, f.bits) as u64;
        // samples per unit
        let lens: Vec<u32> = st
            .units
            .iter()
            .enumerate()
            .map(|(k, u)| match (codec, per_frame) {
                (_, Some(n)) => n,
                (ACodec::LpcmBluray, _) => {
                    let coded = (channels + 1) & !1;
                    let bps = if bits == Some(16) { 2 } else { 3 };
                    (u.size.saturating_sub(4) as usize / (coded * bps)) as u32
                }
                (ACodec::LpcmDvd(f), _) => {
                    let g = group(f);
                    // groups whose first byte is in this unit (and that end before the stream does)
                    let total = byte_starts[st.units.len()] / g;
                    let (g0, g1) = (byte_starts[k].div_ceil(g).min(total), byte_starts[k + 1].div_ceil(g).min(total));
                    ((g1 - g0) * 2) as u32
                }
                _ => 0,
            })
            .collect();
        // place units: contiguous, re-anchored at timestamp gaps of more than half a frame (PCM
        // byte streams are always contiguous)
        let origin = origin.or_else(|| st.units.iter().find_map(|u| u.pts)).unwrap_or(0);
        let to_samples = |pts: i64| ((pts - origin) as i128 * rate as i128).div_euclid(90_000) as i64;
        let contiguous = matches!(codec, ACodec::LpcmDvd(_));
        let mut starts = Vec::with_capacity(lens.len());
        let mut next: Option<i64> = None;
        for (u, &n) in st.units.iter().zip(&lens) {
            let s = match (u.pts.map(to_samples), next) {
                (_, Some(nx)) if contiguous => nx,
                (Some(p), Some(nx)) if (p - nx).abs() * 2 <= n as i64 => nx,
                (Some(p), _) => p,
                (None, Some(nx)) => nx,
                (None, None) => 0,
            };
            starts.push(s);
            next = Some(s + n as i64);
        }
        let total = next.unwrap_or(0).max(0);
        if self.unsupported_audio.take().is_some() {
            self.info.audio_streams.clear();
        }
        if self.info.audio_streams.is_empty() && (self.info.video.is_none() || self.unsupported_video.is_some()) {
            self.info.duration = Tick::from_units(total, rate.max(1) as i64);
            if self.info.video.is_none() {
                self.info.kind = MediaKind::AudioOnly;
            }
        }
        self.info.audio_streams.push(AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits });
        Some(AudioTrack {
            stream: ai,
            codec,
            rate,
            channels: channels.max(1),
            starts,
            lens,
            latm,
            byte_starts,
            state: Mutex::new(AudioState { dec: None, cache: HashMap::new(), order: VecDeque::new(), last: None }),
        })
    }

    /// Decoded samples of audio unit `i` (cached; non-sequential access primes the decoder with
    /// the previous unit).
    fn audio_unit(&self, a: &AudioTrack, st: &mut AudioState, i: usize) -> crate::Result<Arc<Vec<Vec<f32>>>> {
        if let Some(p) = st.cache.get(&i) {
            return Ok(p.clone());
        }
        let file = self.file.as_ref().ok_or_else(|| CodecError::Container("no stream file".into()))?;
        let read = |k: usize| file.read_unit(&self.bytes, a.stream, k).map_err(|e| CodecError::Container(e.to_string()));
        if st.dec.is_none() {
            st.dec = Some(AudioDecoder::new(a.codec, a.rate, a.channels, a.latm.clone())?);
        }
        let dec = st.dec.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?;
        if st.last.is_none_or(|l| l + 1 != i) {
            dec.reset();
            if i > 0
                && let Ok(prev) = read(i - 1)
            {
                let _ = dec.decode(&prev);
            }
        }
        let data = match a.codec {
            ACodec::LpcmDvd(f) if i + 1 < a.byte_starts.len() => {
                // the sample groups starting in this unit (the last may continue in the next)
                let g = crate::audio::dvd_lpcm_group_bytes(f.channels as usize, f.bits) as u64;
                let (b0, b1) = (a.byte_starts[i], a.byte_starts[i + 1]);
                let lo = b0.div_ceil(g) * g;
                let hi = b1.div_ceil(g) * g;
                let mut d = read(i)?;
                if hi > b1
                    && let Ok(next) = read(i + 1)
                {
                    d.extend_from_slice(&next);
                }
                let e = ((hi - b0) as usize).min(d.len());
                d[((lo - b0) as usize).min(e)..e].to_vec()
            }
            _ => read(i)?,
        };
        let out = match dec.decode(&data) {
            Ok(v) => v,
            Err(_) => vec![vec![0.0; a.lens[i] as usize]; a.channels],
        };
        st.last = Some(i);
        let p = Arc::new(out);
        st.cache.insert(i, p.clone());
        st.order.push_back(i);
        if st.order.len() > 2048
            && let Some(old) = st.order.pop_front()
        {
            st.cache.remove(&old);
        }
        Ok(p)
    }
}

/// Fill missing timestamps from their closest preceding timestamp, or the next one when there is
/// no predecessor.  The known positions are sorted, so binary search avoids rescanning them for
/// every missing frame.
fn interpolate_pts(raw: &[Option<i64>], dur: impl Fn(i64) -> i64) -> Vec<Option<i64>> {
    let known: Vec<usize> = (0..raw.len()).filter(|&i| raw[i].is_some()).collect();
    raw.iter()
        .enumerate()
        .map(|(i, &pts)| {
            pts.or_else(|| {
                let next = known.partition_point(|&j| j < i);
                let j = next.checked_sub(1).or_else(|| known.get(next).map(|_| next))?;
                Some(raw[known[j]]? + dur(i as i64 - known[j] as i64))
            })
        })
        .collect()
}

fn chroma_name(c: filmcraft_frame::Chroma) -> &'static str {
    match c {
        filmcraft_frame::Chroma::C420 => "4:2:0",
        filmcraft_frame::Chroma::C422 => "4:2:2",
        filmcraft_frame::Chroma::C444 => "4:4:4",
    }
}

fn codec_is_ac3_unsupported(c: ACodec) -> bool {
    matches!(c, ACodec::Ac3) && !crate::audio::AC3_DECODER
}

/// The audio decoders behind the TS/PS audio codecs.
enum AudioDecoder {
    Packet { dec: crate::audio::PacketDecoder, codec: ACodec, latm: Option<crate::audio::LatmConfig> },
    Pcm { codec: ACodec, channels: usize },
}

impl AudioDecoder {
    fn new(codec: ACodec, rate: u32, channels: usize, latm: Option<crate::audio::LatmConfig>) -> crate::Result<Self> {
        use crate::audio::PacketDecoder;
        Ok(match codec {
            ACodec::Mpa(layer) => AudioDecoder::Packet { dec: PacketDecoder::mpeg_audio(layer, rate)?, codec, latm: None },
            ACodec::Adts | ACodec::Latm => AudioDecoder::Packet { dec: PacketDecoder::lazy_aac(), codec, latm },
            ACodec::Ac3 => AudioDecoder::Packet { dec: PacketDecoder::ac3()?, codec, latm: None },
            ACodec::LpcmBluray | ACodec::LpcmDvd(_) => AudioDecoder::Pcm { codec, channels },
        })
    }
    fn reset(&mut self) {
        if let AudioDecoder::Packet { dec, .. } = self {
            dec.reset();
        }
    }
    fn decode(&mut self, data: &[u8]) -> crate::Result<Vec<Vec<f32>>> {
        match self {
            AudioDecoder::Packet { dec, codec: ACodec::Adts, .. } => {
                let (asc, payload) = crate::audio::adts_split(data).ok_or_else(|| CodecError::Decode("bad ADTS frame".into()))?;
                dec.ensure_aac(&asc)?;
                dec.decode(payload, 0)
            }
            AudioDecoder::Packet { dec, codec: ACodec::Latm, latm } => {
                let (asc, payload) = crate::audio::latm_split(data, latm).ok_or_else(|| CodecError::Decode("bad LATM frame".into()))?;
                dec.ensure_aac(&asc)?;
                dec.decode(&payload, 0)
            }
            AudioDecoder::Packet { dec, .. } => dec.decode(data, 0),
            AudioDecoder::Pcm { codec: ACodec::LpcmBluray, .. } => {
                crate::audio::decode_bluray_lpcm(data).ok_or_else(|| CodecError::Decode("bad LPCM packet".into()))
            }
            AudioDecoder::Pcm { codec: ACodec::LpcmDvd(f), channels } => Ok(crate::audio::decode_dvd_lpcm(data, *channels, f.bits)),
            AudioDecoder::Pcm { .. } => Err(CodecError::Unsupported("PCM format".into())),
        }
    }
}

/// The video samples as a [`VideoSamples`] table.
struct MpegVideo<'a> {
    src: &'a MpegSource,
    v: &'a VideoTrack,
}

impl VideoSamples for MpegVideo<'_> {
    fn count(&self) -> usize {
        self.v.units.len()
    }
    fn pts(&self, i: usize) -> i64 {
        self.v.pts[i]
    }
    fn sync_before(&self, i: usize) -> usize {
        let i = i.min(self.v.keys.len().saturating_sub(1));
        let keys = &self.v.keys;
        let pts = &self.v.pts;
        // a leading picture of an open GOP (shown before its random-access picture) needs the
        // previous GOP's pictures as references
        crate::gop::open_gop_sync(
            i,
            |i| (0..=i).rev().find(|&k| keys.get(k) == Some(&true)).unwrap_or(0),
            |i, key| matches!((pts.get(i), pts.get(key)), (Some(a), Some(b)) if a < b),
        )
    }
    fn sample_at(&self, t: i64) -> Option<usize> {
        usize::try_from(t).ok().and_then(|t| self.v.order.get(t).copied())
    }
    fn read(&self, i: usize) -> crate::Result<Vec<u8>> {
        self.src.read_unit(self.v, i)
    }
    fn make_decoder(&self) -> crate::Result<Box<dyn VideoDecoder>> {
        if let Some(why) = &self.src.unsupported_video {
            return Err(CodecError::Unsupported(why.clone()));
        }
        Ok(match self.v.kind {
            VKind::Mpeg2 => Box::new(Mpeg2Decoder::new(self.v.header.clone())),
            VKind::H264 => Box::new(H264Decoder::annexb()),
            VKind::Hevc => Box::new(HevcDecoder::annexb()),
        })
    }
}

impl MediaSource for MpegSource {
    fn info(&self) -> &MediaInfo {
        &self.info
    }

    fn video_frame(&self, req: FrameRequest) -> Result<Arc<VideoFrame>, MediaError> {
        if let Some(why) = &self.unsupported_video {
            return Err(MediaError::Unsupported(why.clone()));
        }
        let v = self.video.as_ref().ok_or(MediaError::NoStream("video"))?;
        if v.order.is_empty() {
            return Err(MediaError::Decode("no pictures".into()));
        }
        let rate = self.info.frame_rate();
        let f = rate.frame_at(req.time.max(Tick::ZERO)).clamp(0, v.order.len() as i64 - 1);
        let late = filmcraft_media::cancel::catch_up().map(|m| rate.frame_at(req.time - m));
        Ok(self.gop.frame_late(&MpegVideo { src: self, v }, f, late)?)
    }

    fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
        self.audio_stream(0, start, frames, sample_rate)
    }

    fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
        if self.audios.is_empty()
            && let Some(why) = &self.unsupported_audio
        {
            return Err(MediaError::Unsupported(why.clone()));
        }
        let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?;
        let ch = a.channels;
        let ratio = a.rate as f64 / sample_rate as f64;
        let exact = a.rate == sample_rate;
        let s0 = if exact { start } else { (start as f64 * ratio).floor() as i64 };
        let need = if exact { frames as i64 } else { (frames as f64 * ratio).ceil() as i64 + 2 };
        let mut src: Vec<Vec<f32>> = vec![vec![0.0; need.max(0) as usize]; ch];
        let mut st = a.state.lock().unwrap_or_else(|e| e.into_inner());
        // units overlapping [s0, s0 + need)
        let mut i = a.starts.partition_point(|&s| s <= s0).saturating_sub(1);
        while i < a.starts.len() && a.starts[i] < s0 + need {
            if a.starts[i] + a.lens[i] as i64 > s0 {
                let pk = self.audio_unit(a, &mut st, i)?;
                for (c, dst) in src.iter_mut().enumerate() {
                    let Some(chan) = pk.get(c.min(pk.len().saturating_sub(1))) else { continue };
                    for (k, v) in chan.iter().enumerate() {
                        let pos = a.starts[i] + k as i64 - s0;
                        if pos >= 0 && (pos as usize) < dst.len() {
                            dst[pos as usize] = *v;
                        }
                    }
                }
            }
            i += 1;
        }
        drop(st);
        let mut out = AudioBuffer::silence(sample_rate, ch, frames);
        if exact {
            for (c, s) in src.into_iter().enumerate() {
                out.channels[c][..frames.min(s.len())].copy_from_slice(&s[..frames.min(s.len())]);
            }
            return Ok(out);
        }
        filmcraft_media::resample_window_into(&src, s0, a.rate, start, sample_rate, &mut out.channels);
        Ok(out)
    }
}

pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option<Result<SharedSource, MediaError>> {
    if !sniff(&bytes[..bytes.len().min(65_536)]) {
        return None;
    }
    Some(MpegSource::open(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}

/// [`filmcraft_media::ReaderOpener`] for MPEG transport / program / video elementary streams.
pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedReader) -> Option<Result<SharedSource, MediaError>> {
    if !sniff(head) {
        return None;
    }
    Some(MpegSource::open_reader(name, reader.clone()).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}

#[cfg(test)]
mod tests {
    use super::*;
    use filmcraft_media::MediaSource;

    /// A mutated transport stream whose PTS span gave ~0 fps panicked on open with "attempt to
    /// divide by zero" (`FrameRate::tick_of`).
    #[test]
    fn damaged_timestamps_never_give_a_zero_rate() {
        for (coded, p) in
            [(None, Some(1e-9)), (Some(FrameRate::FPS_25), Some(1e-9)), (Some(FrameRate::new(0, 0)), None), (None, Some(0.0)), (None, Some(f64::NAN))]
        {
            let r = stream_rate(coded, p);
            assert!(r.num > 0 && r.den > 0, "{coded:?} {p:?} -> {r:?}");
            let _ = r.tick_of(1000);
        }
        assert_eq!(stream_rate(Some(FrameRate::FPS_25), Some(25.0)), FrameRate::FPS_25);
        assert_eq!(stream_rate(None, Some(50.0)), FrameRate::from_f64(50.0));
    }

    /// MPEG-1 layer II frame, 48 kHz 192 kb/s (576 bytes).
    fn mp2_frame(k: u8) -> Vec<u8> {
        let mut v = vec![0xFF, 0xFD, 0xA4, 0x00];
        v.extend((0..572).map(|i| (i as u8) ^ k));
        v
    }

    fn ts_bytes(prefix: u8, v: i64) -> [u8; 5] {
        let v = v & ((1 << 33) - 1);
        [(prefix << 4) | (((v >> 30) & 7) as u8) << 1 | 1, (v >> 22) as u8, (((v >> 15) & 0x7F) as u8) << 1 | 1, (v >> 7) as u8, ((v & 0x7F) as u8) << 1 | 1]
    }

    fn pes(stream_id: u8, pts: Option<i64>, payload: &[u8]) -> Vec<u8> {
        let mut h = Vec::new();
        if let Some(p) = pts {
            h.extend(ts_bytes(2, p));
        }
        let flags = if pts.is_some() { 0x80 } else { 0 };
        let len = 3 + h.len() + payload.len();
        let mut v = vec![0, 0, 1, stream_id, (len >> 8) as u8, len as u8, 0x80, flags, h.len() as u8];
        v.extend(h);
        v.extend_from_slice(payload);
        v
    }

    /// Tiny TS muxer: PAT, PMT and PES packets in 188-byte packets.
    struct TsMux {
        out: Vec<u8>,
        cc: std::collections::HashMap<u16, u8>,
    }

    impl TsMux {
        fn packet(&mut self, pid: u16, pusi: bool, payload: &[u8]) -> usize {
            let cc = self.cc.entry(pid).or_insert(0);
            let mut p = vec![0x47, (pusi as u8) << 6 | (pid >> 8) as u8, pid as u8];
            let n = payload.len().min(184);
            if n < 184 {
                let al = 183 - n;
                p.push(0x30 | *cc);
                p.push(al as u8);
                if al > 0 {
                    p.push(0);
                    p.extend(std::iter::repeat_n(0xFF, al - 1));
                }
            } else {
                p.push(0x10 | *cc);
            }
            *cc = (*cc + 1) & 15;
            p.extend_from_slice(&payload[..n]);
            self.out.extend(&p);
            n
        }
        fn section(&mut self, pid: u16, body: &[u8]) {
            let mut s = body.to_vec();
            s.extend(crc32_mpeg(&s).to_be_bytes());
            let mut payload = vec![0];
            payload.extend(s);
            self.packet(pid, true, &payload);
        }
        fn psi(&mut self, streams: &[(u8, u16)]) {
            let pat = [0x00, 0xB0, 13, 0, 1, 0xC1, 0, 0, 0, 1, 0xE1, 0x00];
            self.section(0, &pat);
            let mut pmt = vec![0x02, 0xB0, 0, 0, 1, 0xC1, 0, 0, 0xE1, 0x00, 0xF0, 0];
            for &(st, pid) in streams {
                pmt.extend([st, 0xE0 | (pid >> 8) as u8, pid as u8, 0xF0, 0]);
            }
            let len = pmt.len() - 3 + 4;
            pmt[1] = 0xB0 | (len >> 8) as u8;
            pmt[2] = len as u8;
            self.section(0x100, &pmt);
        }
        fn pes(&mut self, pid: u16, data: &[u8]) {
            let mut rest = data;
            let mut first = true;
            while !rest.is_empty() {
                let n = self.packet(pid, first, rest);
                rest = &rest[n..];
                first = false;
            }
        }
    }

    fn crc32_mpeg(data: &[u8]) -> u32 {
        let mut crc = 0xFFFF_FFFFu32;
        for &b in data {
            crc ^= (b as u32) << 24;
            for _ in 0..8 {
                crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 };
            }
        }
        crc
    }

    /// Dual MPEG audio in one transport stream: import used to keep only the first PID.
    #[test]
    fn every_audio_pid_is_a_stream() {
        let mut mux = TsMux { out: Vec::new(), cc: Default::default() };
        mux.psi(&[(0x03, 0x1100), (0x03, 0x1101)]);
        for k in 0..4u16 {
            mux.pes(0x1100, &pes(0xC0, Some(90_000 + 2160 * k as i64), &mp2_frame(k as u8)));
            mux.pes(0x1101, &pes(0xC1, Some(90_000 + 2160 * k as i64), &mp2_frame(k as u8 + 16)));
        }
        let source = MpegSource::open("dual.ts", mux.out.into()).unwrap();
        assert_eq!(source.info().audio_streams.len(), 2, "{:?}", source.info().audio_streams);
        assert_eq!(source.info().audio_streams.iter().map(|a| a.sample_rate).collect::<Vec<_>>(), vec![48_000, 48_000]);
        let a0 = source.audio(0, 64, 48_000).unwrap();
        let a1 = source.audio_stream(1, 0, 64, 48_000).unwrap();
        assert_eq!(a0.channels.len(), 2);
        assert_eq!(a1.channels.len(), 2);
        assert!(source.audio_stream(2, 0, 64, 48_000).is_err());
    }

    /// Resampled audio must not depend on where a read starts (#4673): 16-bit DVD LPCM at 48 kHz in
    /// a program stream.
    #[test]
    fn resampled_audio_does_not_depend_on_the_read_cuts() {
        let signs = filmcraft_testkit::cuts::signs(filmcraft_testkit::cuts::SOURCE_SAMPLES, 6);
        let mut out = vec![0, 0, 1, 0xBA, 0x44, 0x00, 0x04, 0x00, 0x04, 0x01, 0x01, 0x89, 0xC3, 0xF8];
        for (k, block) in signs.chunks(1000).enumerate() {
            // sub-stream 0xA0: frame count, first access unit pointer, emphasis, 16-bit 48 kHz mono, dynamic range
            let mut payload = vec![0xA0, 1, 0, 4, 0, 0, 0x80];
            payload.extend(block.iter().flat_map(|v| ((v * 32767.0) as i16).to_be_bytes()));
            out.extend(pes(0xBD, Some(90_000 + 1875 * k as i64), &payload));
        }
        let source = MpegSource::open("cuts.vob", out.into()).unwrap();
        assert_eq!(source.info().audio().map(|a| (a.sample_rate, a.channels)), Some((48_000, 1)));
        filmcraft_testkit::cuts::assert_cut_independent("DVD LPCM 48 kHz at 1999 Hz", 1_000, filmcraft_testkit::cuts::FRAMES, |s, n| {
            source.audio(s, n, filmcraft_testkit::cuts::RATE).unwrap().channels
        });
    }

    #[test]
    fn presentation_drops_leading_pictures_and_orders_by_key() {
        // decode order I(2) B(0) B(1) P(5) B(3) B(4): the leading B pictures are not presented
        let keys = [2, 0, 1, 5, 3, 4];
        let (pts, order) = presentation(&keys);
        assert_eq!(order, vec![0, 4, 5, 3]);
        assert_eq!(pts, vec![0, -2, -3, 3, 1, 2]);
    }

    #[test]
    fn timestamp_interpolation_handles_long_missing_runs_without_rescanning() {
        let mut raw = vec![None; 10_000];
        raw[4_000] = Some(40_000);
        raw[9_999] = Some(99_990);
        let pts = interpolate_pts(&raw, |frames| frames * 10);
        assert_eq!(pts[0], Some(0));
        assert_eq!(pts[3_999], Some(39_990));
        assert_eq!(pts[4_001], Some(40_010));
        assert_eq!(pts[9_998], Some(99_980));
    }
}