GitCorner

Code

ossartcraftfilmcraft🌐
f67d38b35cc5
//! Random-access byte readers: open media without loading the whole file.
//!
//! A [`ByteReader`] serves reads at any offset (a native file, a web `Blob` read in chunks, an
//! in-memory buffer). Container openers that only need the index and then single samples (MP4/MOV,
//! Matroska) open from a reader ([`ReaderOpener`]), as does WAV (header now, sample ranges on
//! demand); formats decoded in one go (stills, standalone compressed audio) fall back to reading
//! the whole file through it.

use std::io;
use std::sync::Arc;

use crate::{MediaError, Opener, Result, SharedSource};

/// Random-access, read-only bytes.
pub trait ByteReader: Send + Sync {
    /// Total length in bytes.
    fn len(&self) -> u64;
    /// Fill `buf` entirely from `offset`; `UnexpectedEof` past the end. Asynchronous readers fail
    /// with `WouldBlock` (and set [`crate::pending`]) while the range is being fetched.
    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>;
    fn is_empty(&self) -> bool {
        self.len() == 0
    }
}

pub type SharedReader = Arc<dyn ByteReader>;

/// Open `path` for reading without blocking on a FIFO that was swapped in after the metadata check
/// (`O_NONBLOCK`; it changes nothing for a regular file, and the callers check the handle's type).
/// `O_NONBLOCK` has no std constant (no libc dependency here): it is 0o4000 on most Linux targets,
/// 0x80 on Linux MIPS, 0x4000 on Linux SPARC and 4 on the BSD family, and the open stays a plain
/// one on any other target.
fn open_nonblocking(path: &std::path::Path) -> io::Result<std::fs::File> {
    let mut o = std::fs::OpenOptions::new();
    o.read(true);
    #[cfg(unix)]
    {
        use std::os::unix::fs::OpenOptionsExt;
        #[cfg(all(
            any(target_os = "linux", target_os = "android"),
            any(target_arch = "mips", target_arch = "mips32r6", target_arch = "mips64", target_arch = "mips64r6")
        ))]
        o.custom_flags(0x80);
        #[cfg(all(any(target_os = "linux", target_os = "android"), any(target_arch = "sparc", target_arch = "sparc64")))]
        o.custom_flags(0x4000);
        #[cfg(all(
            any(target_os = "linux", target_os = "android"),
            not(any(
                target_arch = "mips",
                target_arch = "mips32r6",
                target_arch = "mips64",
                target_arch = "mips64r6",
                target_arch = "sparc",
                target_arch = "sparc64"
            ))
        ))]
        o.custom_flags(0o4000);
        #[cfg(any(target_os = "macos", target_os = "ios", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly"))]
        o.custom_flags(0x4);
    }
    o.open(path)
}

/// Read a whole regular file. A FIFO, device or folder is refused, checked before opening (a named
/// pipe with no writer would block the open) and again on the open handle, and at most the length
/// the handle reports is read: a link to an endless source cannot fill memory, and a file that
/// grows while it is read is cut at the length it had when opened.
pub fn read_regular_file(path: &std::path::Path) -> io::Result<Vec<u8>> {
    read_regular_file_bounded(path, u64::MAX)
}

/// [`read_regular_file`], refusing a file larger than `max` bytes before it is read.
pub fn read_regular_file_bounded(path: &std::path::Path, max: u64) -> io::Result<Vec<u8>> {
    use std::io::Read;
    let not_regular = || io::Error::new(io::ErrorKind::InvalidInput, format!("{} is not a regular file", path.display()));
    if !std::fs::metadata(path)?.is_file() {
        return Err(not_regular());
    }
    let file = open_nonblocking(path)?;
    let meta = file.metadata()?;
    if !meta.is_file() {
        return Err(not_regular());
    }
    let len = meta.len();
    if len > max {
        return Err(io::Error::new(io::ErrorKind::InvalidData, format!("{}: {len} bytes is over the {max} byte limit", path.display())));
    }
    let mut out = Vec::new();
    file.take(len).read_to_end(&mut out)?;
    Ok(out)
}

fn eof() -> io::Error {
    io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source")
}

/// An in-memory reader.
pub struct MemReader(pub Arc<[u8]>);

impl ByteReader for MemReader {
    fn len(&self) -> u64 {
        self.0.len() as u64
    }
    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
        let a = usize::try_from(offset).map_err(|_| eof())?;
        let src = self.0.get(a..a.checked_add(buf.len()).ok_or_else(eof)?).ok_or_else(eof)?;
        buf.copy_from_slice(src);
        Ok(())
    }
}

/// Files [`FileReader`]s keep open at most, together (#67). Each open media source used to hold
/// its own descriptor, so a project with hundreds of sources hit the soft descriptor limit (256 on
/// macOS, 1024 on Linux) and the rest showed as failed or offline. Past this many, the least
/// recently read file is closed and its reader reopens it on demand.
#[cfg(any(unix, windows))]
pub const MAX_OPEN_FILES: usize = 128;

/// The open files of a set of [`FileReader`]s, least recently used first.
#[cfg(any(unix, windows))]
#[derive(Debug)]
pub(crate) struct FilePool {
    cap: usize,
    state: std::sync::Mutex<PoolState>,
}

#[cfg(any(unix, windows))]
#[derive(Debug, Default)]
struct PoolState {
    next_id: u64,
    open: std::collections::VecDeque<(u64, Arc<std::fs::File>)>,
}

#[cfg(any(unix, windows))]
impl FilePool {
    pub(crate) fn new(cap: usize) -> Self {
        Self { cap: cap.max(1), state: Default::default() }
    }

    fn state(&self) -> std::sync::MutexGuard<'_, PoolState> {
        self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
    }

    fn next_id(&self) -> u64 {
        let mut s = self.state();
        s.next_id = s.next_id.wrapping_add(1);
        s.next_id
    }

    /// The open file of reader `id`, now the most recently used.
    fn get(&self, id: u64) -> Option<Arc<std::fs::File>> {
        let mut s = self.state();
        let at = s.open.iter().position(|(i, _)| *i == id)?;
        let entry = s.open.remove(at)?;
        let file = entry.1.clone();
        s.open.push_back(entry);
        Some(file)
    }

    /// Keep `file` open for reader `id`, closing the least recently used files past the cap. A
    /// file another thread is reading from stays open until that read ends.
    fn insert(&self, id: u64, file: std::fs::File) -> Arc<std::fs::File> {
        let mut s = self.state();
        if let Some((_, open)) = s.open.iter().find(|(i, _)| *i == id) {
            return open.clone(); // another thread reopened it first
        }
        let file = Arc::new(file);
        s.open.push_back((id, file.clone()));
        while s.open.len() > self.cap {
            s.open.pop_front();
        }
        file
    }

    fn remove(&self, id: u64) {
        self.state().open.retain(|(i, _)| *i != id);
    }

    #[cfg(test)]
    fn open_count(&self) -> usize {
        self.state().open.len()
    }
}

#[cfg(any(unix, windows))]
static FILE_POOL: std::sync::LazyLock<Arc<FilePool>> = std::sync::LazyLock::new(|| Arc::new(FilePool::new(MAX_OPEN_FILES)));

/// A file on disk, read in place: only the requested ranges are read, so opening a clip costs its
/// index rather than its size. The file is kept open in a shared pool of at most
/// [`MAX_OPEN_FILES`] and reopened by path when the pool closed it.
#[cfg(any(unix, windows))]
#[derive(Debug)]
pub struct FileReader {
    path: std::path::PathBuf,
    len: u64,
    id: u64,
    pool: Arc<FilePool>,
}

#[cfg(any(unix, windows))]
impl FileReader {
    pub fn open(path: &std::path::Path) -> io::Result<Self> {
        Self::open_in(path, &FILE_POOL)
    }

    pub(crate) fn open_in(path: &std::path::Path, pool: &Arc<FilePool>) -> io::Result<Self> {
        // checked before opening: opening a named pipe with no writer blocks the caller for good
        if !std::fs::metadata(path)?.is_file() {
            return Err(io::Error::new(io::ErrorKind::NotFound, format!("{} is not a file", path.display())));
        }
        let file = open_nonblocking(path)?;
        let m = file.metadata()?;
        if !m.is_file() {
            return Err(io::Error::new(io::ErrorKind::NotFound, format!("{} is not a file", path.display())));
        }
        let id = pool.next_id();
        pool.insert(id, file);
        Ok(Self { path: path.to_path_buf(), len: m.len(), id, pool: pool.clone() })
    }

    /// The open file, reopened when the pool closed it. A file now shorter than when it was opened
    /// (replaced or truncated) is refused rather than read as the old one; a longer one (still
    /// being written) is read up to the opened length, as before.
    fn file(&self) -> io::Result<Arc<std::fs::File>> {
        if let Some(file) = self.pool.get(self.id) {
            return Ok(file);
        }
        let file = open_nonblocking(&self.path)?;
        let m = file.metadata()?;
        if !m.is_file() || m.len() < self.len {
            return Err(io::Error::other(format!("{} changed since it was opened", self.path.display())));
        }
        Ok(self.pool.insert(self.id, file))
    }
}

#[cfg(any(unix, windows))]
impl Drop for FileReader {
    fn drop(&mut self) {
        self.pool.remove(self.id);
    }
}

#[cfg(any(unix, windows))]
impl ByteReader for FileReader {
    fn len(&self) -> u64 {
        self.len
    }
    /// Positional reads: threads decoding different parts of the file do not share a cursor.
    fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
        let file = self.file()?;
        #[cfg(unix)]
        {
            std::os::unix::fs::FileExt::read_exact_at(&*file, buf, offset)
        }
        #[cfg(windows)]
        {
            let (mut done, mut at) = (0usize, offset);
            while let Some(rest) = buf.get_mut(done..).filter(|r| !r.is_empty()) {
                match std::os::windows::fs::FileExt::seek_read(&*file, rest, at) {
                    Ok(0) => return Err(eof()),
                    Ok(n) => {
                        done = done.saturating_add(n);
                        at = at.saturating_add(n as u64);
                    }
                    Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
                    Err(e) => return Err(e),
                }
            }
            Ok(())
        }
    }
}

/// Read `len` bytes at `offset` (fewer at the end of the reader).
pub fn read_range(r: &dyn ByteReader, offset: u64, len: usize) -> io::Result<Vec<u8>> {
    let n = (r.len().saturating_sub(offset)).min(len as u64) as usize;
    let mut v = vec![0u8; n];
    r.read_at(offset, &mut v)?;
    Ok(v)
}

/// Opens a container from a reader: `head` is the first bytes of the file (for sniffing).
/// Returns `None` when the format is not this opener's.
pub type ReaderOpener = fn(name: &str, head: &[u8], reader: &SharedReader) -> Option<Result<SharedSource>>;

/// Bytes sniffed by [`open_reader`].
pub const HEAD_LEN: usize = 64 * 1024;

/// Open media from a reader: reader openers first, then WAV (read in place); otherwise read the
/// whole file and use the byte openers (`extra`, then stills).
pub fn open_reader(name: &str, reader: SharedReader, reader_openers: &[ReaderOpener], extra: &[Opener]) -> Result<SharedSource> {
    open_reader_within(name, reader, reader_openers, extra, u64::MAX)
}

/// Like [`open_reader`], but a file no reader opener takes is read whole only up to `max_whole`
/// bytes; a larger one is refused instead. For looking at files rather than importing them (the
/// Media Browser's properties and thumbnails, #157): reading a multi-gigabyte AVI into
/// memory to show its duration, or to find out it isn't supported, thrashes the disk.
pub fn open_reader_within(name: &str, reader: SharedReader, reader_openers: &[ReaderOpener], extra: &[Opener], max_whole: u64) -> Result<SharedSource> {
    let head = read_range(&*reader, 0, HEAD_LEN).map_err(|e| MediaError::Io(format!("{name}: {e}")))?;
    for o in reader_openers {
        if let Some(r) = o(name, &head, &reader) {
            return r;
        }
    }
    if crate::wav::sniff(&head) {
        return Ok(Arc::new(crate::wav::WavSource::open(name, reader)?));
    }
    if reader.len() > max_whole {
        return Err(MediaError::Unsupported(format!("{name}: no streaming reader for this format, and it is too large to read whole here")));
    }
    let all = read_range(&*reader, 0, usize::try_from(reader.len()).unwrap_or(usize::MAX)).map_err(|e| MediaError::Io(format!("{name}: {e}")))?;
    crate::open_bytes(name, all.into(), extra)
}

#[cfg(test)]
mod regular_file_tests {
    use super::read_regular_file;

    #[test]
    fn reads_a_file_and_refuses_a_folder() {
        let dir = std::env::temp_dir().join(format!("fc-reg-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        std::fs::write(dir.join("a.bin"), [1u8, 2, 3]).unwrap();
        assert_eq!(read_regular_file(&dir.join("a.bin")).unwrap(), [1, 2, 3]);
        assert!(read_regular_file(&dir).is_err());
        assert!(read_regular_file(&dir.join("missing")).is_err());
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// A named pipe with no writer would block `open` for good: both readers refuse it first.
    #[cfg(unix)]
    #[test]
    fn a_fifo_is_refused_without_blocking() {
        let dir = std::env::temp_dir().join(format!("fc-reg-fifo-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        let fifo = dir.join("pipe");
        if !std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()) {
            return; // no mkfifo here
        }
        assert!(read_regular_file(&fifo).is_err());
        assert!(super::FileReader::open(&fifo).is_err());
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// The open itself must not block on a FIFO swapped in after the path check (a FIFO with no writer).
    #[cfg(target_os = "linux")]
    #[test]
    fn opening_a_fifo_with_no_writer_does_not_block() {
        let dir = std::env::temp_dir().join(format!("fc-reg-fifo-open-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        let fifo = dir.join("pipe");
        if std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()) {
            let f = super::open_nonblocking(&fifo).unwrap();
            assert!(!f.metadata().unwrap().is_file());
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// A link to an endless device: the read refuses it instead of reading until memory runs out.
    #[cfg(target_os = "linux")]
    #[test]
    fn refuses_a_link_to_an_endless_device() {
        let dir = std::env::temp_dir().join(format!("fc-reg-dev-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        std::os::unix::fs::symlink("/dev/zero", dir.join("zero")).unwrap();
        assert!(read_regular_file(&dir.join("zero")).is_err());
        let _ = std::fs::remove_dir_all(&dir);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn mem_reader_reads_ranges_and_reports_eof() {
        let r = MemReader(Arc::from(&b"0123456789"[..]));
        let mut b = [0u8; 3];
        r.read_at(4, &mut b).unwrap();
        assert_eq!(&b, b"456");
        assert_eq!(r.read_at(8, &mut b).unwrap_err().kind(), io::ErrorKind::UnexpectedEof);
        assert_eq!(read_range(&r, 8, 100).unwrap(), b"89");
    }

    #[cfg(any(unix, windows))]
    #[test]
    fn file_reader_reads_ranges_in_place() {
        let dir = std::env::temp_dir().join(format!("filmcraft-file-reader-{}", std::process::id()));
        std::fs::create_dir_all(&dir).unwrap();
        let path = dir.join("bytes.bin");
        std::fs::write(&path, b"0123456789").unwrap();
        let r = FileReader::open(&path).unwrap();
        assert_eq!(r.len(), 10);
        let mut b = [0u8; 3];
        r.read_at(4, &mut b).unwrap();
        assert_eq!(&b, b"456");
        // reads are positional: an earlier offset after a later one
        r.read_at(0, &mut b).unwrap();
        assert_eq!(&b, b"012");
        assert_eq!(r.read_at(8, &mut b).unwrap_err().kind(), io::ErrorKind::UnexpectedEof);
        assert_eq!(read_range(&r, 8, 100).unwrap(), b"89");
        assert!(FileReader::open(&dir).is_err(), "a directory is not a media file");
        assert_eq!(FileReader::open(&dir.join("missing")).unwrap_err().kind(), io::ErrorKind::NotFound);
        std::fs::remove_dir_all(&dir).unwrap();
    }

    /// #67: more readers than the pool keeps open stay readable; past the cap the least recently
    /// read file is closed and reopened on demand, and a file truncated meanwhile is refused.
    #[cfg(any(unix, windows))]
    #[test]
    fn more_file_readers_than_the_pool_holds_stay_readable() {
        let dir = std::env::temp_dir().join(format!("filmcraft-file-pool-{}", std::process::id()));
        std::fs::create_dir_all(&dir).unwrap();
        let pool = Arc::new(FilePool::new(2));
        let paths: Vec<_> = (0..5).map(|i| dir.join(format!("{i}.bin"))).collect();
        for (i, p) in paths.iter().enumerate() {
            std::fs::write(p, format!("file{i}-0123456789")).unwrap();
        }
        let readers: Vec<_> = paths.iter().map(|p| FileReader::open_in(p, &pool).unwrap()).collect();
        assert_eq!(pool.open_count(), 2);
        for _ in 0..2 {
            for (i, r) in readers.iter().enumerate() {
                let mut b = [0u8; 5];
                r.read_at(0, &mut b).unwrap();
                assert_eq!(b, *format!("file{i}").as_bytes());
                assert!(pool.open_count() <= 2);
            }
        }
        // reader 0 was closed by the later reads; its file is now shorter than when opened
        std::fs::write(&paths[0], b"short").unwrap();
        let mut b = [0u8; 5];
        assert!(readers[0].read_at(0, &mut b).is_err());
        drop(readers);
        assert_eq!(pool.open_count(), 0, "dropped readers close their files");
        std::fs::remove_dir_all(&dir).unwrap();
    }

    /// A reader that counts the bytes read from it.
    struct Counting(MemReader, std::sync::atomic::AtomicU64);
    impl ByteReader for Counting {
        fn len(&self) -> u64 {
            self.0.len()
        }
        fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
            self.1.fetch_add(buf.len() as u64, std::sync::atomic::Ordering::SeqCst);
            self.0.read_at(offset, buf)
        }
    }

    /// #157: looking at a file no streaming reader takes (an AVI) read all of it. Within a limit
    /// it reads the head only and refuses.
    #[test]
    fn open_within_reads_only_the_head_of_large_unstreamable_files() {
        let mut avi = vec![0u8; 4 * HEAD_LEN];
        avi[..4].copy_from_slice(b"RIFF");
        avi[8..12].copy_from_slice(b"AVI ");
        let r = Arc::new(Counting(MemReader(Arc::from(avi)), Default::default()));
        let e = open_reader_within("a.avi", r.clone(), &[], &[], HEAD_LEN as u64).err().unwrap();
        assert!(matches!(e, MediaError::Unsupported(_)), "{e:?}");
        assert!(r.1.load(std::sync::atomic::Ordering::SeqCst) <= HEAD_LEN as u64);
    }

    /// #279: a WAV is read in place, never whole: opening it reads the sniffing head and its chunk
    /// headers, and audio reads only the requested samples, however large the file (and over the
    /// Media Browser's whole-file limit too).
    #[test]
    fn wav_opens_in_place_and_reads_only_requested_samples() {
        let rate = 48_000usize;
        let samples: Vec<f32> = (0..rate * 2 * 20).map(|i| ((i / 2) % 100) as f32 / 200.0).collect();
        let wav = crate::wav::write_wav16(&samples, 2, rate as u32);
        let size = wav.len() as u64;
        assert!(size > 3 * HEAD_LEN as u64);
        let r = Arc::new(Counting(MemReader(Arc::from(wav)), Default::default()));
        let read = || r.1.load(std::sync::atomic::Ordering::SeqCst);
        let src = open_reader_within("a.wav", r.clone(), &[], &[], HEAD_LEN as u64).unwrap();
        assert_eq!(src.info().audio().unwrap().channels, 2);
        assert_eq!(src.info().duration, filmcraft_time::Tick::from_units(20, 1));
        assert_eq!(src.info().file_size, Some(size));
        let after_open = read();
        assert!(after_open <= HEAD_LEN as u64 + 64, "{after_open}");
        // 10 ms from the middle: about that much is read, and the samples are right
        let a = src.audio(rate as i64 * 10 + 7, 480, rate as u32).unwrap();
        assert!(read() - after_open <= 481 * 4, "{}", read() - after_open);
        let frame = rate * 10 + 7 + 3;
        assert!((a.channels[0][3] - samples[frame * 2]).abs() < 1e-4);
        assert!((a.channels[1][3] - samples[frame * 2 + 1]).abs() < 1e-4);
        // past the end: silence, nothing read
        let before = read();
        let a = src.audio(rate as i64 * 30, 16, rate as u32).unwrap();
        assert!(a.channels[0].iter().all(|&s| s == 0.0));
        assert_eq!(read(), before);
    }

    #[test]
    fn falls_back_to_byte_openers() {
        // a format without a reader opener is read whole and handed to the byte openers
        fn whole(_: &str, bytes: Arc<[u8]>) -> Option<Result<SharedSource>> {
            Some(Err(MediaError::Unsupported(format!("got {} bytes", bytes.len()))))
        }
        let r: SharedReader = Arc::new(MemReader(Arc::from(&b"bytes"[..])));
        let e = open_reader("a.xyz", r, &[], &[whole]).err().unwrap();
        assert!(matches!(&e, MediaError::Unsupported(m) if m == "got 5 bytes"), "{e:?}");
    }
}