GitCorner

Code

ossartcraftcraftanything🌐
642d5a256791
//! Programs and game files that carry other files inside one file: one seam for reading them.
//! A reader says whether a file is its kind, lists what's inside and hands back one member's bytes.
//! Add a kind with a [`PackedFormat`] implementation and one line in [`READERS`]. No I/O and no
//! execution: readers only look at the bytes they're given.

use crate::common::FormatError;
use crate::decompress;
use serde::Serialize;
use serde_json::Value;

/// Most members one listing keeps; a reader sets `truncated` when it stops here.
pub const MAX_ENTRIES: usize = 200_000;

/// One file carried inside a packed file.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct PackedEntry {
    /// The member's path inside the packed file, as the file names it.
    pub path: String,
    /// Where its stored bytes start in the packed file.
    pub offset: u64,
    /// How many bytes it takes up in the packed file.
    pub stored_size: u64,
    /// Its size once unpacked, when the file says.
    pub size: Option<u64>,
    /// How it's stored.
    pub compression: Compression,
    /// What the file says this member is (reader-specific; `None` when it doesn't say).
    pub kind: Option<String>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Compression {
    None,
    Deflate,
    Zlib,
    Gzip,
    Zstd,
    /// Stored in a way we can list but not read back (encrypted, or an unknown method).
    Unreadable,
}

/// What a reader found in a packed file.
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct PackedListing {
    /// Reader-specific facts: versions, flags, entry point.
    pub facts: Value,
    pub entries: Vec<PackedEntry>,
    /// Things the person should know about this listing (a damaged part, a limit reached).
    pub notes: Vec<String>,
    /// True when a limit stopped the listing before its end.
    pub truncated: bool,
}

/// A kind of packed file.
pub trait PackedFormat: Sync {
    /// Stable id, in results and the `format` parameter.
    fn id(&self) -> &'static str;
    /// What people call it.
    fn title(&self) -> &'static str;
    /// Cheap check of the bytes. A yes may still fail in `list`; a no means another reader gets it.
    fn detect(&self, data: &[u8]) -> bool;
    /// What's inside.
    fn list(&self, data: &[u8]) -> Result<PackedListing, FormatError>;
    /// One member's bytes, unpacked, at most `max_bytes`.
    fn read(&self, data: &[u8], entry: &PackedEntry, max_bytes: u64) -> Result<Vec<u8>, FormatError> {
        read_stored(data, entry, max_bytes)
    }
}

/// Every reader, most specific first: the first whose `detect` says yes reads the file.
pub static READERS: &[&dyn PackedFormat] = &[
    &crate::bun_program::BunProgram,
    &crate::node_sea::NodeSea,
    &crate::dotnet_bundle::DotnetBundle,
    &crate::godot_pck::GodotPck,
    &crate::pyinstaller::PyInstaller,
    &crate::pyinstaller::PyzArchive,
];

/// The reader for these bytes, if any.
pub fn detect(data: &[u8]) -> Option<&'static dyn PackedFormat> {
    READERS.iter().copied().find(|r| r.detect(data))
}

/// The reader with this id, if any.
pub fn by_id(id: &str) -> Option<&'static dyn PackedFormat> {
    READERS.iter().copied().find(|r| r.id() == id)
}

/// A member's stored bytes, range-checked.
pub fn stored_bytes<'a>(data: &'a [u8], entry: &PackedEntry) -> Result<&'a [u8], FormatError> {
    let bad = || FormatError::Truncated(format!("{} lies outside the file", entry.path));
    let start = usize::try_from(entry.offset).map_err(|_| bad())?;
    let len = usize::try_from(entry.stored_size).map_err(|_| bad())?;
    let end = start.checked_add(len).ok_or_else(bad)?;
    data.get(start..end).ok_or_else(bad)
}

/// Read a member by its `compression`, stopping at `max_bytes`.
pub fn read_stored(data: &[u8], entry: &PackedEntry, max_bytes: u64) -> Result<Vec<u8>, FormatError> {
    let raw = stored_bytes(data, entry)?;
    let too_large = || FormatError::TooLarge(format!("{} unpacks past {max_bytes} bytes", entry.path));
    if entry.size.is_some_and(|s| s > max_bytes) {
        return Err(too_large());
    }
    let cap = usize::try_from(max_bytes).unwrap_or(usize::MAX);
    let inflate = |r: Result<Vec<u8>, miniz_oxide::inflate::DecompressError>| {
        r.map_err(|e| match e.status {
            miniz_oxide::inflate::TINFLStatus::HasMoreOutput => too_large(),
            _ => FormatError::Malformed(format!("the compressed data of {} is damaged", entry.path)),
        })
    };
    let decoded = match entry.compression {
        Compression::None if raw.len() as u64 > max_bytes => Err(too_large()),
        Compression::None => Ok(raw.to_vec()),
        Compression::Deflate => inflate(miniz_oxide::inflate::decompress_to_vec_with_limit(raw, cap)),
        Compression::Zlib => inflate(miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(raw, cap)),
        Compression::Gzip => read_gzip(raw, max_bytes).map_err(|error| match error {
            crate::firmware::InflateError::TooLarge => too_large(),
            _ => FormatError::Malformed(format!("the gzip data of {} is damaged: {error}", entry.path)),
        }),
        Compression::Zstd => decompress::zstd(raw, max_bytes).map_err(|e| FormatError::Malformed(format!("{}: {e}", entry.path))),
        Compression::Unreadable => Err(FormatError::Unsupported(format!("{} is stored in a way we can't read back", entry.path))),
    }?;
    if entry.size.is_some_and(|size| size != decoded.len() as u64) {
        return Err(FormatError::Malformed(format!("the unpacked size of {} differs from its manifest", entry.path)));
    }
    Ok(decoded)
}

/// Read every gzip member and verify its trailer before returning any bytes.
fn read_gzip(mut raw: &[u8], max_bytes: u64) -> Result<Vec<u8>, crate::firmware::InflateError> {
    if raw.is_empty() {
        return Err(crate::firmware::InflateError::Corrupt);
    }
    let mut output = Vec::new();
    while !raw.is_empty() {
        let remaining = max_bytes.saturating_sub(output.len() as u64);
        let (member, consumed) = crate::firmware::gunzip(raw, remaining)?;
        output.extend_from_slice(&member);
        raw = raw.get(consumed..).ok_or(crate::firmware::InflateError::Truncated)?;
    }
    Ok(output)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn entry(offset: u64, stored_size: u64, compression: Compression) -> PackedEntry {
        PackedEntry { path: "a".into(), offset, stored_size, size: None, compression, kind: None }
    }

    #[test]
    fn member_sizes_must_match_the_manifest() {
        let raw = b"hello";
        let zlib = miniz_oxide::deflate::compress_to_vec_zlib(raw, 6);
        let deflate = miniz_oxide::deflate::compress_to_vec(raw, 6);
        for (bytes, kind) in [(raw.as_slice(), Compression::None), (&zlib, Compression::Zlib), (&deflate, Compression::Deflate)] {
            let mut member = entry(0, bytes.len() as u64, kind);
            for declared in [4, 6] {
                member.size = Some(declared);
                assert!(matches!(read_stored(bytes, &member, 100), Err(FormatError::Malformed(_))));
            }
            member.size = Some(5);
            assert_eq!(read_stored(bytes, &member, 100).unwrap(), raw);
        }
    }

    #[test]
    fn stored_ranges_are_checked() {
        let data = b"hello world";
        assert_eq!(read_stored(data, &entry(6, 5, Compression::None), 100).unwrap(), b"world");
        assert!(read_stored(data, &entry(6, 6, Compression::None), 100).is_err());
        assert!(read_stored(data, &entry(u64::MAX, 2, Compression::None), 100).is_err());
        assert!(read_stored(data, &entry(1, u64::MAX, Compression::None), 100).is_err());
    }

    #[test]
    fn limits_hold_for_every_method() {
        let body = vec![7u8; 4096];
        let z = miniz_oxide::deflate::compress_to_vec_zlib(&body, 6);
        assert_eq!(read_stored(&z, &entry(0, z.len() as u64, Compression::Zlib), 4096).unwrap(), body);
        assert!(matches!(read_stored(&z, &entry(0, z.len() as u64, Compression::Zlib), 100), Err(FormatError::TooLarge(_))));
        let d = miniz_oxide::deflate::compress_to_vec(&body, 6);
        assert_eq!(read_stored(&d, &entry(0, d.len() as u64, Compression::Deflate), 4096).unwrap(), body);
        assert!(matches!(read_stored(&body, &entry(0, 4096, Compression::None), 100), Err(FormatError::TooLarge(_))));
        let mut declared = entry(0, 4096, Compression::None);
        declared.size = Some(u64::MAX);
        assert!(matches!(read_stored(&body, &declared, 8192), Err(FormatError::TooLarge(_))));
    }

    #[test]
    fn gzip_members_skip_every_optional_header_field() {
        let body = b"inside".to_vec();
        let mut g = vec![0x1f, 0x8b, 8, 4 | 8 | 16 | 2, 0, 0, 0, 0, 0, 0];
        g.extend([2, 0, 9, 9]);
        g.extend(b"name\0");
        g.extend(b"comment\0");
        g.extend((crate::firmware::crc32(&g) as u16).to_le_bytes());
        g.extend(miniz_oxide::deflate::compress_to_vec(&body, 6));
        g.extend(crate::firmware::crc32(&body).to_le_bytes());
        g.extend((body.len() as u32).to_le_bytes());
        assert_eq!(read_stored(&g, &entry(0, g.len() as u64, Compression::Gzip), 100).unwrap(), body);
        assert!(read_stored(b"\x1f\x8b\x08\x04", &entry(0, 4, Compression::Gzip), 100).is_err());
    }

    #[test]
    fn gzip_members_require_complete_valid_trailers() {
        let original = craft_anything_testkit::firmware::gzip(b"payload", None);
        for trailer_byte in [original.len() - 8, original.len() - 4] {
            let mut damaged = original.clone();
            damaged[trailer_byte] ^= 1;
            assert!(matches!(read_stored(&damaged, &entry(0, damaged.len() as u64, Compression::Gzip), 100), Err(FormatError::Malformed(_))));
        }
        for missing in 1..=8 {
            let truncated = &original[..original.len() - missing];
            assert!(read_stored(truncated, &entry(0, truncated.len() as u64, Compression::Gzip), 100).is_err());
        }
        let mut trailing = original.clone();
        trailing.extend_from_slice(b"garbage");
        assert!(read_stored(&trailing, &entry(0, trailing.len() as u64, Compression::Gzip), 100).is_err());
        assert!(read_stored(b"", &entry(0, 0, Compression::Gzip), 100).is_err());
    }

    #[test]
    fn gzip_concatenation_shares_the_output_limit() {
        let first = craft_anything_testkit::firmware::gzip(b"first", Some("one"));
        let empty = craft_anything_testkit::firmware::gzip(b"", None);
        let last = craft_anything_testkit::firmware::gzip(b"last", None);
        let data = [first, empty.clone(), last].concat();
        let mut member = entry(0, data.len() as u64, Compression::Gzip);
        assert_eq!(read_stored(&data, &member, 9).unwrap(), b"firstlast");
        assert!(matches!(read_stored(&data, &member, 8), Err(FormatError::TooLarge(_))));
        member.size = Some(5);
        assert!(matches!(read_stored(&data, &member, 100), Err(FormatError::Malformed(_))));
        assert!(read_stored(&empty, &entry(0, empty.len() as u64, Compression::Gzip), 0).unwrap().is_empty());
    }

    #[test]
    fn damaged_and_unreadable_members_are_errors_not_panics() {
        assert!(read_stored(b"not zlib", &entry(0, 8, Compression::Zlib), 100).is_err());
        assert!(read_stored(b"not zstd", &entry(0, 8, Compression::Zstd), 100).is_err());
        assert!(matches!(read_stored(b"x", &entry(0, 1, Compression::Unreadable), 100), Err(FormatError::Unsupported(_))));
    }
}