GitCorner

Code

ossartcraftcraftanything๐ŸŒ
642d5a256791
//! Firmware images with the built-in engine: a signature scan of recognised regions with an
//! entropy map (`firmware_regions_list`), and carving/unpacking those regions into a fresh
//! folder, recursively and within strict limits (`firmware_image_extract`).

use super::artifact_tools::{create_absent_dir, safe_components, write_new_file};
use super::native_common::{page, spec};
use super::static_input::{builder, load};
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of};
use craft_anything_evidence::{Confidence, Coverage, CoverageStatus, ErrorCode, Limitation, Location, ToolError, sha256_hex};
use craft_anything_formats::firmware::{self as fw, Region, ScanOptions, Unpack};
use craft_anything_formats::fsread::{self, FsKind, Limits};
use craft_anything_formats::{decompress, tarcpio, zip};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::collections::BTreeMap;
use std::path::Path;

pub fn specs() -> Vec<ToolSpec> {
    vec![
        spec(
            "firmware_regions_list",
            "Firmware regions",
            "Signature scan of a firmware image or disk dump with the built-in engine: compression streams (gzip, zlib, xz, LZMA, bzip2, Zstandard, LZ4), archives (ZIP, tar, cpio, 7z, RAR, CAB), filesystems (squashfs, cramfs, romfs, JFFS2, UBI/UBIFS, ext2/3/4, FAT, ISO 9660), boot images (legacy boot-loader images, Android boot), device trees/FIT, executables (ELF, PE, Mach-O) and PEM blocks, each checked against its header; plus an entropy map, coverage, gaps and unexplained high-entropy spans. Regions are paged (offset/limit); range limits the scan to part of the file.",
            "Find out what is packed inside a firmware image: compressed parts, filesystems, archives and programs.",
            Toolset::Formats,
            Effects::EVIDENCE,
            Requires::Nothing,
            schema_of::<RegionsParams>,
            firmware_regions_list,
        ),
        spec(
            "firmware_image_extract",
            "Extract firmware",
            "Carve every recognised region of a firmware image into a new folder (output_directory must not exist yet), decompress gzip/zlib/xz/LZMA/bzip2/Zstandard streams, unpack ZIP, tar and cpio members and boot-image payloads, list and extract files from squashfs, cramfs, romfs, ext2/3/4 and (best-effort) JFFS2 filesystems, and repeat on what comes out up to max_depth levels. Bounded by max_output_bytes and max_output_files; never overwrites; members or filesystem paths that would escape the folder and links are refused. Writes manifest.json listing every output with its SHA-256 and how it was unpacked.",
            "Unpack a firmware image into a folder so you can look at the files inside.",
            Toolset::Formats,
            Effects::MUTATES_SESSION.union(Effects::WRITES_FILESYSTEM),
            Requires::Nothing,
            schema_of::<ExtractParams>,
            firmware_image_extract,
        ),
    ]
}

#[derive(Debug, Clone, Copy, Deserialize, Serialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct ByteRange {
    /// First byte of the range.
    pub offset: u64,
    /// Bytes in the range (cut to the end of the file).
    pub length: u64,
}

fn d_max_regions() -> usize {
    2000
}
fn d_block() -> usize {
    4096
}
fn d_limit() -> usize {
    200
}

#[derive(Debug, Deserialize, Serialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct RegionsParams {
    /// Absolute path of the image; the active target when omitted.
    #[serde(default)]
    pub path: Option<String>,
    /// Only scan this part of the file (offsets stay relative to the whole file).
    #[serde(default)]
    pub range: Option<ByteRange>,
    /// Most regions recorded (1..=100000, default 2000).
    #[serde(default = "d_max_regions")]
    pub max_regions: usize,
    /// Entropy block size in bytes (256..=1048576, default 4096). Grows when the map would exceed 4096 blocks.
    #[serde(default = "d_block")]
    pub entropy_block: usize,
    /// First region of the page.
    #[serde(default)]
    pub offset: usize,
    /// Regions per page (1..=5000, default 200).
    #[serde(default = "d_limit")]
    pub limit: usize,
}

fn d_depth() -> u32 {
    3
}
fn d_out_bytes() -> u64 {
    256 << 20
}
fn d_out_files() -> usize {
    10_000
}
fn d_list() -> usize {
    500
}

#[derive(Debug, Deserialize, Serialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct ExtractParams {
    /// Absolute path of the image; the active target when omitted.
    #[serde(default)]
    pub path: Option<String>,
    /// Absolute path of a folder that does not exist yet; it is created (its parent must exist).
    pub output_directory: String,
    /// Only extract from this part of the file.
    #[serde(default)]
    pub range: Option<ByteRange>,
    /// Nested unpacking levels (1..=10, default 3).
    #[serde(default = "d_depth")]
    pub max_depth: u32,
    /// Most bytes written in total (up to 512 MiB, default 256 MiB).
    #[serde(default = "d_out_bytes")]
    pub max_output_bytes: u64,
    /// Most files written (1..=100000, default 10000).
    #[serde(default = "d_out_files")]
    pub max_output_files: usize,
    /// Outputs listed in the evidence (the manifest always lists all of them; 1..=5000, default 500).
    #[serde(default = "d_list")]
    pub limit: usize,
}

/// Apply an optional range to the input: (slice, base offset).
fn apply_range(data: &[u8], range: Option<ByteRange>) -> Result<(&[u8], u64), ToolError> {
    let Some(r) = range else { return Ok((data, 0)) };
    let start = usize::try_from(r.offset)
        .ok()
        .filter(|s| *s < data.len())
        .ok_or_else(|| ToolError::invalid(format!("range.offset {:#x} is past the end of the file ({} bytes)", r.offset, data.len())))?;
    let len = usize::try_from(r.length).unwrap_or(usize::MAX).min(data.len() - start);
    if len == 0 {
        return Err(ToolError::invalid("range.length is 0"));
    }
    Ok((data.get(start..start + len).unwrap_or(&[]), r.offset))
}

fn region_json(r: &Region, base: u64) -> Value {
    let off = r.offset.saturating_add(base);
    let mut v = serde_json::to_value(r).unwrap_or(Value::Null);
    if let Value::Object(m) = &mut v {
        m.insert("offset".into(), json!(off));
        m.insert("offset_hex".into(), json!(format!("{off:#x}")));
        m.insert("end_hex".into(), json!(r.end().map(|e| format!("{:#x}", e.saturating_add(base)))));
    }
    v
}

fn scan_limitations() -> Vec<Limitation> {
    vec![
        Limitation::new(
            "signature_scan",
            "Regions are found by known signatures checked against their headers; formats we don't know, encrypted or vendor-specific containers are not recognised, and absence of a region is not proof.",
        ),
        Limitation::new(
            "nested_hidden",
            "Content inside a region of confirmed length (for example the compressed kernel inside a boot image) is not listed here; firmware_image_extract unpacks and scans it.",
        ),
        Limitation::new(
            "entropy_heuristic",
            "Entropy classes (text, code-or-data, compressed-or-encrypted, padding) are statistical hints, not proof of content.",
        ),
    ]
}

fn firmware_regions_list(ctx: &mut Ctx<'_>, params: Value) -> Result<ToolOutput, ToolError> {
    let p: RegionsParams = parse_params(params)?;
    if !(1..=100_000).contains(&p.max_regions) {
        return Err(ToolError::invalid("max_regions must be 1..=100000"));
    }
    if !(256..=1 << 20).contains(&p.entropy_block) {
        return Err(ToolError::invalid("entropy_block must be 256..=1048576"));
    }
    let input = load(ctx, p.path.as_deref(), fw::MAX_INPUT)?;
    let (data, base) = apply_range(&input.bytes, p.range)?;
    ctx.report(None, format!("Scanning {} ({} bytes)", input.name, data.len()));
    let cancel = ctx.cancel.clone();
    let scan = fw::scan(data, &ScanOptions { max_regions: p.max_regions, ..ScanOptions::default() }, &|| cancel.is_cancelled());
    if scan.cancelled {
        return Err(ToolError::cancelled());
    }
    let (entropy, block) = fw::entropy_map(data, p.entropy_block, 4096);
    let lay = fw::layout(data, &scan.regions, &entropy, 500);
    let regions: Vec<Value> = scan.regions.iter().map(|r| region_json(r, base)).collect();
    let (page_items, cov) = page(&regions, p.offset, p.limit);
    let mut by_kind: BTreeMap<&str, usize> = BTreeMap::new();
    let mut by_family: BTreeMap<String, usize> = BTreeMap::new();
    for r in &scan.regions {
        *by_kind.entry(r.kind).or_default() += 1;
        *by_family.entry(serde_json::to_value(r.family).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_default()).or_default() += 1;
    }
    let shift =
        |s: &fw::Span| json!({"offset": s.offset.saturating_add(base), "offset_hex": format!("{:#x}", s.offset.saturating_add(base)), "length": s.length});
    let mean = if entropy.is_empty() { 0.0 } else { entropy.iter().map(|b| f64::from(b.entropy)).sum::<f64>() / entropy.len() as f64 };
    let mut classes: BTreeMap<&str, usize> = BTreeMap::new();
    for b in &entropy {
        *classes.entry(b.class).or_default() += 1;
    }
    let blocks: Vec<Value> = entropy.iter().map(|b| json!({"offset": b.offset.saturating_add(base), "entropy": b.entropy, "class": b.class})).collect();
    let summary = format!(
        "{}: {} region(s) ({}); {:.1}% of {} bytes recognised; {} unexplained high-entropy span(s).",
        input.name,
        scan.regions.len(),
        if by_kind.is_empty() { "none".to_string() } else { by_kind.iter().map(|(k, n)| format!("{n} {k}")).collect::<Vec<_>>().join(", ") },
        lay.recognised_percent,
        data.len(),
        lay.high_entropy_unrecognised.len()
    );
    let mut lims = scan_limitations();
    if scan.truncated {
        lims.push(Limitation::new("truncated", format!("stopped at max_regions={}; scan a smaller range or raise the limit", p.max_regions)));
    }
    for w in &scan.warnings {
        lims.push(Limitation::new("probe_budget", w.clone()));
    }
    if block != p.entropy_block {
        lims.push(Limitation::new("entropy_block_grown", format!("entropy block grown to {block} bytes to keep the map at 4096 blocks or fewer")));
    }
    let locations: Vec<Location> = scan
        .regions
        .iter()
        .skip(p.offset)
        .take(p.limit.clamp(1, 50))
        .map(|r| Location::new("file_offset", format!("{:#x}", r.offset.saturating_add(base))).with_label(r.kind))
        .collect();
    let ev = builder(
        ctx,
        "firmware_regions_list",
        input.subject.clone(),
        json!({"path": p.path, "range": p.range, "max_regions": p.max_regions, "entropy_block": p.entropy_block, "offset": p.offset, "limit": p.limit}),
    )
    .result(json!({
        "path": input.path,
        "size": input.bytes.len(),
        "scanned": {"offset": base, "length": data.len()},
        "regions_total": regions.len(),
        "by_kind": by_kind,
        "by_family": by_family,
        "regions": page_items,
        "layout": {
            "total_bytes": lay.total_bytes,
            "recognised_bytes": lay.recognised_bytes,
            "recognised_percent": lay.recognised_percent,
            "unknown_length_regions": lay.unknown_length_regions,
            "gaps": lay.gaps.iter().map(shift).collect::<Vec<_>>(),
            "high_entropy_unrecognised": lay.high_entropy_unrecognised.iter().map(shift).collect::<Vec<_>>(),
            "spans_truncated": lay.spans_truncated,
        },
        "entropy": {"block_size": block, "mean": (mean * 1000.0).round() / 1000.0, "classes": classes, "blocks": blocks},
        "truncated": scan.truncated,
        "probes_exhausted": scan.probes_exhausted,
    }))
    .summary(summary)
    .confidence(Confidence::Derived)
    .limitations(lims)
    .coverage(cov)
    .locations(locations)
    .build()?;
    Ok(ToolOutput::evidence(ev))
}

// ---------------------------------------------------------------------------------------------
// Extraction

/// Data waiting to be scanned: where its carvings go and how deep it sits.
struct Job {
    data: Vec<u8>,
    /// Folder (relative components) that receives what is carved from this data.
    dir: Vec<String>,
    /// 0 = the input; each unpacking adds one.
    level: u32,
    /// Relative path of the file this data was written to, or `<input>`.
    parent: String,
    /// Offset added to region offsets (the range start for the input, 0 below).
    base: u64,
}

struct Writer<'a> {
    root: &'a Path,
    files: usize,
    bytes: u64,
    max_files: usize,
    max_bytes: u64,
    outputs: Vec<Value>,
    skipped: Vec<Value>,
    truncated: bool,
    max_level: u32,
}

impl Writer<'_> {
    fn remaining(&self) -> u64 {
        self.max_bytes.saturating_sub(self.bytes)
    }

    fn skip(&mut self, what: &str, reason: impl Into<String>) {
        if self.skipped.len() < 2000 {
            self.skipped.push(json!({"path": what, "reason": reason.into()}));
        }
    }

    fn permits(&mut self, rel: &[String], bytes: u64) -> bool {
        if self.files >= self.max_files || bytes > self.remaining() {
            self.truncated = true;
            self.skip(&rel.join("/"), "output file or byte limit reached");
            return false;
        }
        true
    }

    /// Write one output; false when it was skipped (limits, name clash, unsafe path).
    fn write(&mut self, rel: &[String], data: &[u8], meta: Value, level: u32) -> bool {
        if !self.permits(rel, data.len() as u64) {
            return false;
        }
        let shown = rel.join("/");
        match write_new_file(self.root, rel, data) {
            Ok(_) => {
                self.files += 1;
                self.bytes += data.len() as u64;
                self.max_level = self.max_level.max(level);
                let mut m = meta;
                if let Value::Object(o) = &mut m {
                    o.insert("path".into(), json!(shown));
                    o.insert("size".into(), json!(data.len()));
                    o.insert("sha256".into(), json!(sha256_hex(data)));
                    o.insert("level".into(), json!(level));
                }
                self.outputs.push(m);
                true
            }
            Err(e) => {
                self.skip(&shown, e);
                false
            }
        }
    }
}

/// Unpacked output: (relative path, bytes, action, folder for what is carved out of it).
type Produced = (Vec<String>, Vec<u8>, &'static str, Vec<String>);

fn write_produced(w: &mut Writer<'_>, jobs: &mut Vec<Job>, output: Produced, mut meta: Value, parent: &str, level: u32) {
    let (rel, data, action, scan_dir) = output;
    if let Value::Object(object) = &mut meta {
        object.insert("parent".into(), json!(parent));
        if action != "carved" {
            object.insert("offset".into(), Value::Null);
        }
    }
    let shown = rel.join("/");
    if w.write(&rel, &data, meta, level) && !data.is_empty() {
        jobs.push(Job { data, dir: scan_dir, level, parent: shown, base: 0 });
    }
}

fn child(dir: &[String], name: String) -> Vec<String> {
    let mut v = dir.to_vec();
    v.push(name);
    v
}

/// A member or embedded name as one safe path component (its last component), or a fallback.
fn safe_leaf(name: &str, fallback: &str) -> String {
    safe_components(name).ok().and_then(|c| c.last().cloned()).filter(|s| s.len() <= 200).unwrap_or_else(|| fallback.to_string())
}

fn firmware_image_extract(ctx: &mut Ctx<'_>, params: Value) -> Result<ToolOutput, ToolError> {
    let p: ExtractParams = parse_params(params)?;
    if !(1..=10).contains(&p.max_depth) {
        return Err(ToolError::invalid("max_depth must be 1..=10"));
    }
    if p.max_output_bytes == 0 || p.max_output_bytes > 512 << 20 {
        return Err(ToolError::invalid("max_output_bytes must be 1..=536870912 (512 MiB)"));
    }
    if !(1..=100_000).contains(&p.max_output_files) {
        return Err(ToolError::invalid("max_output_files must be 1..=100000"));
    }
    let input = load(ctx, p.path.as_deref(), fw::MAX_INPUT)?;
    let (data, base) = apply_range(&input.bytes, p.range)?;
    let out = create_absent_dir(&p.output_directory)?;
    ctx.report(None, format!("Extracting {} into {}", input.name, out.display()));
    let mut w = Writer {
        root: &out,
        files: 0,
        bytes: 0,
        max_files: p.max_output_files,
        max_bytes: p.max_output_bytes,
        outputs: Vec::new(),
        skipped: Vec::new(),
        truncated: false,
        max_level: 0,
    };
    let mut not_unpacked: BTreeMap<&'static str, usize> = BTreeMap::new();
    let mut deeper_than_limit = 0usize;
    let mut jobs = vec![Job { data: data.to_vec(), dir: Vec::new(), level: 0, parent: "<input>".into(), base }];
    let mut scanned = 0usize;
    let mut scan_truncated = false;
    let mut member_scan_truncated = false;
    while let Some(job) = jobs.pop() {
        ctx.cancel.check()?;
        scanned += 1;
        let cancel = ctx.cancel.clone();
        let scan = fw::scan(&job.data, &ScanOptions::default(), &|| cancel.is_cancelled());
        if scan.cancelled {
            return Err(ToolError::cancelled());
        }
        scan_truncated |= scan.truncated;
        let regions = scan.regions;
        for (k, r) in regions.iter().enumerate() {
            ctx.cancel.check()?;
            let start = usize::try_from(r.offset).unwrap_or(usize::MAX);
            let next_start = regions.get(k + 1).and_then(|n| usize::try_from(n.offset).ok()).unwrap_or(job.data.len());
            let end = r.end().and_then(|e| usize::try_from(e).ok()).unwrap_or(next_start).min(job.data.len());
            let Some(slice) = job.data.get(start..end) else { continue };
            let label = format!("{:x}-{}", r.offset.saturating_add(job.base), r.kind);
            let meta = |action: &str| json!({"kind": r.kind, "action": action, "offset": r.offset.saturating_add(job.base), "parent": job.parent, "length_known": r.length.is_some()});
            // Data that is exactly one region is already a file on disk; don't write it twice.
            let whole = job.level > 0 && start == 0 && end == job.data.len();
            if !whole {
                w.write(&child(&job.dir, format!("{label}.bin")), slice, meta("carved"), job.level.max(1));
            }
            if r.unpack == Unpack::Carve {
                *not_unpacked.entry(r.kind).or_default() += 1;
                continue;
            }
            let level = job.level + 1;
            if level > p.max_depth {
                deeper_than_limit += 1;
                continue;
            }
            // When this data is exactly the region, its folder already belongs to it.
            let sub = if whole { job.dir.clone() } else { child(&job.dir, format!("{label}.extracted")) };
            let parent_rel = if whole { job.parent.clone() } else { format!("{}/{label}.bin", job.dir.join("/")).trim_start_matches('/').to_string() };
            let mut produced: Vec<Produced> = Vec::new();
            match (r.unpack, r.kind) {
                (Unpack::Decompress, "gzip") => match fw::gunzip(slice, w.remaining()) {
                    Ok((d, _)) => {
                        let name = fw::gzip_header(slice)
                            .and_then(|h| h.original_name)
                            .map(|n| safe_leaf(&n, "decompressed.bin"))
                            .unwrap_or_else(|| "decompressed.bin".into());
                        produced.push((child(&sub, name), d, "decompressed", sub.clone()));
                    }
                    Err(e) => w.skip(&format!("{label}.extracted"), e.to_string()),
                },
                (Unpack::Decompress, "xz") => {
                    decompress_into(&mut produced, &sub, decompress::xz(slice, w.remaining()).map_err(|e| e.to_string()), &mut w, &label)
                }
                (Unpack::Decompress, "lzma") => {
                    decompress_into(&mut produced, &sub, decompress::lzma(slice, w.remaining()).map_err(|e| e.to_string()), &mut w, &label)
                }
                (Unpack::Decompress, "bzip2") => {
                    decompress_into(&mut produced, &sub, decompress::bzip2(slice, w.remaining()).map_err(|e| e.to_string()), &mut w, &label)
                }
                (Unpack::Decompress, "zstd") => {
                    decompress_into(&mut produced, &sub, decompress::zstd(slice, w.remaining()).map_err(|e| e.to_string()), &mut w, &label)
                }
                (Unpack::Decompress, _) => match fw::inflate(slice, true, w.remaining(), true) {
                    Ok(inf) => produced.push((child(&sub, "decompressed.bin".into()), inf.data.unwrap_or_default(), "decompressed", sub.clone())),
                    Err(e) => w.skip(&format!("{label}.extracted"), e.to_string()),
                },
                (Unpack::Filesystem, kind) => {
                    let limits = Limits { max_total_bytes: w.remaining(), max_file_bytes: w.remaining(), ..Limits::default() };
                    match fsread::read(kind, slice, &limits) {
                        Some(fsimg) => {
                            for wmsg in &fsimg.warnings {
                                w.skip(&format!("{label}.extracted"), format!("{kind}: {wmsg}"));
                            }
                            let mut has_files = false;
                            for e in fsimg.entries {
                                ctx.cancel.check()?;
                                if e.kind != FsKind::File {
                                    if e.kind == FsKind::Symlink {
                                        w.skip(&format!("{label}.extracted/{}", e.path), "symlink not created");
                                    }
                                    continue;
                                }
                                has_files = true;
                                let Some(d) = e.data else {
                                    w.skip(&format!("{label}.extracted/{}", e.path), "file data not recovered");
                                    continue;
                                };
                                match safe_components(&e.path) {
                                    Ok(c) => {
                                        let mut rel = sub.clone();
                                        rel.extend(c.iter().cloned());
                                        produced.push((rel, d, "filesystem-file", member_dir(&sub, &c)));
                                    }
                                    Err(why) => w.skip(&format!("{label}.extracted/{}", e.path), format!("refused: {why}")),
                                }
                            }
                            if !has_files {
                                *not_unpacked.entry(r.kind).or_default() += 1;
                            }
                        }
                        None => *not_unpacked.entry(r.kind).or_default() += 1,
                    }
                }
                (Unpack::Members, "zip") => match zip::parse(slice, p.max_output_files.saturating_mul(2)) {
                    Ok(z) => {
                        member_scan_truncated |= z.truncated;
                        for e in z.entries.iter().filter(|e| !e.is_dir) {
                            ctx.cancel.check()?;
                            let shown = format!("{label}.extracted/{}", e.name);
                            let c = match safe_components(&e.name) {
                                Ok(c) => c,
                                Err(why) => {
                                    w.skip(&shown, format!("refused: {why}"));
                                    continue;
                                }
                            };
                            let mut rel = sub.clone();
                            rel.extend(c.iter().cloned());
                            if !w.permits(&rel, e.uncompressed_size) {
                                continue;
                            }
                            match zip::read_entry(slice, e, w.remaining()) {
                                Ok(d) => {
                                    let scan_dir = member_dir(&sub, &c);
                                    // Charge this member before decoding the next one. Failed writes
                                    // leave the budget available to later members.
                                    write_produced(&mut w, &mut jobs, (rel, d, "member", scan_dir), meta("member"), &parent_rel, level);
                                }
                                Err(err) => {
                                    w.truncated |= matches!(&err, zip::ZipError::TooLarge(_));
                                    w.skip(&shown, err.to_string());
                                }
                            }
                        }
                    }
                    Err(e) => w.skip(&format!("{label}.extracted"), e.to_string()),
                },
                (Unpack::Members, kind) => {
                    let listing = if kind == "tar" { tarcpio::tar(slice, p.max_output_files) } else { tarcpio::cpio(slice, p.max_output_files) };
                    match listing {
                        Some(l) => {
                            member_scan_truncated |= l.truncated;
                            for m in &l.members {
                                ctx.cancel.check()?;
                                let shown = format!("{label}.extracted/{}", m.name);
                                match m.kind {
                                    tarcpio::MemberKind::Dir => continue,
                                    tarcpio::MemberKind::File => {}
                                    other => {
                                        w.skip(
                                            &shown,
                                            format!(
                                                "{} not created",
                                                serde_json::to_value(other).ok().and_then(|v| v.as_str().map(String::from)).unwrap_or_default()
                                            ),
                                        );
                                        continue;
                                    }
                                }
                                let c = match safe_components(&m.name) {
                                    Ok(c) => c,
                                    Err(why) => {
                                        w.skip(&shown, format!("refused: {why}"));
                                        continue;
                                    }
                                };
                                let Some(d) = tarcpio::member_data(slice, m) else {
                                    w.skip(&shown, "member data runs past the end of the archive");
                                    continue;
                                };
                                let mut rel = sub.clone();
                                rel.extend(c.iter().cloned());
                                produced.push((rel, d.to_vec(), "member", member_dir(&sub, &c)));
                            }
                        }
                        None => w.skip(&format!("{label}.extracted"), "archive header unreadable"),
                    }
                }
                (Unpack::Payload, "uimage") => {
                    let size = r.details.get("data_size").and_then(Value::as_u64).unwrap_or(0);
                    if let Some(d) = slice.get(64..).map(|s| s.get(..usize::try_from(size).unwrap_or(usize::MAX).min(s.len())).unwrap_or(s)) {
                        produced.push((child(&sub, "payload.bin".into()), d.to_vec(), "payload", sub.clone()));
                    }
                }
                (Unpack::Payload, _) => {
                    for (name, off, len) in boot_parts(r, slice) {
                        if let Some(d) = slice.get(off..off.saturating_add(len)) {
                            produced.push((child(&sub, name.into()), d.to_vec(), "payload", sub.clone()));
                        }
                    }
                }
                _ => {}
            }
            for output in produced {
                let m = meta(output.2);
                write_produced(&mut w, &mut jobs, output, m, &parent_rel, level);
            }
        }
    }
    let manifest = json!({"input": input.path, "sha256": input.subject.sha256, "range": p.range, "max_depth": p.max_depth, "outputs": w.outputs, "skipped": w.skipped, "not_unpacked": not_unpacked});
    let manifest_bytes = serde_json::to_vec_pretty(&manifest).map_err(|e| ToolError::internal(e.to_string()))?;
    let manifest_path = write_new_file(&out, &["manifest.json".to_string()], &manifest_bytes)
        .map_err(|e| ToolError::new(ErrorCode::FileError, format!("could not write manifest.json: {e}")))?;
    let total = w.outputs.len();
    let listed: Vec<&Value> = w.outputs.iter().take(p.limit.clamp(1, 5000)).collect();
    let mut by_action: BTreeMap<String, usize> = BTreeMap::new();
    for o in &w.outputs {
        *by_action.entry(o.get("action").and_then(Value::as_str).unwrap_or("").to_string()).or_default() += 1;
    }
    let summary = format!(
        "Wrote {} file(s), {} bytes, from {} into {} ({}); {} level(s) deep{}.",
        w.files,
        w.bytes,
        input.name,
        out.display(),
        by_action.iter().map(|(k, n)| format!("{n} {k}")).collect::<Vec<_>>().join(", "),
        w.max_level,
        if w.skipped.is_empty() { String::new() } else { format!("; {} skipped", w.skipped.len()) }
    );
    let mut lims = scan_limitations();
    if !not_unpacked.is_empty() {
        lims.push(Limitation::new(
            "carved_only",
            format!(
                "Carved as raw bytes but not unpacked (the built-in engine has no reader for their contents): {}.",
                not_unpacked.iter().map(|(k, n)| format!("{k} ร—{n}")).collect::<Vec<_>>().join(", ")
            ),
        ));
    }
    lims.push(Limitation::new(
        "unknown_length_carves",
        "Regions whose length the format doesn't state are carved up to the next region or the end of the data.",
    ));
    if deeper_than_limit > 0 {
        lims.push(Limitation::new(
            "max_depth",
            format!("{deeper_than_limit} region(s) at the depth limit were carved but not unpacked; raise max_depth to go further."),
        ));
    }
    if w.truncated {
        lims.push(Limitation::new("truncated", "The output byte or file limit stopped extraction; raise max_output_bytes or max_output_files."));
    }
    if scan_truncated {
        lims.push(Limitation::new(
            "truncated",
            format!("A scan stopped at max_regions={}; extract smaller ranges to recover omitted regions.", ScanOptions::default().max_regions),
        ));
    }
    if member_scan_truncated {
        lims.push(Limitation::new(
            "truncated",
            "An archive member scan stopped early; inspect the archive or raise max_output_files if the member limit was reached.",
        ));
    }
    let truncated = w.truncated || scan_truncated || member_scan_truncated;
    let complete = !truncated && w.skipped.is_empty() && deeper_than_limit == 0;
    let ev = builder(
        ctx,
        "firmware_image_extract",
        input.subject.clone(),
        json!({"path": p.path, "output_directory": p.output_directory, "range": p.range, "max_depth": p.max_depth, "max_output_bytes": p.max_output_bytes, "max_output_files": p.max_output_files}),
    )
    .result(json!({
        "output_directory": out,
        "manifest": manifest_path,
        "files_written": w.files,
        "bytes_written": w.bytes,
        "levels": w.max_level,
        "data_scanned": scanned,
        "by_action": by_action,
        "outputs_total": total,
        "outputs": listed,
        "skipped": w.skipped,
        "not_unpacked": not_unpacked,
        "truncated": truncated,
    }))
    .summary(summary)
    .confidence(Confidence::Observed)
    .limitations(lims)
    .coverage(Coverage { status: if complete { CoverageStatus::Complete } else { CoverageStatus::Partial }, examined: Some(total as u64), total: None, next_offset: None })
    .location(Location::new("path", out.display().to_string()).with_label("output folder"))
    .build()?;
    Ok(ToolOutput::evidence(ev))
}

/// Push a decompressed stream as `decompressed.bin`, or record why it could not be decompressed.
fn decompress_into(produced: &mut Vec<Produced>, sub: &[String], result: Result<Vec<u8>, String>, w: &mut Writer<'_>, label: &str) {
    match result {
        Ok(d) => produced.push((child(sub, "decompressed.bin".into()), d, "decompressed", sub.to_vec())),
        Err(e) => w.skip(&format!("{label}.extracted"), e),
    }
}

/// Folder that receives what is carved out of an unpacked member: `<member>.extracted` beside it.
fn member_dir(sub: &[String], comps: &[String]) -> Vec<String> {
    let mut v = sub.to_vec();
    if let Some((last, parents)) = comps.split_last() {
        v.extend(parents.iter().cloned());
        v.push(format!("{last}.extracted"));
    }
    v
}

/// The kernel, ramdisk and second-stage parts of an Android boot image: (name, offset, length).
fn boot_parts(r: &Region, slice: &[u8]) -> Vec<(&'static str, usize, usize)> {
    let get = |k: &str| r.details.get(k).and_then(Value::as_u64).and_then(|v| usize::try_from(v).ok()).unwrap_or(0);
    let page = get("page_size").max(1);
    let align = |v: usize| v.div_ceil(page).saturating_mul(page);
    let kernel = get("kernel_size");
    let ramdisk = get("ramdisk_size");
    let second = get("second_size");
    let k_off = page;
    let r_off = k_off.saturating_add(align(kernel));
    let s_off = r_off.saturating_add(align(ramdisk));
    [("kernel.bin", k_off, kernel), ("ramdisk.bin", r_off, ramdisk), ("second.bin", s_off, second)]
        .into_iter()
        .filter(|(_, o, l)| *l > 0 && o.checked_add(*l).is_some_and(|e| e <= slice.len()))
        .collect()
}

#[cfg(test)]
mod tests {
    use super::super::native_common::test_support::{call, call_err};
    use crate::Engine;
    use craft_anything_testkit::firmware as fx;
    use serde_json::json;
    use std::path::PathBuf;

    fn fresh_out(tag: &str) -> PathBuf {
        let d = tempfile::tempdir().unwrap().keep();
        d.join(format!("out-{tag}"))
    }

    #[test]
    fn inspects_regions_with_paging_and_range() {
        let (img, l) = fx::image();
        let p = craft_anything_testkit::temp_file("fw.bin", &img);
        let e = Engine::new(crate::testing::session());
        let v = call(&e, "firmware_regions_list", json!({"path": p}));
        let res = &v["result"];
        assert_eq!(res["regions_total"], 6, "{}", res["by_kind"]);
        assert_eq!(res["by_kind"]["uimage"], 1);
        assert_eq!(res["regions"][0]["kind"], "uimage");
        assert_eq!(res["regions"][0]["details"]["name"], "Craft Linux-5.10");
        assert!(res["layout"]["high_entropy_unrecognised"].as_array().unwrap().iter().any(|s| s["offset"] == l.noise.0));
        assert!(!res["entropy"]["blocks"].as_array().unwrap().is_empty());
        let page = call(&e, "firmware_regions_list", json!({"path": p, "offset": 1, "limit": 2}));
        assert_eq!(page["coverage"]["next_offset"], 3);
        assert_eq!(page["result"]["regions"].as_array().unwrap().len(), 2);
        let ranged = call(&e, "firmware_regions_list", json!({"path": p, "range": {"offset": l.tar.0, "length": l.tar.1 + l.cpio.1 + 0x200}}));
        let kinds: Vec<_> =
            ranged["result"]["regions"].as_array().unwrap().iter().map(|r| (r["kind"].as_str().unwrap().to_string(), r["offset"].as_u64().unwrap())).collect();
        assert!(kinds.contains(&("tar".into(), l.tar.0 as u64)), "{kinds:?}");
        assert!(kinds.contains(&("cpio-newc".into(), l.cpio.0 as u64)), "{kinds:?}");
        assert!(call_err(&e, "firmware_regions_list", json!({"path": p, "entropy_block": 1})).message.contains("entropy_block"));
        assert!(call_err(&e, "firmware_regions_list", json!({"path": p, "range": {"offset": 1u64 << 40, "length": 1}})).message.contains("past the end"));
        assert!(call_err(&e, "firmware_regions_list", json!({"path": "relative.bin"})).message.contains("absolute"));
    }

    #[test]
    fn decompresses_xz_lzma_bzip2_zstd() {
        let e = Engine::new(crate::testing::session());
        for (name, fixture, plain) in [
            ("xz", fx::XZ_FIXTURE, fx::XZ_PLAIN),
            ("lzma", fx::LZMA_FIXTURE, fx::LZMA_PLAIN),
            ("bzip2", fx::BZIP2_FIXTURE, fx::BZIP2_PLAIN),
            ("zstd", fx::ZSTD_FIXTURE, fx::ZSTD_PLAIN),
        ] {
            let p = craft_anything_testkit::temp_file(&format!("{name}.bin"), fixture);
            let out = fresh_out(name);
            let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out}));
            let got = std::fs::read(out.join(format!("0-{name}.extracted/decompressed.bin")))
                .unwrap_or_else(|_| panic!("{name}: {}", serde_json::to_string_pretty(&v["result"]).unwrap()));
            assert_eq!(got, plain, "{name}");
            std::fs::remove_dir_all(&out).unwrap();
        }
    }

    #[test]
    fn extracts_filesystems() {
        let e = Engine::new(crate::testing::session());
        let cases: Vec<(&str, Vec<u8>, &str, &[u8])> = vec![
            ("romfs", fx::romfs_image(), "etc/hostname", b"craft-router\n"),
            ("cramfs", fx::cramfs_image(), "dir/inner", b"nested"),
            ("ext2", fx::ext_image(false), "sub/note", b"nested note"),
            ("ext4", fx::ext_image(true), "hello.txt", b"Hello ext filesystem!\n"),
            ("squashfs", fx::squashfs_image(), "etc/motd", b"welcome\n"),
        ];
        for (kind, img, path, content) in cases {
            let p = craft_anything_testkit::temp_file(&format!("{kind}.bin"), &img);
            let out = fresh_out(&format!("fs-{kind}"));
            let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out}));
            // The region lands at offset 0; ext images report as ext2/ext4 by their feature bits.
            let region_kind = v["result"]["outputs"].as_array().unwrap().iter().find_map(|o| {
                let pp = o["path"].as_str()?;
                pp.strip_suffix(&format!(".extracted/{path}")).map(|s| s.to_string())
            });
            let dir = region_kind.unwrap_or_else(|| panic!("{kind}: no output for {path} in {}", serde_json::to_string_pretty(&v["result"]).unwrap()));
            let got = std::fs::read(out.join(format!("{dir}.extracted/{path}"))).unwrap();
            assert_eq!(got, content, "{kind}");
            std::fs::remove_dir_all(&out).unwrap();
        }
    }

    #[test]
    fn extracts_recursively_and_never_overwrites() {
        let (img, l) = fx::image();
        let p = craft_anything_testkit::temp_file("fw.bin", &img);
        let e = Engine::new(crate::testing::session());
        let out = fresh_out("a");
        let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out}));
        let res = &v["result"];
        // uImage payload โ†’ gzip โ†’ kernel text.
        let kernel = out.join("0-uimage.extracted/vmlinux");
        let k = std::fs::read(&kernel).unwrap_or_else(|_| panic!("{}", serde_json::to_string_pretty(&res["outputs"]).unwrap()));
        assert!(k.starts_with(fx::KERNEL_TEXT));
        assert_eq!(std::fs::read(out.join(format!("{:x}-tar.extracted/etc/hostname", l.tar.0))).unwrap(), b"craft-router\n");
        assert_eq!(std::fs::read(out.join(format!("{:x}-cpio-newc.extracted/init", l.cpio.0))).unwrap(), b"#!/bin/sh\nexec /sbin/init\n");
        assert_eq!(std::fs::read(out.join(format!("{:x}-zip.extracted/config/settings.json", l.zip.0))).unwrap(), b"{\"wifi\":true}");
        assert_eq!(std::fs::metadata(out.join(format!("{:x}-ext2.bin", l.ext2.0))).unwrap().len(), l.ext2.1 as u64);
        let manifest: serde_json::Value = serde_json::from_slice(&std::fs::read(out.join("manifest.json")).unwrap()).unwrap();
        assert_eq!(manifest["outputs"].as_array().unwrap().len(), res["outputs_total"].as_u64().unwrap() as usize);
        assert!(manifest["outputs"].as_array().unwrap().iter().all(|o| o["sha256"].as_str().unwrap().len() == 64));
        assert_eq!(res["not_unpacked"]["squashfs"], 1);
        assert!(res["levels"].as_u64().unwrap() >= 2);
        assert!(v["limitations"].as_array().unwrap().iter().any(|x| x["code"] == "carved_only"));
        // The folder now exists: a second run must refuse rather than overwrite.
        let err = call_err(&e, "firmware_image_extract", json!({"path": p, "output_directory": out}));
        assert!(err.message.contains("already exists"));
        assert!(call_err(&e, "firmware_image_extract", json!({"path": p, "output_directory": "rel/out"})).message.contains("absolute"));
        std::fs::remove_dir_all(&out).unwrap();
    }

    #[test]
    fn extraction_reports_a_scan_cap_as_partial_coverage() {
        let mut elf = vec![0; 121];
        elf[..7].copy_from_slice(b"\x7fELF\x02\x01\x01");
        for (offset, value) in [(16, 2u16), (18, 62), (52, 64), (54, 56), (56, 1), (58, 64)] {
            elf[offset..offset + 2].copy_from_slice(&value.to_le_bytes());
        }
        for (offset, value) in [(20, 1u32), (64, 1), (68, 5)] {
            elf[offset..offset + 4].copy_from_slice(&value.to_le_bytes());
        }
        for (offset, value) in [(24, 0x400078u64), (32, 64), (80, 0x400000), (88, 0x400000), (96, 121), (104, 121), (112, 4096)] {
            elf[offset..offset + 8].copy_from_slice(&value.to_le_bytes());
        }
        elf[120] = 0xc3;
        let root = tempfile::tempdir().unwrap();
        let input = root.path().join("many-elf.bin");
        std::fs::write(&input, elf.repeat(2001)).unwrap();
        let e = Engine::new(crate::testing::session());
        let listing = call(&e, "firmware_regions_list", json!({"path": input, "max_regions": 2002, "limit": 5000}));
        assert_eq!(listing["result"]["regions_total"], 2001);
        assert_eq!(listing["result"]["truncated"], false);
        let out = root.path().join("out");
        let v = call(&e, "firmware_image_extract", json!({"path": input, "output_directory": out, "limit": 1}));
        assert_eq!(v["result"]["files_written"], 2000);
        assert_eq!(v["result"]["bytes_written"], 242000);
        assert_eq!(v["result"]["skipped"], json!([]));
        assert_eq!(v["result"]["outputs"].as_array().unwrap().len(), 1);
        let manifest: serde_json::Value = serde_json::from_slice(&std::fs::read(out.join("manifest.json")).unwrap()).unwrap();
        assert_eq!(manifest["outputs"].as_array().unwrap().len(), 2000);
        assert_eq!(std::fs::read_dir(&out).unwrap().count(), 2001);
        assert_eq!(std::fs::read(out.join(format!("{:x}-elf.bin", 1999 * 121))).unwrap(), elf);
        assert!(!out.join(format!("{:x}-elf.bin", 2000 * 121)).exists());
        assert_eq!(v["result"]["truncated"], true);
        assert_eq!(v["coverage"]["status"], "partial");
        assert!(v["limitations"].as_array().unwrap().iter().any(|l| l["code"] == "truncated" && l["message"].as_str().unwrap().contains("2000")));
        let ranged_out = root.path().join("ranged-out");
        let ranged = call(
            &e,
            "firmware_image_extract",
            json!({"path": input, "output_directory": ranged_out, "range": {"offset": 2000 * 121, "length": 121}, "limit": 1}),
        );
        assert_eq!(ranged["result"]["files_written"], 1);
        assert_eq!(ranged["result"]["truncated"], false);
        assert_eq!(ranged["coverage"]["status"], "complete");
        assert_eq!(std::fs::read(ranged_out.join(format!("{:x}-elf.bin", 2000 * 121))).unwrap(), elf);
    }

    #[test]
    fn depth_and_caps_are_honoured() {
        let (img, _) = fx::image();
        let p = craft_anything_testkit::temp_file("fw.bin", &img);
        let e = Engine::new(crate::testing::session());
        let out = fresh_out("depth");
        let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out, "max_depth": 1}));
        assert!(out.join("0-uimage.extracted/payload.bin").exists());
        assert!(!out.join("0-uimage.extracted/vmlinux").exists());
        assert!(v["limitations"].as_array().unwrap().iter().any(|x| x["code"] == "max_depth"));
        let out2 = fresh_out("caps");
        let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out2, "max_output_files": 3}));
        assert_eq!(v["result"]["files_written"], 3);
        assert_eq!(v["result"]["truncated"], true);
        assert_eq!(v["coverage"]["status"], "partial");
        let out3 = fresh_out("bytes");
        let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out3, "max_output_bytes": 4096}));
        assert!(v["result"]["bytes_written"].as_u64().unwrap() <= 4096);
        assert!(call_err(&e, "firmware_image_extract", json!({"path": p, "output_directory": fresh_out("x"), "max_depth": 11})).message.contains("max_depth"));
        for d in [out, out2, out3] {
            std::fs::remove_dir_all(d).unwrap();
        }
    }

    #[test]
    fn slip_and_links_are_refused() {
        let t = fx::tar_entries(&[
            ("../../escape.txt", b'0', b"x", ""),
            ("/etc/abs", b'0', b"y", ""),
            ("link", b'2', b"", "/etc/passwd"),
            ("ok/fine.txt", b'0', b"fine", ""),
        ]);
        let z = craft_anything_testkit::electron::zip_stored(&[("../zip-escape", b"z"), ("good.txt", b"g")]);
        let c = fx::cpio_newc(&[("../cpio-escape", b"c", 0o100644), ("sym", b"/bin/sh", 0o120777), ("keep", b"k", 0o100644)]);
        let mut img = t.clone();
        while !img.len().is_multiple_of(512) {
            img.push(0);
        }
        let z_off = img.len();
        img.extend_from_slice(&z);
        img.extend(vec![0u8; 512 - img.len() % 512]);
        let c_off = img.len();
        img.extend_from_slice(&c);
        let p = craft_anything_testkit::temp_file("evil.bin", &img);
        let e = Engine::new(crate::testing::session());
        let out = fresh_out("slip");
        let v = call(&e, "firmware_image_extract", json!({"path": p, "output_directory": out}));
        assert_eq!(std::fs::read(out.join("0-tar.extracted/ok/fine.txt")).unwrap(), b"fine");
        assert_eq!(std::fs::read(out.join(format!("{z_off:x}-zip.extracted/good.txt"))).unwrap(), b"g");
        assert_eq!(std::fs::read(out.join(format!("{c_off:x}-cpio-newc.extracted/keep"))).unwrap(), b"k");
        let skipped = v["result"]["skipped"].as_array().unwrap();
        assert!(skipped.len() >= 6, "{skipped:?}");
        assert!(!out.parent().unwrap().join("escape.txt").exists());
        assert!(!out.join("0-tar.extracted/link").exists());
        assert!(!out.join(format!("{c_off:x}-cpio-newc.extracted/sym")).exists());
        let all: Vec<_> = walk(&out);
        assert!(all.iter().all(|f| f.starts_with(&out)));
        std::fs::remove_dir_all(&out).unwrap();
    }

    fn walk(d: &std::path::Path) -> Vec<PathBuf> {
        let mut v = Vec::new();
        for e in std::fs::read_dir(d).unwrap().flatten() {
            let p = e.path();
            assert!(!e.file_type().unwrap().is_symlink());
            if p.is_dir() {
                v.extend(walk(&p));
            } else {
                v.push(p);
            }
        }
        v
    }

    #[test]
    fn damaged_images_never_panic() {
        let (img, _) = fx::image();
        let e = Engine::new(crate::testing::session());
        for (k, m) in craft_anything_testkit::mutations(&img).into_iter().enumerate().step_by(9) {
            let p = craft_anything_testkit::temp_file("m.bin", &m);
            let _ = e.call("firmware_regions_list", json!({"path": p}), crate::CallOptions::new(crate::Caller::Mcp));
            let out = fresh_out(&format!("m{k}"));
            let _ = e.call(
                "firmware_image_extract",
                json!({"path": p, "output_directory": out, "max_output_bytes": 1 << 20}),
                crate::CallOptions::new(crate::Caller::Mcp),
            );
            let _ = std::fs::remove_dir_all(&out);
        }
    }
}