642d5a256791
rust_bin.rs
//! Rust program fingerprints: rustc version and commit, crates and versions from embedded Cargo
//! paths, standard-library path layout, source paths left by panic locations, well-known panic
//! messages, Rust symbol names (legacy and v0 mangling) and the cargo-auditable dependency list
//! (`.dep-v0`, zlib-compressed JSON). Pure byte scans; never executes anything.
use crate::binary::Binary;
use serde::Serialize;
use serde_json::Value;
use std::collections::BTreeMap;
/// Most entries kept per list.
pub const MAX_ITEMS: usize = 500;
/// Most occurrences of one needle examined.
const MAX_HITS: usize = 200_000;
/// Largest cargo-auditable section read (1 MiB) and largest inflated JSON (8 MiB).
const MAX_DEP_SECTION: u64 = 1 << 20;
const MAX_DEP_JSON: usize = 8 << 20;
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Counted {
pub value: String,
pub count: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct CrateRef {
pub name: String,
pub version: Option<String>,
/// registry, toolchain-vendored, vendored or git
pub source: &'static str,
/// The registry index folder or git checkout it was built from.
pub origin: String,
pub occurrences: usize,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
pub struct SourcePaths {
/// Paths under the Rust toolchain (`/rustc/<commit>/…`).
pub toolchain: usize,
/// Paths inside crates downloaded by Cargo.
pub dependencies: usize,
/// Everything else: usually the program's own source files.
pub local: Vec<Counted>,
pub local_truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct SymbolSample {
pub mangled: String,
pub demangled: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
pub struct RustSymbols {
/// Symbols in the container's symbol tables (exports and imports included).
pub total: usize,
pub rust_legacy: usize,
pub rust_v0: usize,
/// No Rust-mangled names at all (stripped, or names kept in a separate debug file).
pub stripped: bool,
/// Leading path component of demangled names, by frequency.
pub crates: Vec<Counted>,
pub samples: Vec<SymbolSample>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct AuditablePackage {
pub name: String,
pub version: String,
pub source: Option<String>,
pub kind: Option<String>,
pub dependencies: Vec<u64>,
pub root: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct Auditable {
pub section: String,
pub file_offset: u64,
pub section_bytes: u64,
pub uncompressed_bytes: usize,
pub format_version: Option<i64>,
pub packages: Vec<AuditablePackage>,
pub root_package_index: Option<usize>,
pub dependency_edges: usize,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct RustReport {
pub is_rust: bool,
/// strong (symbols, toolchain paths or dependency data), weak (only panic text or crate paths) or none
pub strength: &'static str,
pub signals: Vec<String>,
pub rustc_versions: Vec<Counted>,
pub rustc_commits: Vec<Counted>,
/// `library/` (rustc 1.47 or later) or `src/lib*` (older); from toolchain paths.
pub std_layout: Option<&'static str>,
pub crates: Vec<CrateRef>,
pub source_paths: SourcePaths,
pub panic_messages: Vec<Counted>,
pub symbols: RustSymbols,
pub auditable: Option<Auditable>,
/// Why a present `.dep-v0` section could not be read.
pub auditable_error: Option<String>,
}
/// Offsets of `needle` in `data`, at most `MAX_HITS`.
fn occurrences<'a>(data: &'a [u8], needle: &'a [u8]) -> impl Iterator<Item = usize> + 'a {
let mut i = 0usize;
std::iter::from_fn(move || {
let (&first, rest) = needle.split_first()?;
while let Some(p) = data.get(i..).and_then(|h| h.iter().position(|&b| b == first)) {
let at = i + p;
i = at + 1;
if data.get(at + 1..at + 1 + rest.len()) == Some(rest) {
return Some(at);
}
}
None
})
.take(MAX_HITS)
}
fn is_sep(b: u8) -> bool {
b == b'/' || b == b'\\'
}
/// The path component starting at `at` (up to a separator), when it is plain name characters.
fn component(data: &[u8], at: usize) -> Option<(&str, usize)> {
let rest = data.get(at..)?;
let len = rest.iter().take(129).position(|&b| is_sep(b))?;
let s = std::str::from_utf8(rest.get(..len)?).ok()?;
let ok = !s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.' | b'+'));
ok.then_some((s, at + len + 1))
}
/// A Cargo package directory combines a crate name and a complete semantic version. Splitting
/// at a hyphen only when the suffix parses keeps hyphens in names and prerelease versions.
fn split_crate_dir(dir: &str) -> Option<(&str, &str)> {
dir.match_indices('-').find_map(|(i, _)| {
let name = dir.get(..i)?;
let version = dir.get(i + 1..)?;
let valid_name = !name.is_empty() && name.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-'));
(valid_name && semver::Version::parse(version).is_ok()).then_some((name, version))
})
}
fn bump(map: &mut BTreeMap<String, usize>, key: String) {
if map.len() < MAX_ITEMS * 4 || map.contains_key(&key) {
*map.entry(key).or_insert(0) += 1;
}
}
/// Most frequent first, then by value; at most `MAX_ITEMS`.
fn ranked(map: BTreeMap<String, usize>) -> Vec<Counted> {
let mut v: Vec<Counted> = map.into_iter().map(|(value, count)| Counted { value, count }).collect();
v.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.value.cmp(&b.value)));
v.truncate(MAX_ITEMS);
v
}
/// Printable text from `at`, at most `max` bytes, ending at a control byte or just after `stop`.
fn text_at(data: &[u8], at: usize, max: usize, stop: Option<u8>) -> String {
let rest = data.get(at..).unwrap_or(&[]);
let mut end = 0usize;
for &b in rest.iter().take(max) {
if !(0x20..0x7f).contains(&b) {
break;
}
end += 1;
if Some(b) == stop {
break;
}
}
String::from_utf8_lossy(rest.get(..end).unwrap_or(&[])).into_owned()
}
fn path_char(b: u8) -> bool {
b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.' | b'/' | b'\\' | b':' | b'+' | b'@' | b'~')
}
/// Well-known messages and markers from the Rust standard library's panic machinery.
const PANIC_TEXTS: &[&str] = &[
"called `Option::unwrap()` on a `None` value",
"called `Result::unwrap()` on an `Err` value",
"index out of bounds: the len is ",
"attempt to add with overflow",
"attempt to subtract with overflow",
"attempt to multiply with overflow",
"RUST_BACKTRACE",
"rust_begin_unwind",
"rust_panic",
];
/// Crates that ship inside the standard library; their symbols say little about the program.
const STD_CRATES: &[&str] = &[
"core",
"alloc",
"std",
"compiler_builtins",
"hashbrown",
"panic_unwind",
"panic_abort",
"gimli",
"addr2line",
"miniz_oxide",
"rustc_demangle",
"object",
"memchr",
"adler",
"adler2",
"std_detect",
];
/// Path markers that precede a dependency's folder, with the source they indicate.
const DEP_NEEDLES: &[(&[u8], &str)] = &[
(b"registry/src/", "registry"),
(b"registry\\src\\", "registry"),
(b"/rust/deps/", "toolchain-vendored"),
(b"/rust/deps\\", "toolchain-vendored"),
(b"vendor/", "vendored"),
(b"vendor\\", "vendored"),
(b"git/checkouts/", "git"),
(b"git\\checkouts\\", "git"),
];
type CrateKey = (String, Option<String>, &'static str, String);
/// Look for Rust fingerprints in `data`; `bin` adds symbols and the cargo-auditable section.
pub fn inspect(data: &[u8], bin: Option<&Binary>) -> RustReport {
// Toolchain: `/rustc/<40-hex commit>/library/...` and `rustc version 1.x.y (...)`.
let mut commits = BTreeMap::new();
let (mut layout_new, mut layout_old) = (0usize, 0usize);
for at in occurrences(data, b"/rustc/") {
let Some(h) = data.get(at + 7..at + 47) else { continue };
if h.iter().all(u8::is_ascii_hexdigit) && data.get(at + 47).copied().is_some_and(is_sep) {
bump(&mut commits, String::from_utf8_lossy(h).into_owned());
let tail = data.get(at + 48..).unwrap_or(&[]);
if tail.starts_with(b"library") {
layout_new += 1;
} else if tail.starts_with(b"src") {
layout_old += 1;
}
}
}
let mut versions = BTreeMap::new();
for at in occurrences(data, b"rustc version 1.") {
bump(&mut versions, text_at(data, at + 14, 80, Some(b')')));
}
// Dependencies from Cargo's folders: registry/src/<index>/<crate>-<ver>/, the toolchain's
// remapped /rust/deps/<crate>-<ver>/, vendor/<crate>-<ver>/ and git/checkouts/<repo>-<hash>/<rev>/.
let mut crates: BTreeMap<CrateKey, usize> = BTreeMap::new();
for &(needle, source) in DEP_NEEDLES {
for at in occurrences(data, needle) {
let Some((first, next)) = component(data, at + needle.len()) else { continue };
let key: CrateKey = if source != "git" {
// Only registry paths have an index folder before the crate folder.
let (dir, origin) = if source == "registry" {
let Some((dir, _)) = component(data, next) else { continue };
(dir, first)
} else {
(first, source)
};
let Some((name, ver)) = split_crate_dir(dir) else { continue };
(name.to_string(), Some(ver.to_string()), source, origin.to_string())
} else {
let name = first.rsplit_once('-').map(|(n, _)| n).unwrap_or(first);
(name.to_string(), component(data, next).map(|(r, _)| r.to_string()), source, first.to_string())
};
if crates.len() < MAX_ITEMS * 4 || crates.contains_key(&key) {
*crates.entry(key).or_insert(0) += 1;
}
}
}
let mut crate_list: Vec<CrateRef> =
crates.into_iter().map(|((name, version, source, origin), occurrences)| CrateRef { name, version, source, origin, occurrences }).collect();
crate_list.truncate(MAX_ITEMS);
// Source paths (panic locations, debug info): any path ending in `.rs`.
let mut paths = SourcePaths::default();
let mut local = BTreeMap::new();
for at in occurrences(data, b".rs") {
let mut start = at;
while start > 0 && at - start < 300 && data.get(start - 1).copied().is_some_and(path_char) {
start -= 1;
}
let Some(p) = data.get(start..at + 3).and_then(|b| std::str::from_utf8(b).ok()) else { continue };
let stem = p.trim_end_matches(".rs").rsplit(['/', '\\']).next().unwrap_or("");
if !p.contains(['/', '\\']) || stem.is_empty() || !stem.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') {
continue;
}
if p.contains("/rustc/") {
paths.toolchain += 1;
} else if DEP_NEEDLES.iter().any(|(n, _)| p.as_bytes().windows(n.len()).any(|w| w == *n)) {
paths.dependencies += 1;
} else {
bump(&mut local, p.to_string());
}
}
let local_kinds = local.len();
paths.local = ranked(local);
paths.local_truncated = local_kinds > paths.local.len();
let panic_messages: Vec<Counted> =
PANIC_TEXTS.iter().map(|t| Counted { value: (*t).to_string(), count: occurrences(data, t.as_bytes()).count() }).filter(|c| c.count > 0).collect();
let symbols = bin.map(rust_symbols).unwrap_or(RustSymbols { stripped: true, ..RustSymbols::default() });
let (auditable, auditable_error) = match bin.map(|b| auditable(b, data)) {
Some(Ok(a)) => (a, None),
Some(Err(e)) => (None, Some(e)),
None => (None, None),
};
let mut signals = Vec::new();
if let Some(a) = &auditable {
signals.push(format!("cargo-auditable dependency list ({} packages)", a.packages.len()));
}
if symbols.rust_legacy + symbols.rust_v0 > 0 {
signals.push(format!("{} Rust-mangled symbol names", symbols.rust_legacy + symbols.rust_v0));
}
if !commits.is_empty() {
signals.push("Rust toolchain source paths (/rustc/<commit>/)".into());
}
if !versions.is_empty() {
signals.push("rustc version string".into());
}
let strong = !signals.is_empty();
if !crate_list.is_empty() {
signals.push(format!("{} Cargo dependency paths", crate_list.len()));
}
if !panic_messages.is_empty() {
signals.push("Rust standard-library panic text".into());
}
let strength = match (strong, signals.is_empty()) {
(true, _) => "strong",
(false, true) => "none",
(false, false) => "weak",
};
RustReport {
is_rust: strength != "none",
strength,
signals,
rustc_versions: ranked(versions),
rustc_commits: ranked(commits),
std_layout: match (layout_new, layout_old) {
(0, 0) => None,
(n, o) if n >= o => Some("library/ (rustc 1.47 or later)"),
_ => Some("src/lib* (before rustc 1.47)"),
},
crates: crate_list,
source_paths: paths,
panic_messages,
symbols,
auditable,
auditable_error,
}
}
fn rust_symbols(bin: &Binary) -> RustSymbols {
let names =
bin.symbols.iter().map(|s| s.name.as_str()).chain(bin.imports.iter().map(|i| i.name.as_str())).chain(bin.exports.iter().map(|e| e.name.as_str()));
let mut out = RustSymbols::default();
let mut crates = BTreeMap::new();
let mut std_samples = Vec::new();
for name in names {
out.total += 1;
let Ok(d) = rustc_demangle::try_demangle(name) else { continue };
let full = format!("{d}");
let v0 = name.trim_start_matches('_').starts_with('R');
// Legacy names are Itanium-shaped; only the trailing `::h<16 hex>` hash marks them as Rust.
let legacy = !v0 && full.rsplit_once("::h").is_some_and(|(_, h)| h.len() == 16 && h.bytes().all(|b| b.is_ascii_hexdigit()));
if !(v0 || legacy) {
continue;
}
if v0 {
out.rust_v0 += 1;
} else {
out.rust_legacy += 1;
}
let short = format!("{d:#}");
let krate = short.trim_start_matches('<').split("::").next().unwrap_or("").to_string();
let is_std = STD_CRATES.contains(&krate.as_str());
bump(&mut crates, krate);
let sample = SymbolSample { mangled: name.to_string(), demangled: short };
if !is_std && out.samples.len() < 40 {
out.samples.push(sample);
} else if is_std && std_samples.len() < 10 {
std_samples.push(sample);
}
}
out.samples.extend(std_samples);
out.stripped = out.rust_legacy + out.rust_v0 == 0;
out.crates = ranked(crates);
out
}
/// The cargo-auditable `.dep-v0` section: `Ok(None)` when absent.
fn auditable(bin: &Binary, data: &[u8]) -> Result<Option<Auditable>, String> {
let Some(sec) = bin.sections.iter().find(|s| s.name == ".dep-v0") else { return Ok(None) };
let off = sec.file_offset.ok_or("the .dep-v0 section has no bytes in the file")?;
if sec.file_size > MAX_DEP_SECTION {
return Err(format!("the .dep-v0 section is {} bytes; the limit is {MAX_DEP_SECTION}", sec.file_size));
}
let raw = usize::try_from(off)
.ok()
.zip(usize::try_from(sec.file_size).ok())
.and_then(|(o, n)| data.get(o..o.checked_add(n)?))
.ok_or("the .dep-v0 section runs past the end of the file")?;
let cap = MAX_DEP_JSON.min(raw.len().saturating_mul(64)).max(1024);
let json = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(raw, cap).map_err(|e| format!("the .dep-v0 data did not inflate: {:?}", e.status))?;
let v: Value = serde_json::from_slice(&json).map_err(|e| format!("the .dep-v0 data is not JSON: {e}"))?;
let pkgs = v.get("packages").and_then(Value::as_array).ok_or("the .dep-v0 JSON has no packages list")?;
let s = |p: &Value, k: &str| p.get(k).and_then(Value::as_str).map(str::to_string);
let packages: Vec<AuditablePackage> = pkgs
.iter()
.take(100_000)
.map(|p| AuditablePackage {
name: s(p, "name").unwrap_or_default(),
version: s(p, "version").unwrap_or_default(),
source: s(p, "source"),
kind: s(p, "kind"),
dependencies: p.get("dependencies").and_then(Value::as_array).map(|d| d.iter().filter_map(Value::as_u64).collect()).unwrap_or_default(),
root: p.get("root").and_then(Value::as_bool).unwrap_or(false),
})
.collect();
Ok(Some(Auditable {
section: match &sec.segment {
Some(seg) => format!("{seg},{}", sec.name),
None => sec.name.clone(),
},
file_offset: off,
section_bytes: sec.file_size,
uncompressed_bytes: json.len(),
format_version: v.get("format").and_then(Value::as_i64),
root_package_index: packages.iter().position(|p| p.root),
dependency_edges: packages.iter().map(|p| p.dependencies.len()).sum(),
packages,
}))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn crate_dir_split() {
assert_eq!(split_crate_dir("serde_json-1.0.108"), Some(("serde_json", "1.0.108")));
assert_eq!(split_crate_dir("windows-sys-0.52.0"), Some(("windows-sys", "0.52.0")));
assert_eq!(split_crate_dir("tokio-1.35.0-alpha.1"), Some(("tokio", "1.35.0-alpha.1")));
assert_eq!(split_crate_dir("noversion"), None);
assert_eq!(split_crate_dir("x-1"), None);
}
#[test]
fn invalid_versioned_paths_are_not_dependency_evidence() {
let bytes = b"/vendor/notes-1.2.draft/src/lib.rs\0/vendor/foo-01.2.3/src/lib.rs\0/vendor/folder.name-1.2.3/src/lib.rs\0/vendor/helper-2-1.2.3-rc.1+build.7/src/lib.rs\0";
let report = inspect(bytes, None);
assert_eq!(report.crates.len(), 1);
assert_eq!(report.crates[0].name, "helper-2");
assert_eq!(report.crates[0].version.as_deref(), Some("1.2.3-rc.1+build.7"));
}
#[test]
fn finds_toolchain_crates_paths_and_panics() {
let commit = "82e1608dfa6e0b5569232559e3d385fea5a93112";
let mut d = b"\x00junk".to_vec();
d.extend(format!("/rustc/{commit}/library/core/src/str/mod.rs").as_bytes());
d.extend(b"rustc version 1.75.0 (82e1608df 2023-12-21)\x00");
d.extend(b"/home/u/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.108/src/de.rs\x00");
d.extend(b"C:\\Users\\u\\.cargo\\registry\\src\\index.crates.io-6f17d22bba15001f\\windows-sys-0.52.0\\src\\lib.rs\x00");
d.extend(b"/home/u/.cargo/git/checkouts/egui-1a2b3c4d5e6f7a8b/abc1234/src/lib.rs\x00");
d.extend(b"/rust/deps\\aho-corasick-1.1.4\\src\\dfa.rs\x00");
d.extend(b"\x00src/main.rs\x00called `Option::unwrap()` on a `None` value\x00.rsrc\x00");
let r = inspect(&d, None);
assert!(r.is_rust);
assert_eq!(r.strength, "strong");
assert_eq!(r.rustc_commits[0].value, commit);
assert_eq!(r.rustc_versions[0].value, "1.75.0 (82e1608df 2023-12-21)");
assert_eq!(r.std_layout, Some("library/ (rustc 1.47 or later)"));
let names: Vec<_> = r.crates.iter().map(|c| (c.name.as_str(), c.version.as_deref(), c.source)).collect();
assert!(names.contains(&("serde_json", Some("1.0.108"), "registry")), "{names:?}");
assert!(names.contains(&("windows-sys", Some("0.52.0"), "registry")), "{names:?}");
assert!(names.contains(&("egui", Some("abc1234"), "git")), "{names:?}");
assert!(names.contains(&("aho-corasick", Some("1.1.4"), "toolchain-vendored")), "{names:?}");
assert_eq!(r.source_paths.toolchain, 1);
assert_eq!(r.source_paths.dependencies, 4);
assert_eq!(r.source_paths.local[0].value, "src/main.rs");
assert_eq!(r.panic_messages.len(), 1);
assert!(r.symbols.stripped);
}
#[test]
fn symbols_and_auditable_section() {
let json = br#"{"format":1,"packages":[{"name":"app","version":"0.1.0","source":"local","dependencies":[1],"root":true},{"name":"serde","version":"1.0.193","source":"crates.io"}]}"#;
let packed = miniz_oxide::deflate::compress_to_vec_zlib(json, 6);
let mut data = vec![0u8; 16];
data.extend(&packed);
let mut bin = Binary::raw(crate::Arch::X86_64, 0x1000, packed.len() as u64, 16);
if let Some(s) = bin.sections.first_mut() {
s.name = ".dep-v0".into();
}
let sym = |name: &str| crate::Symbol {
name: name.into(),
demangled: None,
addr: 0,
size: 0,
kind: crate::SymbolKind::Function,
global: true,
weak: false,
defined: true,
dynamic: false,
};
bin.symbols = vec![
sym("_ZN4core9panicking5panic17h0123456789abcdefE"),
sym("_ZN5myapp6config4load17hfedcba9876543210E"),
sym("_RNvCs1234_5myapp4main"),
sym("_ZN3foo3barEv"),
sym("printf"),
];
let r = inspect(&data, Some(&bin));
assert_eq!(r.strength, "strong");
assert_eq!((r.symbols.rust_legacy, r.symbols.rust_v0, r.symbols.total), (2, 1, 5));
assert!(!r.symbols.stripped);
assert_eq!(r.symbols.crates[0].value, "myapp");
assert_eq!(r.symbols.samples[0].demangled, "myapp::config::load");
let a = r.auditable.expect("auditable data");
assert_eq!(a.packages.len(), 2);
assert_eq!(a.root_package_index, Some(0));
assert_eq!(a.dependency_edges, 1);
assert_eq!(a.format_version, Some(1));
assert_eq!(a.packages[1].name, "serde");
// Corrupt compressed data is reported, not fatal.
let r = inspect(&vec![0u8; 16 + packed.len()], Some(&bin));
assert!(r.auditable.is_none());
assert!(r.auditable_error.is_some());
}
#[test]
fn plain_bytes_are_not_rust() {
let r = inspect(b"hello world .rsrc /usr/lib/x.rs.bak \xff", None);
assert_eq!(r.strength, "none");
assert!(!r.is_rust);
assert_eq!(inspect(b"RUST_BACKTRACE", None).strength, "weak");
assert_eq!(inspect(b"", None).strength, "none");
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.