665bfac0b339
aep_corpus.rs
//! `cargo xtask aep-corpus [--dir DIR]`: fetch the After Effects test corpus of `crates/aep`.
//!
//! The corpus is a separate repository (binary `.aep` files stay out of this one). The pin in
//! `crates/aep/tests/fixtures/corpus.lock` names one commit and the SHA-256 of every file. The
//! command fetches exactly that commit into a temporary directory, checks every file against the
//! hashes of the lock file, and only then copies them to `DIR` (default `corpus/aep`, ignored by
//! git). Any mismatch, missing file or malformed line fails the command and leaves `DIR` alone.
//! Then run the tests with `EFFECTCRAFT_AEP_FIXTURES=<DIR>` (add `EFFECTCRAFT_AEP_FIXTURES_REQUIRED=1`
//! so that a missing project fails its test).
//!
//! `EFFECTCRAFT_AEP_CORPUS_REPO` replaces the repository URL (a local clone works); the commit and
//! the hashes still come from the lock file.
use std::path::{Component, Path, PathBuf};
use std::process::Command;
use sha2::{Digest, Sha256};
const LOCK: &str = "crates/aep/tests/fixtures/corpus.lock";
const DEFAULT_DIR: &str = "corpus/aep";
/// Where the fixtures sit inside the corpus repository; the command installs this folder's files.
const PREFIX: &str = "aep/";
#[derive(Debug, PartialEq, Eq)]
struct Lock {
repo: String,
commit: String,
/// (SHA-256 as 64 lower-case hex digits, path relative to the corpus root), in file order.
files: Vec<(String, String)>,
}
fn is_hex(s: &str, len: usize) -> bool {
s.len() == len && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
/// A path that stays inside its folder: relative, no `..`, no root or drive.
fn safe_relative(p: &str) -> bool {
!p.is_empty() && !p.contains('\\') && Path::new(p).components().all(|c| matches!(c, Component::Normal(_)))
}
fn parse_lock(text: &str) -> Result<Lock, String> {
let (mut repo, mut commit, mut files) = (None, None, Vec::new());
for (n, line) in text.lines().enumerate().map(|(i, l)| (i + 1, l.trim())) {
if line.is_empty() || line.starts_with('#') {
continue;
}
let bad = |why: &str| format!("{LOCK}:{n}: {why}");
if let Some(v) = line.strip_prefix("repo ") {
if repo.replace(v.trim().to_string()).is_some() {
return Err(bad("`repo` given twice"));
}
} else if let Some(v) = line.strip_prefix("commit ") {
let v = v.trim();
if !is_hex(v, 40) {
return Err(bad("`commit` must be 40 lower-case hex digits (no branch or tag)"));
}
if commit.replace(v.to_string()).is_some() {
return Err(bad("`commit` given twice"));
}
} else {
let (hash, path) = line.split_once(" ").ok_or_else(|| bad("expected `<sha256> <path>`"))?;
let path = path.trim();
if !is_hex(hash, 64) {
return Err(bad("hash must be 64 lower-case hex digits"));
}
if !safe_relative(path) || !path.starts_with(PREFIX) {
return Err(bad(&format!("path must be relative, without `..`, and under `{PREFIX}`")));
}
if files.iter().any(|(_, p)| p == path) {
return Err(bad("path listed twice"));
}
files.push((hash.to_string(), path.to_string()));
}
}
let repo = repo.filter(|r| !r.is_empty()).ok_or_else(|| format!("{LOCK}: no `repo` line"))?;
let commit = commit.ok_or_else(|| format!("{LOCK}: no `commit` line"))?;
if files.is_empty() {
return Err(format!("{LOCK}: no files listed"));
}
Ok(Lock { repo, commit, files })
}
fn sha256_hex(bytes: &[u8]) -> String {
Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect()
}
/// Reads every listed file from `root` and checks its hash. Returns the bytes in lock order, or
/// every problem found.
fn verify(root: &Path, files: &[(String, String)]) -> Result<Vec<Vec<u8>>, String> {
let (mut data, mut errors) = (Vec::new(), Vec::new());
for (hash, path) in files {
match std::fs::read(root.join(path)) {
Err(e) => errors.push(format!("{path}: {e}")),
Ok(bytes) => {
let got = sha256_hex(&bytes);
if &got == hash {
data.push(bytes);
} else {
errors.push(format!("{path}: SHA-256 is {got}, the lock file says {hash}"));
}
}
}
}
if errors.is_empty() { Ok(data) } else { Err(errors.join("\n")) }
}
/// Copies verified files below `dest` (the `aep/` prefix dropped), via a temporary name so a
/// failure never leaves a half-written file under its real name.
fn install(dest: &Path, files: &[(String, String)], data: &[Vec<u8>]) -> Result<(), String> {
for ((_, path), bytes) in files.iter().zip(data) {
let rel = path.strip_prefix(PREFIX).unwrap_or(path);
let target = dest.join(rel);
if let Some(dir) = target.parent() {
std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
}
let tmp = target.with_extension("part");
std::fs::write(&tmp, bytes).map_err(|e| format!("{}: {e}", tmp.display()))?;
std::fs::rename(&tmp, &target).map_err(|e| format!("{}: {e}", target.display()))?;
}
Ok(())
}
/// True when every listed file is already in `dest` with the right hash.
fn up_to_date(dest: &Path, files: &[(String, String)]) -> bool {
files.iter().all(|(hash, path)| {
let rel = path.strip_prefix(PREFIX).unwrap_or(path);
std::fs::read(dest.join(rel)).is_ok_and(|b| &sha256_hex(&b) == hash)
})
}
fn git(dir: &Path, args: &[&str]) -> Result<(), String> {
let out = Command::new("git")
.current_dir(dir)
.args(["-c", "core.autocrlf=false", "-c", "core.safecrlf=false"])
.args(args)
.output()
.map_err(|e| format!("git {}: {e}", args.first().copied().unwrap_or("")))?;
if out.status.success() { Ok(()) } else { Err(format!("git {}: {}", args.first().copied().unwrap_or(""), String::from_utf8_lossy(&out.stderr).trim())) }
}
/// Fetches exactly `commit` of `repo` into `into` (an empty directory).
fn fetch(repo: &str, commit: &str, into: &Path) -> Result<(), String> {
git(into, &["init", "-q"])?;
git(into, &["fetch", "-q", "--depth", "1", repo, commit])?;
git(into, &["checkout", "-q", "--detach", "FETCH_HEAD"])?;
let head = Command::new("git").current_dir(into).args(["rev-parse", "HEAD"]).output().map_err(|e| format!("git rev-parse: {e}"))?;
let head = String::from_utf8_lossy(&head.stdout).trim().to_string();
if head == commit { Ok(()) } else { Err(format!("fetched {head}, not the pinned {commit}")) }
}
pub fn run(root: &Path, args: &[String]) -> Result<(), String> {
let mut dir = PathBuf::from(DEFAULT_DIR);
let mut it = args.iter();
while let Some(a) = it.next() {
match a.as_str() {
"--dir" => dir = PathBuf::from(it.next().ok_or("--dir needs a folder")?),
other => return Err(format!("unknown argument `{other}`; usage: cargo xtask aep-corpus [--dir DIR]")),
}
}
let dest = if dir.is_absolute() { dir } else { root.join(dir) };
let text = std::fs::read_to_string(root.join(LOCK)).map_err(|e| format!("{LOCK}: {e}"))?;
let lock = parse_lock(&text)?;
if up_to_date(&dest, &lock.files) {
println!("aep-corpus: {} files in {} already match the pin {}", lock.files.len(), dest.display(), &lock.commit[..12]);
} else {
let repo = std::env::var("EFFECTCRAFT_AEP_CORPUS_REPO").ok().filter(|r| !r.is_empty()).unwrap_or_else(|| lock.repo.clone());
let tmp = tempfile::Builder::new().prefix("effectcraft-aep-corpus-").tempdir().map_err(|e| format!("temporary folder: {e}"))?;
fetch(&repo, &lock.commit, tmp.path())?;
let data = verify(tmp.path(), &lock.files)?;
install(&dest, &lock.files, &data)?;
println!("aep-corpus: {} files verified against {} and installed in {}", lock.files.len(), &lock.commit[..12], dest.display());
}
println!("run the tests with:\n EFFECTCRAFT_AEP_FIXTURES={} EFFECTCRAFT_AEP_FIXTURES_REQUIRED=1 cargo test -p effectcraft-aep", dest.display());
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
const H: &str = "4a51318c4b37fa95e2c02682183d3982fdfbe1870dc50ba729b776d1826f982e";
const C: &str = "72539568193239fadb3ccd0d7e70f6cd622a650a";
fn lock(body: &str) -> String {
format!("# c\nrepo https://example.org/c.git\ncommit {C}\n{body}")
}
#[test]
fn parses_a_lock_file() {
let l = parse_lock(&lock(&format!("{H} aep/a.aep\n"))).unwrap();
assert_eq!(l.repo, "https://example.org/c.git");
assert_eq!(l.commit, C);
assert_eq!(l.files, vec![(H.to_string(), "aep/a.aep".to_string())]);
}
#[test]
fn rejects_malformed_locks() {
for (body, why) in [
(format!("{H} ../x.aep\n"), "parent folder"),
(format!("{H} /abs/x.aep\n"), "absolute path"),
(format!("{H} aep/../../x.aep\n"), "escaping path"),
(format!("{H} other/x.aep\n"), "outside aep/"),
(format!("{H} aep/a.aep\n{H} aep/a.aep\n"), "duplicate"),
("abc aep/a.aep\n".to_string(), "short hash"),
(format!("{} aep/a.aep\n", H.to_uppercase()), "upper-case hash"),
(format!("{H} aep/a.aep\n"), "one space"),
(String::new(), "no files"),
] {
assert!(parse_lock(&lock(&body)).is_err(), "{why} must be rejected");
}
assert!(parse_lock(&format!("repo r\ncommit main\n{H} aep/a.aep\n")).is_err(), "a branch name is not a pin");
assert!(parse_lock(&format!("commit {C}\n{H} aep/a.aep\n")).is_err(), "no repo");
assert!(parse_lock(&format!("repo r\n{H} aep/a.aep\n")).is_err(), "no commit");
}
#[test]
fn the_checked_in_lock_file_is_valid() {
let text = std::fs::read_to_string(crate::root().join(LOCK)).unwrap();
let l = parse_lock(&text).unwrap();
assert_eq!(l.files.len(), 20);
assert!(l.files.windows(2).all(|w| w[0].1 < w[1].1), "files sorted by path");
}
#[test]
fn hashes_match_known_vectors() {
assert_eq!(sha256_hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
assert_eq!(sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
}
fn corpus(files: &[(&str, &[u8])]) -> (tempfile::TempDir, Vec<(String, String)>) {
let dir = tempfile::tempdir().unwrap();
let mut list = Vec::new();
for (path, bytes) in files {
let full = dir.path().join(path);
std::fs::create_dir_all(full.parent().unwrap()).unwrap();
std::fs::write(full, bytes).unwrap();
list.push((sha256_hex(bytes), (*path).to_string()));
}
(dir, list)
}
#[test]
fn verify_accepts_matching_files_and_install_copies_them() {
let (src, list) = corpus(&[("aep/a.aep", b"one"), ("aep/inputs/p.ffx", b"two")]);
let data = verify(src.path(), &list).unwrap();
let dest = tempfile::tempdir().unwrap();
install(dest.path(), &list, &data).unwrap();
assert_eq!(std::fs::read(dest.path().join("a.aep")).unwrap(), b"one");
assert_eq!(std::fs::read(dest.path().join("inputs/p.ffx")).unwrap(), b"two");
assert!(up_to_date(dest.path(), &list));
}
#[test]
fn verify_reports_a_changed_and_a_missing_file_and_installs_nothing() {
let (src, mut list) = corpus(&[("aep/a.aep", b"one"), ("aep/b.aep", b"two")]);
std::fs::write(src.path().join("aep/a.aep"), b"changed").unwrap();
list.push((H.to_string(), "aep/gone.aep".to_string()));
let err = verify(src.path(), &list).unwrap_err();
assert!(err.contains("aep/a.aep: SHA-256"), "{err}");
assert!(err.contains("aep/gone.aep"), "{err}");
assert!(!err.contains("aep/b.aep"), "{err}");
}
#[test]
fn up_to_date_is_false_for_missing_or_changed_files() {
let (_src, list) = corpus(&[("aep/a.aep", b"one")]);
let dest = tempfile::tempdir().unwrap();
assert!(!up_to_date(dest.path(), &list), "missing");
std::fs::write(dest.path().join("a.aep"), b"other").unwrap();
assert!(!up_to_date(dest.path(), &list), "changed");
}
#[test]
fn fetch_checks_out_exactly_the_pinned_commit() {
let repo = tempfile::tempdir().unwrap();
git(repo.path(), &["init", "-q"]).unwrap();
std::fs::create_dir_all(repo.path().join("aep")).unwrap();
std::fs::write(repo.path().join("aep/a.aep"), b"v1").unwrap();
git(repo.path(), &["add", "."]).unwrap();
git(repo.path(), &["-c", "user.name=t", "-c", "user.email=t@example.org", "commit", "-q", "-m", "one"]).unwrap();
let rev =
|d: &Path| String::from_utf8(Command::new("git").current_dir(d).args(["rev-parse", "HEAD"]).output().unwrap().stdout).unwrap().trim().to_string();
let first = rev(repo.path());
std::fs::write(repo.path().join("aep/a.aep"), b"v2").unwrap();
git(repo.path(), &["-c", "user.name=t", "-c", "user.email=t@example.org", "commit", "-q", "-am", "two"]).unwrap();
let into = tempfile::tempdir().unwrap();
fetch(repo.path().to_str().unwrap(), &first, into.path()).unwrap();
assert_eq!(std::fs::read(into.path().join("aep/a.aep")).unwrap(), b"v1", "the pinned commit, not the newer one");
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.