GitCorner

Code

ossartcraftfilmcraft๐ŸŒ
f67d38b35cc5
//! EXIF Orientation on stills (#1583): a phone photo stored sideways with Orientation 6 must
//! decode upright, its size swapped. The files are built here: EXIF from the EXIF 2.32 / TIFF 6.0
//! IFD layout (tag 0x0112, SHORT, count 1), carried in a JPEG APP1, a PNG eXIf chunk, a WebP EXIF
//! chunk, or as tag 274 of a TIFF.
//!
//! The picture is 3 ร— 2 blocks of distinct colours, so each of the eight orientations puts them
//! somewhere different. Stored block (x, y) of a W ร— H picture ends up, for Orientation
//! 1: (x, y) 2: (W-1-x, y) 3: (W-1-x, H-1-y) 4: (x, H-1-y) 5: (y, x) 6: (H-1-y, x)
//! 7: (H-1-y, W-1-x) 8: (y, W-1-x), the picture H ร— W for 5โ€“8.

use std::sync::Arc;

use filmcraft_media::sequence::{FrameLoader, ImageSequenceSource};
use filmcraft_media::still::{Raster, decode_raster};
use filmcraft_media::{FrameRequest, MediaSource, open_bytes};
use filmcraft_time::{FrameRate, Tick};
use image::ImageEncoder;

const W: u32 = 3;
const H: u32 = 2;
const COLOURS: [[u8; 3]; 6] = [[230, 20, 20], [20, 200, 20], [20, 20, 230], [230, 230, 20], [230, 20, 230], [20, 230, 230]];

/// The stored picture: `W` ร— `H` blocks of `block` ร— `block` pixels, RGB8.
fn picture(block: u32) -> Vec<u8> {
    let (w, h) = (W * block, H * block);
    (0..w * h).flat_map(|i| COLOURS[((i / w / block) * W + (i % w) / block) as usize]).collect()
}

/// Where stored block (x, y) lands for EXIF Orientation `o`.
fn oriented(o: u16, x: u32, y: u32) -> (u32, u32) {
    match o {
        2 => (W - 1 - x, y),
        3 => (W - 1 - x, H - 1 - y),
        4 => (x, H - 1 - y),
        5 => (y, x),
        6 => (H - 1 - y, x),
        7 => (H - 1 - y, W - 1 - x),
        8 => (y, W - 1 - x),
        _ => (x, y),
    }
}

/// `r` is the stored picture under Orientation `o` (1 for "unchanged"), each block's centre pixel
/// within `tol` of its colour.
fn assert_oriented(r: &Raster, o: u16, block: u32, tol: u8, what: &str) {
    let (w, h) = if (5..=8).contains(&o) { (H, W) } else { (W, H) };
    assert_eq!((r.width, r.height, r.rgba.len()), (w * block, h * block, (w * h * block * block * 4) as usize), "{what}: size");
    for y in 0..H {
        for x in 0..W {
            let (ox, oy) = oriented(o, x, y);
            let (px, py) = (ox * block + block / 2, oy * block + block / 2);
            let i = ((py * r.width + px) * 4) as usize;
            let got = &r.rgba[i..i + 4];
            let want = COLOURS[(y * W + x) as usize];
            let close = got[..3].iter().zip(want).all(|(a, b)| a.abs_diff(b) <= tol) && got[3] == 255;
            assert!(close, "{what}: stored block ({x}, {y}) should be at ({ox}, {oy}), found {got:?} there, want {want:?}");
        }
    }
}

/// A little-endian EXIF block (a TIFF header and one IFD) with `entries` (tag, type, count,
/// value field).
fn exif(entries: &[(u16, u16, u32, [u8; 4])]) -> Vec<u8> {
    let mut out = b"II*\0".to_vec();
    out.extend(8u32.to_le_bytes());
    out.extend((entries.len() as u16).to_le_bytes());
    for (tag, ty, count, value) in entries {
        out.extend(tag.to_le_bytes());
        out.extend(ty.to_le_bytes());
        out.extend(count.to_le_bytes());
        out.extend(value);
    }
    out.extend(0u32.to_le_bytes());
    out
}

/// EXIF with only an Orientation tag of value `o`.
fn orientation_exif(o: u16) -> Vec<u8> {
    let [a, b] = o.to_le_bytes();
    exif(&[(0x0112, 3, 1, [a, b, 0, 0])])
}

fn jpeg(exif: Option<Vec<u8>>) -> Vec<u8> {
    // quality 100 (all quantisers 1) and 4:4:4, flat 8 ร— 8 blocks: within a few levels
    let mut out = Vec::new();
    let mut enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, 100);
    if let Some(e) = exif {
        enc.set_exif_metadata(e).unwrap();
    }
    enc.write_image(&picture(8), W * 8, H * 8, image::ExtendedColorType::Rgb8).unwrap();
    out
}

fn png(o: u16) -> Vec<u8> {
    let mut out = Vec::new();
    let mut enc = image::codecs::png::PngEncoder::new(&mut out);
    enc.set_exif_metadata(orientation_exif(o)).unwrap();
    enc.write_image(&picture(1), W, H, image::ExtendedColorType::Rgb8).unwrap();
    out
}

fn webp(o: u16) -> Vec<u8> {
    let mut out = Vec::new();
    let mut enc = image::codecs::webp::WebPEncoder::new_lossless(&mut out);
    enc.set_exif_metadata(orientation_exif(o)).unwrap();
    enc.write_image(&picture(1), W, H, image::ExtendedColorType::Rgb8).unwrap();
    out
}

/// An uncompressed, single-strip, little-endian TIFF of `width` ร— `height`, 8 bits ร— 3 samples,
/// PhotometricInterpretation `photometric` (2 RGB, 6 YCbCr), plus `extra` entries (tag, SHORT
/// values).
fn tiff(width: u32, height: u32, photometric: u16, data: &[u8], extra: &[(u16, Vec<u16>)]) -> Vec<u8> {
    let mut entries: Vec<(u16, u16, Vec<u32>)> = vec![
        (256, 4, vec![width]),
        (257, 4, vec![height]),
        (258, 3, vec![8, 8, 8]),
        (259, 3, vec![1]),
        (262, 3, vec![u32::from(photometric)]),
        (273, 4, vec![0]), // patched below
        (277, 3, vec![3]),
        (278, 4, vec![height]),
        (279, 4, vec![data.len() as u32]),
    ];
    entries.extend(extra.iter().map(|(tag, v)| (*tag, 3, v.iter().map(|&v| u32::from(v)).collect())));
    entries.sort_by_key(|e| e.0);
    let ifd_at = 8u32;
    let data_at = ifd_at + 2 + 12 * entries.len() as u32 + 4;
    let mut values_at = data_at + data.len() as u32;
    let mut ifd: Vec<u8> = (entries.len() as u16).to_le_bytes().to_vec();
    let mut extra_data: Vec<u8> = Vec::new();
    for (tag, ty, vals) in &entries {
        let vals = if *tag == 273 { vec![data_at] } else { vals.clone() };
        let mut bytes: Vec<u8> = Vec::new();
        for v in &vals {
            if *ty == 3 { bytes.extend((*v as u16).to_le_bytes()) } else { bytes.extend(v.to_le_bytes()) }
        }
        ifd.extend(tag.to_le_bytes());
        ifd.extend(ty.to_le_bytes());
        ifd.extend((vals.len() as u32).to_le_bytes());
        if bytes.len() <= 4 {
            bytes.resize(4, 0);
            ifd.extend(&bytes);
        } else {
            ifd.extend(values_at.to_le_bytes());
            values_at += bytes.len() as u32;
            extra_data.extend(bytes);
        }
    }
    ifd.extend(0u32.to_le_bytes());
    let mut out = b"II*\0".to_vec();
    out.extend(ifd_at.to_le_bytes());
    out.extend(ifd);
    out.extend(data);
    out.extend(extra_data);
    out
}

fn rgb_tiff(o: u16) -> Vec<u8> {
    tiff(W, H, 2, &picture(1), &[(274, vec![o])])
}

#[test]
fn jpeg_exif_orientation_is_applied() {
    for o in 1..=8 {
        let r = decode_raster("photo.jpg", &jpeg(Some(orientation_exif(o)))).unwrap();
        assert_oriented(&r, o, 8, 8, &format!("JPEG orientation {o}"));
        assert!(!r.has_alpha);
    }
}

#[test]
fn big_endian_exif_is_read_too() {
    let mut e = b"MM\0*".to_vec();
    e.extend(8u32.to_be_bytes());
    e.extend(1u16.to_be_bytes());
    e.extend(0x0112u16.to_be_bytes());
    e.extend(3u16.to_be_bytes());
    e.extend(1u32.to_be_bytes());
    e.extend([0, 6, 0, 0]);
    e.extend(0u32.to_be_bytes());
    let r = decode_raster("photo.jpg", &jpeg(Some(e))).unwrap();
    assert_oriented(&r, 6, 8, 8, "big-endian JPEG orientation 6");
}

#[test]
fn png_exif_chunk_orientation_is_applied() {
    for o in 1..=8 {
        assert_oriented(&decode_raster("shot.png", &png(o)).unwrap(), o, 1, 0, &format!("PNG orientation {o}"));
    }
}

#[test]
fn webp_exif_chunk_orientation_is_applied() {
    for o in 1..=8 {
        assert_oriented(&decode_raster("shot.webp", &webp(o)).unwrap(), o, 1, 0, &format!("WebP orientation {o}"));
    }
}

#[test]
fn rgb_tiff_orientation_tag_is_applied() {
    for o in 1..=8 {
        assert_oriented(&decode_raster("scan.tiff", &rgb_tiff(o)).unwrap(), o, 1, 0, &format!("TIFF orientation {o}"));
    }
}

#[test]
fn opened_still_reports_the_upright_size() {
    let s = open_bytes("portrait.jpg", Arc::from(jpeg(Some(orientation_exif(6)))), &[]).unwrap();
    let v = s.info().video.as_ref().unwrap();
    assert_eq!((v.width, v.height), (H * 8, W * 8));
    let f = s.video_frame(FrameRequest::full(Tick::ZERO)).unwrap();
    assert_eq!((f.width, f.height), (v.width, v.height));
    let filmcraft_frame::PixelData::Rgba8(p) = &f.data else { panic!("not RGBA8") };
    assert_oriented(&Raster { width: f.width, height: f.height, rgba: p.to_vec(), has_alpha: false }, 6, 8, 8, "opened JPEG");
}

#[test]
fn image_sequence_frames_are_upright() {
    let files: std::collections::HashMap<String, Vec<u8>> = [("s1.png".to_string(), png(8)), ("s2.png".to_string(), png(8))].into_iter().collect();
    let loader: FrameLoader = Arc::new(move |p: &str| files.get(p).cloned().ok_or_else(|| std::io::Error::other("missing")));
    let s = ImageSequenceSource::new("s1.png", vec![Some("s1.png".into()), Some("s2.png".into())], FrameRate::FPS_24, loader).unwrap();
    let v = s.info().video.as_ref().unwrap();
    assert_eq!((v.width, v.height), (H, W));
    for i in 0..2 {
        let f = s.video_frame(FrameRequest::full(FrameRate::FPS_24.tick_of(i))).unwrap();
        let filmcraft_frame::PixelData::Rgba8(p) = &f.data else { panic!("not RGBA8") };
        assert_oriented(&Raster { width: f.width, height: f.height, rgba: p.to_vec(), has_alpha: false }, 8, 1, 0, &format!("sequence frame {i}"));
    }
}

/// Frames of one sequence may carry different orientations: each decodes upright at its own size,
/// and the sequence keeps the first frame's size (frames of another size are scaled to fit, as for
/// any mixed-size sequence).
#[test]
fn sequence_frames_with_different_orientations_each_decode_upright() {
    let files: std::collections::HashMap<String, Vec<u8>> = [("m1.png".to_string(), png(1)), ("m2.png".to_string(), png(6))].into_iter().collect();
    let loader: FrameLoader = Arc::new(move |p: &str| files.get(p).cloned().ok_or_else(|| std::io::Error::other("missing")));
    let s = ImageSequenceSource::new("m1.png", vec![Some("m1.png".into()), Some("m2.png".into())], FrameRate::FPS_24, loader).unwrap();
    let v = s.info().video.as_ref().unwrap();
    assert_eq!((v.width, v.height), (W, H));
    for (i, o) in [(0, 1), (1, 6)] {
        let f = s.video_frame(FrameRequest::full(FrameRate::FPS_24.tick_of(i))).unwrap();
        let filmcraft_frame::PixelData::Rgba8(p) = &f.data else { panic!("not RGBA8") };
        assert_oriented(&Raster { width: f.width, height: f.height, rgba: p.to_vec(), has_alpha: false }, o, 1, 0, &format!("sequence frame {i}"));
    }
}

/// No EXIF, Orientation 1, out-of-range values and damaged EXIF or Orientation entries leave the
/// picture as stored, and never fail the decode.
#[test]
fn missing_invalid_or_damaged_orientation_leaves_the_picture_alone() {
    let six = orientation_exif(6);
    let cases: Vec<(&str, Option<Vec<u8>>)> = vec![
        ("no EXIF", None),
        ("orientation 1", Some(orientation_exif(1))),
        ("orientation 0", Some(orientation_exif(0))),
        ("orientation 9", Some(orientation_exif(9))),
        ("orientation 255", Some(orientation_exif(255))),
        ("orientation 65535", Some(orientation_exif(65_535))),
        ("orientation as a LONG", Some(exif(&[(0x0112, 4, 1, [6, 0, 0, 0])]))),
        ("two orientation values", Some(exif(&[(0x0112, 3, 2, [6, 0, 6, 0])]))),
        ("EXIF cut inside the entry", Some(six[..14].to_vec())),
        ("EXIF cut before the value", Some(six[..18].to_vec())),
        ("IFD offset past the end", Some([&b"II*\0"[..], &u32::MAX.to_le_bytes()].concat())),
        ("not a TIFF header", Some(b"garbage".to_vec())),
    ];
    for (what, e) in cases {
        let r = decode_raster("photo.jpg", &jpeg(e)).unwrap_or_else(|err| panic!("{what}: {err}"));
        assert_oriented(&r, 1, 8, 8, what);
    }
    for (what, o) in [("PNG orientation 0", 0), ("PNG orientation 9", 9)] {
        assert_oriented(&decode_raster("shot.png", &png(o)).unwrap(), 1, 1, 0, what);
    }
    for (what, o) in [("TIFF orientation 0", 0), ("TIFF orientation 9", 9), ("TIFF orientation 255", 255)] {
        assert_oriented(&decode_raster("scan.tiff", &rgb_tiff(o)).unwrap(), 1, 1, 0, what);
    }
    for (what, o) in [("YCbCr TIFF orientation 0", 0), ("YCbCr TIFF orientation 9", 9), ("YCbCr TIFF orientation 65535", 65_535)] {
        assert_oriented(&decode_raster("frame.tiff", &ycbcr_tiff(o)).unwrap(), 1, 1, 3, what);
    }
    // TIFF Orientation is one SHORT: an entry with more values is damaged and ignored
    assert_oriented(&decode_raster("frame.tiff", &ycbcr_tiff_with(vec![6, 6])).unwrap(), 1, 1, 3, "YCbCr TIFF two orientation values");
}

/// The stored picture as an uncompressed 4:4:4 YCbCr TIFF (ITU-R BT.601 coefficients and full
/// range, the TIFF 6.0 defaults) with Orientation `o`: our own decoder reads these, not the image
/// crate.
fn ycbcr_tiff(o: u16) -> Vec<u8> {
    ycbcr_tiff_with(vec![o])
}

/// [`ycbcr_tiff`] with an Orientation entry holding all of `orientation` (a valid one holds one).
fn ycbcr_tiff_with(orientation: Vec<u16>) -> Vec<u8> {
    let data: Vec<u8> = picture(1)
        .as_chunks::<3>()
        .0
        .iter()
        .flat_map(|p| {
            let [r, g, b] = p.map(f64::from);
            let y = 0.299 * r + 0.587 * g + 0.114 * b;
            [y, 128.0 + (b - y) / 1.772, 128.0 + (r - y) / 1.402].map(|v| v.round().clamp(0.0, 255.0) as u8)
        })
        .collect();
    tiff(W, H, 6, &data, &[(274, orientation), (530, vec![1, 1])])
}

/// An uncompressed YCbCr TIFF's Orientation 1โ€“8 turns the picture upright.
#[test]
fn ycbcr_tiff_orientation_tag_is_applied() {
    for o in 1..=8 {
        assert_oriented(&decode_raster("frame.tiff", &ycbcr_tiff(o)).unwrap(), o, 1, 3, &format!("YCbCr TIFF orientation {o}"));
    }
}

/// Truncated and bit-flipped oriented files of every format decode or fail with an error, never a panic.
#[test]
fn damaged_oriented_files_fail_without_panicking() {
    let good_jpeg = jpeg(Some(orientation_exif(6)));
    let mut cases: Vec<Vec<u8>> = Vec::new();
    for good in [&good_jpeg, &png(6), &webp(6), &rgb_tiff(6), &ycbcr_tiff(6)] {
        cases.extend((0..good.len()).map(|n| good[..n].to_vec()));
        for i in 0..good.len() {
            for bit in [0, 3, 7] {
                let mut b = good.clone();
                b[i] ^= 1 << bit;
                cases.push(b);
            }
        }
    }
    // EXIF with a huge entry count and an IFD offset pointing at itself
    cases.push(jpeg(Some([&b"II*\0"[..], &8u32.to_le_bytes(), &u16::MAX.to_le_bytes()].concat())));
    cases.push(jpeg(Some([&b"II*\0"[..], &0u32.to_le_bytes()].concat())));
    for (i, b) in cases.into_iter().enumerate() {
        assert!(std::panic::catch_unwind(|| decode_raster("damaged.jpg", &b).map(|r| r.rgba.len())).is_ok(), "case {i} panicked");
    }

    // a 65535 ร— 65535 SOF behind the orientation: an error, not a 12 GB allocation
    let mut huge = good_jpeg.clone();
    let sof = huge.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap();
    huge[sof + 5..sof + 9].copy_from_slice(&[0xFF; 4]);
    assert!(decode_raster("huge.jpg", &huge).is_err());
}