5a24289898fd
pano.rs
//! Panorama stitching on synthetic input: a procedural scene is used as a spherical environment
//! (longitude/latitude map) and photographed by pinhole cameras with known rotations and focal
//! length. The stitch must recover the geometry and reproduce the environment without seams.
use lightcraft_merge::frame::Frame;
use lightcraft_merge::linalg::{self, M3};
use lightcraft_merge::no_progress;
use lightcraft_merge::pano::{PanoOptions, Projection, stitch};
use lightcraft_raster::Rgb32f;
const THETA: f64 = 200.0; // degrees of longitude covered by the environment map
const PHI: f64 = 70.0;
fn env() -> Rgb32f {
let scene = lightcraft_scenes::demo_library().into_iter().find(|s| s.kind == lightcraft_scenes::Kind::Canyon).unwrap();
let img = scene.render(2400, 840);
let mut l: Vec<f32> = img.data.iter().map(|p| p[1]).collect();
l.sort_by(|a, b| a.total_cmp(b));
let s = 0.25 / l[l.len() / 2].max(1e-6);
img.map(|p| p.map(|v| (v * s).min(0.98)))
}
fn sample_env(e: &Rgb32f, d: [f64; 3]) -> Option<[f32; 3]> {
let n = linalg::norm(d);
let t = d[0].atan2(d[2]).to_degrees();
let p = (d[1] / n).asin().to_degrees();
if t.abs() > THETA / 2.0 || p.abs() > PHI / 2.0 {
return None;
}
let x = (t / THETA + 0.5) * e.width as f64;
let y = (p / PHI + 0.5) * e.height as f64;
Some(e.sample_bilinear(x as f32, y as f32))
}
fn rot(yaw: f64, pitch: f64, roll: f64) -> M3 {
// camera-to-world = Ry(yaw) · Rx(pitch) · Rz(roll); world→camera is its transpose
let ry = linalg::rodrigues([0.0, yaw.to_radians(), 0.0]);
let rx = linalg::rodrigues([pitch.to_radians(), 0.0, 0.0]);
let rz = linalg::rodrigues([0.0, 0.0, roll.to_radians()]);
linalg::transpose3(&linalg::mul3(&linalg::mul3(&ry, &rx), &rz))
}
fn shoot(e: &Rgb32f, r: &M3, f: f64, w: usize, h: usize, gain: f32) -> Frame {
let rt = linalg::transpose3(r);
let img = Rgb32f::from_fn(w, h, |x, y| {
let d = [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0];
sample_env(e, linalg::apply3(&rt, d)).map(|p| p.map(|v| v * gain)).unwrap_or([0.0; 3])
});
Frame::from_linear_rec2020(img, None)
}
fn angle_between(a: &M3, b: &M3) -> f64 {
let d = linalg::mul3(a, &linalg::transpose3(b));
linalg::norm(linalg::rotation_vector(&d)).to_degrees()
}
struct Shot {
yaw: f64,
pitch: f64,
roll: f64,
gain: f32,
}
fn run(shots: &[Shot], f: f64, opts: &PanoOptions) -> (lightcraft_merge::pano::PanoResult, Vec<M3>, Rgb32f) {
let e = env();
let rs: Vec<M3> = shots.iter().map(|s| rot(s.yaw, s.pitch, s.roll)).collect();
let frames: Vec<Frame> = shots.iter().zip(&rs).map(|(s, r)| shoot(&e, r, f, 800, 600, s.gain)).collect();
let t0 = std::time::Instant::now();
let res = stitch(frames, opts, &no_progress).unwrap();
eprintln!(
"stitch {} frames → {}×{} ({:?}) in {:?}; BA rms {:.3} → {:.3} px; fov {:.0}°×{:.0}°",
shots.len(),
res.image.width,
res.image.height,
res.projection,
t0.elapsed(),
res.rms_before,
res.rms,
res.fov.0,
res.fov.1
);
(res, rs, e)
}
/// Check the recovered geometry and compare the panorama with the environment rendered through the
/// recovered projection; returns (median, p99) of the log error over covered pixels.
fn check(res: &lightcraft_merge::pano::PanoResult, rs: &[M3], e: &Rgb32f, f: f64) -> (f32, f32, f32) {
let cams: Vec<_> = res.cameras.iter().map(|c| c.expect("all frames used")).collect();
for c in &cams {
assert!((c.f - f).abs() / f < 0.01, "focal {} vs {f}", c.f);
}
for i in 0..cams.len() {
for j in i + 1..cams.len() {
let truth = linalg::mul3(&rs[i], &linalg::transpose3(&rs[j]));
let got = linalg::mul3(&cams[i].r, &linalg::transpose3(&cams[j].r));
let err = angle_between(&truth, &got);
assert!(err < 0.1, "relative rotation {i}-{j} off by {err:.3}°");
}
}
// world alignment: recovered world → environment world
let mut acc = [[0.0; 3]; 3];
for (c, r) in cams.iter().zip(rs) {
let m = linalg::mul3(&linalg::transpose3(r), &c.r);
for a in 0..3 {
for b in 0..3 {
acc[a][b] += m[a][b];
}
}
}
let wmap = linalg::orthonormalize(&acc);
let (w, h) = (res.image.width, res.image.height);
let scale = res.focal;
let (bx, by) = res.origin;
// the environment rendered through the recovered geometry (alpha 0 where it has no data)
let mut truth = Rgb32f::new(w, h);
let mut talpha = lightcraft_raster::Plane::new(w, h);
for y in 0..h {
for x in 0..w {
let d = res.projection.inverse(x as f64 + 0.5 + bx, y as f64 + 0.5 + by, scale);
if let Some(t) = sample_env(e, linalg::apply3(&wmap, d))
&& res.alpha.get(x, y) >= 1.0
{
truth.set(x, y, t);
talpha.set(x, y, 1.0);
}
}
}
// global level (exposure compensation fixes relative, not absolute, exposure)
let mut lr: Vec<f32> =
(0..w * h).filter(|&k| talpha.data[k] > 0.0).map(|k| (res.image.data[k][1] + 0.01).ln() - (truth.data[k][1] + 0.01).ln()).collect();
lr.sort_by(|a, b| a.total_cmp(b));
let offset = lr[lr.len() / 2];
let mut errs = Vec::new();
let mut emap = Rgb32f::new(w, h);
for k in 0..w * h {
if talpha.data[k] == 0.0 {
continue;
}
let (g, t) = (res.image.data[k], truth.data[k]);
let er = (0..3).map(|c| ((g[c] + 0.01).ln() - offset - (t[c] + 0.01).ln()).abs()).fold(0.0, f32::max);
errs.push(er);
emap.data[k] = [er * 5.0, 1.0 - er * 5.0, 0.0];
}
if let Ok(dir) = std::env::var("LC_MERGE_DUMP") {
let tag = format!("{:?}-{}", res.projection, w);
ppm(&format!("{dir}/{tag}.ppm"), &res.image);
ppm(&format!("{dir}/{tag}-err.ppm"), &emap);
}
// seams: the column-to-column change of mean log luminance must follow the truth's
let steps = column_steps(&res.image, &talpha);
let tsteps = column_steps(&truth, &talpha);
// summed over 5 columns: resampling legitimately spreads a sharp scene edge over neighbours
let win = |v: &[Option<f32>], x: usize| -> Option<f32> { (x..x + 5).map(|k| v.get(k).copied().flatten()).sum() };
let seam = (0..steps.len().saturating_sub(5)).filter_map(|x| Some((win(&steps, x)? - win(&tsteps, x)?).abs())).fold(0.0, f32::max);
errs.sort_by(|a, b| a.total_cmp(b));
(errs[errs.len() / 2], errs[errs.len() * 99 / 100], seam)
}
fn ppm(path: &str, img: &Rgb32f) {
let mut b = format!("P6 {} {} 255\n", img.width, img.height).into_bytes();
for p in &img.data {
for c in 0..3 {
b.push((p[c].clamp(0.0, 1.0).powf(1.0 / 2.2) * 255.0) as u8);
}
}
std::fs::write(path, b).unwrap();
}
/// Per column: mean change of log luminance to the next column (where both are covered).
fn column_steps(img: &Rgb32f, alpha: &lightcraft_raster::Plane) -> Vec<Option<f32>> {
let (w, h) = (img.width, img.height);
(0..w - 1)
.map(|x| {
let mut s = 0.0;
let mut n = 0;
for y in 0..h {
if alpha.get(x, y) >= 1.0 && alpha.get(x + 1, y) >= 1.0 {
s += (img.get(x + 1, y)[1] + 0.01).ln() - (img.get(x, y)[1] + 0.01).ln();
n += 1;
}
}
(n > h / 3).then(|| s / n as f32)
})
.collect()
}
#[test]
fn spherical_panorama_recovers_geometry_without_seams() {
let shots = [
Shot { yaw: -60.0, pitch: 1.5, roll: 0.8, gain: 1.0 },
Shot { yaw: -30.0, pitch: -1.0, roll: -0.5, gain: 1.0 },
Shot { yaw: 0.0, pitch: 0.5, roll: 1.0, gain: 1.0 },
Shot { yaw: 31.0, pitch: -1.5, roll: 0.0, gain: 1.3 }, // auto exposure drifted
Shot { yaw: 61.0, pitch: 1.0, roll: -1.0, gain: 1.0 },
];
let f = 700.0;
let opts = PanoOptions { projection: Projection::Spherical, max_megapixels: 100.0, ..Default::default() };
let (res, rs, e) = run(&shots, f, &opts);
assert_eq!(res.used.len(), 5);
assert!(res.rms < 0.5, "BA rms {}", res.rms);
assert!((res.fov.0 - 182.0).abs() < 8.0, "fov {:?}", res.fov);
// exposure compensation undid the 1.3× drift (relative to its neighbours)
let g = &res.gains;
let rel = g[3] / ((g[2] + g[4]) / 2.0);
assert!((rel - 1.0 / 1.3).abs() < 0.03, "gains {g:?}");
let (med, p99, seam) = check(&res, &rs, &e, f);
eprintln!("log error vs environment: median {med:.4}, p99 {p99:.4}, seam step {seam:.4}");
assert!(med < 0.02 && p99 < 0.15, "median {med}, p99 {p99}");
// (an unblended 1.3× exposure seam would be a step of ln 1.3 ≈ 0.26)
assert!(seam < 0.05, "a column step of {seam} beyond the scene's suggests a seam");
}
#[test]
fn auto_projection_and_finishing() {
let shots = [
Shot { yaw: -22.0, pitch: 0.5, roll: 0.3, gain: 1.0 },
Shot { yaw: 0.0, pitch: -0.4, roll: 0.0, gain: 1.0 },
Shot { yaw: 21.0, pitch: 0.8, roll: -0.4, gain: 1.0 },
];
let f = 800.0;
let (res, rs, e) = run(&shots, f, &PanoOptions { projection: Projection::Auto, auto_crop: true, max_megapixels: 100.0, ..Default::default() });
assert_eq!(res.projection, Projection::Perspective, "a ~90° set is shown rectilinear");
let (med, p99, seam) = check(&res, &rs, &e, f);
eprintln!("perspective: median {med:.4}, p99 {p99:.4}, seam step {seam:.4}");
assert!(med < 0.02 && p99 < 0.1 && seam < 0.05);
let c = res.crop.expect("auto crop");
assert!(c[2] * c[3] > 0.6, "crop {c:?}");
let (w, h) = (res.image.width as f64, res.image.height as f64);
for y in (c[1] * h) as usize..((c[1] + c[3]) * h) as usize {
for x in (c[0] * w) as usize..((c[0] + c[2]) * w) as usize {
assert!(res.alpha.get(x, y) >= 0.999, "crop must be inside the image data");
}
}
// boundary warp fills the canvas; cylindrical also works
let (warped, _, _) =
run(&shots, f, &PanoOptions { projection: Projection::Cylindrical, boundary_warp: 100.0, fill_edges: true, ..Default::default() });
let cover = warped.alpha.data.iter().filter(|a| **a >= 0.5).count() as f64 / warped.alpha.data.len() as f64;
assert!(cover > 0.97, "boundary warp coverage {cover}");
assert!(warped.image.data.iter().all(|p| p.iter().all(|v| v.is_finite())));
}
#[test]
fn unrelated_photos_do_not_stitch() {
let a = lightcraft_scenes::demo_library();
let frames: Vec<Frame> = a.iter().take(2).map(|s| Frame::from_linear_rec2020(s.render(400, 300), None)).collect();
assert!(stitch(frames, &PanoOptions::default(), &no_progress).is_err());
}
/// Pair matching compares every pair of frames before anything is known about them: an unbounded
/// photo count is quadratic CPU and memory (issue #1519). More than `MAX_FRAMES` is refused at once,
/// before any frame is worked on.
#[test]
fn a_stitch_of_more_than_the_frame_cap_is_refused_up_front() {
let frames: Vec<Frame> = (0..lightcraft_merge::pano::MAX_FRAMES + 1)
.map(|i| Frame::from_linear_rec2020(Rgb32f::from_fn(8, 8, |_, _| [(i % 255) as f32 / 255.0, 0.0, 0.0]), None))
.collect();
let err = stitch(frames, &PanoOptions::default(), &no_progress).unwrap_err();
assert!(err.to_string().contains("refused"), "the frame cap errors, it does not begin the quadratic match: {err}");
}
/// Issue #1455: a panorama came out at 40 MP whatever its photos' resolution. It now keeps their
/// own resolution unless a size is asked for (up to Lightroom's 512 MP); a size scales it down,
/// and a NaN one (from an agent) is no size at all, not a 0.1 MP thumbnail.
#[test]
fn panoramas_keep_the_photos_resolution_unless_asked() {
assert_eq!(PanoOptions::default().max_megapixels, lightcraft_merge::pano::MAX_MEGAPIXELS);
let shots = [
Shot { yaw: -22.0, pitch: 0.5, roll: 0.3, gain: 1.0 },
Shot { yaw: 0.0, pitch: -0.4, roll: 0.0, gain: 1.0 },
Shot { yaw: 21.0, pitch: 0.8, roll: -0.4, gain: 1.0 },
];
let opts = |max_megapixels: f64| PanoOptions { projection: Projection::Perspective, max_megapixels, ..Default::default() };
let mp = |r: &lightcraft_merge::pano::PanoResult| (r.image.width * r.image.height) as f64 / 1e6;
let full = run(&shots, 800.0, &PanoOptions { projection: Projection::Perspective, ..Default::default() }).0;
assert!((mp(&full) / full.native_megapixels - 1.0).abs() < 0.02, "{} MP for {} MP", mp(&full), full.native_megapixels);
let quarter = run(&shots, 800.0, &opts(full.native_megapixels / 4.0)).0;
assert!((quarter.native_megapixels / full.native_megapixels - 1.0).abs() < 1e-6);
let k = mp(&quarter) / (full.native_megapixels / 4.0);
assert!(k > 0.9 && k <= 1.02, "a quarter of the size asked: {} MP", mp(&quarter));
let nan = run(&shots, 800.0, &opts(f64::NAN)).0;
assert_eq!((nan.image.width, nan.image.height), (full.image.width, full.image.height));
}
/// Issue #2320: a few wide frames can settle the fitted focal length below the lens's, and a canvas
/// sized from it holds fewer pixels than the photos. EXIF's focal length sets the native scale.
#[test]
fn the_lens_focal_length_sets_the_native_scale() {
let e = env();
let f = 2000.0; // what the frames were shot with: the fit settles near it
let lens = 2400.0; // what EXIF says, on a 4000-pixel long edge
let shots = [
Shot { yaw: -20.0, pitch: 0.0, roll: 0.0, gain: 1.0 },
Shot { yaw: 0.0, pitch: 0.0, roll: 0.0, gain: 1.0 },
Shot { yaw: 20.0, pitch: 0.0, roll: 0.0, gain: 1.0 },
];
let rs: Vec<M3> = shots.iter().map(|s| rot(s.yaw, s.pitch, s.roll)).collect();
let mut frames: Vec<Frame> = shots.iter().zip(&rs).map(|(s, r)| shoot(&e, r, f, 4000, 3000, s.gain)).collect();
for frame in &mut frames {
frame.metadata.focal_length_35mm = Some(lens / 4000.0 * 36.0);
}
let res = stitch(frames, &PanoOptions { projection: Projection::Spherical, ..Default::default() }, &no_progress).unwrap();
assert_eq!(res.used.len(), 3, "all three frames must connect: {:?}", res.used);
assert!((res.focal / lens - 1.0).abs() < 0.01, "output focal {} for a {lens} px lens", res.focal);
let mp = (res.image.width * res.image.height) as f64 / 1e6;
assert!((mp / res.native_megapixels - 1.0).abs() < 0.02, "canvas {mp} MP for {} MP native", res.native_megapixels);
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.