5a24289898fd
segmask.rs
//! Model segmentations stored with a mask ([`SegMask`]), so rendering never needs the model.
use std::cell::RefCell;
use std::collections::HashMap;
use serde::{Deserialize, Serialize, Serializer};
/// Logits are stored in steps of 1/`SCALE`, clamped to ±127/`SCALE` (≈ ±15.9: a sigmoid of
/// 1 − 1.2e-7, i.e. fully in or out).
const SCALE: f32 = 8.0;
/// Largest grid a stored segmentation may claim (hostile input caps the allocation).
pub const MAX_SIDE: usize = 1024;
/// A segmentation computed by a model (SAM 3 in `lightcraft-segment`): `side × side` logits
/// over the uncropped, oriented image stretched to a square (row `y`, column `x` covers
/// normalized `((x + 0.5) / side, (y + 0.5) / side)`), quantized to `i8`, deflated and base64
/// encoded. Coverage at a point is the sigmoid of the bilinearly sampled logit, so edges stay
/// smooth at any output size.
#[derive(Clone, Debug, PartialEq, Deserialize)]
pub struct SegMask {
pub side: u32,
/// The grid's bytes (quantized, deflated, base64), unless this is a reference into the
/// snapshot's shared grid table (`grid` is set and `data` is empty).
#[serde(default)]
pub data: String,
/// The part of the image the grid covers, normalized `[x0, y0, x1, y1]` (`None` = all of
/// it). A zoomed-in pass over one object covers just its surroundings, at a higher
/// resolution than a pass over the whole photo.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rect: Option<[f64; 4]>,
/// Index into the catalog snapshot's shared grid table (`Catalog::grids`) this mask refers
/// to instead of carrying its own `data`. A serialization detail of snapshots only: the
/// snapshot writer sets it (through the interner, into its own table), `Catalog::repair`
/// resolves it when a snapshot is read, and no other serialization writes it. A `grid` that
/// arrives from any other document (a command, a preset, XMP, a log record) is read but
/// never written back, so it cannot point a snapshot at another mask's grid.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub grid: Option<u32>,
}
impl SegMask {
/// From `side × side` logits over the whole image.
pub fn from_logits(side: usize, logits: &[f32]) -> SegMask {
let q: Vec<u8> = logits.iter().take(side * side).map(|l| ((l * SCALE).round().clamp(-127.0, 127.0) as i8) as u8).collect();
let z = miniz_oxide::deflate::compress_to_vec(&q, 8);
SegMask { side: side as u32, data: base64_encode(&z), rect: None, grid: None }
}
/// From `side × side` logits over the part `rect` of the image.
pub fn from_logits_in(side: usize, logits: &[f32], rect: [f64; 4]) -> SegMask {
SegMask { rect: Some(rect), ..SegMask::from_logits(side, logits) }
}
/// The covered part, normalized (the whole image without a `rect`); `None` when damaged.
pub fn bounds(&self) -> Option<[f64; 4]> {
let r = self.rect.unwrap_or([0.0, 0.0, 1.0, 1.0]);
(r.iter().all(|v| v.is_finite()) && r[2] > r[0] && r[3] > r[1]).then_some(r)
}
/// The logits, or `None` when the data is damaged or claims an absurd size.
pub fn logits(&self) -> Option<Vec<f32>> {
let side = self.side as usize;
if side == 0 || side > MAX_SIDE {
return None;
}
let z = base64_decode(&self.data)?;
let q = miniz_oxide::inflate::decompress_to_vec_with_limit(&z, side * side).ok()?;
(q.len() == side * side).then(|| q.iter().map(|b| f32::from(*b as i8) / SCALE).collect())
}
}
// The grid table a snapshot serialization refers to: each distinct grid's data → its index.
thread_local! {
static INTERNER: RefCell<Option<HashMap<String, u32>>> = const { RefCell::new(None) };
}
/// Serializes masks as references into a shared grid table (`grid` = index, `data` omitted)
/// while `f` runs, one thread at a time (snapshot writes). Nested calls are ignored (`None`):
/// the outermost writer owns the table.
pub fn with_grid_interner<T>(map: HashMap<String, u32>, f: impl FnOnce() -> T) -> Option<T> {
INTERNER.with(|i| {
if i.borrow().is_some() {
return None;
}
struct Guard;
impl Drop for Guard {
fn drop(&mut self) {
INTERNER.with(|j| *j.borrow_mut() = None);
}
}
*i.borrow_mut() = Some(map);
let _guard = Guard;
Some(f())
})
}
impl Serialize for SegMask {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeMap;
// While a snapshot is being written with a shared grid table, a mask whose exact bytes
// are in the table refers to it by index instead of repeating the blob (issue #2154).
if !self.data.is_empty()
&& let Some(id) = INTERNER.with(|i| i.borrow().as_ref().and_then(|m| m.get(&self.data).copied()))
{
let len = if self.rect.is_some() { 3 } else { 2 };
let mut map = serializer.serialize_map(Some(len))?;
map.serialize_entry("side", &self.side)?;
map.serialize_entry("grid", &id)?;
if let Some(rect) = self.rect {
map.serialize_entry("rect", &rect)?;
}
return map.end();
}
// Without a table hit: exactly the pre-v10 derived form (field order and skips included),
// so every serialization that doesn't dedup keeps its bytes (`hash64`, cache keys). `grid`
// is never written here: a value read from some document is dropped on save, and only the
// interner above can write a reference (into the table of the snapshot being written).
let len = 2 + usize::from(self.rect.is_some());
let mut map = serializer.serialize_map(Some(len))?;
map.serialize_entry("side", &self.side)?;
map.serialize_entry("data", &self.data)?;
if let Some(rect) = self.rect {
map.serialize_entry("rect", &rect)?;
}
map.end()
}
}
/// Most zoomed-in detail patches one AI mask component keeps (the app makes one; each costs a
/// decode per render, so a hostile document can't make rendering explode).
pub const MAX_DETAIL: usize = 4;
/// Deserialize a component's detail patches, keeping at most [`MAX_DETAIL`] (the rest are
/// skipped without being decoded).
pub(crate) fn de_detail<'de, D: serde::Deserializer<'de>>(d: D) -> Result<Vec<SegMask>, D::Error> {
struct Capped;
impl<'de> serde::de::Visitor<'de> for Capped {
type Value = Vec<SegMask>;
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str("a list of segmentations")
}
fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<Vec<SegMask>, A::Error> {
let mut out = Vec::new();
while out.len() < MAX_DETAIL {
match seq.next_element::<SegMask>()? {
Some(m) => out.push(m),
None => return Ok(out),
}
}
while seq.next_element::<serde::de::IgnoredAny>()?.is_some() {}
Ok(out)
}
}
d.deserialize_seq(Capped)
}
const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
use craft_mime::transfer::base64 as base64_encode;
fn base64_decode(s: &str) -> Option<Vec<u8>> {
let mut out = Vec::with_capacity(s.len() / 4 * 3);
let (mut acc, mut bits) = (0u32, 0u32);
let mut padded = false;
for c in s.bytes().filter(|c| !c.is_ascii_whitespace()) {
if c == b'=' {
padded = true;
continue;
}
if padded {
return None;
}
let v = B64.iter().position(|b| *b == c)? as u32;
acc = (acc << 6) | v;
bits += 6;
if bits >= 8 {
bits -= 8;
out.push((acc >> bits) as u8);
acc &= (1 << bits) - 1;
}
}
if bits == 6 {
return None;
}
Some(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn base64_round_trips() {
for n in 0..10 {
let v: Vec<u8> = (0..n).map(|i| (i * 37 + 11) as u8).collect();
assert_eq!(base64_decode(&base64_encode(&v)).unwrap(), v);
}
assert_eq!(base64_encode(b"Man"), "TWFu");
assert_eq!(base64_encode(b"Ma"), "TWE=");
assert!(base64_decode("TW!u").is_none());
assert!(base64_decode("Z").is_none());
assert!(base64_decode("Zm9vY").is_none());
assert!(base64_decode("Zg==!!!not-base64").is_none());
assert_eq!(base64_decode("Zg==\n").unwrap(), b"f");
}
#[test]
fn logits_round_trip_quantized() {
let side = 16;
let l: Vec<f32> = (0..side * side).map(|i| (i as f32 - 128.0) / 4.0).collect();
let m = SegMask::from_logits(side, &l);
let back = m.logits().unwrap();
for (a, b) in l.iter().zip(&back) {
assert!((a.clamp(-127.0 / SCALE, 127.0 / SCALE) - b).abs() <= 0.5 / SCALE + 1e-6);
}
}
#[test]
fn a_hostile_document_keeps_at_most_max_detail_patches() {
let patch = serde_json::to_value(SegMask::from_logits(4, &[1.0; 16])).unwrap();
let shape = serde_json::json!({"kind": "object", "hint": [{"x": 0.5, "y": 0.5}], "detail": vec![patch; 10_000]});
let s: crate::MaskShape = serde_json::from_value(shape).unwrap();
let crate::MaskShape::Object { detail, .. } = s else { panic!("an Object") };
assert_eq!(detail.len(), MAX_DETAIL);
// and a whole develop document with them still loads
let mut d = crate::DevelopSettings::default();
d.masks.push(crate::Mask {
id: 1,
components: vec![crate::MaskComponent {
name: None,
op: crate::MaskOp::Add,
invert: false,
shape: crate::MaskShape::Prompt { text: "x".into(), seg: None, detail: vec![SegMask::from_logits(4, &[1.0; 16]); 50], edge: 0.0 },
}],
..Default::default()
});
let back: crate::DevelopSettings = serde_json::from_str(&serde_json::to_string(&d).unwrap()).unwrap();
let crate::MaskShape::Prompt { detail, .. } = &back.masks[0].components[0].shape else { panic!("a Prompt") };
assert_eq!(detail.len(), MAX_DETAIL);
}
#[test]
fn damaged_data_is_none() {
let mut m = SegMask::from_logits(8, &[1.0; 64]);
m.side = 9;
assert!(m.logits().is_none(), "size mismatch");
m.side = 100_000;
assert!(m.logits().is_none(), "absurd size");
assert!(SegMask { side: 8, data: "not base64 !".into(), rect: None, grid: None }.logits().is_none());
assert!(SegMask { rect: Some([0.5, 0.0, 0.2, 1.0]), ..SegMask::from_logits(2, &[0.0; 4]) }.bounds().is_none());
}
/// A `grid` read from a document is never written outside the snapshot interner, with or
/// without data: the bytes stay the pre-v10 form, so `hash64` and cache keys don't depend on
/// a stray value (issue #2154).
#[test]
fn stray_grid_is_never_written_outside_the_interner() {
let empty = SegMask { side: 4, data: String::new(), rect: None, grid: None };
let stray = SegMask { grid: Some(3), ..empty.clone() };
assert_eq!(serde_json::to_string(&stray).unwrap(), "{\"side\":4,\"data\":\"\"}");
assert_eq!(serde_json::to_string(&stray).unwrap(), serde_json::to_string(&empty).unwrap());
let with_data = SegMask { grid: Some(3), ..SegMask::from_logits(4, &[1.0; 16]) };
assert!(!serde_json::to_string(&with_data).unwrap().contains("grid"), "no grid key outside the interner");
}
/// Without the snapshot interner the JSON is exactly the derived form it always was; with
/// it, a pooled grid serializes as an index and the data is left to the table (issue #2154).
#[test]
fn serialization_refs_only_under_the_interner() {
let m = SegMask::from_logits(4, &[1.0; 16]);
let plain = serde_json::to_string(&m).unwrap();
assert_eq!(plain, format!("{{\"side\":4,\"data\":\"{}\"}}", m.data), "no interner: unchanged bytes");
let with_rect = SegMask { rect: Some([0.0, 0.0, 0.5, 0.5]), ..m.clone() };
assert_eq!(serde_json::to_string(&with_rect).unwrap(), format!("{{\"side\":4,\"data\":\"{}\",\"rect\":[0.0,0.0,0.5,0.5]}}", m.data));
let mut table = std::collections::HashMap::new();
table.insert(m.data.clone(), 3u32);
let referenced = with_grid_interner(table, || serde_json::to_string(&m).unwrap()).unwrap();
assert_eq!(referenced, "{\"side\":4,\"grid\":3}", "pooled grid: an index, no data");
// a grid the table doesn't hold, and an empty-data mask, stay inline
let other = SegMask::from_logits(2, &[0.0; 4]);
let mixed = with_grid_interner(std::collections::HashMap::new(), || serde_json::to_string(&other).unwrap()).unwrap();
assert_eq!(mixed, format!("{{\"side\":2,\"data\":\"{}\"}}", other.data));
// the interner is gone afterwards: serialization is inline again
assert_eq!(serde_json::to_string(&m).unwrap(), plain);
// nested calls are ignored: the outer writer keeps its table
let outer_table = std::collections::HashMap::from([(m.data.clone(), 0u32)]);
let nested = with_grid_interner(outer_table, || {
let inner_table = std::collections::HashMap::from([(other.data.clone(), 1u32)]);
(with_grid_interner(inner_table, || serde_json::to_string(&other).unwrap()), serde_json::to_string(&m).unwrap())
})
.unwrap();
assert_eq!(nested.0, None, "the nested writer is refused");
assert_eq!(nested.1, "{\"side\":4,\"grid\":0}");
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.