GitCorner

Code

ossartcraftdesigncraft๐ŸŒ
88f5a00e389b
//! Right-to-left text in exported PDFs: content streams list it in reading order with marks after
//! their letters, ActualText never spans more than a cluster, and glyphs stay where composition
//! put them.

use designcraft_compose::{Cache, ComposeOptions, compose_story};
use designcraft_doc::build::NewDocument;
use designcraft_doc::{Align, Document, ParaAttrs, ParaFormat, SpreadRef, TextDirection};
use designcraft_geom::Rect;

use crate::tests::extract_text;
use crate::{PdfOptions, export_pdf};

const HEBREW: &str = "DC Test PDF Hebrew";
const ARABIC: &str = "DC Test PDF Arabic";

/// A stand-in Hebrew font: each letter and point has a glyph of its own (Latin capitals and
/// combining accents of the bundled face), so the ToUnicode map is unambiguous.
fn hebrew_font() {
    let letters = ['ื“', 'ื•', 'ืฉ', 'ืจ', 'ื”', 'ื™', 'ื', 'ืœ', 'ื‘', 'ื', 'ืฆ', 'ื—', 'ืง', 'ืข'];
    let points = ['\u{5B7}', '\u{5B0}', '\u{5B8}', '\u{5B4}', '\u{5B2}', '\u{5B9}'];
    let accents = ['\u{300}', '\u{301}', '\u{302}', '\u{303}', '\u{304}', '\u{308}'];
    let mut map: Vec<(char, char)> = letters.iter().zip('A'..).map(|(h, l)| (*h, l)).collect();
    map.extend(points.iter().copied().zip(accents));
    map.extend(" &abcdef".chars().map(|c| (c, c)));
    designcraft_fonts::FontDb::global().add_font(designcraft_fonts::testing::font_mapping(HEBREW, &map).expect("font"));
}

/// A stand-in Arabic font with a tatweel, so justification inserts kashidas.
fn arabic_font() {
    let map = [('ุจ', 'b'), ('ุณ', 's'), ('ู…', 'm'), ('ุง', 'a'), ('ู„', 'l'), ('ู‡', 'h'), ('\u{640}', '_'), (' ', ' ')];
    designcraft_fonts::FontDb::global().add_font(designcraft_fonts::testing::font_mapping(ARABIC, &map).expect("font"));
}

/// One frame of `text` in `family` with paragraph attributes `para`.
fn doc(text: &str, family: &str, r: Rect, para: ParaAttrs) -> Document {
    let mut d = Document::new(&NewDocument::default());
    let lid = d.default_layer();
    d.add_text_frame(SpreadRef::Doc(0), r, lid, text, ParaFormat { para, ..ParaFormat::default() }).unwrap();
    let sid = *d.stories.keys().next().unwrap();
    d.story_mut(sid).unwrap().format_chars(0..text.len(), |f| f.over.font_family = Some(family.into()));
    d
}

fn rtl() -> ParaAttrs {
    ParaAttrs { direction: Some(TextDirection::RightToLeft), ..ParaAttrs::default() }
}

/// The ActualText of every marked-content span on the first page.
fn actual_texts(bytes: &[u8]) -> Vec<String> {
    use craft_pdf::syntax::content::ops::TypedInstruction;
    use craft_pdf::syntax::object::{Dict, Object};
    let pdf = craft_pdf::syntax::Pdf::new(bytes.to_vec()).expect("parse");
    let page = &pdf.pages()[0];
    let mut out = Vec::new();
    let mut ops = page.typed_operations();
    while let Some(op) = ops.next() {
        let TypedInstruction::BeginMarkedContentWithProperties(op) = op else { continue };
        let Object::Dict(props) = op.1 else { continue };
        let props: &Dict<'_> = props;
        let Some(s) = props.get::<craft_pdf::syntax::object::String>(b"ActualText") else { continue };
        let units: Vec<u16> = s.as_bytes().as_chunks::<2>().0.iter().map(|&b| u16::from_be_bytes(b)).collect();
        let text = String::from_utf16(&units).expect("UTF-16");
        out.push(text.trim_start_matches('\u{FEFF}').to_string());
    }
    out
}

#[test]
fn hebrew_extracts_in_reading_order_with_points_on_their_letters() {
    hebrew_font();
    let lines = ["ื“ื•ื“ & ืฉืจื”", "ื™ืฉืจืืœ ืึทื‘ึฐืจึธื”ึธื ื™ึดืฆึฐื—ึธืง ื•ึฐื™ึทืขึฒืงึนื‘", "ื“ื•ื“ abc ืฉืจื”"];
    let d = doc(&lines.join("\n"), HEBREW, Rect::new(36.0, 36.0, 576.0, 300.0), rtl());
    let bytes = export_pdf(&d, &Cache::new(), &PdfOptions::default()).unwrap();
    // Glyph by glyph (ToUnicode, content order): each point right after its letter, once.
    assert_eq!(extract_text(&bytes)[0], lines.concat());
    assert!(actual_texts(&bytes).is_empty());
    let tagged = export_pdf(&d, &Cache::new(), &PdfOptions { tagged: true, ..PdfOptions::default() }).unwrap();
    assert!(actual_texts(&tagged).is_empty());
    assert_eq!(extract_text(&tagged)[0], lines.concat());
}

#[test]
fn hebrew_words_in_left_to_right_lines_extract_in_reading_order() {
    hebrew_font();
    let text = "abc ื“ื•ื“ & ืฉืจื” def";
    let d = doc(text, HEBREW, Rect::new(36.0, 36.0, 576.0, 300.0), ParaAttrs::default());
    let bytes = export_pdf(&d, &Cache::new(), &PdfOptions::default()).unwrap();
    assert_eq!(extract_text(&bytes)[0], text);
    assert!(actual_texts(&bytes).is_empty());
}

/// Device origin of every glyph drawn on the first page.
fn glyph_origins(bytes: &[u8]) -> Vec<kurbo::Point> {
    use hayro_interpret::font::Glyph;
    use hayro_interpret::{
        BlendMode, ClipPath, Context, Device, GlyphDrawMode, Image, InterpreterCache, InterpreterSettings, Paint, PathDrawMode, SoftMask,
        interpret_page,
    };
    struct Rec(Vec<kurbo::Point>);
    impl Device<'_> for Rec {
        fn set_soft_mask(&mut self, _: Option<SoftMask<'_>>) {}
        fn set_blend_mode(&mut self, _: BlendMode) {}
        fn draw_path(&mut self, _: &kurbo::BezPath, _: kurbo::Affine, _: &Paint<'_>, _: &PathDrawMode) {}
        fn push_clip_path(&mut self, _: &ClipPath) {}
        fn push_transparency_group(&mut self, _: f32, _: Option<SoftMask<'_>>, _: BlendMode) {}
        fn draw_glyph(&mut self, g: &Glyph<'_>, xf: kurbo::Affine, gxf: kurbo::Affine, _: &Paint<'_>, _: &GlyphDrawMode) {
            if let Glyph::Outline(_) = g {
                self.0.push((xf * gxf) * kurbo::Point::ZERO);
            }
        }
        fn draw_image(&mut self, _: Image<'_, '_>, _: kurbo::Affine) {}
        fn pop_clip_path(&mut self) {}
        fn pop_transparency_group(&mut self) {}
    }
    let pdf = craft_pdf::syntax::Pdf::new(bytes.to_vec()).expect("parse");
    let cache = InterpreterCache::new();
    let mut ctx = Context::new(kurbo::Affine::IDENTITY, kurbo::Rect::new(0.0, 0.0, 1.0, 1.0), &cache, pdf.xref(), InterpreterSettings::default());
    let mut rec = Rec(Vec::new());
    interpret_page(&pdf.pages()[0], &mut ctx, &mut rec);
    rec.0
}

#[test]
fn pointed_letters_are_drawn_where_composed() {
    hebrew_font();
    let text = "ื™ึดืฆึฐื—ึธืง ืึทื‘ึฐืจึธื”ึธื\nื™ึดืฆึฐื—ึธืง ืึทื‘ึฐืจึธื”ึธื";
    let mut d = doc(text, HEBREW, Rect::new(36.0, 36.0, 576.0, 300.0), rtl());
    let sid = *d.stories.keys().next().unwrap();
    // The second paragraph is skewed: points keep their place relative to their letters.
    let second = text.find('\n').unwrap() + 1;
    d.story_mut(sid).unwrap().format_chars(second..text.len(), |f| f.over.skew = Some(12.0));
    let cs = compose_story(&d, sid, &ComposeOptions::default());
    let composed: Vec<kurbo::Point> =
        cs.frames[0].lines.iter().flat_map(|l| l.glyphs.iter().filter(|g| g.visible).map(|g| kurbo::Point::new(g.x, l.baseline + g.y))).collect();
    assert!(composed.iter().any(|p| (p.y - composed[0].y).abs() > 1.0 && (p.y - composed[0].y).abs() < 20.0), "the points sit off the baseline");
    let drawn = glyph_origins(&export_pdf(&d, &Cache::new(), &PdfOptions::default()).unwrap());
    assert_eq!(drawn.len(), composed.len());
    // The page places the frame (y grows upwards on the page): match the two through their means.
    let mean = |ps: &[kurbo::Point]| ps.iter().fold(kurbo::Vec2::ZERO, |a, p| a + p.to_vec2()) / ps.len() as f64;
    let (c, k) = (mean(&composed), mean(&drawn));
    let key = |x: f64, y: f64| ((x * 100.0).round() as i64, (y * 100.0).round() as i64);
    let mut want: Vec<(i64, i64)> = composed.iter().map(|p| key(p.x - c.x, c.y - p.y)).collect();
    let mut got: Vec<(i64, i64)> = drawn.iter().map(|p| key(p.x - k.x, p.y - k.y)).collect();
    want.sort_unstable();
    got.sort_unstable();
    let off: Vec<_> = want.iter().zip(&got).filter(|(w, g)| (w.0 - g.0).abs() > 1 || (w.1 - g.1).abs() > 1).collect();
    assert!(off.is_empty(), "{off:?}");
}

#[test]
fn kashidas_extract_as_no_text() {
    arabic_font();
    let text = "ุจุณู… ุงู„ู„ู‡ ุจุณู… ุงู„ู„ู‡ ุจุณู… ุงู„ู„ู‡ ุจุณู… ุงู„ู„ู‡ ุจุณู… ุงู„ู„ู‡ ุจุณู… ุงู„ู„ู‡ ุจุณู… ุงู„ู„ู‡";
    let para = ParaAttrs { align: Some(Align::FullyJustified), ..rtl() };
    let d = doc(text, ARABIC, Rect::new(36.0, 36.0, 186.0, 300.0), para);
    let sid = *d.stories.keys().next().unwrap();
    let cs = compose_story(&d, sid, &ComposeOptions::default());
    let kashida = |g: &designcraft_compose::PlacedGlyph| g.len == 0 && g.gid == g.face.glyph_for('\u{640}');
    assert!(cs.frames[0].lines.iter().any(|l| l.glyphs.iter().any(kashida)), "justification inserts kashidas");
    let bytes = export_pdf(&d, &Cache::new(), &PdfOptions::default()).unwrap();
    let words: Vec<String> = extract_text(&bytes)[0].split([' ', '\u{FFFD}']).filter(|w| !w.is_empty()).map(str::to_string).collect();
    assert!(words.iter().all(|w| w == "ุจุณู…" || w == "ุงู„ู„ู‡"), "{words:?}");
    let spans = actual_texts(&bytes);
    assert!(spans.iter().all(|t| !t.contains('\u{640}') && !t.contains(' ')), "{spans:?}");
}

#[test]
fn arabic_ligatures_extract_their_letters() {
    // Lam-alef in a system font with Arabic (skipped without one).
    let text = "ู„ุง ุณู„ุงู…";
    let mut d = doc(text, designcraft_fonts::DEFAULT_FAMILY, Rect::new(36.0, 36.0, 576.0, 300.0), rtl());
    d.settings.glyph_fallback = true;
    let sid = *d.stories.keys().next().unwrap();
    let cs = compose_story(&d, sid, &ComposeOptions::default());
    let glyphs = &cs.frames[0].lines[0].glyphs;
    if glyphs.iter().any(|g| g.gid == 0) || !glyphs.iter().any(|g| g.byte == 0 && g.len == "ู„ุง".len()) {
        return;
    }
    let bytes = export_pdf(&d, &Cache::new(), &PdfOptions::default()).unwrap();
    assert_eq!(extract_text(&bytes)[0], text);
}

/// How many marked-content sections the first page opens while an ActualText span is open.
/// Viewers such as poppler drop the text before an ActualText span nested in another.
fn opened_inside_actual_text(bytes: &[u8]) -> usize {
    use craft_pdf::syntax::content::ops::TypedInstruction;
    use craft_pdf::syntax::object::Object;
    let pdf = craft_pdf::syntax::Pdf::new(bytes.to_vec()).expect("parse");
    let page = &pdf.pages()[0];
    let (mut open, mut nested) = (Vec::new(), 0);
    let mut ops = page.typed_operations();
    while let Some(op) = ops.next() {
        let actual = match op {
            TypedInstruction::BeginMarkedContentWithProperties(op) => matches!(op.1, Object::Dict(d) if d.contains_key(b"ActualText")),
            TypedInstruction::BeginMarkedContent(_) => false,
            TypedInstruction::EndMarkedContent(_) => {
                open.pop();
                continue;
            }
            _ => continue,
        };
        nested += usize::from(open.contains(&true));
        open.push(actual);
    }
    nested
}

#[test]
fn right_to_left_lines_carry_actual_text_only_on_their_clusters() {
    // Shadda clusters on contextual forms and pointed letters, in a system font with Arabic and
    // Hebrew (skipped without one): krilla gives such clusters an ActualText span of their own,
    // which must not nest in another (poppler drops the text before a nested one).
    let lines = ["ุณูŠู‘ุฏ ุงู„ูƒู„ู‘ ู…ูุญูŽู…ูŽู‘ุฏ", "ื™ืฉืจืืœ ืึทื‘ึฐืจึธื”ึธื ื™ึดืฆึฐื—ึธืง"];
    let mut d = doc(&lines.join("\n"), designcraft_fonts::DEFAULT_FAMILY, Rect::new(36.0, 36.0, 576.0, 300.0), rtl());
    d.settings.glyph_fallback = true;
    let sid = *d.stories.keys().next().unwrap();
    let cs = compose_story(&d, sid, &ComposeOptions::default());
    let glyphs: Vec<_> = cs.frames[0].lines.iter().flat_map(|l| &l.glyphs).collect();
    if glyphs.iter().any(|g| g.gid == 0) || !glyphs.windows(2).any(|w| w[1].len == 0 && w[1].byte == w[0].byte) {
        return;
    }
    for tagged in [false, true] {
        let bytes = export_pdf(&d, &Cache::new(), &PdfOptions { tagged, ..PdfOptions::default() }).unwrap();
        assert_eq!(opened_inside_actual_text(&bytes), 0, "tagged: {tagged}");
        let spans = actual_texts(&bytes);
        assert!(!spans.is_empty() && spans.iter().all(|t| !t.contains(' ') && t.chars().count() <= 3), "tagged: {tagged}: {spans:?}");
    }
}

/// A slanted (shatai) glyph in a right-to-left line is drawn through its own tagged run; tagged
/// exports must not nest marked content (krilla panics on it).
#[test]
fn shatai_glyphs_in_right_to_left_lines_export_in_reading_order() {
    hebrew_font();
    let text = "ื“ื•ื“ & ืฉืจื”";
    let mut d = doc(text, HEBREW, Rect::new(36.0, 36.0, 576.0, 300.0), rtl());
    let sid = *d.stories.keys().next().unwrap();
    d.story_mut(sid).unwrap().format_chars(0.."ื“ื•ื“".len(), |f| f.over.shatai_magnification = Some(30.0));
    for tagged in [false, true] {
        let bytes = export_pdf(&d, &Cache::new(), &PdfOptions { tagged, ..PdfOptions::default() }).unwrap();
        assert_eq!(opened_inside_actual_text(&bytes), 0, "tagged: {tagged}");
        assert!(actual_texts(&bytes).iter().all(|t| !t.contains(' ')), "tagged: {tagged}");
        assert_eq!(extract_text(&bytes)[0], text, "tagged: {tagged}");
    }
}