a770fa5a5e5f
bidi.rs
//! Right-to-left text at the code-point level, independent of any font:
//! - the Unicode Bidirectional Algorithm (`unicode-bidi`) for one line: visual runs, per-character
//! embedding levels, mirrored brackets;
//! - Arabic-script joining (Arabic, Persian, Urdu): each letter's contextual form (isolated,
//! initial, medial, final), with ZWJ / tatweel joining and ZWNJ breaking, and the lam-alef
//! ligatures, mapped to Unicode presentation forms. Fonts with OpenType Arabic features are
//! shaped by `harfrust` instead (the `complex` module); the presentation forms serve fonts
//! without them (older fonts that only map the presentation-form code points).
use std::ops::Range;
use unicode_bidi::{BidiClass, Level, ParagraphBidiInfo, bidi_class};
/// Whether `c` is a strong right-to-left character or an explicit right-to-left control.
pub(crate) fn is_rtl(c: char) -> bool {
matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::RLE | BidiClass::RLO | BidiClass::RLI)
}
/// Whether a line needs the right-to-left path (bidi reordering and shaping).
pub fn has_rtl(s: &str) -> bool {
s.chars().any(is_rtl)
}
/// Invisible formatting characters: joiners, directional marks, embeddings and isolates. They
/// take no space and draw nothing (a shaper still sees the joiners).
pub(crate) fn is_format_control(c: char) -> bool {
matches!(c, '\u{200B}'..='\u{200F}' | '\u{202A}'..='\u{202E}' | '\u{2060}'..='\u{2069}' | '\u{FEFF}' | '\u{061C}')
}
/// A combining mark (drawn on the preceding character).
pub(crate) fn is_mark(c: char) -> bool {
bidi_class(c) == BidiClass::NSM
}
/// Base direction of a paragraph from its first strong character (rules P2/P3): `Some(true)`
/// right-to-left, `Some(false)` left-to-right, `None` when it has no strong character. As rule
/// P2 requires, characters between an isolate initiator and its matching PDI (or the end of the
/// paragraph, when unmatched) are skipped.
pub fn base_rtl(s: &str) -> Option<bool> {
let mut isolate_depth: usize = 0;
for c in s.chars() {
match bidi_class(c) {
BidiClass::LRI | BidiClass::RLI | BidiClass::FSI => isolate_depth = isolate_depth.saturating_add(1),
BidiClass::PDI => isolate_depth = isolate_depth.saturating_sub(1),
BidiClass::L if isolate_depth == 0 => return Some(false),
BidiClass::R | BidiClass::AL if isolate_depth == 0 => return Some(true),
_ => {}
}
}
None
}
fn para_level(base: Option<bool>) -> Option<Level> {
base.map(|rtl| if rtl { Level::rtl() } else { Level::ltr() })
}
/// Map character indices of `text` to byte offsets (one extra entry for the end).
fn byte_starts(text: &str) -> Vec<usize> {
let mut v: Vec<usize> = text.char_indices().map(|(b, _)| b).collect();
v.push(text.len());
v
}
/// The visual runs of one line of `chars`: character-index ranges in left-to-right display order,
/// each flagged right-to-left or not. The paragraph direction comes from the first strong
/// character (or `base` when given). Characters inside a right-to-left run are still in logical
/// order: the caller lays them out from right to left.
pub fn visual_runs(chars: &[char], base: Option<bool>) -> Vec<(Range<usize>, bool)> {
if chars.is_empty() {
return Vec::new();
}
let text: String = chars.iter().collect();
let starts = byte_starts(&text);
let char_at = |byte: usize| starts.partition_point(|b| *b < byte);
let info = ParagraphBidiInfo::new(&text, para_level(base));
let (levels, runs) = info.visual_runs(0..text.len());
runs.into_iter()
.filter(|r| !r.is_empty())
.map(|r| {
let rtl = levels.get(r.start).is_some_and(Level::is_rtl);
(char_at(r.start)..char_at(r.end), rtl)
})
.collect()
}
/// Resolved embedding level of every character of one line (after rule L1), for reordering
/// pieces of the line with [`reorder`].
pub fn char_levels(text: &str, base: Option<bool>) -> Vec<u8> {
if text.is_empty() {
return Vec::new();
}
let info = ParagraphBidiInfo::new(text, para_level(base));
info.reordered_levels_per_char(0..text.len()).into_iter().map(|l| l.number()).collect()
}
/// Display order (rule L2) of items with the given embedding levels: indices into `levels`,
/// left to right.
pub fn reorder(levels: &[u8]) -> Vec<usize> {
let lv: Vec<Level> = levels.iter().map(|l| Level::new(*l).unwrap_or_else(|_| Level::ltr())).collect();
ParagraphBidiInfo::reorder_visual(&lv)
}
/// The mirrored glyph of a paired bracket in right-to-left text (rule L4, common pairs).
pub fn mirror(c: char) -> char {
match c {
'(' => ')',
')' => '(',
'[' => ']',
']' => '[',
'{' => '}',
'}' => '{',
'<' => '>',
'>' => '<',
'«' => '»',
'»' => '«',
'‹' => '›',
'›' => '‹',
'≤' => '≥',
'≥' => '≤',
_ => c,
}
}
/// A line in display order, code point by code point: right-to-left runs reversed and their
/// brackets mirrored. Shows what the layout draws, before glyph shaping.
pub fn visual_order(s: &str) -> String {
let chars: Vec<char> = s.chars().collect();
let mut out = String::with_capacity(s.len());
for (r, rtl) in visual_runs(&chars, None) {
let run = chars.get(r).unwrap_or_default();
if rtl {
out.extend(run.iter().rev().map(|c| mirror(*c)));
} else {
out.extend(run.iter());
}
}
out
}
/// Arabic joining type (Unicode `ArabicShaping.txt` classes, the letters in common use).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Jt {
/// Non-joining (also ZWNJ, spaces, Latin).
U,
/// Right-joining: joins only the preceding letter (alef, dal, reh, waw …).
R,
/// Dual-joining.
D,
/// Join-causing (ZWJ, tatweel).
C,
/// Transparent (combining marks).
T,
}
fn joining_type(c: char) -> Jt {
let u = u32::from(c);
match u {
0x0640 | 0x200D => Jt::C,
0x0622..=0x0625
| 0x0627
| 0x0629
| 0x062F..=0x0632
| 0x0648
| 0x0671..=0x0673
| 0x0675..=0x0677
| 0x0688..=0x0699
| 0x06C0
| 0x06C3..=0x06CB
| 0x06CD
| 0x06CF
| 0x06D2
| 0x06D3
| 0x06D5
| 0x06EE
| 0x06EF
| 0x0759..=0x075B
| 0x076B
| 0x076C
| 0x0771
| 0x0773
| 0x0774
| 0x0778
| 0x0779 => Jt::R,
0x0620
| 0x0626
| 0x0628
| 0x062A..=0x062E
| 0x0633..=0x063F
| 0x0641..=0x0647
| 0x0649
| 0x064A
| 0x066E
| 0x066F
| 0x0678..=0x0687
| 0x069A..=0x06BF
| 0x06C1
| 0x06C2
| 0x06CC
| 0x06CE
| 0x06D0
| 0x06D1
| 0x06FA..=0x06FC
| 0x06FF
| 0x0750..=0x0758
| 0x075C..=0x076A
| 0x076D..=0x0770
| 0x0772
| 0x0775..=0x0777
| 0x077A..=0x077F => Jt::D,
_ if is_mark(c) => Jt::T,
_ => Jt::U,
}
}
/// Contextual form of a joining letter.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Form {
Isolated,
Final,
Initial,
Medial,
}
/// The contextual form of every character of a logical-order run (`None` for characters that
/// don't join: spaces, digits, Latin, marks, ZWNJ).
pub fn joining_forms(chars: &[char]) -> Vec<Option<Form>> {
let jt: Vec<Jt> = chars.iter().map(|c| joining_type(*c)).collect();
// Neighbour joining types, skipping transparent marks.
let prev = |i: usize| jt.get(..i).and_then(|s| s.iter().rev().find(|t| **t != Jt::T)).copied().unwrap_or(Jt::U);
let next = |i: usize| jt.get(i + 1..).and_then(|s| s.iter().find(|t| **t != Jt::T)).copied().unwrap_or(Jt::U);
jt.iter()
.enumerate()
.map(|(i, t)| {
if !matches!(t, Jt::R | Jt::D) {
return None;
}
let joins_prev = matches!(prev(i), Jt::D | Jt::C);
let joins_next = *t == Jt::D && matches!(next(i), Jt::R | Jt::D | Jt::C);
Some(match (joins_prev, joins_next) {
(true, true) => Form::Medial,
(true, false) => Form::Final,
(false, true) => Form::Initial,
(false, false) => Form::Isolated,
})
})
.collect()
}
/// Presentation forms of the basic Arabic letters (block FE70–FEFF, which starts at U+FE80):
/// each letter takes 1 (hamza), 2 (isolated, final) or 4 (isolated, final, initial, medial)
/// consecutive code points, in this order.
const FORMS_B: &[(u32, u32)] = &[
(0x0621, 1),
(0x0622, 2),
(0x0623, 2),
(0x0624, 2),
(0x0625, 2),
(0x0626, 4),
(0x0627, 2),
(0x0628, 4),
(0x0629, 2),
(0x062A, 4),
(0x062B, 4),
(0x062C, 4),
(0x062D, 4),
(0x062E, 4),
(0x062F, 2),
(0x0630, 2),
(0x0631, 2),
(0x0632, 2),
(0x0633, 4),
(0x0634, 4),
(0x0635, 4),
(0x0636, 4),
(0x0637, 4),
(0x0638, 4),
(0x0639, 4),
(0x063A, 4),
(0x0641, 4),
(0x0642, 4),
(0x0643, 4),
(0x0644, 4),
(0x0645, 4),
(0x0646, 4),
(0x0647, 4),
(0x0648, 2),
(0x0649, 2),
(0x064A, 4),
];
/// Presentation forms of Persian and Urdu letters (block FB50–FDFF): letter, first form, count.
const FORMS_A: &[(u32, u32, u32)] = &[
(0x0671, 0xFB50, 2), // alef wasla
(0x067E, 0xFB56, 4), // peh پ
(0x0679, 0xFB66, 4), // tteh
(0x0686, 0xFB7A, 4), // tcheh چ
(0x0688, 0xFB88, 2), // ddal
(0x0698, 0xFB8A, 2), // jeh ژ
(0x0691, 0xFB8C, 2), // rreh
(0x06A9, 0xFB8E, 4), // keheh ک
(0x06AF, 0xFB92, 4), // gaf گ
(0x06BA, 0xFB9E, 2), // noon ghunna
(0x06C0, 0xFBA4, 2), // heh with yeh above
(0x06C1, 0xFBA6, 4), // heh goal
(0x06BE, 0xFBAA, 4), // heh doachashmee
(0x06D2, 0xFBAE, 2), // yeh barree
(0x06CC, 0xFBFC, 4), // farsi yeh ی
];
/// The presentation-form code point of `c` in `form`, when Unicode has one.
pub fn presentation_form(c: char, form: Form) -> Option<char> {
let u = u32::from(c);
let slot = match form {
Form::Isolated => 0,
Form::Final => 1,
Form::Initial => 2,
Form::Medial => 3,
};
let mut at = 0xFE80;
let mut found = None;
for (base, n) in FORMS_B {
if *base == u {
found = Some((at, *n));
break;
}
at += n;
}
let (first, n) = found.or_else(|| FORMS_A.iter().find(|(b, _, _)| *b == u).map(|(_, f, n)| (*f, *n)))?;
// A letter with only isolated and final forms keeps them in initial / medial position.
let slot = if slot >= n { slot % 2 } else { slot };
if slot >= n {
return None;
}
char::from_u32(first + slot)
}
/// Lam-alef ligature for lam followed by an alef, isolated or final.
fn lam_alef(alef: char, final_form: bool) -> Option<char> {
let first = match alef {
'\u{0622}' => 0xFEF5,
'\u{0623}' => 0xFEF7,
'\u{0625}' => 0xFEF9,
'\u{0627}' => 0xFEFB,
_ => return None,
};
char::from_u32(first + u32::from(final_form))
}
/// Arabic joining with Unicode presentation forms, for a logical-order run: each joining letter
/// becomes its contextual form and lam + alef becomes one ligature. Returns the output code
/// points with the index of the source character each comes from (a ligature takes the lam's).
/// Joiners stay in place (a shaper hides them); `keep` decides whether a form may be used (a font
/// lacking it keeps the base letter).
pub fn presentation_forms(chars: &[char], keep: impl Fn(char) -> bool) -> Vec<(char, usize)> {
let forms = joining_forms(chars);
let mut out = Vec::with_capacity(chars.len());
let mut i = 0;
while let Some(&c) = chars.get(i) {
let form = forms.get(i).copied().flatten();
// Combining marks (harakat) between lam and alef are transparent for joining, so they
// don't block the ligature; they stay after it, in logical order.
let alef_at =
if c == '\u{0644}' { chars.get(i + 1..).and_then(|rest| rest.iter().position(|ch| !is_mark(*ch))).map(|p| i + 1 + p) } else { None };
if c == '\u{0644}'
&& let Some(f) = form
&& let Some(j) = alef_at
&& let Some(&a) = chars.get(j)
&& let Some(lig) = lam_alef(a, matches!(f, Form::Medial | Form::Final))
&& keep(lig)
{
out.push((lig, i));
for k in i + 1..j {
if let Some(&m) = chars.get(k) {
out.push((m, k));
}
}
i = j + 1;
continue;
}
let shaped = form.and_then(|f| presentation_form(c, f)).filter(|p| keep(*p)).unwrap_or(c);
out.push((shaped, i));
i += 1;
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn forms(s: &str) -> String {
let chars: Vec<char> = s.chars().collect();
presentation_forms(&chars, |_| true).into_iter().map(|(c, _)| c).collect()
}
#[test]
fn salam_takes_contextual_forms_and_the_lam_alef_ligature() {
// س initial, ل + ا ligature (final, since lam joins seen), م isolated (alef doesn't join
// forward).
let chars: Vec<char> = "سلام".chars().collect();
assert_eq!(joining_forms(&chars), vec![Some(Form::Initial), Some(Form::Medial), Some(Form::Final), Some(Form::Isolated)]);
assert_eq!(forms("سلام"), "\u{FEB3}\u{FEFC}\u{FEE1}");
let src: Vec<usize> = presentation_forms(&chars, |_| true).into_iter().map(|(_, i)| i).collect();
assert_eq!(src, vec![0, 1, 3]);
// A word-initial lam-alef is the isolated ligature.
assert_eq!(forms("لا"), "\u{FEFB}");
}
#[test]
fn persian_letters_join() {
// پ چ ژ گ ی ک: initial, medial, final and isolated Persian forms.
assert_eq!(forms("پی"), "\u{FB58}\u{FBFD}");
assert_eq!(forms("چک"), "\u{FB7C}\u{FB8F}");
assert_eq!(forms("گژ"), "\u{FB94}\u{FB8B}");
assert_eq!(forms("کیک"), "\u{FB90}\u{FBFF}\u{FB8F}");
// Jeh is right-joining: it never joins the next letter.
assert_eq!(forms("ژی"), "\u{FB8A}\u{FBFC}");
}
#[test]
fn zwnj_breaks_and_zwj_forces_joining() {
// میروم ("I go"): ZWNJ after meem keeps it from joining yeh.
let chars: Vec<char> = "می\u{200C}روم".chars().collect();
let f = joining_forms(&chars);
assert_eq!(f.get(1), Some(&Some(Form::Final)), "yeh before ZWNJ is final");
assert_eq!(f.get(2), Some(&None), "ZWNJ itself does not join");
assert_eq!(f.get(3), Some(&Some(Form::Isolated)), "reh after ZWNJ starts afresh");
// ZWJ makes an isolated letter join forward.
let z: Vec<char> = "ب\u{200D}".chars().collect();
assert_eq!(joining_forms(&z).first(), Some(&Some(Form::Initial)));
// Marks are transparent: بَب joins across the fatha.
let m: Vec<char> = "ب\u{064E}ب".chars().collect();
assert_eq!(joining_forms(&m), vec![Some(Form::Initial), None, Some(Form::Final)]);
}
#[test]
fn mixed_line_visual_order() {
// "Price 25 toman": a right-to-left paragraph; the number stays left-to-right.
let v = visual_order("قیمت 25 تومان");
assert_eq!(v, "ناموت 25 تمیق");
let chars: Vec<char> = "قیمت 25 تومان".chars().collect();
let runs = visual_runs(&chars, None);
assert_eq!(runs, vec![(7..13, true), (5..7, false), (0..5, true)]);
// Latin inside Farsi, and Farsi inside a Latin paragraph.
assert_eq!(visual_order("فایل CAD باز"), "زاب CAD لیاف");
assert_eq!(visual_order("open فایل now"), "open لیاف now");
// Brackets mirror in right-to-left runs.
assert_eq!(visual_order("(سلام)"), "(مالس)");
}
#[test]
fn levels_and_reorder() {
assert!(has_rtl("abc سلام") && !has_rtl("abc 123"));
assert_eq!(base_rtl(" 123 سلام abc"), Some(true));
assert_eq!(base_rtl("123"), None);
let lv = char_levels("ab سل", None);
assert_eq!(lv, vec![0, 0, 0, 1, 1]);
assert_eq!(reorder(&[1, 1, 2, 1]), vec![3, 2, 1, 0]);
assert_eq!(reorder(&[0, 1, 1, 0]), vec![0, 2, 1, 3]);
assert!(visual_runs(&[], None).is_empty() && char_levels("", None).is_empty());
}
#[test]
fn paragraph_direction_skips_isolate_contents() {
// RLI שלום PDI abc: the Hebrew sits inside the isolate, so the first strong character is "a".
assert_eq!(base_rtl("\u{2067}שלום\u{2069} abc"), Some(false));
assert_eq!(base_rtl("\u{2066}abc\u{2069} שלום"), Some(true));
assert_eq!(base_rtl("\u{2068}שלום\u{2069}abc"), Some(false));
// Nested isolates: only the outermost PDI closes the outer one.
assert_eq!(base_rtl("\u{2066}a\u{2067}ב\u{2069}a\u{2069}ב"), Some(true));
assert_eq!(base_rtl("\u{2067}\u{2066}a\u{2069}ב\u{2069}a"), Some(false));
// An unmatched initiator isolates the rest of the paragraph.
assert_eq!(base_rtl("\u{2067}שלום"), None);
assert_eq!(base_rtl("\u{2067}שלום\u{2069}"), None);
// A stray PDI does not close anything and does not underflow.
assert_eq!(base_rtl("\u{2069}\u{2069}שלום"), Some(true));
// No strong character outside the isolates.
assert_eq!(base_rtl("12 \u{2066}abc\u{2069} 34"), None);
// Without isolates nothing changes.
assert_eq!(base_rtl(" 123 سلام abc"), Some(true));
}
#[test]
fn harakat_between_lam_and_alef_keep_the_ligature() {
let run = |s: &str| -> Vec<(char, usize)> {
let chars: Vec<char> = s.chars().collect();
presentation_forms(&chars, |_| true)
};
// Lam + fatha + alef: the ligature, then the fatha (logical order kept).
assert_eq!(run("ل\u{064E}ا"), vec![('\u{FEFB}', 0), ('\u{064E}', 1)]);
// Word-medial: س + lam + fatha + shadda + alef gives the final ligature, marks after it.
assert_eq!(run("سل\u{064E}\u{0651}ا"), vec![('\u{FEB3}', 0), ('\u{FEFC}', 1), ('\u{064E}', 2), ('\u{0651}', 3)]);
// Plain lam + alef is unchanged.
assert_eq!(run("لا"), vec![('\u{FEFB}', 0)]);
// Mark after the alef is not between them: unchanged behaviour.
assert_eq!(run("لا\u{064E}"), vec![('\u{FEFB}', 0), ('\u{064E}', 2)]);
// Lam + mark + a non-alef letter: no ligature, the lam joins forward as before.
assert_eq!(run("ل\u{064E}م"), vec![('\u{FEDF}', 0), ('\u{064E}', 1), ('\u{FEE2}', 2)]);
// Lam + mark at the end: no alef, nothing breaks.
assert_eq!(run("ل\u{064E}"), vec![('\u{FEDD}', 0), ('\u{064E}', 1)]);
// A non-mark between lam and alef blocks the ligature.
assert_eq!(run("ل-ا").len(), 3);
}
#[test]
fn hostile_input_never_panics() {
for s in [
"a\nسلام\u{2029}b\r",
"\u{202E}\u{202E}\u{202E}abc",
"\u{2067}\u{2069}\u{2069}",
"\u{0644}",
"\u{0644}\u{200D}",
"\u{064E}\u{064E}",
"\u{FEFF}",
] {
let _ = visual_order(s);
let chars: Vec<char> = s.chars().collect();
let _ = presentation_forms(&chars, |_| true);
let _ = char_levels(s, Some(true));
}
let long: String = "سلام abc 123 ".repeat(2000);
assert_eq!(visual_order(&long).chars().count(), long.chars().count());
assert_eq!(reorder(&[200, 0, 255]).len(), 3);
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.