6ef13b2cc8a6
packs.rs
//! Language packs (CP4.3): spelling dictionaries and hyphenation patterns for languages beyond
//! the embedded English, one pack per locale, downloaded and verified by the font store
//! (craft-libs `craft-font-store`, item kind `language-pack`) and read by `craft-proof` from the
//! store's `language-packs/` folder.
//!
//! The commands (`proof.packs`, `proof.installPack`, `proof.removePack`), Options › Language,
//! the missing-pack bar and About › Language data all go through this module. It never asks
//! anything: an install needs the font store's single consent setting (downloading on), which only
//! a person in the app can turn on ([`Backend::enable_network`], called from an Install click).
//! Agents (MCP, the CLI, the control channel) install only while it is on.
//!
//! The [`Backend`] is set once at start-up by the desktop app and the CLI ([`use_font_store`]).
//! Without one (tests, the web build, `ui_shot`) nothing is available and nothing is downloaded.
use std::collections::BTreeSet;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, PoisonError, RwLock};
use serde::Serialize;
use crate::{CmdError, Session};
#[cfg(not(target_arch = "wasm32"))]
mod store;
/// Longest locale tag taken, in bytes (BCP 47 tags in practice stay under 20).
pub const MAX_LOCALE_LEN: usize = 35;
/// Most packs an index may list that are read.
const MAX_PACKS: usize = 1000;
/// Paragraphs of a document looked at for its languages (the missing-pack bar).
const MAX_PARAGRAPHS_SCANNED: usize = 20_000;
/// Distinct languages of a document collected.
const MAX_DOC_LANGUAGES: usize = 64;
/// A pack the pinned index lists.
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct Pack {
/// Lowercase BCP 47 (`nl`, `pt-pt`).
pub locale: String,
/// The language's own name (`Nederlands`).
pub name: String,
/// The craft-i18n commit the pack was built from.
pub version: String,
/// The archive.
pub url: String,
/// The upstream folder the data comes from, at its pinned commit (the index's optional
/// `source`), shown before install; the archive when the index gives none.
pub source: Option<String>,
/// Bytes of the archive.
pub size: u64,
/// SPDX expression.
pub licence: String,
/// `Embeddable` or `Download-only`.
pub class: String,
/// Has a spelling dictionary.
pub spelling: bool,
/// Has hyphenation patterns.
pub hyphenation: bool,
}
/// An installed pack.
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct Installed {
pub locale: String,
pub name: String,
pub licence: String,
pub path: PathBuf,
}
/// Stops a download. On the desktop it is the font store's own flag, which a download watches
/// even while the server sends nothing.
#[derive(Clone, Debug, Default)]
pub struct Cancel(#[cfg(not(target_arch = "wasm32"))] pub(crate) craft_font_store::Cancel, #[cfg(target_arch = "wasm32")] Arc<AtomicBool>);
impl Cancel {
pub fn new() -> Self {
Self::default()
}
#[cfg(not(target_arch = "wasm32"))]
pub fn cancel(&self) {
self.0.cancel();
}
#[cfg(not(target_arch = "wasm32"))]
pub fn is_cancelled(&self) -> bool {
self.0.is_cancelled()
}
#[cfg(target_arch = "wasm32")]
pub fn cancel(&self) {
self.0.store(true, Ordering::Relaxed);
}
#[cfg(target_arch = "wasm32")]
pub fn is_cancelled(&self) -> bool {
self.0.load(Ordering::Relaxed)
}
}
/// Where packs come from: the font store on the desktop, a fixed list in tests and `ui_shot`.
pub trait Backend: Send + Sync {
/// The pinned index as last fetched (no request); empty when it never was.
fn index(&self) -> Result<Vec<Pack>, String>;
/// Fetch the pinned index (a request; only while downloading is on).
fn fetch_index(&self, cancel: &Cancel) -> Result<Vec<Pack>, String>;
/// Whether the font store may download.
fn network_on(&self) -> bool;
/// Turn downloading on. Only from the person's own action in the app (an Install click):
/// the commands never call it, and a backend for an agent refuses.
fn enable_network(&self) -> Result<(), String>;
/// Download, verify and unpack `pack` into the packs folder. `progress` gets (done, total) bytes.
fn install(&self, pack: &Pack, cancel: &Cancel, progress: &mut dyn FnMut(u64, u64)) -> Result<(), String>;
/// Remove the installed pack for `locale`.
fn remove(&self, locale: &str) -> Result<(), String>;
/// The installed-packs folder (`<font store>/language-packs`).
fn packs_dir(&self) -> Option<PathBuf>;
/// The packs installed now.
fn installed(&self) -> Vec<Installed> {
installed_from_proof()
}
/// The test index in use instead of the pinned one (`CRAFT_LANGUAGE_PACK_INDEX`), if any.
fn index_override(&self) -> Option<String> {
None
}
}
static BACKEND: RwLock<Option<Arc<dyn Backend>>> = RwLock::new(None);
/// The index last read, so the bar and the list don't read the store's cache every frame.
static INDEX: Mutex<Option<Vec<Pack>>> = Mutex::new(None);
/// Bumped whenever [`INDEX`] changes.
static INDEX_GENERATION: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
fn set_index(index: Option<Vec<Pack>>) {
*INDEX.lock().unwrap_or_else(PoisonError::into_inner) = index;
INDEX_GENERATION.fetch_add(1, Ordering::Relaxed);
}
/// Changes whenever the list of available packs does (a fetch, another backend).
pub fn index_generation() -> u64 {
INDEX_GENERATION.load(Ordering::Relaxed)
}
/// Use `backend` for packs from now on (`None`: no packs), and point proofing at its folder.
pub fn set_backend(backend: Option<Arc<dyn Backend>>) {
let dir = backend.as_ref().and_then(|b| b.packs_dir());
*BACKEND.write().unwrap_or_else(PoisonError::into_inner) = backend;
set_index(None);
FETCHED.store(false, Ordering::Relaxed);
AUTO_REFRESHED.store(false, Ordering::Relaxed);
WANTED.lock().unwrap_or_else(PoisonError::into_inner).clear();
jobs().errors.clear();
wordcraft_proof::packs::set_packs_dir(dir);
}
/// Use the suite's font store for packs (`<font store>/language-packs`), as the person in the app
/// (`user`: an Install click may turn downloading on) or as an agent (the CLI, headless MCP: it
/// never can). The pack index is the pinned one, or `CRAFT_LANGUAGE_PACK_INDEX` (a URL or a path,
/// for testing packs before they are published), which the font store reads itself. Returns
/// whether there is a store folder.
#[cfg(not(target_arch = "wasm32"))]
pub fn use_font_store(user: bool) -> bool {
match store::FontStoreBackend::new(user) {
Some(b) => {
set_backend(Some(Arc::new(b)));
true
}
None => false,
}
}
/// [`use_font_store`] with the store in `dir` instead of the person's (`ui_shot`, tests).
#[cfg(not(target_arch = "wasm32"))]
pub fn use_font_store_in(dir: PathBuf, user: bool) {
set_backend(Some(Arc::new(store::FontStoreBackend::in_dir(dir, user))));
}
/// The backend in use.
pub fn backend() -> Option<Arc<dyn Backend>> {
BACKEND.read().unwrap_or_else(PoisonError::into_inner).clone()
}
fn installed_from_proof() -> Vec<Installed> {
wordcraft_proof::packs::installed().into_iter().map(|p| Installed { locale: p.locale, name: p.name, licence: p.licence, path: p.path }).collect()
}
/// The test source packs come from instead of the pinned index (`CRAFT_LANGUAGE_PACK_INDEX`):
/// Options › Language says so.
pub fn index_override() -> Option<String> {
backend().and_then(|b| b.index_override())
}
/// The packs installed now (none without a backend).
pub fn installed() -> Vec<Installed> {
backend().map(|b| b.installed()).unwrap_or_default()
}
/// The available packs: the index as last read, read from the store's cache the first time.
pub fn available() -> Vec<Pack> {
let Some(b) = backend() else { return Vec::new() };
let mut cached = INDEX.lock().unwrap_or_else(PoisonError::into_inner);
if let Some(i) = cached.as_ref() {
return i.clone();
}
let index = clean_index(b.index().unwrap_or_default());
*cached = Some(index.clone());
drop(cached);
INDEX_GENERATION.fetch_add(1, Ordering::Relaxed);
index
}
/// How many packs [`available`] lists, without copying them (the window asks every frame).
pub fn available_len() -> usize {
if let Some(i) = INDEX.lock().unwrap_or_else(PoisonError::into_inner).as_ref() {
return i.len();
}
available().len()
}
/// Fetch the index again (downloading must be on).
pub fn refresh(cancel: &Cancel) -> Result<Vec<Pack>, String> {
let b = backend().ok_or_else(no_store)?;
if !b.network_on() {
return Err(network_off());
}
let index = clean_index(b.fetch_index(cancel)?);
set_index(Some(index.clone()));
FETCHED.store(true, Ordering::Relaxed);
install_wanted(&index);
Ok(index)
}
/// Drop packs with a malformed locale, duplicates and English (embedded), and cap the list.
fn clean_index(mut packs: Vec<Pack>) -> Vec<Pack> {
packs.truncate(MAX_PACKS);
let mut seen = BTreeSet::new();
packs.retain(|p| valid_locale(&p.locale) && !is_embedded(&p.locale) && seen.insert(p.locale.clone()));
packs.sort_by(|a, b| a.locale.cmp(&b.locale));
packs
}
/// English is embedded in every build; it never comes as a pack.
fn is_embedded(locale: &str) -> bool {
locale == "en" || locale.starts_with("en-")
}
/// A lowercase BCP 47-shaped tag: letters, digits and `-`, starting with a letter.
pub fn valid_locale(locale: &str) -> bool {
!locale.is_empty()
&& locale.len() <= MAX_LOCALE_LEN
&& locale.starts_with(|c: char| c.is_ascii_lowercase())
&& locale.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
&& !locale.contains("--")
&& !locale.ends_with('-')
}
/// A locale from a command's params: trimmed, lowercased, `_` as `-`.
pub fn parse_locale(raw: &str) -> Result<String, CmdError> {
let l = raw.trim().replace('_', "-").to_ascii_lowercase();
if valid_locale(&l) { Ok(l) } else { Err(CmdError::Params(format!("`locale` must be a language tag like nl or pt-PT, not {raw:?}"))) }
}
fn no_store() -> String {
if cfg!(target_arch = "wasm32") {
"Language packs aren't available in the browser".into()
} else {
"there is no font store folder for language packs".into()
}
}
fn network_off() -> String {
"downloading is turned off: turn it on in WordCraft (File › Options › Language, or an Install button there)".into()
}
/// The pack for `locale` in the index.
pub fn find(locale: &str) -> Option<Pack> {
available().into_iter().find(|p| p.locale == locale)
}
/// Whether a pack is installed for `locale`.
pub fn is_installed(locale: &str) -> bool {
installed().iter().any(|p| p.locale == locale)
}
/// Install the pack for `locale` (downloading must be on): the commands and the Install button.
/// Proofing picks it up at once.
pub fn install(locale: &str, cancel: &Cancel, progress: &mut dyn FnMut(u64, u64)) -> Result<Installed, String> {
let b = backend().ok_or_else(no_store)?;
let pack = find(locale).ok_or_else(|| format!("there is no language pack for {locale}"))?;
if !b.network_on() {
return Err(network_off());
}
if let Some(dir) = b.packs_dir() {
crate::file_access::check_write(&dir.to_string_lossy())?;
}
b.install(&pack, cancel, progress)?;
reload(&b);
b.installed().into_iter().find(|p| p.locale == locale).ok_or_else(|| format!("the {locale} pack didn't install"))
}
/// Remove the installed pack for `locale`.
pub fn remove(locale: &str) -> Result<(), String> {
let b = backend().ok_or_else(no_store)?;
if let Some(job) = jobs().installs.iter().find(|j| j.locale == locale) {
return Err(if job.stopping() {
format!("the {locale} pack's cancelled install is still stopping: try again in a moment")
} else {
format!("the {locale} pack is being installed: cancel it first")
});
}
if !b.installed().iter().any(|p| p.locale == locale) {
return Err(format!("no language pack is installed for {locale}"));
}
if let Some(dir) = b.packs_dir() {
crate::file_access::check_write(&dir.to_string_lossy())?;
}
b.remove(locale)?;
reload(&b);
Ok(())
}
/// What the app does with a long pack request (an install or a fetch of the list): `None` runs it
/// in the caller (the CLI, headless MCP, tests); `Some(notify)` runs it on a thread, so the window
/// keeps drawing whoever asked (the person, the control channel, MCP), and calls `notify` on each
/// change (the window repaints). Set once by the desktop app.
pub type Notify = Arc<dyn Fn() + Send + Sync>;
static BACKGROUND: RwLock<Option<Notify>> = RwLock::new(None);
/// An install running on a thread.
struct InstallJob {
locale: String,
/// Bytes done.
done: u64,
/// Bytes in all.
total: u64,
/// Stops this run. A cancelled job stays listed (hidden from [`installing`]) until its thread
/// notices and ends, so two downloads of a locale never write its folder at once.
cancel: Cancel,
/// Install was asked again after a cancel, before the cancelled run ended: the thread starts
/// over (with the fresh `cancel`) instead of ending.
restart: bool,
}
impl InstallJob {
/// Cancelled, and its thread hasn't ended yet.
fn stopping(&self) -> bool {
self.cancel.is_cancelled()
}
}
/// Installs and fetches running on a thread, and how the finished ones ended.
#[derive(Default)]
struct Jobs {
/// Running installs, one per locale.
installs: Vec<InstallJob>,
fetching: bool,
/// The last failure of each locale's install ("" for the list), until it is tried again.
errors: Vec<(String, String)>,
/// Bumped as each job ends: the window relays out and shows the result.
finished: u64,
}
static JOBS: Mutex<Jobs> = Mutex::new(Jobs { installs: Vec::new(), fetching: false, errors: Vec::new(), finished: 0 });
fn jobs() -> std::sync::MutexGuard<'static, Jobs> {
JOBS.lock().unwrap_or_else(PoisonError::into_inner)
}
/// Run long pack requests on a thread (`Some`: the desktop app), or in the caller (`None`).
pub fn set_background(notify: Option<Notify>) {
*BACKGROUND.write().unwrap_or_else(PoisonError::into_inner) = notify;
}
fn background() -> Option<Notify> {
if cfg!(target_arch = "wasm32") {
return None;
}
BACKGROUND.read().unwrap_or_else(PoisonError::into_inner).clone()
}
/// Whether installs and fetches run on a thread ([`set_background`]).
pub fn runs_in_background() -> bool {
background().is_some()
}
/// A running install: (locale, bytes done, bytes in all).
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct Progress {
pub locale: String,
pub done: u64,
pub total: u64,
}
/// The installs running now (not the cancelled ones still stopping).
pub fn installing() -> Vec<Progress> {
jobs().installs.iter().filter(|j| !j.stopping()).map(|j| Progress { locale: j.locale.clone(), done: j.done, total: j.total }).collect()
}
/// Whether the list is being fetched.
pub fn fetching() -> bool {
jobs().fetching
}
/// The last failure of `locale`'s install ("" for the list), until it is tried again.
pub fn error(locale: &str) -> Option<String> {
jobs().errors.iter().find(|(l, _)| l == locale).map(|(_, e)| e.clone())
}
/// Every recorded failure, (locale or "" for the list, message).
pub fn errors() -> Vec<(String, String)> {
jobs().errors.clone()
}
/// Counts finished jobs: when it changes, the window lays the documents out again.
pub fn finished() -> u64 {
jobs().finished
}
/// Stop `locale`'s running install. The person's own choice: it ends without an error, and
/// [`installing`] stops listing it at once (Install shows again).
pub fn cancel(locale: &str) {
if let Some(job) = jobs().installs.iter_mut().find(|j| j.locale == locale) {
job.restart = false;
job.cancel.cancel();
}
}
fn set_error(locale: &str, e: Option<String>) {
let mut j = jobs();
j.errors.retain(|(l, _)| l != locale);
if let Some(e) = e {
j.errors.push((locale.to_string(), e));
}
}
/// Start installing `locale` ([`install`]): on a thread when [`runs_in_background`], else now.
/// What's wrong up front (no pack, downloading off, outside the write root) is an error at once.
/// Returns whether it is still running (`false`: done). Asked again while it runs, it carries on;
/// right after a cancel, before the cancelled download has stopped, it starts over once that
/// one ends (never two downloads of a pack at once).
pub fn start_install(locale: &str) -> Result<bool, String> {
let b = backend().ok_or_else(no_store)?;
let pack = find(locale).ok_or_else(|| format!("there is no language pack for {locale}"))?;
if !b.network_on() {
return Err(network_off());
}
if let Some(dir) = b.packs_dir() {
crate::file_access::check_write(&dir.to_string_lossy())?;
}
let Some(notify) = background() else {
let r = install(locale, &Cancel::new(), &mut |_, _| {});
set_error(locale, r.as_ref().err().cloned());
return r.map(|_| false);
};
let cancel = Cancel::new();
{
let mut j = jobs();
j.errors.retain(|(l, _)| l != locale);
if let Some(job) = j.installs.iter_mut().find(|j| j.locale == locale) {
if job.stopping() {
// Cancelled and not ended yet: its thread starts over when it does.
job.done = 0;
job.total = pack.size;
job.cancel = cancel;
job.restart = true;
}
return Ok(true);
}
j.installs.push(InstallJob { locale: locale.to_string(), done: 0, total: pack.size, cancel: cancel.clone(), restart: false });
}
let key = locale.to_string();
let locale = locale.to_string();
let started = spawn(move || {
let mut cancel = cancel;
loop {
let progress_notify = notify.clone();
let r = install(&locale, &cancel, &mut |done, total| {
if let Some(job) = jobs().installs.iter_mut().find(|j| j.locale == locale) {
job.done = done;
job.total = total;
}
progress_notify();
});
let mut j = jobs();
if let Some(job) = j.installs.iter_mut().find(|j| j.locale == locale && j.restart) {
job.restart = false;
cancel = job.cancel.clone();
drop(j);
notify();
continue;
}
j.installs.retain(|j| j.locale != locale);
// The person's own cancel isn't a failure: nothing to show.
if let Err(e) = r
&& !cancel.is_cancelled()
{
j.errors.push((locale, e));
}
j.finished = j.finished.wrapping_add(1);
drop(j);
notify();
break;
}
});
if let Err(e) = started {
jobs().installs.retain(|j| j.locale != key);
return Err(e);
}
Ok(true)
}
/// Start fetching the list again ([`refresh`]): on a thread when [`runs_in_background`], else now.
/// Returns whether it is still running.
pub fn start_refresh() -> Result<bool, String> {
let b = backend().ok_or_else(no_store)?;
if !b.network_on() {
return Err(network_off());
}
let Some(notify) = background() else {
let r = refresh(&Cancel::new());
set_error("", r.as_ref().err().cloned());
return r.map(|_| false);
};
{
let mut j = jobs();
if j.fetching {
return Ok(true);
}
j.fetching = true;
j.errors.retain(|(l, _)| !l.is_empty());
}
let started = spawn(move || {
let r = refresh(&Cancel::new());
let mut j = jobs();
j.fetching = false;
if let Err(e) = r {
j.errors.push((String::new(), e));
}
j.finished = j.finished.wrapping_add(1);
drop(j);
notify();
});
if let Err(e) = started {
jobs().fetching = false;
return Err(e);
}
Ok(true)
}
#[cfg(not(target_arch = "wasm32"))]
fn spawn(f: impl FnOnce() + Send + 'static) -> Result<(), String> {
std::thread::Builder::new().name("language-pack".into()).spawn(f).map(|_| ()).map_err(|e| format!("couldn't start the download: {e}"))
}
#[cfg(target_arch = "wasm32")]
fn spawn(_: impl FnOnce() + Send + 'static) -> Result<(), String> {
Err(no_store())
}
/// Have proofing look at the packs folder again (its dictionaries are reloaded).
fn reload(b: &Arc<dyn Backend>) {
if wordcraft_proof::packs::packs_dir() == b.packs_dir() {
wordcraft_proof::packs::reload();
} else {
wordcraft_proof::packs::set_packs_dir(b.packs_dir());
}
}
/// The full licence texts for About › Language data: the embedded English data, then each
/// installed pack.
pub fn licence_texts() -> Vec<(String, String)> {
let mut texts = wordcraft_proof::packs::licence_texts();
// SCOWL's notices (with the Ispell BSD licence of its affix files), when English spells from
// the embedded SCOWL data; after the Moby and supplement entries.
if let Some(notice) = wordcraft_proof::scowl::notice() {
let at = texts.len().min(2);
texts.insert(at, (SCOWL_TITLE.into(), notice.into()));
}
texts
}
/// About › Language data's title for SCOWL's notices.
pub const SCOWL_TITLE: &str = "English spelling dictionaries (SCOWL)";
/// The pack locale for a document language tag: the whole tag (`pt-pt`), else each shorter
/// prefix of it (`sr-latn` for `sr-Latn-RS`, `nl` for `nl-NL`), the first the index has. A
/// variant after the region is also tried without the region, before the region alone
/// (`ca-ES-valencia` is the `ca-valencia` pack's), as craft-proof looks its dictionary up. `None`
/// for English and tags with no pack.
pub fn pack_for_tag<'a>(tag: &str, packs: &'a [Pack]) -> Option<&'a Pack> {
pack_candidates(tag).iter().find_map(|c| packs.iter().find(|p| &p.locale == c))
}
/// Is `subtag` a region, as it follows a tag's language (and script): two letters (`NL`) or
/// three digits (`419`)? The one test of what a region is; the lists, the menus and
/// [`pack_for_tag`] all use it.
pub fn is_region(subtag: &str) -> bool {
(subtag.len() == 2 && subtag.bytes().all(|b| b.is_ascii_alphabetic())) || (subtag.len() == 3 && subtag.bytes().all(|b| b.is_ascii_digit()))
}
/// `tag` without its region subtags (`nl-NL` is `nl`, `sr-Latn-RS` is `sr-Latn`, `ca-ES-valencia`
/// is `ca-valencia`), joined with `-` whichever of `-` and `_` separated them; the case is kept.
/// The language itself (the first subtag) is never a region.
pub fn without_region(tag: &str) -> String {
tag.split(['-', '_']).enumerate().filter(|(i, p)| *i == 0 || !is_region(p)).map(|(_, p)| p).collect::<Vec<&str>>().join("-")
}
/// The pack locales [`pack_for_tag`] tries for `tag`, best first.
fn pack_candidates(tag: &str) -> Vec<String> {
// A tag is a handful of short subtags; anything longer is junk and only its start is read.
let tag: String = tag.trim().chars().take(64).collect::<String>().replace('_', "-").to_ascii_lowercase();
let parts: Vec<&str> = tag.split('-').collect();
let base = parts.first().copied().unwrap_or("");
if base.is_empty() || base == "en" {
return Vec::new();
}
let mut out: Vec<String> = Vec::new();
for n in (1..=parts.len()).rev() {
let prefix = parts.get(..n).unwrap_or_default().join("-");
// The prefix, then without its region: the variant's pack before the region alone.
let stripped = without_region(&prefix);
for c in [prefix, stripped] {
if !out.contains(&c) {
out.push(c);
}
}
}
out
}
/// The languages (`w:lang`, resolved through the styles) of the body text, in first-seen order.
pub fn document_languages(s: &Session) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
for path in s.doc.para_paths(wordcraft_doc::StoryRef::Body).into_iter().take(MAX_PARAGRAPHS_SCANNED) {
let Some(p) = s.doc.para(wordcraft_doc::StoryRef::Body, &path) else { continue };
if p.text.trim().is_empty() {
continue;
}
let style = p.props.style.as_deref();
for (range, c) in p.run_ranges() {
let Some(text) = p.text.get(range) else { continue };
if !text.chars().any(char::is_alphabetic) {
continue;
}
let rc = s.doc.styles.resolve_char_in(style, None, c);
if rc.no_proof {
continue;
}
let tag = rc.lang.unwrap_or_else(|| "en-US".into());
if !out.contains(&tag) {
out.push(tag);
if out.len() >= MAX_DOC_LANGUAGES {
return out;
}
}
}
}
out
}
/// Whether proofing reads a pack for text in `tag`: every language craft-proof can name, its own
/// (Polish…) and any other (`ProofLang::Pack`, read from the pack alone); not `und`, `zxx` or a
/// malformed tag. The one place the bar's language rule lives.
pub fn proofable(tag: &str) -> bool {
wordcraft_proof::ProofLang::from_tag(Some(tag)) != wordcraft_proof::ProofLang::Other
}
/// The Hunspell dictionary `tag`'s language needs and doesn't have, for the status bar's
/// "(no dictionary)": craft-proof's [`wordcraft_proof::missing_dictionary`], except for a
/// language craft-proof has no rules of its own for (`ProofLang::Pack`: Japanese, Chinese,
/// Thai…) when no pack for it is available either, as no Hunspell dictionary may exist at all.
pub fn missing_dictionary(tag: Option<&str>) -> Option<&'static str> {
let name = wordcraft_proof::missing_dictionary(tag)?;
let lang = wordcraft_proof::ProofLang::from_tag(tag);
if lang.pack_locale().is_some() && !has_pack_for_tag(tag.unwrap_or("")) {
return None;
}
Some(name)
}
/// The last answer of whether the index has a pack for a tag, for the index generation it was
/// worked out for: the status bar asks every frame while the caret is in a pack language, and
/// working it out copies the whole index.
#[derive(Default)]
struct PackLookup {
/// (index generation, tag, has a pack).
entry: Option<(u64, String, bool)>,
}
impl PackLookup {
/// The remembered answer for `tag` at `generation`, else `compute`'s (then remembered).
fn get(&mut self, generation: u64, tag: &str, compute: impl FnOnce() -> bool) -> bool {
if let Some((g, t, has)) = &self.entry
&& *g == generation
&& t == tag
{
return *has;
}
let has = compute();
self.entry = Some((generation, tag.to_string(), has));
has
}
}
static PACK_LOOKUP: Mutex<PackLookup> = Mutex::new(PackLookup { entry: None });
/// Whether the index has a pack for `tag` ([`pack_for_tag`]), worked out again only when the
/// index changes ([`index_generation`]) or the tag does.
fn has_pack_for_tag(tag: &str) -> bool {
// Read before [`available`], which may fill the index and bump it: the answer is then worked
// out once more, never kept for a list it wasn't worked out from.
let generation = index_generation();
PACK_LOOKUP.lock().unwrap_or_else(PoisonError::into_inner).get(generation, tag, || pack_for_tag(tag, &available()).is_some())
}
/// The language new text at the caret gets (its typing language, resolved through the styles
/// and the document default), unless it is marked Do not check spelling or grammar: choosing a
/// language with nothing selected asks for its pack before anything is typed in it.
pub fn caret_language(s: &Session) -> Option<String> {
let style = s.doc.para_at(&s.sel.focus).and_then(|p| p.props.style.as_deref());
let rc = s.doc.styles.resolve_char(style, &s.typing_props());
if rc.no_proof {
return None;
}
Some(rc.lang.unwrap_or_else(|| "en-US".into()))
}
/// The languages the missing-pack bar looks at: the body text's ([`document_languages`]), then
/// the caret's typing language ([`caret_language`]).
pub fn wanted_languages(s: &Session) -> Vec<String> {
let mut out = document_languages(s);
if let Some(tag) = caret_language(s)
&& !out.contains(&tag)
{
out.push(tag);
}
out
}
/// The packs the document's text (or the caret's typing language) asks for that are available
/// and not installed: what the missing-pack bar offers. Never a language without a pack, nor one
/// proofing can't use.
pub fn missing(s: &Session) -> Vec<Pack> {
let packs = available();
if packs.is_empty() {
return Vec::new();
}
let installed = installed();
let mut out: Vec<Pack> = Vec::new();
for tag in wanted_languages(s) {
if proofable(&tag)
&& let Some(p) = pack_for_tag(&tag, &packs)
&& !installed.iter().any(|i| i.locale == p.locale)
&& !out.iter().any(|o| o.locale == p.locale)
{
out.push(p.clone());
}
}
out
}
/// Where a language's proofing comes from (Options › Language › Your languages).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Proofing {
/// English: the embedded dictionaries.
BuiltIn,
/// An installed Language pack.
Installed,
/// A Hunspell dictionary in the person's own dictionaries folder.
OwnDictionary,
/// A Hunspell dictionary the system has.
SystemDictionary,
/// No dictionary yet; a Language pack can be installed.
Install,
/// Grammar and hyphenation rules built in; a Language pack adds spelling.
GrammarBuiltIn,
/// No dictionary and no pack for it.
NotAvailable,
}
/// A row of Options › Language › Your languages.
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct LanguageRow {
/// The tags the default proofing language or the open document use for this language (the
/// first is what Set as default sets); empty for a language listed for its installed pack.
pub tags: Vec<String>,
/// The pack locale: the installed pack, else the one the index offers.
pub pack: Option<String>,
/// Is the pack installed?
pub installed: bool,
/// Bytes of the offered pack (0 when the index has none).
pub size: u64,
pub proofing: Proofing,
/// Is this the default proofing language?
pub default: bool,
}
impl LanguageRow {
/// Is `tag` (the insertion point's language) one of this row's languages?
pub fn is_tag(&self, tag: &str) -> bool {
self.tags.iter().any(|t| t.eq_ignore_ascii_case(tag))
}
}
/// The first locale `has` accepts for `tag`, in [`pack_for_tag`]'s order. `None` for English and junk.
fn locale_for_tag(tag: &str, has: &dyn Fn(&str) -> bool) -> Option<String> {
pack_candidates(tag).into_iter().find(|c| has(c))
}
/// [`your_languages_with`] against the dictionaries on this computer.
pub fn your_languages(s: &Session) -> Vec<LanguageRow> {
let folder = s.prefs.dictionaries_folder.clone();
your_languages_with(s, folder.as_deref(), &dictionary_file)
}
/// The Hunspell `.dic` file proofing reads for `tag`'s language now (a built-in-rules language's
/// own, or a pack language's), if any.
fn dictionary_file(tag: &str) -> Option<PathBuf> {
let lang = wordcraft_proof::ProofLang::from_tag(Some(tag));
if let Some((_, f)) = wordcraft_proof::installed_dictionaries().into_iter().find(|(l, _)| *l == lang) {
return f;
}
wordcraft_proof::pack_dictionary_file(lang)
}
/// Options › Language › Your languages (unsorted; the window names and sorts them): each English
/// variant in use (the default proofing language, American English and any in the open document), every language
/// with an installed Language pack, and every proofing language of the open document. One row per
/// language: tags one pack proofs share a row. `folder` is the person's own dictionaries folder;
/// `dictionary` finds the `.dic` file proofing reads for a tag.
pub fn your_languages_with(s: &Session, folder: Option<&str>, dictionary: &dyn Fn(&str) -> Option<PathBuf>) -> Vec<LanguageRow> {
let index = available();
let installed = installed();
let default_tag = s
.prefs
.default_language
.clone()
.or_else(|| s.editing_lang.clone())
.or_else(|| s.doc.styles.default_chr.lang.clone())
.unwrap_or_else(|| "en-US".to_string());
// American English stays listed whatever the default is, so it can always be set back.
let mut tags = vec![default_tag.clone(), "en-US".to_string()];
tags.dedup_by(|a, b| a.eq_ignore_ascii_case(b));
for t in document_languages(s) {
if !tags.iter().any(|x| x.eq_ignore_ascii_case(&t)) {
tags.push(t);
}
}
let mut rows: Vec<LanguageRow> = Vec::new();
let mut add = |tag: Option<&str>, locale: Option<String>| {
let english = tag.is_some_and(|t| wordcraft_proof::ProofLang::from_tag(Some(t)).is_english());
// English variants stay apart (en-GB and en-US are two rows); a pack's tags share one.
let existing = rows.iter().position(|r| match (&locale, &r.pack) {
(Some(l), Some(p)) if !english => l == p,
_ => tag.is_some_and(|t| r.is_tag(t)),
});
let i = existing.unwrap_or_else(|| {
rows.push(LanguageRow {
tags: Vec::new(),
pack: locale.clone(),
installed: false,
size: 0,
proofing: Proofing::NotAvailable,
default: false,
});
rows.len() - 1
});
if let (Some(t), Some(row)) = (tag, rows.get_mut(i))
&& !row.is_tag(t)
{
row.tags.push(t.to_string());
}
};
for t in &tags {
if !proofable(t) {
continue;
}
// The most specific pack, installed or offered, as the bar picks it ([`pack_for_tag`]): with
// `ca` installed, Valencian text is still the `ca-valencia` pack's row.
let locale = locale_for_tag(t, &|l| installed.iter().any(|i| i.locale == l) || index.iter().any(|p| p.locale == l));
add(Some(t), locale);
}
for i in &installed {
add(None, Some(i.locale.clone()));
}
for row in &mut rows {
let installed_here = row.pack.as_ref().is_some_and(|l| installed.iter().any(|i| &i.locale == l));
let offered = row.pack.as_ref().and_then(|l| index.iter().find(|p| &p.locale == l));
row.installed = installed_here;
row.size = offered.map_or(0, |p| p.size);
row.default = row.is_tag(&default_tag);
let tag = row.tags.first().cloned().or_else(|| row.pack.clone()).unwrap_or_default();
let lang = wordcraft_proof::ProofLang::from_tag(Some(&tag));
let file = if lang.is_english() { None } else { dictionary(&tag) };
let own = file.as_ref().is_some_and(|f| folder.is_some_and(|d| !d.trim().is_empty() && f.starts_with(d.trim())));
row.proofing = if lang.is_english() {
Proofing::BuiltIn
} else if own {
Proofing::OwnDictionary
} else if installed_here {
Proofing::Installed
} else if file.is_some() {
Proofing::SystemDictionary
} else if offered.is_some() {
if matches!(lang, wordcraft_proof::ProofLang::Pack(_)) { Proofing::Install } else { Proofing::GrammarBuiltIn }
} else {
Proofing::NotAvailable
};
}
rows
}
/// The index was fetched this session (since the backend was set): what it lists is then the
/// whole answer, even when it lists nothing.
static FETCHED: AtomicBool = AtomicBool::new(false);
/// The automatic fetch of a missing or stale list ([`refresh_if_stale`]) was tried.
static AUTO_REFRESHED: AtomicBool = AtomicBool::new(false);
/// Languages (document tags) whose pack the person asked for before the list was known
/// ([`start_get`]): installed once a fetch lists one.
static WANTED: Mutex<Vec<String>> = Mutex::new(Vec::new());
/// Whether the index was fetched since the backend was set.
pub fn fetched() -> bool {
FETCHED.load(Ordering::Relaxed)
}
/// The languages the document (or the caret) asks for while there is no list of packs yet (a
/// fresh install, or a list cached for an older pin): proofable, not English, with no dictionary
/// and no pack installed, one per proofing language, as the document tags them (`tr-TR`). The
/// bar offers to get them: the list is fetched first, and the language then installed if it has
/// a pack. Nothing once a list is known or was fetched this session ([`missing`] then answers).
pub fn unlisted(s: &Session) -> Vec<String> {
if backend().is_none() || fetched() || available_len() > 0 {
return Vec::new();
}
let installed = installed();
let mut seen: Vec<wordcraft_proof::ProofLang> = Vec::new();
let mut out = Vec::new();
for tag in wanted_languages(s) {
let base = tag.split(['-', '_']).next().unwrap_or("").to_ascii_lowercase();
let lang = wordcraft_proof::ProofLang::from_tag(Some(&tag));
if base.is_empty()
|| base == "en"
|| !proofable(&tag)
|| wordcraft_proof::missing_dictionary(Some(&tag)).is_none()
|| seen.contains(&lang)
|| installed.iter().any(|i| i.locale == base || tag.eq_ignore_ascii_case(&i.locale))
{
continue;
}
seen.push(lang);
out.push(tag);
}
out
}
/// The person asked for `tag`'s pack before the list was known (the bar's Install, after their
/// consent turned downloading on): fetch the list ([`start_refresh`]), then install the pack it
/// lists for `tag`, if any; with none, the bar stops offering it. Downloading must be on.
/// Returns whether it is still running.
pub fn start_get(tag: &str) -> Result<bool, String> {
let tag = tag.trim();
if tag.is_empty() || tag.len() > MAX_LOCALE_LEN {
return Err(format!("`{tag}` isn't a language tag"));
}
if let Some(p) = pack_for_tag(tag, &available()) {
return start_install(&p.locale);
}
let b = backend().ok_or_else(no_store)?;
if !b.network_on() {
// Never queued: a later fetch mustn't install what was refused.
return Err(network_off());
}
let queued = {
let mut w = WANTED.lock().unwrap_or_else(PoisonError::into_inner);
let new = !w.iter().any(|t| t == tag) && w.len() < MAX_DOC_LANGUAGES;
if new {
w.push(tag.to_string());
}
new
};
let r = start_refresh();
if r.is_err() && queued {
WANTED.lock().unwrap_or_else(PoisonError::into_inner).retain(|t| t != tag);
}
r
}
/// After a fetch: install the packs asked for before the list was known ([`start_get`]).
fn install_wanted(index: &[Pack]) {
let wanted = std::mem::take(&mut *WANTED.lock().unwrap_or_else(PoisonError::into_inner));
for tag in wanted {
let Some(p) = pack_for_tag(&tag, index) else { continue };
if is_installed(&p.locale) {
continue;
}
if let Err(e) = start_install(&p.locale) {
set_error(&p.locale, Some(e));
}
}
}
/// Once a session (the first time it is asked once requests run on a thread, [`runs_in_background`]:
/// the window), with downloading already on and no list (none cached, or one cached for an older
/// pin): fetch it in the background, so the bar can offer packs. The person's consent is the
/// downloading setting itself; nothing is asked. Its failure is recorded for Options (""). The
/// window asks every frame: after the first time this reads nothing (the downloading setting is
/// a file). Returns whether a fetch was started.
pub fn refresh_if_stale() -> bool {
let Some(b) = backend() else { return false };
if !runs_in_background() || AUTO_REFRESHED.swap(true, Ordering::Relaxed) {
return false;
}
if !b.network_on() || fetched() || fetching() || available_len() > 0 {
return false;
}
start_refresh().is_ok()
}
/// A backend with a fixed list and installs that only record themselves: tests and `ui_shot`.
#[derive(Default)]
pub struct FixedBackend {
pub packs: Vec<Pack>,
pub network: AtomicBool,
/// Agents can't turn the network on.
pub agent: bool,
pub installed: Mutex<Vec<Installed>>,
}
impl Pack {
/// Where the data comes from: the upstream source when the index names it, else the archive.
pub fn source_link(&self) -> &str {
self.source.as_deref().filter(|s| !s.is_empty()).unwrap_or(&self.url)
}
}
impl FixedBackend {
/// A few packs shaped like the published ones, for screenshots and tests.
pub fn sample() -> Self {
let pack = |locale: &str, name: &str, size: u64, licence: &str, class: &str| Pack {
locale: locale.into(),
name: name.into(),
version: "0000000".into(),
url: format!("https://github.com/storytold/craft-i18n/releases/download/language-packs-2/{locale}.tar.zst"),
source: Some("https://github.com/LibreOffice/dictionaries".into()),
size,
licence: licence.into(),
class: class.into(),
spelling: true,
hyphenation: true,
};
FixedBackend {
packs: vec![
pack("ca", "Català", 1_480_000, "GPL-2.0-or-later OR LGPL-2.1-or-later", "Download-only"),
pack("el", "Ελληνικά", 2_310_000, "MPL-1.1 OR GPL-2.0-or-later OR LGPL-2.1-or-later", "Embeddable"),
pack("fr", "Français", 1_120_000, "MPL-2.0", "Embeddable"),
pack("nl", "Nederlands", 1_950_000, "BSD-3-Clause", "Embeddable"),
pack("pl", "Polski", 4_620_000, "GPL-2.0-or-later OR LGPL-2.1-or-later OR MPL-1.1 OR Apache-2.0 OR CC-BY-4.0", "Embeddable"),
pack("sk", "Slovenčina", 2_040_000, "GPL-2.0-only OR LGPL-2.1-only OR MPL-1.1", "Download-only"),
],
..Default::default()
}
}
}
impl Backend for FixedBackend {
fn index(&self) -> Result<Vec<Pack>, String> {
Ok(self.packs.clone())
}
fn fetch_index(&self, _: &Cancel) -> Result<Vec<Pack>, String> {
Ok(self.packs.clone())
}
fn network_on(&self) -> bool {
self.network.load(Ordering::Relaxed)
}
fn enable_network(&self) -> Result<(), String> {
if self.agent {
return Err("only the user can turn on downloading".into());
}
self.network.store(true, Ordering::Relaxed);
Ok(())
}
fn install(&self, pack: &Pack, cancel: &Cancel, progress: &mut dyn FnMut(u64, u64)) -> Result<(), String> {
if cancel.is_cancelled() {
return Err("cancelled".into());
}
progress(pack.size, pack.size);
let mut i = self.installed.lock().unwrap_or_else(PoisonError::into_inner);
i.retain(|p| p.locale != pack.locale);
i.push(Installed { locale: pack.locale.clone(), name: pack.name.clone(), licence: pack.licence.clone(), path: PathBuf::new() });
Ok(())
}
fn remove(&self, locale: &str) -> Result<(), String> {
self.installed.lock().unwrap_or_else(PoisonError::into_inner).retain(|p| p.locale != locale);
Ok(())
}
fn packs_dir(&self) -> Option<PathBuf> {
None
}
fn installed(&self) -> Vec<Installed> {
self.installed.lock().unwrap_or_else(PoisonError::into_inner).clone()
}
}
#[cfg(test)]
mod tests {
/// CP4.4 (`cargo xtask language`): a build embeds English Language data and nothing it
/// wasn't asked to. A Language build names the locales it embeds in `CRAFT_PROOF_EMBED`
/// (`en` alone: English only); the default embeds the carry-over locales too, until Language
/// packs are published and pinned (craft-proof's `DEFAULT_EMBED`; TODO(CP4.4): English only
/// then).
/// About › Language data shows SCOWL's notices (the Ispell BSD licence among them) whenever
/// English spells from the embedded SCOWL data.
#[test]
fn about_shows_the_scowl_notice_when_scowl_is_embedded() {
let texts = super::licence_texts();
match wordcraft_proof::scowl::notice() {
Some(notice) => {
let entry = texts.iter().find(|(t, _)| t == super::SCOWL_TITLE).map(|(_, x)| x.as_str());
assert_eq!(entry, Some(notice));
assert!(notice.contains("Ispell") || notice.contains("ispell"), "the Ispell notice is part of it");
}
None => assert!(texts.iter().all(|(t, _)| t != super::SCOWL_TITLE), "no SCOWL data: no SCOWL entry"),
}
}
/// FU.3: the status bar's "(no dictionary)" asks every frame whether the index has a pack
/// for the caret's language; the answer is reused until the index changes (a new
/// generation) or the tag does, so the index isn't copied every frame.
#[test]
fn the_pack_lookup_is_reused_until_the_index_changes() {
let mut lookup = super::PackLookup::default();
let mut computed = 0;
let mut ask = |lookup: &mut super::PackLookup, generation: u64, tag: &str, answer: bool| {
lookup.get(generation, tag, || {
computed += 1;
answer
})
};
assert!(ask(&mut lookup, 1, "ja-JP", true));
// Same generation and tag: the remembered answer, even where the list would now say no.
for _ in 0..10 {
assert!(ask(&mut lookup, 1, "ja-JP", false));
}
assert!(!ask(&mut lookup, 2, "ja-JP", false), "a new generation works it out again");
assert!(ask(&mut lookup, 2, "th-TH", true), "another tag too");
assert_eq!(computed, 3);
}
#[test]
fn the_default_build_embeds_only_english() {
const CARRY_OVER: [&str; 5] = ["de", "pl", "pt", "it", "ru"];
let named: Vec<String> = std::env::var("CRAFT_PROOF_EMBED")
.unwrap_or_default()
.split(',')
.map(|l| l.trim().to_ascii_lowercase())
.filter(|l| !l.is_empty())
.collect();
let allowed = |base: &str| base == "en" || if named.is_empty() { CARRY_OVER.contains(&base) } else { named.iter().any(|n| n == base) };
let embedded = wordcraft_proof::packs::embedded_locales();
let extra: Vec<&&str> = embedded.iter().filter(|l| !allowed(l.split('-').next().unwrap_or(l))).collect();
assert!(extra.is_empty(), "embedded beyond English and {named:?} (or the carry-over by default): {extra:?}");
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.