f67d38b35cc5
kokoro.rs
//! Kokoro-82M neural voices (feature `kokoro`): pure-Rust inference on candle (CPU).
//!
//! Implemented from the papers the model card cites (StyleTTS 2, iSTFTNet; with ALBERT, HiFi-GAN,
//! the neural source-filter model and the Snake activation they build on), the released
//! `config.json` and the tensor names and shapes in the checkpoint. No reference code was read.
//! Weights (Apache-2.0, hexgrad) are downloaded on first use and never bundled.
//!
//! [`Kokoro::speak`] takes Kokoro phonemes (from `filmcraft-tts-text`), at most
//! [`MAX_PHONEMES`] per call, and a voice pack.
mod dsp;
mod layers;
mod model;
mod weights;
use std::path::Path;
use crate::TtsError;
pub use model::SAMPLE_RATE as KOKORO_RATE;
/// Longest phoneme string one forward pass takes (the position table has 512 rows, two of them
/// for the boundary tokens).
pub const MAX_PHONEMES: usize = 510;
/// A loaded Kokoro model.
pub struct Kokoro {
model: model::Model,
vocab: std::collections::HashMap<char, u32>,
}
/// A loaded voice pack.
pub struct VoicePack(Vec<f32>);
impl VoicePack {
pub fn load(path: &Path) -> Result<VoicePack, TtsError> {
weights::read_voice(path).map(VoicePack)
}
}
impl Kokoro {
/// Load `config.json` and `kokoro-v1_0.pth` from `dir`.
pub fn load(dir: &Path) -> Result<Kokoro, TtsError> {
let cfg = weights::read_config(&dir.join("config.json"))?;
let w = weights::Weights::load(&dir.join("kokoro-v1_0.pth"))?;
Ok(Kokoro { model: model::Model::load(&w)?, vocab: cfg.vocab })
}
/// Phoneme ids of `phonemes`; symbols outside the vocabulary are dropped.
pub fn ids(&self, phonemes: &str) -> Vec<u32> {
phonemes.chars().filter_map(|c| self.vocab.get(&c).copied()).collect()
}
/// Speak `phonemes` with `voice` at `speed` (0.5–2). Deterministic for a given `seed`.
pub fn speak(&self, phonemes: &str, voice: &VoicePack, speed: f64, seed: u64) -> Result<Vec<f32>, TtsError> {
if !(0.5..=2.0).contains(&speed) {
return Err(TtsError::Invalid("speed must be between 0.5 and 2".into()));
}
let ids = self.ids(phonemes);
if ids.is_empty() {
return Err(TtsError::Empty);
}
if ids.len() > MAX_PHONEMES {
return Err(TtsError::Invalid(format!("at most {MAX_PHONEMES} phonemes per pass")));
}
let style = weights::style_for(&voice.0, ids.len(), self.model.device())?;
let out = self.model.forward(&ids, &style, speed, seed)?;
if out.iter().any(|s| !s.is_finite()) {
return Err(TtsError::Model("the model produced invalid samples".into()));
}
Ok(out)
}
}
/// Tests with the real model run when `FILMCRAFT_KOKORO_DIR` names a directory holding
/// `config.json`, `kokoro-v1_0.pth` and `af_heart.pt` (the downloaded package); they skip otherwise.
#[cfg(test)]
mod tests {
use super::*;
fn model() -> Option<(Kokoro, VoicePack)> {
let dir = std::path::PathBuf::from(std::env::var_os("FILMCRAFT_KOKORO_DIR")?);
let voice = [dir.join("af_heart.pt"), dir.join("voices/af_heart.pt")].into_iter().find(|p| p.exists())?;
Some((Kokoro::load(&dir).unwrap(), VoicePack::load(&voice).unwrap()))
}
#[test]
fn speaks_deterministically_and_pace_changes_the_length() {
let Some((k, v)) = model() else { return };
let ph = "həlˈO wˈɜɹld.";
let a = k.speak(ph, &v, 1.0, 1).unwrap();
assert_eq!(a, k.speak(ph, &v, 1.0, 1).unwrap(), "deterministic");
assert_eq!(a.len() % 600, 0, "whole frames");
let secs = a.len() as f64 / f64::from(KOKORO_RATE);
assert!((0.5..3.0).contains(&secs), "{secs}");
let peak = a.iter().fold(0f32, |m, x| m.max(x.abs()));
assert!(peak > 0.05 && peak < 1.0, "{peak}");
let fast = k.speak(ph, &v, 2.0, 1).unwrap();
let ratio = fast.len() as f64 / a.len() as f64;
assert!((0.35..0.7).contains(&ratio), "{ratio}");
}
#[test]
fn refuses_bad_input() {
let Some((k, v)) = model() else { return };
assert_eq!(k.speak("", &v, 1.0, 1).err(), Some(TtsError::Empty));
assert_eq!(k.speak("日本語👍", &v, 1.0, 1).err(), Some(TtsError::Empty), "nothing in the vocabulary");
assert!(k.speak("a", &v, 3.0, 1).is_err());
assert!(k.speak("a", &v, f64::NAN, 1).is_err());
assert!(k.speak(&"ə".repeat(MAX_PHONEMES + 1), &v, 1.0, 1).is_err());
assert_eq!(k.ids("a日b"), k.ids("ab"));
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.