f67d38b35cc5
weights.rs
//! Loading the Kokoro-82M checkpoint, its config and voice packs.
//!
//! The official release is a PyTorch zip checkpoint (`kokoro-v1_0.pth`) holding five state
//! dictionaries (`bert`, `bert_encoder`, `predictor`, `text_encoder`, `decoder`), each with a
//! `module.` prefix. They are read with candle's restricted pickle reader, which only rebuilds
//! tensors and never runs code from the file. Weight-normalised layers (`weight_g`, `weight_v`) are
//! fused into a plain weight on load. Voice packs (`voices/<id>.pt`) hold one bare float32 tensor
//! `[510, 1, 256]`, read straight from the zip archive.
use std::collections::HashMap;
use std::io::Read;
use std::path::Path;
use candle_core::{DType, Device, Tensor};
use sha2::{Digest, Sha256};
use crate::TtsError;
pub(crate) fn err(e: impl std::fmt::Display) -> TtsError {
TtsError::Model(e.to_string())
}
/// The architecture this code implements; a checkpoint whose config differs is refused.
pub(crate) struct Config {
pub vocab: HashMap<char, u32>,
}
/// Values of `config.json` this implementation is written for.
const EXPECT: &[(&str, &str)] = &[
("hidden_dim", "512"),
("style_dim", "128"),
("n_token", "178"),
("max_dur", "50"),
("n_layer", "3"),
("text_encoder_kernel_size", "5"),
("istftnet.gen_istft_n_fft", "20"),
("istftnet.gen_istft_hop_size", "5"),
("istftnet.upsample_rates", "[10,6]"),
("istftnet.upsample_kernel_sizes", "[20,12]"),
("istftnet.resblock_kernel_sizes", "[3,7,11]"),
("istftnet.upsample_initial_channel", "512"),
("plbert.hidden_size", "768"),
("plbert.num_attention_heads", "12"),
("plbert.num_hidden_layers", "12"),
("plbert.intermediate_size", "2048"),
];
pub(crate) fn read_config(path: &Path) -> Result<Config, TtsError> {
let bytes = std::fs::read(path).map_err(|e| err(format!("{}: {e}", path.display())))?;
if bytes.len() > 1 << 20 {
return Err(err("config.json is too large"));
}
let v: serde_json::Value = serde_json::from_slice(&bytes).map_err(err)?;
for (key, want) in EXPECT {
let mut cur = &v;
for part in key.split('.') {
cur = cur.get(part).ok_or_else(|| err(format!("config.json: missing `{key}`")))?;
}
let got = serde_json::to_string(cur).map_err(err)?;
if got != *want {
return Err(err(format!("config.json: `{key}` is {got}, this build supports {want}")));
}
}
let table = v.get("vocab").and_then(|t| t.as_object()).ok_or_else(|| err("config.json: missing `vocab`"))?;
let mut vocab = HashMap::new();
for (sym, id) in table {
let mut chars = sym.chars();
let (Some(c), None) = (chars.next(), chars.next()) else { continue };
let Some(id) = id.as_u64().filter(|i| *i > 0 && *i < 178) else { continue };
vocab.insert(c, id as u32);
}
if vocab.len() < 100 {
return Err(err("config.json: the phoneme vocabulary is incomplete"));
}
Ok(Config { vocab })
}
/// All model tensors by full name (`decoder.generator.conv_post.weight`…), weight norm fused.
pub(crate) struct Weights {
map: HashMap<String, Tensor>,
}
const PARTS: [&str; 5] = ["bert", "bert_encoder", "predictor", "text_encoder", "decoder"];
impl Weights {
pub(crate) fn load(path: &Path) -> Result<Weights, TtsError> {
verify_checkpoint(path)?;
let mut raw: HashMap<String, Tensor> = HashMap::new();
for part in PARTS {
let tensors = candle_core::pickle::read_all_with_key(path, Some(part)).map_err(|e| err(format!("{}: {e}", path.display())))?;
if tensors.is_empty() {
return Err(err(format!("{}: no `{part}` weights", path.display())));
}
for (name, t) in tensors {
let name = name.strip_prefix("module.").unwrap_or(&name);
let t = t.to_dtype(DType::F32).map_err(err)?;
raw.insert(format!("{part}.{name}"), t);
}
}
// fuse weight norm: w = g · v / ‖v‖ (norm over every dim but the first)
let mut map = HashMap::with_capacity(raw.len());
let gs: Vec<String> = raw.keys().filter(|k| k.ends_with(".weight_g")).cloned().collect();
for g_name in gs {
let base = g_name.trim_end_matches("_g").to_string();
let v_name = format!("{base}_v");
let (Some(g), Some(v)) = (raw.remove(&g_name), raw.remove(&v_name)) else {
return Err(err(format!("{g_name} without {v_name}")));
};
let norm = v.sqr().map_err(err)?.sum_keepdim(1).map_err(err)?.sum_keepdim(2).map_err(err)?.sqrt().map_err(err)?;
let w = v.broadcast_div(&norm).map_err(err)?.broadcast_mul(&g).map_err(err)?;
map.insert(base, w);
}
map.extend(raw);
Ok(Weights { map })
}
pub(crate) fn get(&self, name: &str) -> Result<Tensor, TtsError> {
self.map.get(name).cloned().ok_or_else(|| err(format!("missing weight `{name}`")))
}
/// `get` with an expected shape.
pub(crate) fn shaped(&self, name: &str, dims: &[usize]) -> Result<Tensor, TtsError> {
let t = self.get(name)?;
if t.dims() != dims {
return Err(err(format!("weight `{name}` is {:?}, expected {dims:?}", t.dims())));
}
Ok(t)
}
}
/// Candle's pickle reader is used only for the fixed, catalogued checkpoint.
/// Verify its immutable size and digest before any pickle object is expanded.
fn verify_checkpoint(path: &Path) -> Result<(), TtsError> {
let file = crate::catalog::FILES
.iter()
.find(|file| file.name == "kokoro-v1_0.pth")
.ok_or_else(|| err("Kokoro checkpoint is missing from the package catalogue"))?;
let metadata = std::fs::metadata(path).map_err(|e| err(format!("{}: {e}", path.display())))?;
if metadata.len() != file.size {
return Err(err(format!("{}: checkpoint size does not match the official package", path.display())));
}
let mut reader = std::fs::File::open(path).map_err(|e| err(format!("{}: {e}", path.display())))?;
let mut hasher = Sha256::new();
let mut buf = [0u8; 64 * 1024];
loop {
let n = reader.read(&mut buf).map_err(|e| err(format!("{}: {e}", path.display())))?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
let actual = format!("{:x}", hasher.finalize());
if actual != file.sha256 {
return Err(err(format!("{}: checkpoint digest does not match the official package", path.display())));
}
Ok(())
}
/// Number of style vectors in a voice pack (one per input length 1..=510).
pub(crate) const PACK_LEN: usize = 510;
pub(crate) const STYLE: usize = 256;
/// Read a voice pack: a zip whose `<name>/data/0` holds `[510, 1, 256]` little-endian float32.
pub(crate) fn read_voice(path: &Path) -> Result<Vec<f32>, TtsError> {
let f = std::fs::File::open(path).map_err(|e| err(format!("{}: {e}", path.display())))?;
let mut z = zip::ZipArchive::new(f).map_err(|e| err(format!("{}: {e}", path.display())))?;
let name = (0..z.len())
.filter_map(|i| z.by_index(i).ok().map(|e| e.name().to_string()))
.find(|n| n.ends_with("/data/0"))
.ok_or_else(|| err(format!("{}: not a voice pack", path.display())))?;
let order = (0..z.len()).filter_map(|i| z.by_index(i).ok().map(|e| e.name().to_string())).find(|n| n.ends_with("/byteorder"));
if let Some(o) = order {
let mut s = String::new();
z.by_name(&o).map_err(err)?.take(16).read_to_string(&mut s).map_err(err)?;
if s.trim() != "little" {
return Err(err(format!("{}: unsupported byte order `{s}`", path.display())));
}
}
let want = PACK_LEN * STYLE * 4;
let mut e = z.by_name(&name).map_err(err)?;
if e.size() != want as u64 {
return Err(err(format!("{}: voice data is {} bytes, expected {want}", path.display(), e.size())));
}
let mut buf = Vec::with_capacity(want);
e.by_ref().take(want as u64).read_to_end(&mut buf).map_err(err)?;
if buf.len() != want {
return Err(err(format!("{}: truncated voice data", path.display())));
}
let v: Vec<f32> = buf.as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect();
if v.iter().any(|x| !x.is_finite()) {
return Err(err(format!("{}: voice data is not finite", path.display())));
}
Ok(v)
}
/// The style vector for an input of `n` phonemes: `[1, 256]`.
pub(crate) fn style_for(pack: &[f32], n: usize, dev: &Device) -> Result<Tensor, TtsError> {
let row = n.clamp(1, PACK_LEN) - 1;
let s = pack.get(row * STYLE..(row + 1) * STYLE).ok_or_else(|| err("voice pack too short"))?;
Tensor::from_slice(s, (1, STYLE), dev).map_err(err)
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
fn tmp(name: &str) -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!("filmcraft-kokoro-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
/// A voice-pack-shaped zip with `data` as `x/data/0`.
fn pack(dir: &Path, data: &[u8], order: &str) -> std::path::PathBuf {
let p = dir.join("v.pt");
let mut z = zip::ZipWriter::new(std::fs::File::create(&p).unwrap());
let o = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
z.start_file("x/data.pkl", o).unwrap();
z.write_all(b"not used").unwrap();
z.start_file("x/byteorder", o).unwrap();
z.write_all(order.as_bytes()).unwrap();
z.start_file("x/data/0", o).unwrap();
z.write_all(data).unwrap();
z.finish().unwrap();
p
}
fn floats(n: usize, v: f32) -> Vec<u8> {
std::iter::repeat_n(v, n).flat_map(f32::to_le_bytes).collect()
}
#[test]
fn reads_a_well_formed_voice_pack_and_picks_the_row_by_length() {
let d = tmp("ok");
let mut data = floats(PACK_LEN * STYLE, 0.0);
// row 4 (inputs of 5 phonemes) starts with 7.0
data[4 * STYLE * 4..4 * STYLE * 4 + 4].copy_from_slice(&7.0f32.to_le_bytes());
let v = read_voice(&pack(&d, &data, "little")).unwrap();
assert_eq!(v.len(), PACK_LEN * STYLE);
let s = style_for(&v, 5, &Device::Cpu).unwrap();
assert_eq!(s.dims(), &[1, STYLE]);
assert_eq!(s.to_vec2::<f32>().unwrap()[0][0], 7.0);
// lengths outside 1..=510 clamp instead of failing
assert!(style_for(&v, 0, &Device::Cpu).is_ok());
assert!(style_for(&v, 100_000, &Device::Cpu).is_ok());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn hostile_voice_packs_are_errors_not_panics() {
let d = tmp("bad");
let cases: Vec<(&str, std::path::PathBuf)> =
vec![("short", pack(&d, &floats(10, 0.0), "little")), ("long", pack(&d.join(".."), &floats(PACK_LEN * STYLE + 1, 0.0), "little"))];
for (what, p) in cases {
assert!(read_voice(&p).is_err(), "{what}");
}
let p = pack(&d, &floats(PACK_LEN * STYLE, 0.0), "big");
assert!(read_voice(&p).is_err(), "big endian");
let p = pack(&d, &floats(PACK_LEN * STYLE, f32::NAN), "little");
assert!(read_voice(&p).is_err(), "NaN");
let junk = d.join("junk.pt");
std::fs::write(&junk, b"PK\x03\x04 definitely not a zip").unwrap();
assert!(read_voice(&junk).is_err());
std::fs::write(&junk, b"").unwrap();
assert!(read_voice(&junk).is_err());
assert!(read_voice(&d.join("missing.pt")).is_err());
// a checkpoint that isn't one
assert!(Weights::load(&junk).is_err());
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn config_must_match_the_implemented_architecture() {
let d = tmp("cfg");
let p = d.join("config.json");
let good = r#"{"hidden_dim":512,"style_dim":128,"n_token":178,"max_dur":50,"n_layer":3,"text_encoder_kernel_size":5,
"istftnet":{"gen_istft_n_fft":20,"gen_istft_hop_size":5,"upsample_rates":[10,6],"upsample_kernel_sizes":[20,12],"resblock_kernel_sizes":[3,7,11],"upsample_initial_channel":512},
"plbert":{"hidden_size":768,"num_attention_heads":12,"num_hidden_layers":12,"intermediate_size":2048},"vocab":VOCAB}"#;
let vocab: String = format!("{{{}}}", (0..110).map(|i| format!("\"{}\":{}", char::from_u32(0x100 + i).unwrap(), i + 1)).collect::<Vec<_>>().join(","));
std::fs::write(&p, good.replace("VOCAB", &vocab)).unwrap();
assert_eq!(read_config(&p).unwrap().vocab.len(), 110);
std::fs::write(&p, good.replace("VOCAB", &vocab).replace("\"hidden_dim\":512", "\"hidden_dim\":256")).unwrap();
assert!(read_config(&p).err().unwrap().to_string().contains("hidden_dim"));
std::fs::write(&p, good.replace("VOCAB", "{\"a\":1}")).unwrap();
assert!(read_config(&p).is_err(), "tiny vocabulary");
for junk in ["", "{", "[]", "null", "{\"vocab\":5}"] {
std::fs::write(&p, junk).unwrap();
assert!(read_config(&p).is_err(), "{junk}");
}
std::fs::write(&p, vec![b' '; (1 << 20) + 1]).unwrap();
assert!(read_config(&p).is_err(), "oversized");
let _ = std::fs::remove_dir_all(&d);
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.