f67d38b35cc5
lib.rs
//! Speech-to-text for FilmCraft's Text panel ▸ Transcript.
//!
//! - [`Transcriber`]: the trait every speech recogniser implements. It takes mono 16 kHz samples
//! and returns a [`Transcript`] whose words carry media-time bounds (`Tick`s; one 16 kHz sample
//! is exactly 15 876 000 ticks).
//! - [`models`]: the catalogue of downloadable Whisper models (source URL pinned to a revision,
//! SHA-256, size, licence), where they live in the per-user data directory, and (feature
//! `download`) the verified downloader. Weights are **never** bundled or committed.
//! - [`whisper`] (feature `whisper`): Whisper inference in pure Rust on candle, with timestamp
//! decoding, language detection and word-level timestamps from cross-attention alignment (DTW).
//! - [`diarize`]: speaker labelling by clustering per-chunk MFCC statistics (classical, no model).
//! - [`mel`]: the log-mel front end shared by Whisper and diarization.
//! - [`vad`]: energy-based tightening of word bounds (keeps pauses out of words).
//!
//! See `docs/transcripts.md` for the user-facing behaviour and accuracy numbers.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
pub mod diarize;
pub mod mel;
pub mod models;
pub mod vad;
#[cfg(feature = "whisper")]
pub mod whisper;
use filmcraft_project::Transcript;
use filmcraft_time::{TICKS_PER_SECOND, Tick};
/// The sample rate every transcriber takes.
pub const SAMPLE_RATE: u32 = 16_000;
/// Ticks per 16 kHz sample (exact).
pub const TICKS_PER_SAMPLE: i64 = TICKS_PER_SECOND / SAMPLE_RATE as i64;
/// Media time of a 16 kHz sample index.
pub fn sample_tick(n: i64) -> Tick {
Tick(n * TICKS_PER_SAMPLE)
}
/// Media time of `s` seconds (rounded to the nearest 16 kHz sample).
pub fn seconds_tick(s: f64) -> Tick {
sample_tick((s * SAMPLE_RATE as f64).round() as i64)
}
#[derive(Debug, Clone, PartialEq, thiserror::Error)]
pub enum SpeechError {
#[error("stopped")]
Cancelled,
#[error("the speech model `{0}` is not downloaded")]
NotInstalled(String),
#[error("unknown speech model `{0}`")]
UnknownModel(String),
#[error("speech-to-text is not available in this build ({0})")]
Unavailable(String),
#[error("model: {0}")]
Model(String),
#[error("download: {0}")]
Download(String),
#[error("{0}")]
Io(String),
}
impl From<std::io::Error> for SpeechError {
fn from(e: std::io::Error) -> Self {
SpeechError::Io(e.to_string())
}
}
/// Transcription options.
#[derive(Clone, Debug, PartialEq)]
pub struct Options {
/// ISO 639-1 language code; `None` = detect from the first 30 s.
pub language: Option<String>,
/// Label speakers ([`diarize`]); otherwise every word gets speaker 0.
pub diarize: bool,
/// Upper bound for the number of speakers found.
pub max_speakers: usize,
}
impl Default for Options {
fn default() -> Self {
Self { language: None, diarize: true, max_speakers: 6 }
}
}
/// Progress callback: `(fraction 0..1, status)`; return `false` to cancel.
pub type ProgressFn<'a> = &'a mut dyn FnMut(f32, &str) -> bool;
/// A speech recogniser.
pub trait Transcriber: Send + Sync {
/// Stable id of the model (`"whisper-base"`), stored in the transcript's `source`.
fn id(&self) -> String;
/// Transcribe mono 16 kHz `audio`. Word times are relative to the first sample.
fn transcribe(&self, audio: &[f32], opts: &Options, progress: ProgressFn) -> Result<Transcript, SpeechError>;
}
/// A transcriber that returns a fixed transcript (tests, demos and agents that bring their own
/// transcript); words beyond the audio's length are dropped.
pub struct FixedTranscriber {
pub transcript: Transcript,
pub id: String,
}
impl Transcriber for FixedTranscriber {
fn id(&self) -> String {
self.id.clone()
}
fn transcribe(&self, audio: &[f32], _: &Options, progress: ProgressFn) -> Result<Transcript, SpeechError> {
if !progress(0.5, "Transcribing") {
return Err(SpeechError::Cancelled);
}
let end = sample_tick(audio.len() as i64);
let mut t = self.transcript.clone();
t.words.retain(|w| w.start < end);
t.source = self.id.clone();
progress(1.0, "Done");
Ok(t)
}
}
/// Load the transcriber for catalogue model `id` from `models_dir` (feature `whisper`).
pub fn load(models_dir: &std::path::Path, id: &str) -> Result<std::sync::Arc<dyn Transcriber>, SpeechError> {
let m = models::find(id).ok_or_else(|| SpeechError::UnknownModel(id.into()))?;
if !models::installed(models_dir, m) {
return Err(SpeechError::NotInstalled(id.into()));
}
#[cfg(feature = "whisper")]
{
Ok(std::sync::Arc::new(whisper::Whisper::load(&models::model_dir(models_dir, m), m.id)?))
}
#[cfg(not(feature = "whisper"))]
{
Err(SpeechError::Unavailable("built without the `whisper` feature".into()))
}
}
/// Whether this build can run speech models.
pub const fn available() -> bool {
cfg!(feature = "whisper")
}
/// Mix a multi-channel buffer down to mono.
pub fn downmix(channels: &[Vec<f32>]) -> Vec<f32> {
let n = channels.iter().map(Vec::len).min().unwrap_or(0);
if channels.len() == 1 {
return channels[0][..n].to_vec();
}
let k = 1.0 / channels.len().max(1) as f32;
(0..n).map(|i| channels.iter().map(|c| c[i]).sum::<f32>() * k).collect()
}
/// Word error rate between a reference and a hypothesis (words compared case-insensitively
/// without punctuation, hyphenated words split): (substitutions + deletions + insertions) / reference words.
pub fn word_error_rate(reference: &str, hypothesis: &str) -> f64 {
let norm = |s: &str| -> Vec<String> {
s.split(|c: char| c.is_whitespace() || c == '-').map(filmcraft_project::transcript::normalize_word).filter(|w| !w.is_empty()).collect()
};
let (r, h) = (norm(reference), norm(hypothesis));
if r.is_empty() {
return if h.is_empty() { 0.0 } else { 1.0 };
}
let mut prev: Vec<usize> = (0..=h.len()).collect();
for i in 1..=r.len() {
let mut cur = vec![i; h.len() + 1];
for j in 1..=h.len() {
let sub = prev[j - 1] + usize::from(r[i - 1] != h[j - 1]);
cur[j] = sub.min(prev[j] + 1).min(cur[j - 1] + 1);
}
prev = cur;
}
prev[h.len()] as f64 / r.len() as f64
}
#[cfg(test)]
mod tests {
use super::*;
use filmcraft_project::Word;
#[test]
fn sample_ticks_are_exact() {
assert_eq!(TICKS_PER_SAMPLE * SAMPLE_RATE as i64, TICKS_PER_SECOND);
assert_eq!(seconds_tick(1.5), Tick(TICKS_PER_SECOND * 3 / 2));
}
#[test]
fn wer_counts_edits() {
assert_eq!(word_error_rate("the cat sat", "The cat, sat."), 0.0);
assert!((word_error_rate("the cat sat on the mat", "the cat sat on mat") - 1.0 / 6.0).abs() < 1e-9);
assert!((word_error_rate("a b", "a x b y") - 1.0).abs() < 1e-9);
}
#[test]
fn fixed_transcriber_trims_and_cancels() {
let mut t = Transcript::default();
t.words.push(Word::new("hi", seconds_tick(0.1), seconds_tick(0.3)));
t.words.push(Word::new("late", seconds_tick(5.0), seconds_tick(5.5)));
let f = FixedTranscriber { transcript: t, id: "fixed".into() };
let out = f.transcribe(&vec![0.0; 16_000], &Options::default(), &mut |_, _| true).unwrap();
assert_eq!(out.words.len(), 1);
assert_eq!(out.source, "fixed");
assert_eq!(f.transcribe(&[], &Options::default(), &mut |_, _| false), Err(SpeechError::Cancelled));
}
#[test]
fn downmix_averages() {
assert_eq!(downmix(&[vec![1.0, 0.0], vec![0.0, 0.0, 9.0]]), vec![0.5, 0.0]);
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.