f67d38b35cc5
model.rs
//! The Whisper network (encoder–decoder transformer), written against candle from the published
//! architecture (Radford et al. 2022, "Robust Speech Recognition via Large-Scale Weak
//! Supervision", and OpenAI's MIT-licensed model description), reading the tensor names of the
//! Hugging Face safetensors release.
//!
//! Encoder: two 1-D convolutions (k = 3, the second with stride 2) with GELU, learned/sinusoidal
//! position table, pre-LayerNorm self-attention blocks, final LayerNorm. Decoder: token and
//! position embeddings, pre-LayerNorm blocks of causal self-attention, cross-attention to the
//! audio and an MLP, final LayerNorm, logits tied to the token embedding.
//!
//! The decoder keeps a key/value cache for incremental decoding and can return the cross-attention
//! logits of chosen heads (word alignment).
use candle_core::{DType, Device, Module, Result, Tensor};
use candle_nn::{Conv1d, Conv1dConfig, Embedding, LayerNorm, Linear, VarBuilder};
/// `config.json` fields we use.
#[derive(Clone, Debug, serde::Deserialize)]
pub struct Config {
pub num_mel_bins: usize,
pub d_model: usize,
pub encoder_layers: usize,
pub encoder_attention_heads: usize,
pub decoder_layers: usize,
pub decoder_attention_heads: usize,
pub max_source_positions: usize,
pub max_target_positions: usize,
pub vocab_size: usize,
}
/// Most attention heads a model may declare.
const MAX_HEADS: usize = 64;
struct Attention {
q: Linear,
k: Linear,
v: Linear,
out: Linear,
heads: usize,
/// cached keys/values (self-attention: grows; cross-attention: the audio's, computed once)
cache: Option<(Tensor, Tensor)>,
}
impl Attention {
fn new(d: usize, heads: usize, vb: VarBuilder) -> Result<Self> {
// from config.json: zero heads divided by zero in `forward`, and the count sizes every
// attention-logit tensor (released Whisper models use 6 to 20)
if heads == 0 || heads > MAX_HEADS {
candle_core::bail!("config.json: {heads} attention heads is outside 1 to {MAX_HEADS}");
}
if !d.is_multiple_of(heads) {
candle_core::bail!("config.json: d_model {d} is not divisible by {heads} attention heads");
}
Ok(Self {
q: candle_nn::linear(d, d, vb.pp("q_proj"))?,
k: candle_nn::linear_no_bias(d, d, vb.pp("k_proj"))?,
v: candle_nn::linear(d, d, vb.pp("v_proj"))?,
out: candle_nn::linear(d, d, vb.pp("out_proj"))?,
heads,
cache: None,
})
}
fn split(&self, x: &Tensor) -> Result<Tensor> {
let (b, t, d) = x.dims3()?;
x.reshape((b, t, self.heads, d / self.heads))?.transpose(1, 2)?.contiguous()
}
/// `x`: queries (b, t, d). `kv`: None = self-attention on `x` (appending to the cache when
/// `cached`), Some(audio) = cross-attention (keys/values cached after the first call when
/// `cached`). Returns the output and the attention logits (b, heads, t, tk).
fn forward(&mut self, x: &Tensor, kv: Option<&Tensor>, mask: Option<&Tensor>, cached: bool) -> Result<(Tensor, Tensor)> {
let (_, t, d) = x.dims3()?;
let dh = d / self.heads;
let q = (self.split(&self.q.forward(x)?)? * (dh as f64).powf(-0.5))?;
let (k, v) = match kv {
None => {
let k = self.split(&self.k.forward(x)?)?;
let v = self.split(&self.v.forward(x)?)?;
if cached {
let (k, v) = match &self.cache {
Some((pk, pv)) => (Tensor::cat(&[pk, &k], 2)?, Tensor::cat(&[pv, &v], 2)?),
None => (k, v),
};
self.cache = Some((k.clone(), v.clone()));
(k, v)
} else {
(k, v)
}
}
Some(xa) => match (&self.cache, cached) {
(Some((k, v)), true) => (k.clone(), v.clone()),
_ => {
let k = self.split(&self.k.forward(xa)?)?;
let v = self.split(&self.v.forward(xa)?)?;
if cached {
self.cache = Some((k.clone(), v.clone()));
}
(k, v)
}
},
};
let mut qk = q.matmul(&k.transpose(2, 3)?.contiguous()?)?;
if let Some(m) = mask {
qk = qk.broadcast_add(m)?;
}
let w = candle_nn::ops::softmax_last_dim(&qk)?;
let o = w.matmul(&v)?.transpose(1, 2)?.contiguous()?;
let (b, _, _, _) = o.dims4()?;
Ok((self.out.forward(&o.reshape((b, t, d))?)?, qk))
}
}
struct Block {
attn: Attention,
attn_ln: LayerNorm,
cross: Option<(Attention, LayerNorm)>,
fc1: Linear,
fc2: Linear,
mlp_ln: LayerNorm,
}
impl Block {
fn new(d: usize, heads: usize, cross: bool, vb: VarBuilder) -> Result<Self> {
let cross = if cross {
Some((Attention::new(d, heads, vb.pp("encoder_attn"))?, candle_nn::layer_norm(d, 1e-5, vb.pp("encoder_attn_layer_norm"))?))
} else {
None
};
Ok(Self {
attn: Attention::new(d, heads, vb.pp("self_attn"))?,
attn_ln: candle_nn::layer_norm(d, 1e-5, vb.pp("self_attn_layer_norm"))?,
cross,
fc1: candle_nn::linear(d, 4 * d, vb.pp("fc1"))?,
fc2: candle_nn::linear(4 * d, d, vb.pp("fc2"))?,
mlp_ln: candle_nn::layer_norm(d, 1e-5, vb.pp("final_layer_norm"))?,
})
}
fn forward(&mut self, x: &Tensor, xa: Option<&Tensor>, mask: Option<&Tensor>, cached: bool) -> Result<(Tensor, Option<Tensor>)> {
let (a, _) = self.attn.forward(&self.attn_ln.forward(x)?, None, mask, cached)?;
let mut x = (x + a)?;
let mut qk = None;
if let (Some((c, ln)), Some(xa)) = (&mut self.cross, xa) {
let (a, w) = c.forward(&ln.forward(&x)?, Some(xa), None, cached)?;
x = (x + a)?;
qk = Some(w);
}
let h = self.fc2.forward(&self.fc1.forward(&self.mlp_ln.forward(&x)?)?.gelu_erf()?)?;
Ok(((x + h)?, qk))
}
fn reset(&mut self) {
self.attn.cache = None;
if let Some((c, _)) = &mut self.cross {
c.cache = None;
}
}
}
pub struct Encoder {
conv1: Conv1d,
conv2: Conv1d,
pos: Tensor,
blocks: Vec<Block>,
ln: LayerNorm,
}
impl Encoder {
fn new(cfg: &Config, vb: VarBuilder) -> Result<Self> {
let d = cfg.d_model;
let c1 = Conv1dConfig { padding: 1, stride: 1, ..Default::default() };
let c2 = Conv1dConfig { padding: 1, stride: 2, ..Default::default() };
Ok(Self {
conv1: candle_nn::conv1d(cfg.num_mel_bins, d, 3, c1, vb.pp("conv1"))?,
conv2: candle_nn::conv1d(d, d, 3, c2, vb.pp("conv2"))?,
pos: vb.get((cfg.max_source_positions, d), "embed_positions.weight")?,
blocks: (0..cfg.encoder_layers).map(|i| Block::new(d, cfg.encoder_attention_heads, false, vb.pp(format!("layers.{i}")))).collect::<Result<_>>()?,
ln: candle_nn::layer_norm(d, 1e-5, vb.pp("layer_norm"))?,
})
}
/// `mel`: (1, n_mels, 3000) → (1, 1500, d).
pub fn forward(&mut self, mel: &Tensor) -> Result<Tensor> {
let x = self.conv1.forward(mel)?.gelu_erf()?;
let x = self.conv2.forward(&x)?.gelu_erf()?;
let x = x.transpose(1, 2)?;
let t = x.dim(1)?;
let mut x = x.broadcast_add(&self.pos.narrow(0, 0, t)?)?;
for b in &mut self.blocks {
x = b.forward(&x, None, None, false)?.0;
}
self.ln.forward(&x)
}
}
pub struct Decoder {
tok: Embedding,
/// The token embedding transposed (d, vocab), contiguous: the output projection.
tok_t: Tensor,
pos: Tensor,
blocks: Vec<Block>,
ln: LayerNorm,
/// tokens already in the self-attention cache
offset: usize,
}
impl Decoder {
fn new(cfg: &Config, vb: VarBuilder) -> Result<Self> {
let d = cfg.d_model;
let tok = candle_nn::embedding(cfg.vocab_size, d, vb.pp("embed_tokens"))?;
Ok(Self {
tok_t: tok.embeddings().t()?.contiguous()?,
tok,
pos: vb.get((cfg.max_target_positions, d), "embed_positions.weight")?,
blocks: (0..cfg.decoder_layers).map(|i| Block::new(d, cfg.decoder_attention_heads, true, vb.pp(format!("layers.{i}")))).collect::<Result<_>>()?,
ln: candle_nn::layer_norm(d, 1e-5, vb.pp("layer_norm"))?,
offset: 0,
})
}
/// Forget the cached keys/values (new audio window or new token sequence).
pub fn reset(&mut self) {
self.offset = 0;
for b in &mut self.blocks {
b.reset();
}
}
fn mask(t: usize, offset: usize, dev: &Device) -> Result<Tensor> {
let m: Vec<f32> = (0..t).flat_map(|i| (0..offset + t).map(move |j| if j > offset + i { f32::NEG_INFINITY } else { 0.0 })).collect();
Tensor::from_vec(m, (t, offset + t), dev)
}
/// Logits (1, t, vocab) for `tokens` appended to the cache. With `qk_heads`, also returns the
/// cross-attention logits of those (layer, head) pairs: (heads, t, audio frames).
pub fn forward(&mut self, tokens: &[u32], xa: &Tensor, cached: bool, qk_heads: Option<&[(usize, usize)]>) -> Result<(Tensor, Option<Tensor>)> {
let dev = xa.device();
let t = tokens.len();
let offset = if cached { self.offset } else { 0 };
let ids = Tensor::new(tokens, dev)?.unsqueeze(0)?;
let mut x = self.tok.forward(&ids)?.broadcast_add(&self.pos.narrow(0, offset, t)?)?;
let mask = if t > 1 { Some(Self::mask(t, offset, dev)?) } else { None };
let mut picked = Vec::new();
for (li, b) in self.blocks.iter_mut().enumerate() {
let (y, qk) = b.forward(&x, Some(xa), mask.as_ref(), cached)?;
x = y;
if let (Some(heads), Some(qk)) = (qk_heads, qk) {
for &(l, h) in heads {
if l == li {
picked.push(qk.get(0)?.get(h)?);
}
}
}
}
if cached {
self.offset += t;
}
let x = self.ln.forward(&x)?;
let logits = x.broadcast_matmul(&self.tok_t)?;
let qk = if picked.is_empty() { None } else { Some(Tensor::stack(&picked, 0)?) };
Ok((logits, qk))
}
}
pub struct Model {
pub cfg: Config,
pub encoder: Encoder,
pub decoder: Decoder,
pub device: Device,
}
impl Model {
pub fn load(cfg: Config, weights: Vec<u8>) -> Result<Self> {
let device = Device::Cpu;
let vb = VarBuilder::from_buffered_safetensors(weights, DType::F32, &device)?;
let vb = vb.pp("model");
Ok(Self { encoder: Encoder::new(&cfg, vb.pp("encoder"))?, decoder: Decoder::new(&cfg, vb.pp("decoder"))?, cfg, device })
}
pub fn last_logits(logits: &Tensor) -> Result<Vec<f32>> {
let t = logits.dim(1)?;
logits.get(0)?.get(t - 1)?.to_dtype(DType::F32)?.to_vec1()
}
}
#[cfg(test)]
mod head_tests {
use super::*;
#[test]
fn head_counts_that_cannot_split_d_model_are_errors() {
let vb = VarBuilder::zeros(DType::F32, &Device::Cpu);
for heads in [0, 7, 1 << 20] {
assert!(Attention::new(384, heads, vb.clone()).is_err(), "{heads} heads");
}
let why = |d: usize, heads: usize| Attention::new(d, heads, vb.clone()).err().map(|e| e.to_string()).unwrap_or_default();
assert!(why(65, 65).contains("outside"), "{}", why(65, 65));
assert!(why(384, 7).contains("not divisible"), "{}", why(384, 7));
assert!(Attention::new(384, 6, vb).is_ok());
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.